pax_global_header00006660000000000000000000000064152576204770014531gustar00rootroot0000000000000052 comment=fa142d049d08eac45a263f5abc250c8877ae076e timescale-pg_textsearch-b63fe58/000077500000000000000000000000001525762047700167615ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/.clang-format000066400000000000000000000050211525762047700213320ustar00rootroot00000000000000--- # PostgreSQL-style C formatting for pg_textsearch # Optimized for 79-character line limit with proper alignment BasedOnStyle: LLVM # Basic formatting ColumnLimit: 79 IndentWidth: 4 TabWidth: 4 UseTab: Always ContinuationIndentWidth: 8 # Function formatting (key for PostgreSQL style) AllowAllParametersOfDeclarationOnNextLine: true AllowAllArgumentsOnNextLine: true AlwaysBreakAfterReturnType: AllDefinitions BinPackArguments: false BinPackParameters: false IndentWrappedFunctionNames: false # Alignment (PostgreSQL style with macro alignment) AlignAfterOpenBracket: AlwaysBreak AlignConsecutiveAssignments: Consecutive AlignConsecutiveDeclarations: Consecutive AlignConsecutiveMacros: Consecutive AlignEscapedNewlines: Left AlignOperands: AlignAfterOperator AlignTrailingComments: true # Braces (PostgreSQL style: opening brace on new line) BreakBeforeBraces: Allman AllowShortBlocksOnASingleLine: Never AllowShortCaseLabelsOnASingleLine: false AllowShortFunctionsOnASingleLine: None AllowShortIfStatementsOnASingleLine: Never AllowShortLoopsOnASingleLine: false # Spacing SpaceAfterCStyleCast: false SpaceBeforeAssignmentOperators: true SpaceBeforeParens: ControlStatements SpaceInEmptyParentheses: false SpacesBeforeTrailingComments: 2 SpacesInCStyleCastParentheses: false SpacesInParentheses: false SpacesInSquareBrackets: false # Comments and strings ReflowComments: true BreakStringLiterals: true # Pointer alignment PointerAlignment: Right DerivePointerAlignment: false # Include handling (postgres.h first, then system, then local with blank lines) SortIncludes: CaseSensitive IncludeBlocks: Regroup # Disable main file priority to treat all quoted includes equally IncludeIsMainRegex: '$^' IncludeCategories: - Regex: '^' Priority: -3 SortPriority: -3 - Regex: '^<.*\.h>' Priority: 1 SortPriority: 1 - Regex: '^".*\.h"' Priority: 2 SortPriority: 2 # PostgreSQL-specific macros ForEachMacros: - foreach - forboth - for_each_cell - for_both_cell - forthree - MemSet - MemSetAligned - MemSetLoop # Switch/case formatting IndentCaseLabels: false # Penalty tuning for better line breaks PenaltyBreakAssignment: 100 PenaltyBreakBeforeFirstCallParameter: 200 PenaltyBreakComment: 300 PenaltyBreakFirstLessLess: 120 PenaltyBreakString: 1000 PenaltyExcessCharacter: 1000000 PenaltyReturnTypeOnItsOwnLine: 75 # Other formatting KeepEmptyLinesAtTheStartOfBlocks: false MaxEmptyLinesToKeep: 1 CompactNamespaces: false # Encourage blank lines around function declarations SeparateDefinitionBlocks: Always timescale-pg_textsearch-b63fe58/.clang-format-ignore000066400000000000000000000005661525762047700226240ustar00rootroot00000000000000# Files to exclude from clang-format # Generated files and external dependencies # Generated build files *.o *.so *.dylib Makefile config.log config.status # PostgreSQL generated files sql/*.sql expected/*.out results/ # Test output test/results/ test/expected/ # Any external or vendored code vendor/ third_party/ # Backup files from manual formatting *.orig *~ *.bak timescale-pg_textsearch-b63fe58/.codecov.yml000066400000000000000000000007501525762047700212060ustar00rootroot00000000000000# Codecov configuration for pg_textsearch # https://docs.codecov.com/docs/codecovyml-reference coverage: precision: 2 round: down range: "50...90" status: project: default: target: 85% threshold: 1% patch: default: informational: true # Ignore test files in coverage reports ignore: - "test/**" - "benchmarks/**" # Comment settings for PRs comment: layout: "reach,diff,flags,files" behavior: default require_changes: true timescale-pg_textsearch-b63fe58/.github/000077500000000000000000000000001525762047700203215ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/.github/ISSUE_TEMPLATE/000077500000000000000000000000001525762047700225045ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/.github/ISSUE_TEMPLATE/bug_report.md000066400000000000000000000012741525762047700252020ustar00rootroot00000000000000--- name: Bug Report about: Report a bug in pg_textsearch title: '' labels: bug assignees: '' --- ## Bug Description A clear and concise description of the bug. ## Environment - **PostgreSQL version**: (e.g., 17.2) - **pg_textsearch version**: (e.g., 0.0.6) - **Operating system**: (e.g., Ubuntu 24.04, macOS 15) ## Steps to Reproduce 1. Create table/index... 2. Run query... 3. See error... ```sql -- Include SQL to reproduce the issue ``` ## Expected Behavior What you expected to happen. ## Actual Behavior What actually happened. Include error messages if applicable. ``` -- Error output here ``` ## Additional Context Any other context about the problem (logs, screenshots, etc.). timescale-pg_textsearch-b63fe58/.github/ISSUE_TEMPLATE/feature_request.md000066400000000000000000000010351525762047700262300ustar00rootroot00000000000000--- name: Feature Request about: Suggest a new feature or enhancement title: '' labels: enhancement assignees: '' --- ## Problem Statement Describe the problem you're trying to solve. What use case isn't currently supported? ## Proposed Solution Describe your proposed solution. How would this feature work? ```sql -- Example usage (if applicable) ``` ## Alternatives Considered What alternatives have you considered? Why is this approach preferred? ## Additional Context Any other context, references, or examples that would help. timescale-pg_textsearch-b63fe58/.github/scripts/000077500000000000000000000000001525762047700220105ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/.github/scripts/build-postgres-injection.sh000077500000000000000000000013041525762047700272700ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail version=${1:?PostgreSQL version is required} prefix=${2:?installation prefix is required} source_dir=${3:-"${RUNNER_TEMP:-/tmp}/postgresql-${version}"} rm -rf "${source_dir}" mkdir -p "${source_dir}" curl -fsSL \ "https://ftp.postgresql.org/pub/source/v${version}/postgresql-${version}.tar.bz2" | tar -xj -C "${source_dir}" --strip-components=1 cd "${source_dir}" ./configure \ --prefix="${prefix}" \ --enable-debug \ --enable-cassert \ --enable-injection-points \ --without-icu \ --without-readline \ --without-zlib make -j"$(nproc)" make install make -C src/test/modules/injection_points install make -C contrib/dblink install timescale-pg_textsearch-b63fe58/.github/scripts/build-postgres-sanitizer.sh000077500000000000000000000023611525762047700273220ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail version=${1:?PostgreSQL version is required} source_dir=${2:?source directory is required} prefix=${3:?installation prefix is required} randomize_memory=${4:-false} build_walinspect=${5:-false} repo_root=$(pwd) rm -rf "${source_dir}" mkdir -p "${source_dir}" wget -qO- \ "https://ftp.postgresql.org/pub/source/v${version}/postgresql-${version}.tar.bz2" | tar -xj -C "${source_dir}" --strip-components=1 major=${version%%.*} if ((major < 18)); then patch_file="${repo_root}/test/postgres-asan-instrumentation.patch" else patch_file="${repo_root}/test/postgres-asan-instrumentation-PG18GE.patch" fi patch -F5 -p1 -d "${source_dir}" <"${patch_file}" cd "${source_dir}" configure_args=( "--prefix=${prefix}" --enable-debug --enable-cassert --enable-injection-points --with-openssl --without-readline --without-zlib --without-libxml ) if [[ "${randomize_memory}" == "true" ]]; then CPPFLAGS="-DRANDOMIZE_ALLOCATED_MEMORY" ./configure "${configure_args[@]}" else ./configure "${configure_args[@]}" fi make -j"$(nproc)" make -C src/test/modules/injection_points -j"$(nproc)" make -C contrib/dblink -j"$(nproc)" if [[ "${build_walinspect}" == "true" ]]; then make -C contrib/pg_walinspect -j"$(nproc)" fi timescale-pg_textsearch-b63fe58/.github/workflows/000077500000000000000000000000001525762047700223565ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/.github/workflows/benchmark-concurrent-insert.yml000066400000000000000000000357701525762047700305310ustar00rootroot00000000000000name: Concurrent INSERT Benchmark permissions: contents: write deployments: write env: PARADEDB_VERSION: "0.25.2" on: # TODO: remove push trigger before merging push: branches: - benchmark/concurrent-insert-runner schedule: - cron: '0 7 * * *' # Daily at 7 AM UTC (after main benchmarks) workflow_dispatch: inputs: runner: description: 'GitHub Actions runner label' required: false default: 'ubuntu-latest-8-cores-TJ' type: choice options: - ubuntu-latest-8-cores-TJ - ubuntu-latest-8-cores-TJ msmarco_size: description: 'MS MARCO subset size (must exceed duration * max_tps * num_levels)' required: false default: 'full' type: choice options: - '500K' - '1M' - 'full' duration: description: 'Seconds per concurrency level' required: false default: '60' type: string dry_run: description: 'Dry run - skip publishing to GitHub Pages' required: false default: false type: boolean jobs: pg-textsearch: name: "pg_textsearch concurrent INSERT" runs-on: ${{ inputs.runner || 'ubuntu-latest-8-cores-TJ' }} timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift df -h / - name: Install PostgreSQL run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential jq wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Build and install pg_textsearch run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean && make -j$(nproc) sudo make install - name: Configure and start PostgreSQL run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB effective_cache_size = 4GB maintenance_work_mem = 512MB work_mem = 64MB max_connections = 100 checkpoint_completion_target = 0.9 wal_buffers = 16MB max_parallel_maintenance_workers = $NCPUS shared_preload_libraries = 'pg_textsearch' EOF pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_textsearch" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Download MS MARCO dataset run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh ${{ inputs.msmarco_size || 'full' }} - name: Run concurrent insert setup run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" cd benchmarks/datasets/msmarco psql -f insert/concurrent-setup.sql - name: Run concurrent insert benchmark run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test DURATION="${{ inputs.duration || '60' }}" # Cap client counts to available vCPUs NCPUS=$(nproc) CLIENTS="" for c in 1 2 4 8 16 32; do if [ "$c" -le "$NCPUS" ]; then CLIENTS="$CLIENTS $c" fi done CLIENTS="${CLIENTS# }" STAGING_ROWS=$(psql -qtAc \ "SELECT count(*) FROM msmarco_staging;") echo "==============================================" echo " pg_textsearch Concurrent INSERT Benchmark" echo "==============================================" echo "Runner: ${{ inputs.runner || 'ubuntu-latest-8-cores-TJ' }}" echo "vCPUs: $NCPUS" echo "RAM: $(free -h | awk '/Mem:/{print $2}')" echo "Dataset: MS MARCO ${{ inputs.msmarco_size || 'full' }} ($STAGING_ROWS rows)" echo "Duration: ${DURATION}s per level" echo "Clients: $CLIENTS" echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" echo "" cd benchmarks/datasets/msmarco # Collect results as JSON for reporting echo '[]' > /tmp/tapir_results.json printf "%-10s %10s %12s %12s %15s\n" \ "CLIENTS" "TPS" "AVG_LAT_MS" "ROWS" "CUMULATIVE" echo "--------------------------------------------------------------" for c in $CLIENTS; do # Check remaining rows before each level remaining=$(psql -qtAc \ "SELECT count(*) FROM msmarco_staging s WHERE s.staging_id > (SELECT last_value FROM insert_seq);" \ 2>/dev/null || echo "$STAGING_ROWS") if [ "$remaining" -lt 1000 ]; then echo "STOPPING: only $remaining rows remaining in staging" break fi output=$(pgbench -n -c "$c" -j "$c" -T "$DURATION" \ -f insert/pgbench-insert.sql 2>&1) tps=$(echo "$output" | grep "^tps" | \ grep -oE '[0-9]+\.[0-9]+' | head -1) lat=$(echo "$output" | grep "latency average" | \ grep -oE '[0-9]+\.[0-9]+') txns=$(echo "$output" | \ grep "number of transactions actually processed" | \ grep -oE '[0-9]+' | head -1) failed=$(echo "$output" | \ grep "number of failed transactions" | \ grep -oE '[0-9]+' | head -1) cumulative=$(psql -qtAc \ "SELECT count(*) FROM msmarco_passages;") printf "%-10s %10s %12s %12s %15s\n" \ "$c" "$tps" "$lat" "$txns" "$cumulative" if [ "${failed:-0}" != "0" ]; then echo " WARNING: $failed failed transactions at c=$c" fi # Append to JSON results jq --arg c "$c" --arg tps "$tps" --arg lat "$lat" \ '. += [ {"name": "pg_textsearch INSERT TPS (c=\($c))", "unit": "tps", "value": ($tps | tonumber)}, {"name": "pg_textsearch INSERT latency (c=\($c))", "unit": "ms", "value": ($lat | tonumber)} ]' /tmp/tapir_results.json > /tmp/tapir_results_tmp.json mv /tmp/tapir_results_tmp.json /tmp/tapir_results.json done cp /tmp/tapir_results.json "$GITHUB_WORKSPACE/tapir_concurrent_insert.json" echo "" echo "--- Results JSON ---" cat "$GITHUB_WORKSPACE/tapir_concurrent_insert.json" echo "" echo "--- Index State ---" psql -c "SELECT * FROM bm25_summarize_index('msmarco_bm25_idx');" psql -c "SELECT pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_size_pretty(pg_table_size('msmarco_passages')) as table_size;" - name: Publish pg_textsearch TPS results if: always() && hashFiles('tapir_concurrent_insert.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'Concurrent INSERT (pg_textsearch)' tool: 'customBiggerIsBetter' output-file-path: tapir_concurrent_insert.json auto-push: true github-token: ${{ secrets.GITHUB_TOKEN }} gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks/concurrent-insert/pg_textsearch - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true paradedb: name: "ParadeDB concurrent INSERT" runs-on: ${{ inputs.runner || 'ubuntu-latest-8-cores-TJ' }} timeout-minutes: 120 steps: - uses: actions/checkout@v4 - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift df -h / - name: Install PostgreSQL and ParadeDB run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential jq wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 # pg_search (>= 0.24) requires the vector (pgvector) extension sudo apt-get install -y postgresql-17-pgvector # Install ParadeDB wget -q "https://github.com/paradedb/paradedb/releases/download/v${PARADEDB_VERSION}/postgresql-17-pg-search_${PARADEDB_VERSION}-1PARADEDB-noble_amd64.deb" sudo apt-get install -y "./postgresql-17-pg-search_${PARADEDB_VERSION}-1PARADEDB-noble_amd64.deb" - name: Configure and start PostgreSQL run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB effective_cache_size = 4GB maintenance_work_mem = 512MB work_mem = 64MB max_connections = 100 checkpoint_completion_target = 0.9 wal_buffers = 16MB max_parallel_maintenance_workers = $NCPUS shared_preload_libraries = 'pg_search' EOF pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_search CASCADE" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Download MS MARCO dataset run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh ${{ inputs.msmarco_size || 'full' }} - name: Run concurrent insert setup run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" cd benchmarks/datasets/msmarco psql -f paradedb/concurrent-setup.sql - name: Run concurrent insert benchmark run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test DURATION="${{ inputs.duration || '60' }}" # Cap client counts to available vCPUs NCPUS=$(nproc) CLIENTS="" for c in 1 2 4 8 16 32; do if [ "$c" -le "$NCPUS" ]; then CLIENTS="$CLIENTS $c" fi done CLIENTS="${CLIENTS# }" STAGING_ROWS=$(psql -qtAc \ "SELECT count(*) FROM msmarco_paradedb_staging;") echo "==============================================" echo " ParadeDB Concurrent INSERT Benchmark" echo "==============================================" echo "Runner: ${{ inputs.runner || 'ubuntu-latest-8-cores-TJ' }}" echo "vCPUs: $NCPUS" echo "RAM: $(free -h | awk '/Mem:/{print $2}')" echo "Dataset: MS MARCO ${{ inputs.msmarco_size || 'full' }} ($STAGING_ROWS rows)" echo "Duration: ${DURATION}s per level" echo "Clients: $CLIENTS" echo "Date: $(date -u '+%Y-%m-%d %H:%M:%S UTC')" echo "" cd benchmarks/datasets/msmarco # Collect results as JSON for reporting echo '[]' > /tmp/paradedb_results.json printf "%-10s %10s %12s %12s %15s\n" \ "CLIENTS" "TPS" "AVG_LAT_MS" "ROWS" "CUMULATIVE" echo "--------------------------------------------------------------" for c in $CLIENTS; do # Check remaining rows before each level remaining=$(psql -qtAc \ "SELECT count(*) FROM msmarco_paradedb_staging s WHERE s.staging_id > (SELECT last_value FROM insert_seq);" \ 2>/dev/null || echo "$STAGING_ROWS") if [ "$remaining" -lt 1000 ]; then echo "STOPPING: only $remaining rows remaining in staging" break fi output=$(pgbench -n -c "$c" -j "$c" -T "$DURATION" \ -f paradedb/pgbench-insert.sql 2>&1) tps=$(echo "$output" | grep "^tps" | \ grep -oE '[0-9]+\.[0-9]+' | head -1) lat=$(echo "$output" | grep "latency average" | \ grep -oE '[0-9]+\.[0-9]+') txns=$(echo "$output" | \ grep "number of transactions actually processed" | \ grep -oE '[0-9]+' | head -1) failed=$(echo "$output" | \ grep "number of failed transactions" | \ grep -oE '[0-9]+' | head -1) cumulative=$(psql -qtAc \ "SELECT count(*) FROM msmarco_passages_paradedb;") printf "%-10s %10s %12s %12s %15s\n" \ "$c" "$tps" "$lat" "$txns" "$cumulative" if [ "${failed:-0}" != "0" ]; then echo " WARNING: $failed failed transactions at c=$c" fi # Append to JSON results jq --arg c "$c" --arg tps "$tps" --arg lat "$lat" \ '. += [ {"name": "ParadeDB INSERT TPS (c=\($c))", "unit": "tps", "value": ($tps | tonumber)}, {"name": "ParadeDB INSERT latency (c=\($c))", "unit": "ms", "value": ($lat | tonumber)} ]' /tmp/paradedb_results.json > /tmp/paradedb_results_tmp.json mv /tmp/paradedb_results_tmp.json /tmp/paradedb_results.json done cp /tmp/paradedb_results.json "$GITHUB_WORKSPACE/paradedb_concurrent_insert.json" echo "" echo "--- Results JSON ---" cat "$GITHUB_WORKSPACE/paradedb_concurrent_insert.json" - name: Publish ParadeDB TPS results if: always() && hashFiles('paradedb_concurrent_insert.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'Concurrent INSERT (ParadeDB)' tool: 'customBiggerIsBetter' output-file-path: paradedb_concurrent_insert.json auto-push: true github-token: ${{ secrets.GITHUB_TOKEN }} gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks/concurrent-insert/paradedb - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true timescale-pg_textsearch-b63fe58/.github/workflows/benchmark.yml000066400000000000000000003771741525762047700250560ustar00rootroot00000000000000name: Benchmarks permissions: contents: write deployments: write env: # ParadeDB (competitive baseline) version for benchmarking PARADEDB_VERSION: "0.25.2" on: # Nightly benchmark schedule: - cron: '0 6 * * *' # Every day at 6 AM UTC # Manual trigger workflow_dispatch: inputs: dataset: description: 'Dataset to benchmark' required: true default: 'all' type: choice options: - cranfield - msmarco - wikipedia - all wikipedia_size: description: 'Wikipedia subset size' required: false default: '100K' type: choice options: - '10K' - '100K' - '1M' - 'full' enable_profile: description: 'Enable CPU profiling (generates flamegraph)' required: false default: false type: boolean dry_run: description: 'Dry run - skip publishing to GitHub Pages' required: false default: false type: boolean compress_segments: description: 'Enable segment compression (delta + bitpacking)' required: false default: false type: boolean validate_compression: description: 'Run compression validation (compare compressed vs uncompressed)' required: false default: false type: boolean parallel_scaling: description: 'Run parallel build scaling benchmark (varies worker count)' required: false default: false type: boolean benchmark_suite: description: 'Which benchmark suite(s) to run (manual dispatch only). Scheduled runs always execute everything.' required: false default: 'all' type: choice options: - all - insert - full - paradedb - baseline jobs: # CPU profiling job (separate from timing benchmarks) profile: runs-on: ubuntu-latest timeout-minutes: 120 # Allow 2 hours for full dataset profiling if: github.event_name == 'workflow_dispatch' && github.event.inputs.enable_profile == 'true' steps: - uses: actions/checkout@v4 - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift df -h / - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc \ linux-tools-common linux-tools-generic linux-tools-$(uname -r) || true # Install FlameGraph for visualization git clone --depth 1 https://github.com/brendangregg/FlameGraph.git # PostgreSQL wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 \ postgresql-17-dbgsym || sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Build extension (release with debug symbols) run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean # Release optimization but keep frame pointers for profiling CFLAGS="-O2 -fno-omit-frame-pointer -g" make sudo make install - name: Configure PostgreSQL run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" LIB_NAME="pg_textsearch" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust # Parallel build requires maintenance_work_mem >= 64MB # (ubuntu-latest: 2-4 vCPUs, ~7GB RAM) NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB maintenance_work_mem = 512MB max_parallel_maintenance_workers = $NCPUS shared_preload_libraries = '${LIB_NAME}' EOF pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_textsearch" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Download MS MARCO dataset run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh full - name: Load data (without index) run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" # Load data using the standard load script, but stop before index creation # Line 91 is after sample queries, before index creation (line 96) head -n 91 benchmarks/datasets/msmarco/load.sql > /tmp/load_data_only.sql psql -f /tmp/load_data_only.sql psql -c "SELECT COUNT(*) as num_rows FROM msmarco_passages" - name: Profile index creation with perf run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # Enable perf for non-root (may fail on some systems) echo -1 | sudo tee /proc/sys/kernel/perf_event_paranoid || true echo 0 | sudo tee /proc/sys/kernel/kptr_restrict || true # Start index creation in background psql -c "CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english')" & PSQL_PID=$! # Wait a moment for the backend to start sleep 2 # Find the postgres backend process PG_PID=$(pgrep -f "postgres.*bench_test" | head -1) echo "Profiling PID: $PG_PID" if [ -n "$PG_PID" ]; then # Record profile for duration of index build (up to 60 min for full dataset) sudo perf record -F 99 -p $PG_PID -g --call-graph dwarf -o perf.data -- sleep 3600 & PERF_PID=$! # Wait for index creation to complete wait $PSQL_PID || true # Stop perf recording sudo kill -INT $PERF_PID 2>/dev/null || true sleep 2 # Generate perf report for indexing sudo perf report -i perf.data --stdio --no-children > index_profile.txt 2>&1 || true # Generate flamegraph for indexing sudo perf script -i perf.data > perf.script 2>/dev/null || true if [ -s perf.script ]; then ./FlameGraph/stackcollapse-perf.pl perf.script > index_perf.folded ./FlameGraph/flamegraph.pl --title "Index Build CPU Profile" index_perf.folded > index_flamegraph.svg echo "Index flamegraph generated successfully" fi else echo "Could not find postgres backend PID" wait $PSQL_PID || true fi - name: Profile queries with perf run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # Disable autovacuum to avoid profiling vacuum instead of queries psql -c "ALTER SYSTEM SET autovacuum = off" pg_ctl reload -D tmp_bench/data # Create a query script that runs many iterations for profiling cat > /tmp/profile_queries.sql << 'EOSQL' -- Run queries repeatedly to get meaningful profile samples -- Test both: queries WITHOUT scores (index scan) and WITH scores (per-row operator) \timing off DO $$ DECLARE i INT; dummy RECORD; BEGIN -- Part 1: Queries WITHOUT scores (300 iterations to get good samples) FOR i IN 1..300 LOOP -- Short query - no score FOR dummy IN SELECT ctid FROM msmarco_passages ORDER BY passage_text <@> 'search engine'::bm25query LIMIT 10 LOOP END LOOP; -- Medium query - no score FOR dummy IN SELECT ctid FROM msmarco_passages ORDER BY passage_text <@> 'machine learning algorithms'::bm25query LIMIT 10 LOOP END LOOP; -- Long query - no score FOR dummy IN SELECT ctid FROM msmarco_passages ORDER BY passage_text <@> 'what is the best way to learn programming'::bm25query LIMIT 10 LOOP END LOOP; END LOOP; -- Part 2: Queries WITH scores (per-row operator call) FOR i IN 1..300 LOOP -- Short query - with score FOR dummy IN SELECT ctid, passage_text <@> 'search engine'::bm25query as score FROM msmarco_passages ORDER BY passage_text <@> 'search engine'::bm25query LIMIT 10 LOOP END LOOP; -- Medium query - with score FOR dummy IN SELECT ctid, passage_text <@> 'machine learning algorithms'::bm25query as score FROM msmarco_passages ORDER BY passage_text <@> 'machine learning algorithms'::bm25query LIMIT 10 LOOP END LOOP; -- Long query - with score FOR dummy IN SELECT ctid, passage_text <@> 'what is the best way to learn programming'::bm25query as score FROM msmarco_passages ORDER BY passage_text <@> 'what is the best way to learn programming'::bm25query LIMIT 10 LOOP END LOOP; END LOOP; END $$; EOSQL # Start queries in background psql -f /tmp/profile_queries.sql & PSQL_PID=$! # Wait for backend to start, then find it # With autovacuum disabled, we should only see our query backend sleep 2 PG_PID=$(ps aux | grep "postgres:.*bench_test" | grep -v "autovacuum\|checkpointer\|walwriter\|bgwriter\|logical\|startup" | grep -v grep | awk '{print $2}' | head -1) echo "Profiling query backend PID: $PG_PID" if [ -n "$PG_PID" ]; then # Record profile for query duration (up to 10 min) sudo perf record -F 499 -p $PG_PID -g --call-graph dwarf -o query_perf.data -- sleep 600 & PERF_PID=$! # Wait for queries to complete wait $PSQL_PID || true # Stop perf recording sudo kill -INT $PERF_PID 2>/dev/null || true sleep 2 # Generate perf report for queries sudo perf report -i query_perf.data --stdio --no-children > query_profile.txt 2>&1 || true # Generate flamegraph for queries sudo perf script -i query_perf.data > query_perf.script 2>/dev/null || true if [ -s query_perf.script ]; then ./FlameGraph/stackcollapse-perf.pl query_perf.script > query_perf.folded ./FlameGraph/flamegraph.pl --title "Query CPU Profile" query_perf.folded > query_flamegraph.svg echo "Query flamegraph generated successfully" fi else echo "Could not find postgres backend PID for queries" wait $PSQL_PID || true fi # Re-enable autovacuum psql -c "ALTER SYSTEM RESET autovacuum" pg_ctl reload -D tmp_bench/data - name: Generate profile summary run: | echo "# CPU Profile Summary" > profile_summary.md echo "" >> profile_summary.md echo "**Date:** $(date -u)" >> profile_summary.md echo "**Commit:** ${{ github.sha }}" >> profile_summary.md echo "**Branch:** ${{ github.ref_name }}" >> profile_summary.md echo "**Dataset:** MS MARCO (full)" >> profile_summary.md echo "" >> profile_summary.md echo "## Index Build Profile" >> profile_summary.md if [ -f index_profile.txt ]; then echo '```' >> profile_summary.md head -80 index_profile.txt >> profile_summary.md echo '```' >> profile_summary.md else echo "Index profiling failed" >> profile_summary.md fi echo "" >> profile_summary.md echo "## Query Profile" >> profile_summary.md if [ -f query_profile.txt ]; then echo '```' >> profile_summary.md head -80 query_profile.txt >> profile_summary.md echo '```' >> profile_summary.md else echo "Query profiling failed" >> profile_summary.md fi echo "" >> profile_summary.md echo "## Artifacts" >> profile_summary.md echo "- index_flamegraph.svg - Index build CPU flamegraph" >> profile_summary.md echo "- query_flamegraph.svg - Query execution CPU flamegraph" >> profile_summary.md echo "- index_profile.txt - Index build perf report" >> profile_summary.md echo "- query_profile.txt - Query execution perf report" >> profile_summary.md - name: Upload profile artifacts uses: actions/upload-artifact@v4 with: name: cpu-profile-${{ github.run_id }} path: | index_flamegraph.svg query_flamegraph.svg index_profile.txt query_profile.txt profile_summary.md index_perf.folded query_perf.folded retention-days: 30 - name: Publish flamegraphs to GitHub Pages if: always() && github.event.inputs.dry_run != 'true' run: | # Only publish if we have flamegraphs if [ ! -f index_flamegraph.svg ] && [ ! -f query_flamegraph.svg ]; then echo "No flamegraphs to publish" exit 0 fi git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git fetch origin gh-pages # Create a temporary directory for gh-pages content mkdir -p /tmp/gh-pages git worktree add /tmp/gh-pages gh-pages # Create profiles directory structure: profiles/// SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) PROFILE_DIR="/tmp/gh-pages/benchmarks/profiles/${{ github.ref_name }}/${SHORT_SHA}" mkdir -p "$PROFILE_DIR" # Copy flamegraphs [ -f index_flamegraph.svg ] && cp index_flamegraph.svg "$PROFILE_DIR/" [ -f query_flamegraph.svg ] && cp query_flamegraph.svg "$PROFILE_DIR/" [ -f index_profile.txt ] && cp index_profile.txt "$PROFILE_DIR/" [ -f query_profile.txt ] && cp query_profile.txt "$PROFILE_DIR/" # Get commit message for metadata COMMIT_MSG=$(git log -1 --format='%s' ${{ github.sha }} | head -c 80) COMMIT_DATE=$(date -u +%Y-%m-%dT%H:%M:%SZ) # Create metadata file for this profile cat > "$PROFILE_DIR/meta.json" << EOF { "sha": "${SHORT_SHA}", "full_sha": "${{ github.sha }}", "branch": "${{ github.ref_name }}", "date": "${COMMIT_DATE}", "message": "${COMMIT_MSG}", "dataset": "MS MARCO (full)" } EOF # Create an index.html for this profile cat > "$PROFILE_DIR/index.html" << EOF CPU Profiles - ${SHORT_SHA}

CPU Profiles for ${{ github.ref_name }} @ ${SHORT_SHA}

Commit: ${COMMIT_MSG}

Dataset: MS MARCO (full)

Date: $(date -u)

Flamegraphs

Text Reports

EOF # Update profiles index with sorted entries and commit messages cat > /tmp/gh-pages/benchmarks/profiles/index.html << 'INDEXEOF' CPU Profiles

CPU Profiles

Sorted by date (newest first)

INDEXEOF # Collect all profiles with metadata, sort by date for branch_dir in /tmp/gh-pages/benchmarks/profiles/*/; do [ -d "$branch_dir" ] || continue branch=$(basename "$branch_dir") [ "$branch" = "index.html" ] && continue echo "

$branch

" >> /tmp/gh-pages/benchmarks/profiles/index.html # Collect and sort profiles by date (newest first) for commit_dir in "$branch_dir"*/; do [ -d "$commit_dir" ] || continue commit=$(basename "$commit_dir") meta="$commit_dir/meta.json" if [ -f "$meta" ]; then date=$(grep '"date"' "$meta" | sed 's/.*": *"\([^"]*\)".*/\1/') msg=$(grep '"message"' "$meta" | sed 's/.*": *"\([^"]*\)".*/\1/') echo "$date|$commit|$msg" else echo "1970-01-01T00:00:00Z|$commit|" fi done | sort -r | while IFS='|' read -r date commit msg; do echo "
" >> /tmp/gh-pages/benchmarks/profiles/index.html echo " $commit" >> /tmp/gh-pages/benchmarks/profiles/index.html [ -n "$msg" ] && echo " - $msg" >> /tmp/gh-pages/benchmarks/profiles/index.html [ "$date" != "1970-01-01T00:00:00Z" ] && echo " ($date)" >> /tmp/gh-pages/benchmarks/profiles/index.html echo "
" >> /tmp/gh-pages/benchmarks/profiles/index.html done done echo "" >> /tmp/gh-pages/benchmarks/profiles/index.html # Commit and push cd /tmp/gh-pages git add benchmarks/profiles/ if ! git diff --cached --quiet; then git commit -m "Add CPU profiles for ${{ github.ref_name }}@${SHORT_SHA}" git push origin gh-pages echo "Flamegraphs published to: https://timescale.github.io/pg_textsearch/benchmarks/profiles/${{ github.ref_name }}/${SHORT_SHA}/" else echo "No changes to publish" fi # Cleanup worktree cd "$GITHUB_WORKSPACE" git worktree remove /tmp/gh-pages --force || true - name: Post profile summary if: always() run: | SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) PROFILE_URL="https://timescale.github.io/pg_textsearch/benchmarks/profiles/${{ github.ref_name }}/${SHORT_SHA}/" echo "### CPU Profile Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "**[View Flamegraphs on GitHub Pages](${PROFILE_URL})**" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "#### Index Build Profile" >> $GITHUB_STEP_SUMMARY if [ -f index_profile.txt ]; then echo '```' >> $GITHUB_STEP_SUMMARY head -40 index_profile.txt >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY else echo "Index profiling failed - check logs" >> $GITHUB_STEP_SUMMARY fi echo "" >> $GITHUB_STEP_SUMMARY echo "#### Query Profile" >> $GITHUB_STEP_SUMMARY if [ -f query_profile.txt ]; then echo '```' >> $GITHUB_STEP_SUMMARY head -40 query_profile.txt >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY else echo "Query profiling failed - check logs" >> $GITHUB_STEP_SUMMARY fi - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true rm -rf tmp_bench # ParadeDB benchmark (competitive baseline) # NOTE: pg_textsearch and pg_search cannot coexist in the same PostgreSQL # instance due to conflicting 'bm25' access method names. This job runs # in a separate PostgreSQL instance with identical configuration. paradedb-benchmark: runs-on: ubuntu-latest timeout-minutes: 360 # Run for any dataset selection or scheduled runs if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.benchmark_suite == 'all' || github.event.inputs.benchmark_suite == 'paradedb' || github.event.inputs.benchmark_suite == '')) steps: - uses: actions/checkout@v4 - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL /usr/share/swift df -h / - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install PostgreSQL and ParadeDB run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc pip install 'wikiextractor==3.0.6' # Install PostgreSQL 17 wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 # pg_search (>= 0.24) requires the vector (pgvector) extension sudo apt-get install -y postgresql-17-pgvector # Install ParadeDB extension from GitHub releases # ubuntu-latest is Ubuntu 24.04 (Noble) wget -q "https://github.com/paradedb/paradedb/releases/download/v${PARADEDB_VERSION}/postgresql-17-pg-search_${PARADEDB_VERSION}-1PARADEDB-noble_amd64.deb" sudo apt-get install -y "./postgresql-17-pg-search_${PARADEDB_VERSION}-1PARADEDB-noble_amd64.deb" - name: Configure PostgreSQL run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust # Use identical configuration to pg_textsearch benchmark # (ubuntu-latest: 2-4 vCPUs, ~7GB RAM) NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF shared_preload_libraries = 'pg_search' unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB effective_cache_size = 4GB maintenance_work_mem = 512MB work_mem = 64MB max_connections = 20 checkpoint_completion_target = 0.9 wal_buffers = 16MB random_page_cost = 1.1 effective_io_concurrency = 200 max_parallel_maintenance_workers = $NCPUS max_parallel_workers_per_gather = $NCPUS max_parallel_workers = $NCPUS EOF if ! pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w; then echo "PostgreSQL failed to start. Log:" cat tmp_bench/postgres.log exit 1 fi export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_search CASCADE" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Determine datasets to run id: datasets run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "dataset=${{ github.event.inputs.dataset }}" >> $GITHUB_OUTPUT echo "wiki_size=${{ github.event.inputs.wikipedia_size }}" >> $GITHUB_OUTPUT else echo "dataset=all" >> $GITHUB_OUTPUT echo "wiki_size=100K" >> $GITHUB_OUTPUT fi - name: Download MS MARCO dataset if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh full - name: Download Wikipedia dataset if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/wikipedia chmod +x download.sh ./download.sh ${{ steps.datasets.outputs.wiki_size }} - name: Run ParadeDB Cranfield benchmark if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== ParadeDB Cranfield Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt cd benchmarks/sql/cranfield time psql -f paradedb/01-load.sql 2>&1 | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" time psql -f paradedb/02-queries.sql 2>&1 | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" - name: Run ParadeDB MS MARCO benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== ParadeDB MS MARCO Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/paradedb/load.sql 2>&1 | tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/paradedb/queries.sql 2>&1 | tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run ParadeDB Wikipedia benchmark if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== ParadeDB Wikipedia Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/wikipedia/paradedb/load.sql 2>&1 | tee -a benchmark_results.txt time psql -f benchmarks/datasets/wikipedia/paradedb/queries.sql 2>&1 | tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run ParadeDB Cranfield insert benchmark if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== ParadeDB Cranfield Insert Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt cd benchmarks/sql/cranfield time psql -f paradedb/insert-load.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" time psql -f paradedb/02-queries.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" - name: Run ParadeDB MS MARCO insert benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== ParadeDB MS MARCO Insert Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/paradedb/insert-load.sql 2>&1 | \ tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/paradedb/queries.sql 2>&1 | \ tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run ParadeDB Wikipedia insert benchmark if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== ParadeDB Wikipedia Insert Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/wikipedia/paradedb/insert-load.sql 2>&1 | \ tee -a benchmark_results.txt time psql -f benchmarks/datasets/wikipedia/paradedb/queries.sql 2>&1 | \ tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run ParadeDB Cranfield concurrent insert if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== ParadeDB Cranfield Concurrent Insert Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt cd benchmarks/sql/cranfield psql -f paradedb/concurrent-setup.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" ROWS=$(psql -tA -c "SELECT count(*) FROM cranfield_paradedb_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f paradedb/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | \ tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" - name: Run ParadeDB Cranfield concurrent insert queries if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test cd benchmarks/sql/cranfield time psql -f paradedb/02-queries.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" - name: Run ParadeDB MS MARCO concurrent insert if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== ParadeDB MS MARCO Concurrent Insert Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt psql -f benchmarks/datasets/msmarco/paradedb/concurrent-setup.sql 2>&1 | \ tee -a benchmark_results.txt ROWS=$(psql -tA -c "SELECT count(*) FROM msmarco_paradedb_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f benchmarks/datasets/msmarco/paradedb/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a benchmark_results.txt END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run ParadeDB MS MARCO concurrent insert queries if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test time psql -f benchmarks/datasets/msmarco/paradedb/queries.sql 2>&1 | \ tee -a benchmark_results.txt - name: Run ParadeDB Wikipedia concurrent insert if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== ParadeDB Wikipedia Concurrent Insert Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt psql -f benchmarks/datasets/wikipedia/paradedb/concurrent-setup.sql 2>&1 | \ tee -a benchmark_results.txt ROWS=$(psql -tA -c "SELECT count(*) FROM wikipedia_paradedb_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f benchmarks/datasets/wikipedia/paradedb/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a benchmark_results.txt END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run ParadeDB Wikipedia concurrent insert queries if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test time psql -f benchmarks/datasets/wikipedia/paradedb/queries.sql 2>&1 | \ tee -a benchmark_results.txt - name: Extract metrics run: | chmod +x benchmarks/runner/extract_metrics.sh DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then # Extract each dataset separately with section filter ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_cranfield_metrics.json "paradedb_cranfield" "ParadeDB Cranfield" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_msmarco_metrics.json "paradedb_msmarco" "ParadeDB MS MARCO" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_wikipedia_metrics.json "paradedb_wikipedia" "ParadeDB Wikipedia" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_cranfield_insert_metrics.json "paradedb_cranfield_insert" "ParadeDB Cranfield Insert" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_msmarco_insert_metrics.json "paradedb_msmarco_insert" "ParadeDB MS MARCO Insert" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_wikipedia_insert_metrics.json "paradedb_wikipedia_insert" "ParadeDB Wikipedia Insert" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_cranfield_concurrent_metrics.json "paradedb_cranfield_concurrent" "ParadeDB Cranfield Concurrent" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_msmarco_concurrent_metrics.json "paradedb_msmarco_concurrent" "ParadeDB MS MARCO Concurrent" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_wikipedia_concurrent_metrics.json "paradedb_wikipedia_concurrent" "ParadeDB Wikipedia Concurrent" || true else # Map dataset slug to display name for section matching case "$DATASET" in cranfield) DD="Cranfield" ;; msmarco) DD="MS MARCO" ;; wikipedia) DD="Wikipedia" ;; *) DD="$DATASET" ;; esac ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_metrics.json "paradedb_${DATASET}" "ParadeDB ${DD}" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_${DATASET}_insert_metrics.json "paradedb_${DATASET}_insert" "ParadeDB ${DD} Insert" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ paradedb_${DATASET}_concurrent_metrics.json "paradedb_${DATASET}_concurrent" "ParadeDB ${DD} Concurrent" || true fi - name: Upload results uses: actions/upload-artifact@v4 with: name: paradedb-benchmark-${{ github.run_id }} path: | benchmark_results.txt paradedb_*_metrics.json paradedb_metrics.json *_action.json tmp_bench/postgres.log retention-days: 90 - name: Post benchmark summary if: always() run: | # Helper function to output metrics table for a dataset output_paradedb_table() { local file=$1 local name=$2 if [ ! -f "$file" ]; then return fi echo "### ParadeDB ${name} Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY echo "| Documents | $(jq -r '.metrics.num_documents // "N/A"' "$file") |" >> $GITHUB_STEP_SUMMARY echo "| Index Build | $(jq -r '.metrics.index_build_time_ms // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| Index Size | $(jq -r '.metrics.index_size // "N/A"' "$file") |" >> $GITHUB_STEP_SUMMARY echo "| 1 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_1.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 2 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_2.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 3 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_3.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 4 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_4.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 5 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_5.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 6 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_6.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 7 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_7.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 8+ Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_8.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| Throughput | $(jq -r '.metrics.throughput.avg_ms_per_query // "N/A"' "$file") ms/query |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY } DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then # Show per-dataset summaries output_paradedb_table "paradedb_cranfield_metrics.json" "Cranfield" output_paradedb_table "paradedb_msmarco_metrics.json" "MS MARCO" output_paradedb_table "paradedb_wikipedia_metrics.json" "Wikipedia" elif [ -f paradedb_metrics.json ]; then output_paradedb_table "paradedb_metrics.json" "$DATASET" fi - name: Format ParadeDB metrics for benchmark tracking if: always() run: | chmod +x benchmarks/runner/format_for_action.sh for f in paradedb_*_metrics.json; do if [ -f "$f" ]; then base="${f%_metrics.json}" ./benchmarks/runner/format_for_action.sh "$f" "${base}_action.json" fi done if [ -f paradedb_metrics.json ]; then ./benchmarks/runner/format_for_action.sh \ paradedb_metrics.json paradedb_action.json fi - name: Publish ParadeDB Cranfield benchmark if: always() && hashFiles('paradedb_cranfield_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_cranfield Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_cranfield_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB MS MARCO benchmark if: always() && hashFiles('paradedb_msmarco_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_msmarco Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_msmarco_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB Wikipedia benchmark if: always() && hashFiles('paradedb_wikipedia_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_wikipedia Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_wikipedia_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB Cranfield insert benchmark if: always() && hashFiles('paradedb_cranfield_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_cranfield_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_cranfield_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB MS MARCO insert benchmark if: always() && hashFiles('paradedb_msmarco_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_msmarco_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_msmarco_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB Wikipedia insert benchmark if: always() && hashFiles('paradedb_wikipedia_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_wikipedia_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_wikipedia_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB Cranfield concurrent benchmark if: always() && hashFiles('paradedb_cranfield_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_cranfield_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_cranfield_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB MS MARCO concurrent benchmark if: always() && hashFiles('paradedb_msmarco_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_msmarco_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_msmarco_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish ParadeDB Wikipedia concurrent benchmark if: always() && hashFiles('paradedb_wikipedia_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'paradedb_wikipedia_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: paradedb_wikipedia_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true rm -rf tmp_bench # Full benchmark suite full-benchmark: runs-on: ubuntu-latest # Standard runner (2-4 vCPUs) timeout-minutes: 360 # 6 hours max for full Wikipedia if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.benchmark_suite == 'all' || github.event.inputs.benchmark_suite == 'full' || github.event.inputs.benchmark_suite == '')) steps: - uses: actions/checkout@v4 - name: Free disk space run: | echo "Initial disk space:" df -h / # Remove unnecessary packages sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo rm -rf /usr/share/swift sudo rm -rf /usr/local/share/powershell sudo rm -rf /usr/local/.ghcup echo "After cleanup:" df -h / # wikiextractor requires Python 3.10 (unmaintained, breaks on newer versions). # Pinned to 3.0.6: 3.0.8+ changes extraction output and breaks the committed # Wikipedia ground_truth.tsv (see benchmarks/datasets/wikipedia/download.sh). - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc pip install 'wikiextractor==3.0.6' # PostgreSQL wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Build extension (release mode) run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean CFLAGS="-O3 -DNDEBUG" make sudo make install - name: Configure PostgreSQL for benchmarks run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" LIB_NAME="pg_textsearch" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust # Tune for benchmarking (ubuntu-latest: 2-4 vCPUs, ~7GB RAM) # Parallel build requires maintenance_work_mem >= 64MB NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB effective_cache_size = 4GB maintenance_work_mem = 512MB work_mem = 64MB max_connections = 20 checkpoint_completion_target = 0.9 wal_buffers = 16MB random_page_cost = 1.1 effective_io_concurrency = 200 max_parallel_maintenance_workers = $NCPUS max_parallel_workers_per_gather = $NCPUS max_parallel_workers = $NCPUS pg_textsearch.segments_per_level = 16 shared_preload_libraries = '${LIB_NAME}' EOF # Enable compression if requested if [ "${{ github.event.inputs.compress_segments }}" = "true" ]; then echo "pg_textsearch.compress_segments = on" >> tmp_bench/data/postgresql.conf echo "Compression enabled for this benchmark run" fi if ! pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w; then echo "PostgreSQL failed to start. Log:" cat tmp_bench/postgres.log exit 1 fi export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_textsearch" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Determine datasets to run id: datasets run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "dataset=${{ github.event.inputs.dataset }}" >> $GITHUB_OUTPUT echo "wiki_size=${{ github.event.inputs.wikipedia_size }}" >> $GITHUB_OUTPUT else echo "dataset=all" >> $GITHUB_OUTPUT echo "wiki_size=100K" >> $GITHUB_OUTPUT fi - name: Download MS MARCO dataset if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh full - name: Download Wikipedia dataset if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/wikipedia chmod +x download.sh ./download.sh ${{ steps.datasets.outputs.wiki_size }} - name: Run Cranfield benchmark if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== Cranfield Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt cd benchmarks/sql/cranfield time psql -f 01-load.sql 2>&1 | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" time psql -f 02-queries.sql 2>&1 | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" - name: Run MS MARCO benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== MS MARCO Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/load.sql 2>&1 | \ tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/queries.sql 2>&1 | \ tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Validate MS MARCO results if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # pipefail: catch psql ON_ERROR_STOP exits that would otherwise be # masked by the `| tee` chain and let the step silently pass. set -o pipefail echo "=== MS MARCO Validation ===" | tee -a benchmark_results.txt # Select version-specific ground truth based on PG major version. # PG17 and PG18 have different Snowball stemmers which affect df values. PG_MAJOR=$(pg_config --version | sed 's/PostgreSQL //' | cut -d. -f1) GT_FILE="benchmarks/datasets/msmarco/ground_truth_pg${PG_MAJOR}.tsv" if [ -f "$GT_FILE" ]; then echo "Using ground truth for PG${PG_MAJOR}" | tee -a benchmark_results.txt cp "$GT_FILE" benchmarks/datasets/msmarco/ground_truth.tsv else echo "ERROR: No ground truth file for PG${PG_MAJOR} ($GT_FILE not found)" | tee -a benchmark_results.txt exit 1 fi psql -f benchmarks/datasets/msmarco/validate_queries.sql 2>&1 | \ tee msmarco_validation.log | tee -a benchmark_results.txt if grep -q "VALIDATION FAILED" msmarco_validation.log; then echo "ERROR: MS MARCO validation FAILED!" | tee -a benchmark_results.txt exit 1 fi # Require an explicit PASSED marker -- absent it the script errored # out mid-run (e.g. SQL error in validate_queries.sql) and grep for # FAILED above would silently miss the regression. if ! grep -q "VALIDATION PASSED" msmarco_validation.log; then echo "ERROR: MS MARCO validation did not complete (no PASSED or FAILED marker found)" | tee -a benchmark_results.txt exit 1 fi echo "MS MARCO validation PASSED" | tee -a benchmark_results.txt - name: Run MS MARCO VACUUM benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== MS MARCO VACUUM Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/msmarco/vacuum.sql 2>&1 | \ tee msmarco_vacuum_results.txt | tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Run parallel scaling benchmark if: github.event.inputs.parallel_scaling == 'true' && (steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all') run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== Parallel Scaling Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt # Data should already be loaded from MS MARCO benchmark # Just run the parallel scaling test psql -f benchmarks/datasets/msmarco/parallel_scaling.sql 2>&1 | \ tee parallel_scaling_results.txt | tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt # Extract parallel scaling metrics echo "" echo "=== Parallel Scaling Summary ===" grep -E "PARALLEL_(BUILD_WORKERS|QUERY_RESULT)" parallel_scaling_results.txt || true - name: Run Wikipedia benchmark if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== Wikipedia Benchmark ===" | tee -a benchmark_results.txt echo "Start: $(date)" | tee -a benchmark_results.txt time psql -f benchmarks/datasets/wikipedia/load.sql 2>&1 | \ tee -a benchmark_results.txt time psql -f benchmarks/datasets/wikipedia/queries.sql 2>&1 | \ tee -a benchmark_results.txt echo "End: $(date)" | tee -a benchmark_results.txt - name: Validate Wikipedia results if: | (steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all') && steps.datasets.outputs.wiki_size == '100K' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # pipefail: catch psql ON_ERROR_STOP exits that would otherwise be # masked by the `| tee` chain and let the step silently pass. set -o pipefail echo "=== Wikipedia Validation ===" | tee -a benchmark_results.txt psql -f benchmarks/datasets/wikipedia/validate_queries.sql 2>&1 | \ tee wikipedia_validation.log | tee -a benchmark_results.txt if grep -q "VALIDATION FAILED" wikipedia_validation.log; then echo "ERROR: Wikipedia validation FAILED!" | tee -a benchmark_results.txt exit 1 fi # Require an explicit PASSED marker -- absent it the script errored # out mid-run (e.g. SQL error in validate_queries.sql) and grep for # FAILED above would silently miss the regression. if ! grep -q "VALIDATION PASSED" wikipedia_validation.log; then echo "ERROR: Wikipedia validation did not complete (no PASSED or FAILED marker found)" | tee -a benchmark_results.txt exit 1 fi echo "Wikipedia validation PASSED" | tee -a benchmark_results.txt - name: Generate summary and metrics run: | echo "# Benchmark Summary" > benchmark_summary.md echo "" >> benchmark_summary.md echo "**Date:** $(date -u)" >> benchmark_summary.md echo "**Commit:** ${{ github.sha }}" >> benchmark_summary.md echo "**Dataset:** ${{ steps.datasets.outputs.dataset }}" >> benchmark_summary.md echo "**Compression:** ${{ github.event.inputs.compress_segments == 'true' && 'enabled' || 'disabled' }}" >> benchmark_summary.md echo "" >> benchmark_summary.md chmod +x benchmarks/runner/extract_metrics.sh # Extract metrics per dataset when running "all" DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then # Extract each dataset separately (with section filter for accurate metrics) ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ cranfield_metrics.json "cranfield" "Cranfield" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ msmarco_metrics.json "msmarco" "MS MARCO" || true ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ wikipedia_metrics.json "wikipedia" "Wikipedia" || true else ./benchmarks/runner/extract_metrics.sh benchmark_results.txt \ benchmark_metrics.json "$DATASET" || true fi # Extract VACUUM benchmark metrics (from dedicated log) if [ -f msmarco_vacuum_results.txt ]; then ./benchmarks/runner/extract_metrics.sh \ msmarco_vacuum_results.txt \ msmarco_vacuum_metrics.json "msmarco_vacuum" || true fi # Add metrics to summary echo "## Key Metrics" >> benchmark_summary.md for f in *_metrics.json; do if [ -f "$f" ]; then echo "### $f" >> benchmark_summary.md echo '```json' >> benchmark_summary.md cat "$f" >> benchmark_summary.md echo '```' >> benchmark_summary.md fi done # Extract timing info echo "" >> benchmark_summary.md echo "## Timing" >> benchmark_summary.md grep -E "^(real|user|sys|Time:)" benchmark_results.txt >> benchmark_summary.md || true # Extract query latencies echo "" >> benchmark_summary.md echo "## Query Latencies" >> benchmark_summary.md grep -E "Execution Time:" benchmark_results.txt >> benchmark_summary.md || true # Extract throughput echo "" >> benchmark_summary.md echo "## Throughput" >> benchmark_summary.md grep -E "THROUGHPUT_RESULT:" benchmark_results.txt >> benchmark_summary.md || true echo "" >> benchmark_summary.md echo "## Full Results (last 300 lines)" >> benchmark_summary.md echo '```' >> benchmark_summary.md tail -300 benchmark_results.txt >> benchmark_summary.md echo '```' >> benchmark_summary.md - name: Upload benchmark results uses: actions/upload-artifact@v4 with: name: benchmark-results-${{ github.run_id }} path: | benchmark_results.txt benchmark_summary.md *_metrics.json *_action.json *_validation.log tmp_bench/postgres.log retention-days: 90 - name: Post benchmark summary to job if: always() run: | # Helper function to output metrics table for a dataset output_metrics_table() { local file=$1 local name=$2 if [ ! -f "$file" ]; then return fi echo "### pg_textsearch ${name} Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY if [ "${{ github.event.inputs.compress_segments }}" = "true" ]; then echo "**Compression: enabled**" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY fi echo "| Metric | Value |" >> $GITHUB_STEP_SUMMARY echo "|--------|-------|" >> $GITHUB_STEP_SUMMARY echo "| Documents | $(jq -r '.metrics.num_documents // "N/A"' "$file") |" >> $GITHUB_STEP_SUMMARY echo "| Index Build | $(jq -r '.metrics.index_build_time_ms // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| Index Size | $(jq -r '.metrics.index_size // "N/A"' "$file") |" >> $GITHUB_STEP_SUMMARY echo "| 1 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_1.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 2 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_2.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 3 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_3.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 4 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_4.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 5 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_5.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 6 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_6.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 7 Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_7.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| 8+ Token (p50) | $(jq -r '.metrics.latency_by_tokens.bucket_8.p50 // "N/A"' "$file") ms |" >> $GITHUB_STEP_SUMMARY echo "| Throughput | $(jq -r '.metrics.throughput.avg_ms_per_query // "N/A"' "$file") ms/query |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY } DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then # Show per-dataset summaries for accurate comparison output_metrics_table "cranfield_metrics.json" "Cranfield" output_metrics_table "msmarco_metrics.json" "MS MARCO" output_metrics_table "wikipedia_metrics.json" "Wikipedia" elif [ -f benchmark_metrics.json ]; then output_metrics_table "benchmark_metrics.json" "$DATASET" fi # VACUUM benchmark summary if [ -f msmarco_vacuum_metrics.json ]; then echo "### VACUUM Benchmark Results" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "| Scenario | VACUUM Time | Index Size | Query Latency |" >> $GITHUB_STEP_SUMMARY echo "|----------|------------|------------|---------------|" >> $GITHUB_STEP_SUMMARY echo "| Baseline | - | $(jq -r 'if .metrics.vacuum.baseline_index_bytes != null then ((.metrics.vacuum.baseline_index_bytes / 1048576 | floor | tostring) + " MB") else "N/A" end' msmarco_vacuum_metrics.json) | $(jq -r '.metrics.vacuum.baseline_query_avg_ms // "N/A"' msmarco_vacuum_metrics.json) ms |" >> $GITHUB_STEP_SUMMARY echo "| Partial (concentrated delete) | $(jq -r '.metrics.vacuum.partial_vacuum_ms // "N/A"' msmarco_vacuum_metrics.json) ms | $(jq -r 'if .metrics.vacuum.partial_post_index_bytes != null then ((.metrics.vacuum.partial_post_index_bytes / 1048576 | floor | tostring) + " MB") else "N/A" end' msmarco_vacuum_metrics.json) | $(jq -r '.metrics.vacuum.partial_query_avg_ms // "N/A"' msmarco_vacuum_metrics.json) ms |" >> $GITHUB_STEP_SUMMARY echo "| Full (uniform delete) | $(jq -r '.metrics.vacuum.full_vacuum_ms // "N/A"' msmarco_vacuum_metrics.json) ms | $(jq -r 'if .metrics.vacuum.full_post_index_bytes != null then ((.metrics.vacuum.full_post_index_bytes / 1048576 | floor | tostring) + " MB") else "N/A" end' msmarco_vacuum_metrics.json) | $(jq -r '.metrics.vacuum.full_query_avg_ms // "N/A"' msmarco_vacuum_metrics.json) ms |" >> $GITHUB_STEP_SUMMARY echo "| Full (uniform update) | $(jq -r '.metrics.vacuum.update_vacuum_ms // "N/A"' msmarco_vacuum_metrics.json) ms | $(jq -r 'if .metrics.vacuum.update_post_index_bytes != null then ((.metrics.vacuum.update_post_index_bytes / 1048576 | floor | tostring) + " MB") else "N/A" end' msmarco_vacuum_metrics.json) | $(jq -r '.metrics.vacuum.update_query_avg_ms // "N/A"' msmarco_vacuum_metrics.json) ms |" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY fi - name: Format metrics for benchmark tracking if: always() id: format_metrics run: | chmod +x benchmarks/runner/format_for_action.sh DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then # Format each dataset separately for name in cranfield msmarco wikipedia; do if [ -f "${name}_metrics.json" ]; then ./benchmarks/runner/format_for_action.sh \ "${name}_metrics.json" "${name}_action.json" fi done echo "is_all=true" >> $GITHUB_OUTPUT else if [ -f benchmark_metrics.json ]; then ./benchmarks/runner/format_for_action.sh \ benchmark_metrics.json benchmark_action.json echo "dataset=$DATASET" >> $GITHUB_OUTPUT fi echo "is_all=false" >> $GITHUB_OUTPUT fi # Format VACUUM metrics if available if [ -f msmarco_vacuum_metrics.json ]; then ./benchmarks/runner/format_for_action.sh \ msmarco_vacuum_metrics.json msmarco_vacuum_action.json fi - name: Publish Cranfield benchmark if: always() && hashFiles('cranfield_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'cranfield Benchmarks' tool: 'customSmallerIsBetter' output-file-path: cranfield_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO benchmark if: always() && hashFiles('msmarco_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO VACUUM benchmark if: always() && hashFiles('msmarco_vacuum_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco_vacuum Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_vacuum_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '200%' comment-on-alert: true fail-on-alert: false - name: Publish Wikipedia benchmark if: always() && hashFiles('wikipedia_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'wikipedia Benchmarks' tool: 'customSmallerIsBetter' output-file-path: wikipedia_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish single dataset benchmark if: always() && steps.format_metrics.outputs.is_all == 'false' && hashFiles('benchmark_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: '${{ steps.format_metrics.outputs.dataset }} Benchmarks' tool: 'customSmallerIsBetter' output-file-path: benchmark_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Update benchmark dashboard if: always() && github.event.inputs.dry_run != 'true' run: | # Save the new pages before switching branches cp benchmarks/gh-pages/index.html /tmp/new_index.html cp benchmarks/gh-pages/methodology.html /tmp/methodology.html 2>/dev/null || true cp benchmarks/gh-pages/comparison.html /tmp/comparison.html 2>/dev/null || true # Update pages and branch info on gh-pages branch git config user.name "github-actions[bot]" git config user.email "github-actions[bot]@users.noreply.github.com" git fetch origin gh-pages git checkout gh-pages cp /tmp/new_index.html benchmarks/index.html [ -f /tmp/methodology.html ] && cp /tmp/methodology.html benchmarks/methodology.html [ -f /tmp/comparison.html ] && cp /tmp/comparison.html benchmarks/comparison.html # Update branch_info.js with this commit's branch BRANCH_FILE="benchmarks/branch_info.js" SHORT_SHA=$(echo "${{ github.sha }}" | cut -c1-7) BRANCH="${{ github.ref_name }}" # Create file if it doesn't exist if [ ! -f "$BRANCH_FILE" ]; then echo "window.BRANCH_INFO = {};" > "$BRANCH_FILE" fi # Add this commit's branch info (keep last 500 entries) node -e " const fs = require('fs'); const file = '$BRANCH_FILE'; let content = fs.readFileSync(file, 'utf8'); // Extract existing object const match = content.match(/window\.BRANCH_INFO\s*=\s*(\{[\s\S]*\});?/); let info = match ? JSON.parse(match[1]) : {}; // Add new entry info['${{ github.sha }}'] = '$BRANCH'; // Keep only last 500 entries const keys = Object.keys(info); if (keys.length > 500) { const toRemove = keys.slice(0, keys.length - 500); toRemove.forEach(k => delete info[k]); } fs.writeFileSync(file, 'window.BRANCH_INFO = ' + JSON.stringify(info, null, 2) + ';\\n'); " git add benchmarks/index.html benchmarks/methodology.html benchmarks/comparison.html "$BRANCH_FILE" 2>/dev/null || git add benchmarks/index.html "$BRANCH_FILE" if ! git diff --cached --quiet; then git commit -m "Update benchmark dashboard for ${SHORT_SHA} on ${BRANCH}" git push origin gh-pages fi # Return to original branch git checkout - - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true rm -rf tmp_bench # Insert benchmark (single-txn + concurrent pgbench inserts) insert-benchmark: runs-on: ubuntu-latest timeout-minutes: 360 if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.benchmark_suite == 'all' || github.event.inputs.benchmark_suite == 'insert' || github.event.inputs.benchmark_suite == '')) steps: - uses: actions/checkout@v4 - name: Free disk space run: | echo "Initial disk space:" df -h / sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo rm -rf /usr/share/swift sudo rm -rf /usr/local/share/powershell sudo rm -rf /usr/local/.ghcup echo "After cleanup:" df -h / - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc gdb pip install 'wikiextractor==3.0.6' # PostgreSQL wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Build extension (release mode) run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean CFLAGS="-O3 -DNDEBUG" make sudo make install - name: Configure PostgreSQL for benchmarks run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" LIB_NAME="pg_textsearch" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB effective_cache_size = 4GB maintenance_work_mem = 512MB work_mem = 64MB max_connections = 20 checkpoint_completion_target = 0.9 wal_buffers = 16MB random_page_cost = 1.1 effective_io_concurrency = 200 max_parallel_maintenance_workers = $NCPUS max_parallel_workers_per_gather = $NCPUS max_parallel_workers = $NCPUS pg_textsearch.segments_per_level = 16 shared_preload_libraries = '${LIB_NAME}' EOF if ! pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w; then echo "PostgreSQL failed to start. Log:" cat tmp_bench/postgres.log exit 1 fi export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_textsearch" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Determine datasets to run id: datasets run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "dataset=${{ github.event.inputs.dataset }}" >> $GITHUB_OUTPUT echo "wiki_size=${{ github.event.inputs.wikipedia_size }}" >> $GITHUB_OUTPUT else echo "dataset=all" >> $GITHUB_OUTPUT echo "wiki_size=100K" >> $GITHUB_OUTPUT fi - name: Download MS MARCO dataset if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh full - name: Download Wikipedia dataset if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/wikipedia chmod +x download.sh ./download.sh ${{ steps.datasets.outputs.wiki_size }} # Phase 1: Single-transaction inserts # # NOTE: Each long-running step has a step-level `timeout-minutes` so a # hang fails the step rather than cancelling the whole job. Cancellation # skips `if: always()` cleanup/upload steps, so we lose artifacts; a # step failure does not. See PR adding these timeouts for context. - name: Run Cranfield insert benchmark if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 10 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== Cranfield Insert Benchmark ===" | tee -a insert_results.txt echo "Start: $(date)" | tee -a insert_results.txt cd benchmarks/sql/cranfield time psql -f insert/load.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" time psql -f 02-queries.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" time psql -f insert/size_report.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/insert_results.txt" - name: Run MS MARCO insert benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 45 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== MS MARCO Insert Benchmark ===" | tee -a insert_results.txt echo "Start: $(date)" | tee -a insert_results.txt time psql -f benchmarks/datasets/msmarco/insert/load.sql 2>&1 | \ tee -a insert_results.txt time psql -f benchmarks/datasets/msmarco/queries.sql 2>&1 | \ tee -a insert_results.txt time psql -f benchmarks/datasets/msmarco/insert/size_report.sql 2>&1 | \ tee -a insert_results.txt echo "End: $(date)" | tee -a insert_results.txt - name: Validate MS MARCO insert results if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # pipefail: catch psql ON_ERROR_STOP exits that would otherwise be # masked by the `| tee` chain and let the step silently pass. set -o pipefail echo "=== MS MARCO Insert Validation ===" | tee -a insert_results.txt PG_MAJOR=$(pg_config --version | sed 's/PostgreSQL //' | cut -d. -f1) GT_FILE="benchmarks/datasets/msmarco/ground_truth_pg${PG_MAJOR}.tsv" if [ -f "$GT_FILE" ]; then echo "Using ground truth for PG${PG_MAJOR}" | tee -a insert_results.txt cp "$GT_FILE" benchmarks/datasets/msmarco/ground_truth.tsv else echo "ERROR: No ground truth file for PG${PG_MAJOR} ($GT_FILE not found)" | tee -a insert_results.txt exit 1 fi psql -f benchmarks/datasets/msmarco/validate_queries.sql 2>&1 | \ tee msmarco_insert_validation.log | tee -a insert_results.txt if grep -q "VALIDATION FAILED" msmarco_insert_validation.log; then echo "ERROR: MS MARCO insert validation FAILED!" | tee -a insert_results.txt exit 1 fi # Require an explicit PASSED marker -- absent it the script errored # out mid-run (e.g. SQL error in validate_queries.sql) and grep for # FAILED above would silently miss the regression. if ! grep -q "VALIDATION PASSED" msmarco_insert_validation.log; then echo "ERROR: MS MARCO insert validation did not complete (no PASSED or FAILED marker found)" | tee -a insert_results.txt exit 1 fi echo "MS MARCO insert validation PASSED" | tee -a insert_results.txt - name: Run Wikipedia insert benchmark if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 30 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== Wikipedia Insert Benchmark ===" | tee -a insert_results.txt echo "Start: $(date)" | tee -a insert_results.txt time psql -f benchmarks/datasets/wikipedia/insert/load.sql 2>&1 | \ tee -a insert_results.txt time psql -f benchmarks/datasets/wikipedia/queries.sql 2>&1 | \ tee -a insert_results.txt time psql -f benchmarks/datasets/wikipedia/insert/size_report.sql 2>&1 | \ tee -a insert_results.txt echo "End: $(date)" | tee -a insert_results.txt - name: Validate Wikipedia insert results if: | (steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all') && steps.datasets.outputs.wiki_size == '100K' timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # pipefail: catch psql ON_ERROR_STOP exits that would otherwise be # masked by the `| tee` chain and let the step silently pass. set -o pipefail echo "=== Wikipedia Insert Validation ===" | tee -a insert_results.txt psql -f benchmarks/datasets/wikipedia/validate_queries.sql 2>&1 | \ tee wikipedia_insert_validation.log | tee -a insert_results.txt if grep -q "VALIDATION FAILED" wikipedia_insert_validation.log; then echo "ERROR: Wikipedia insert validation FAILED!" | tee -a insert_results.txt exit 1 fi # Require an explicit PASSED marker -- absent it the script errored # out mid-run (e.g. SQL error in validate_queries.sql) and grep for # FAILED above would silently miss the regression. if ! grep -q "VALIDATION PASSED" wikipedia_insert_validation.log; then echo "ERROR: Wikipedia insert validation did not complete (no PASSED or FAILED marker found)" | tee -a insert_results.txt exit 1 fi echo "Wikipedia insert validation PASSED" | tee -a insert_results.txt # Phase 2: Concurrent inserts (pgbench) - name: Run Cranfield concurrent insert if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== Cranfield Concurrent Insert Benchmark ===" | tee -a insert_results.txt echo "Start: $(date)" | tee -a insert_results.txt cd benchmarks/sql/cranfield psql -f insert/concurrent-setup.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" ROWS=$(psql -tA -c "SELECT count(*) FROM cranfield_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f insert/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/insert_results.txt" - name: Run Cranfield concurrent insert queries if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test cd benchmarks/sql/cranfield time psql -f 02-queries.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/insert_results.txt" - name: Run MS MARCO concurrent insert if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 60 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== MS MARCO Concurrent Insert Benchmark ===" | tee -a insert_results.txt echo "Start: $(date)" | tee -a insert_results.txt psql -f benchmarks/datasets/msmarco/insert/concurrent-setup.sql 2>&1 | \ tee -a insert_results.txt ROWS=$(psql -tA -c "SELECT count(*) FROM msmarco_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f benchmarks/datasets/msmarco/insert/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a insert_results.txt END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | tee -a insert_results.txt echo "End: $(date)" | tee -a insert_results.txt # NOTE: This step has historically been the most common hang (bucket-7 # MS MARCO queries hang indefinitely on certain segment topologies, and # the BMW scan loop has no CHECK_FOR_INTERRUPTS so neither # statement_timeout nor pg_cancel_backend can stop it). The 30-minute # step timeout lets GitHub Actions kill the runner cleanly so the # diagnostic / upload steps below still run with `if: always()`. - name: Run MS MARCO concurrent insert queries if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 30 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test time psql -f benchmarks/datasets/msmarco/queries.sql 2>&1 | \ tee -a insert_results.txt - name: Validate MS MARCO concurrent results if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # pipefail: catch psql ON_ERROR_STOP exits that would otherwise be # masked by the `| tee` chain and let the step silently pass. set -o pipefail psql -f benchmarks/datasets/msmarco/validate_queries.sql 2>&1 | \ tee msmarco_concurrent_validation.log | tee -a insert_results.txt if grep -q "VALIDATION FAILED" msmarco_concurrent_validation.log; then echo "ERROR: MS MARCO concurrent validation FAILED!" | tee -a insert_results.txt exit 1 fi # Require an explicit PASSED marker -- absent it the script errored # out mid-run (e.g. SQL error in validate_queries.sql) and grep for # FAILED above would silently miss the regression. if ! grep -q "VALIDATION PASSED" msmarco_concurrent_validation.log; then echo "ERROR: MS MARCO concurrent validation did not complete (no PASSED or FAILED marker found)" | tee -a insert_results.txt exit 1 fi echo "MS MARCO concurrent validation PASSED" | tee -a insert_results.txt - name: Run Wikipedia concurrent insert if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 30 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== Wikipedia Concurrent Insert Benchmark ===" | tee -a insert_results.txt echo "Start: $(date)" | tee -a insert_results.txt psql -f benchmarks/datasets/wikipedia/insert/concurrent-setup.sql 2>&1 | \ tee -a insert_results.txt ROWS=$(psql -tA -c "SELECT count(*) FROM wikipedia_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f benchmarks/datasets/wikipedia/insert/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a insert_results.txt END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | tee -a insert_results.txt echo "End: $(date)" | tee -a insert_results.txt - name: Run Wikipedia concurrent insert queries if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' timeout-minutes: 30 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test time psql -f benchmarks/datasets/wikipedia/queries.sql 2>&1 | \ tee -a insert_results.txt - name: Validate Wikipedia concurrent results if: | (steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all') && steps.datasets.outputs.wiki_size == '100K' timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # pipefail: catch psql ON_ERROR_STOP exits that would otherwise be # masked by the `| tee` chain and let the step silently pass. set -o pipefail psql -f benchmarks/datasets/wikipedia/validate_queries.sql 2>&1 | \ tee wikipedia_concurrent_validation.log | tee -a insert_results.txt if grep -q "VALIDATION FAILED" wikipedia_concurrent_validation.log; then echo "ERROR: Wikipedia concurrent validation FAILED!" | tee -a insert_results.txt exit 1 fi # Require an explicit PASSED marker -- absent it the script errored # out mid-run (e.g. SQL error in validate_queries.sql) and grep for # FAILED above would silently miss the regression. if ! grep -q "VALIDATION PASSED" wikipedia_concurrent_validation.log; then echo "ERROR: Wikipedia concurrent validation did not complete (no PASSED or FAILED marker found)" | tee -a insert_results.txt exit 1 fi echo "Wikipedia concurrent validation PASSED" | tee -a insert_results.txt # If anything above failed (e.g. a step-level timeout), capture # diagnostics so we can root-cause hangs from the artifact alone. # Runs only on failure to avoid noise on successful runs. # # `timeout-minutes` here is critical: without it, a hanging gdb attach # or unresponsive psql could consume the remaining job budget, the # job would be CANCELLED (not failed), and `if: always()` upload # would be skipped — defeating the entire point of this PR. - name: Capture diagnostics on failure if: failure() timeout-minutes: 5 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test # Bound each sub-tool individually so one stuck call can't burn # the whole 5-minute budget. export PGCONNECT_TIMEOUT=5 echo "=== Failure diagnostics @ $(date -Iseconds) ===" \ | tee diagnostics.log # pg_stat_activity: which queries are running, for how long, are # they waiting on a lock? `datname IS NULL` includes background # workers (checkpointer / walwriter / autovacuum launcher / # logical replication launcher / parallel workers) which have # NULL datname; we deliberately scope to the bench_test database # plus those bgworkers rather than dumping every database. timeout 30 psql -P pager=off \ -v ON_ERROR_STOP=0 \ -c "SET statement_timeout = '10s'; SELECT pid, backend_type, state, wait_event_type, wait_event, xact_start, query_start, now() - query_start AS query_runtime, left(query, 200) AS query FROM pg_stat_activity WHERE datname = current_database() OR datname IS NULL ORDER BY query_start NULLS LAST;" \ >> diagnostics.log 2>&1 || \ echo "(pg_stat_activity dump failed/timed out)" \ >> diagnostics.log # pg_locks: blockers / waiters timeout 30 psql -P pager=off \ -v ON_ERROR_STOP=0 \ -c "SET statement_timeout = '10s'; SELECT locktype, relation::regclass, mode, granted, pid, fastpath FROM pg_locks WHERE NOT granted OR locktype IN ('relation', 'extend', 'page', 'tuple') ORDER BY granted, pid;" \ >> diagnostics.log 2>&1 || \ echo "(pg_locks dump failed/timed out)" >> diagnostics.log # postgres.log tail (full file is uploaded too, but tail makes # the GitHub UI summary readable without downloading the # artifact). echo "" >> diagnostics.log echo "=== tmp_bench/postgres.log (last 500 lines) ===" \ >> diagnostics.log if [ -f tmp_bench/postgres.log ]; then tail -n 500 tmp_bench/postgres.log >> diagnostics.log else echo "(no postgres.log found)" >> diagnostics.log fi # Stack traces from any postgres processes still running. Useful # when the BMW scan loop is hung in C code (no # CHECK_FOR_INTERRUPTS) and statement_timeout cannot abort it. # `pgrep -x postgres` matches by exact process name, so it # captures the postmaster, all backends, and aux processes # (walwriter / checkpointer / bgwriter / autovacuum / parallel # workers) — not just user backends whose proc title contains # the database name. if command -v gdb >/dev/null && command -v pgrep >/dev/null; then echo "" >> diagnostics.log echo "=== Backend stack traces ===" >> diagnostics.log for pid in $(pgrep -x postgres || true); do echo "--- pid $pid ---" >> diagnostics.log # Bound each gdb attach so a stuck process can't stall # the whole diagnostic capture. timeout 30 sudo gdb -p "$pid" -batch \ -ex "set pagination off" \ -ex "thread apply all bt 30" 2>/dev/null \ >> diagnostics.log || \ echo "(gdb attach failed or timed out)" \ >> diagnostics.log done fi echo "Diagnostics captured: $(wc -l < diagnostics.log) lines" - name: Extract insert metrics if: always() run: | chmod +x benchmarks/runner/extract_metrics.sh DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then ./benchmarks/runner/extract_metrics.sh insert_results.txt \ cranfield_insert_metrics.json "cranfield_insert" "Cranfield Insert" || true ./benchmarks/runner/extract_metrics.sh insert_results.txt \ msmarco_insert_metrics.json "msmarco_insert" "MS MARCO Insert" || true ./benchmarks/runner/extract_metrics.sh insert_results.txt \ wikipedia_insert_metrics.json "wikipedia_insert" "Wikipedia Insert" || true ./benchmarks/runner/extract_metrics.sh insert_results.txt \ cranfield_concurrent_metrics.json "cranfield_concurrent" "Cranfield Concurrent" || true ./benchmarks/runner/extract_metrics.sh insert_results.txt \ msmarco_concurrent_metrics.json "msmarco_concurrent" "MS MARCO Concurrent" || true ./benchmarks/runner/extract_metrics.sh insert_results.txt \ wikipedia_concurrent_metrics.json "wikipedia_concurrent" "Wikipedia Concurrent" || true else # Map dataset slug to display name for section matching case "$DATASET" in cranfield) DD="Cranfield" ;; msmarco) DD="MS MARCO" ;; wikipedia) DD="Wikipedia" ;; *) DD="$DATASET" ;; esac ./benchmarks/runner/extract_metrics.sh insert_results.txt \ ${DATASET}_insert_metrics.json "${DATASET}_insert" "${DD} Insert" || true ./benchmarks/runner/extract_metrics.sh insert_results.txt \ ${DATASET}_concurrent_metrics.json "${DATASET}_concurrent" "${DD} Concurrent" || true fi # Format runs BEFORE Upload so generated *_action.json files are # included in the artifact. (Was previously after Upload, which meant # the *_action.json files were never archived.) - name: Format insert metrics if: always() run: | chmod +x benchmarks/runner/format_for_action.sh for f in *_insert_metrics.json *_concurrent_metrics.json; do if [ -f "$f" ]; then base="${f%_metrics.json}" ./benchmarks/runner/format_for_action.sh "$f" "${base}_action.json" fi done # `if: always()` is critical: when a step above times out (or fails) # we still need postgres.log + diagnostics.log to root-cause the hang. # Without it, GitHub Actions skips this step and we lose all evidence. - name: Upload insert benchmark results if: always() uses: actions/upload-artifact@v4 with: name: insert-benchmark-results-${{ github.run_id }} path: | insert_results.txt *_insert_metrics.json *_concurrent_metrics.json *_action.json *_validation.log diagnostics.log tmp_bench/postgres.log retention-days: 90 if-no-files-found: warn - name: Publish Cranfield insert benchmark if: always() && hashFiles('cranfield_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'cranfield_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: cranfield_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO insert benchmark if: always() && hashFiles('msmarco_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Wikipedia insert benchmark if: always() && hashFiles('wikipedia_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'wikipedia_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: wikipedia_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Cranfield concurrent benchmark if: always() && hashFiles('cranfield_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'cranfield_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: cranfield_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO concurrent benchmark if: always() && hashFiles('msmarco_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Wikipedia concurrent benchmark if: always() && hashFiles('wikipedia_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'wikipedia_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: wikipedia_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true rm -rf tmp_bench # GIN+tsvector baseline benchmark (built-in Postgres, no extensions) baseline-benchmark: runs-on: ubuntu-latest timeout-minutes: 360 if: github.event_name == 'schedule' || (github.event_name == 'workflow_dispatch' && (github.event.inputs.benchmark_suite == 'all' || github.event.inputs.benchmark_suite == 'baseline' || github.event.inputs.benchmark_suite == '')) steps: - uses: actions/checkout@v4 - name: Free disk space run: | echo "Initial disk space:" df -h / sudo rm -rf /usr/share/dotnet sudo rm -rf /usr/local/lib/android sudo rm -rf /opt/ghc sudo rm -rf /opt/hostedtoolcache/CodeQL sudo rm -rf /usr/share/swift sudo rm -rf /usr/local/share/powershell sudo rm -rf /usr/local/.ghcup echo "After cleanup:" df -h / - name: Set up Python 3.10 uses: actions/setup-python@v5 with: python-version: '3.10' - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc pip install 'wikiextractor==3.0.6' # PostgreSQL wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Configure PostgreSQL for benchmarks run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust NCPUS=$(nproc) cat >> tmp_bench/data/postgresql.conf << EOF unix_socket_directories = '$PWD/tmp_bench/socket' shared_buffers = 1GB effective_cache_size = 4GB maintenance_work_mem = 512MB work_mem = 64MB max_connections = 20 checkpoint_completion_target = 0.9 wal_buffers = 16MB random_page_cost = 1.1 effective_io_concurrency = 200 max_parallel_maintenance_workers = $NCPUS max_parallel_workers_per_gather = $NCPUS max_parallel_workers = $NCPUS EOF if ! pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w; then echo "PostgreSQL failed to start. Log:" cat tmp_bench/postgres.log exit 1 fi export PGHOST="$PWD/tmp_bench/socket" createdb bench_test echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Determine datasets to run id: datasets run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "dataset=${{ github.event.inputs.dataset }}" >> $GITHUB_OUTPUT echo "wiki_size=${{ github.event.inputs.wikipedia_size }}" >> $GITHUB_OUTPUT else echo "dataset=all" >> $GITHUB_OUTPUT echo "wiki_size=100K" >> $GITHUB_OUTPUT fi - name: Download MS MARCO dataset if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/msmarco chmod +x download.sh ./download.sh full - name: Download Wikipedia dataset if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | cd benchmarks/datasets/wikipedia chmod +x download.sh ./download.sh ${{ steps.datasets.outputs.wiki_size }} # Phase 1: GIN CREATE INDEX on populated table - name: Run Cranfield GIN benchmark if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== Cranfield GIN Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt cd benchmarks/sql/cranfield time psql -f gin/load.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/gin_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/gin_results.txt" - name: Run MS MARCO GIN benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== MS MARCO GIN Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt time psql -f benchmarks/datasets/msmarco/gin/load.sql 2>&1 | \ tee -a gin_results.txt echo "End: $(date)" | tee -a gin_results.txt - name: Run Wikipedia GIN benchmark if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== Wikipedia GIN Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt time psql -f benchmarks/datasets/wikipedia/gin/load.sql 2>&1 | \ tee -a gin_results.txt echo "End: $(date)" | tee -a gin_results.txt # Phase 2: GIN single-txn insert - name: Run Cranfield GIN insert benchmark if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== Cranfield GIN Insert Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt cd benchmarks/sql/cranfield time psql -f gin/insert-load.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/gin_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/gin_results.txt" - name: Run MS MARCO GIN insert benchmark if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== MS MARCO GIN Insert Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt time psql -f benchmarks/datasets/msmarco/gin/insert-load.sql 2>&1 | \ tee -a gin_results.txt echo "End: $(date)" | tee -a gin_results.txt - name: Run Wikipedia GIN insert benchmark if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== Wikipedia GIN Insert Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt time psql -f benchmarks/datasets/wikipedia/gin/insert-load.sql 2>&1 | \ tee -a gin_results.txt echo "End: $(date)" | tee -a gin_results.txt # Phase 3: GIN concurrent insert - name: Run Cranfield GIN concurrent insert if: steps.datasets.outputs.dataset == 'cranfield' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test echo "=== Cranfield GIN Concurrent Insert Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt cd benchmarks/sql/cranfield psql -f gin/concurrent-setup.sql 2>&1 | \ tee -a "$GITHUB_WORKSPACE/gin_results.txt" ROWS=$(psql -tA -c "SELECT count(*) FROM cranfield_gin_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f gin/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a "$GITHUB_WORKSPACE/gin_results.txt" END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | \ tee -a "$GITHUB_WORKSPACE/gin_results.txt" echo "End: $(date)" | tee -a "$GITHUB_WORKSPACE/gin_results.txt" - name: Run MS MARCO GIN concurrent insert if: steps.datasets.outputs.dataset == 'msmarco' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/msmarco/data" echo "=== MS MARCO GIN Concurrent Insert Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt psql -f benchmarks/datasets/msmarco/gin/concurrent-setup.sql 2>&1 | \ tee -a gin_results.txt ROWS=$(psql -tA -c "SELECT count(*) FROM msmarco_gin_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f benchmarks/datasets/msmarco/gin/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a gin_results.txt END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | tee -a gin_results.txt echo "End: $(date)" | tee -a gin_results.txt - name: Run Wikipedia GIN concurrent insert if: steps.datasets.outputs.dataset == 'wikipedia' || steps.datasets.outputs.dataset == 'all' run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test export DATA_DIR="$PWD/benchmarks/datasets/wikipedia/data" echo "=== Wikipedia GIN Concurrent Insert Benchmark ===" | tee -a gin_results.txt echo "Start: $(date)" | tee -a gin_results.txt psql -f benchmarks/datasets/wikipedia/gin/concurrent-setup.sql 2>&1 | \ tee -a gin_results.txt ROWS=$(psql -tA -c "SELECT count(*) FROM wikipedia_gin_staging") CLIENTS=$(nproc) TXN=$((ROWS / CLIENTS)) echo "Inserting $ROWS rows with $CLIENTS clients ($TXN txns each)" START=$(date +%s.%N) pgbench -f benchmarks/datasets/wikipedia/gin/pgbench-insert.sql \ -c $CLIENTS -j $CLIENTS -t $TXN -n bench_test 2>&1 | \ tee -a gin_results.txt END=$(date +%s.%N) MS=$(echo "($END - $START) * 1000" | bc) echo "CONCURRENT_INSERT_TIME: ${MS}ms" | tee -a gin_results.txt echo "End: $(date)" | tee -a gin_results.txt - name: Extract GIN metrics if: always() run: | chmod +x benchmarks/runner/extract_metrics.sh DATASET="${{ steps.datasets.outputs.dataset }}" if [ "$DATASET" = "all" ]; then ./benchmarks/runner/extract_metrics.sh gin_results.txt \ cranfield_gin_metrics.json "cranfield_gin" "Cranfield GIN" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ msmarco_gin_metrics.json "msmarco_gin" "MS MARCO GIN" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ wikipedia_gin_metrics.json "wikipedia_gin" "Wikipedia GIN" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ cranfield_gin_insert_metrics.json "cranfield_gin_insert" "Cranfield GIN Insert" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ msmarco_gin_insert_metrics.json "msmarco_gin_insert" "MS MARCO GIN Insert" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ wikipedia_gin_insert_metrics.json "wikipedia_gin_insert" "Wikipedia GIN Insert" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ cranfield_gin_concurrent_metrics.json "cranfield_gin_concurrent" "Cranfield GIN Concurrent" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ msmarco_gin_concurrent_metrics.json "msmarco_gin_concurrent" "MS MARCO GIN Concurrent" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ wikipedia_gin_concurrent_metrics.json "wikipedia_gin_concurrent" "Wikipedia GIN Concurrent" || true else # Map dataset slug to display name for section matching case "$DATASET" in cranfield) DD="Cranfield" ;; msmarco) DD="MS MARCO" ;; wikipedia) DD="Wikipedia" ;; *) DD="$DATASET" ;; esac ./benchmarks/runner/extract_metrics.sh gin_results.txt \ ${DATASET}_gin_metrics.json "${DATASET}_gin" "${DD} GIN" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ ${DATASET}_gin_insert_metrics.json "${DATASET}_gin_insert" "${DD} GIN Insert" || true ./benchmarks/runner/extract_metrics.sh gin_results.txt \ ${DATASET}_gin_concurrent_metrics.json "${DATASET}_gin_concurrent" "${DD} GIN Concurrent" || true fi - name: Upload GIN benchmark results uses: actions/upload-artifact@v4 with: name: baseline-benchmark-results-${{ github.run_id }} path: | gin_results.txt *_gin_metrics.json *_gin_insert_metrics.json *_gin_concurrent_metrics.json *_action.json tmp_bench/postgres.log retention-days: 90 - name: Format GIN metrics for benchmark tracking if: always() run: | chmod +x benchmarks/runner/format_for_action.sh for f in *_gin_metrics.json *_gin_insert_metrics.json *_gin_concurrent_metrics.json; do if [ -f "$f" ]; then base="${f%_metrics.json}" ./benchmarks/runner/format_for_action.sh "$f" "${base}_action.json" fi done - name: Publish Cranfield GIN benchmark if: always() && hashFiles('cranfield_gin_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'cranfield_gin Benchmarks' tool: 'customSmallerIsBetter' output-file-path: cranfield_gin_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO GIN benchmark if: always() && hashFiles('msmarco_gin_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco_gin Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_gin_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Wikipedia GIN benchmark if: always() && hashFiles('wikipedia_gin_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'wikipedia_gin Benchmarks' tool: 'customSmallerIsBetter' output-file-path: wikipedia_gin_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Cranfield GIN insert benchmark if: always() && hashFiles('cranfield_gin_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'cranfield_gin_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: cranfield_gin_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO GIN insert benchmark if: always() && hashFiles('msmarco_gin_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco_gin_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_gin_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Wikipedia GIN insert benchmark if: always() && hashFiles('wikipedia_gin_insert_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'wikipedia_gin_insert Benchmarks' tool: 'customSmallerIsBetter' output-file-path: wikipedia_gin_insert_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Cranfield GIN concurrent benchmark if: always() && hashFiles('cranfield_gin_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'cranfield_gin_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: cranfield_gin_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish MS MARCO GIN concurrent benchmark if: always() && hashFiles('msmarco_gin_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'msmarco_gin_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: msmarco_gin_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Publish Wikipedia GIN concurrent benchmark if: always() && hashFiles('wikipedia_gin_concurrent_action.json') != '' && github.event.inputs.dry_run != 'true' uses: benchmark-action/github-action-benchmark@v1 with: name: 'wikipedia_gin_concurrent Benchmarks' tool: 'customSmallerIsBetter' output-file-path: wikipedia_gin_concurrent_action.json github-token: ${{ secrets.GITHUB_TOKEN }} auto-push: true gh-pages-branch: gh-pages benchmark-data-dir-path: benchmarks alert-threshold: '150%' comment-on-alert: true fail-on-alert: false - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true rm -rf tmp_bench # Compression validation job - compares compressed vs uncompressed results validate-compression: runs-on: ubuntu-latest timeout-minutes: 60 if: github.event_name == 'workflow_dispatch' && github.event.inputs.validate_compression == 'true' steps: - uses: actions/checkout@v4 - name: Free disk space run: | sudo rm -rf /usr/share/dotnet /usr/local/lib/android /opt/ghc df -h / - name: Install PostgreSQL 17 run: | sudo apt-get update wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Build extension run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make -j$(nproc) sudo make install - name: Setup database run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" LIB_NAME="pg_textsearch" mkdir -p tmp_validate/data tmp_validate/socket initdb -D tmp_validate/data echo "shared_preload_libraries = '${LIB_NAME}'" >> tmp_validate/data/postgresql.conf echo "shared_buffers = '1GB'" >> tmp_validate/data/postgresql.conf echo "work_mem = '256MB'" >> tmp_validate/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_validate/socket'" >> tmp_validate/data/postgresql.conf pg_ctl -D tmp_validate/data -l tmp_validate/logfile start -w export PGHOST="$PWD/tmp_validate/socket" createdb validation echo "PGHOST=$PWD/tmp_validate/socket" >> $GITHUB_ENV - name: Download MS MARCO dataset run: | # Compression validation uses a 100K subset — the job only needs # enough documents to exercise the compression code paths, not # a realistic throughput benchmark. echo "Downloading MS MARCO 100K subset..." mkdir -p benchmarks/datasets/msmarco cd benchmarks/datasets/msmarco wget -q https://msmarco.z22.web.core.windows.net/msmarcoranking/collection.tar.gz tar -xzf collection.tar.gz head -100000 collection.tsv > collection_subset.tsv mv collection_subset.tsv collection.tsv rm collection.tar.gz wc -l collection.tsv - name: Run compression validation run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" echo "=== Phase 1: Build COMPRESSED index and capture results ===" psql -d validation << 'EOSQL' CREATE EXTENSION pg_textsearch; -- Create table CREATE TABLE passages ( id INT PRIMARY KEY, passage_text TEXT ); -- Load data \copy passages(id, passage_text) FROM 'benchmarks/datasets/msmarco/collection.tsv' SELECT count(*) as doc_count FROM passages; -- Build compressed index SET pg_textsearch.compress_segments = on; CREATE INDEX passages_bm25_idx ON passages USING bm25(passage_text) WITH (text_config = 'english'); SELECT bm25_spill_index('passages_bm25_idx'); -- Capture compressed results: top-50 per query via the BM25 -- index scan. Using ORDER BY ... LIMIT (rather than -- WHERE score < 0) exercises the compression-aware segment -- decoders — which is the code path this job exists to -- validate. CREATE TABLE compressed_results AS SELECT q AS query, id, score FROM (VALUES ('what is'), ('how to'), ('search engine'), ('machine learning'), ('database query'), ('information retrieval'), ('text search'), ('full text'), ('ranking algorithm'), ('document search') ) AS queries(q) CROSS JOIN LATERAL ( SELECT id, passage_text <@> to_bm25query(queries.q, 'passages_bm25_idx') AS score FROM passages ORDER BY passage_text <@> to_bm25query(queries.q, 'passages_bm25_idx') LIMIT 50 ) hits; SELECT count(*) as compressed_count FROM compressed_results; DROP INDEX passages_bm25_idx; EOSQL echo "=== Phase 2: Build UNCOMPRESSED index and capture results ===" psql -d validation << 'EOSQL' -- Build uncompressed index SET pg_textsearch.compress_segments = off; CREATE INDEX passages_bm25_idx ON passages USING bm25(passage_text) WITH (text_config = 'english'); SELECT bm25_spill_index('passages_bm25_idx'); -- Capture uncompressed results (same queries, same pattern) CREATE TABLE uncompressed_results AS SELECT q AS query, id, score FROM (VALUES ('what is'), ('how to'), ('search engine'), ('machine learning'), ('database query'), ('information retrieval'), ('text search'), ('full text'), ('ranking algorithm'), ('document search') ) AS queries(q) CROSS JOIN LATERAL ( SELECT id, passage_text <@> to_bm25query(queries.q, 'passages_bm25_idx') AS score FROM passages ORDER BY passage_text <@> to_bm25query(queries.q, 'passages_bm25_idx') LIMIT 50 ) hits; SELECT count(*) as uncompressed_count FROM uncompressed_results; EOSQL echo "=== Phase 3: Compare results ===" psql -d validation << 'EOSQL' -- Find any differences SELECT 'DIFFERENCES FOUND:' as status; SELECT c.query, c.id as compressed_id, u.id as uncompressed_id, c.score as compressed_score, u.score as uncompressed_score, abs(c.score - u.score) as score_diff FROM compressed_results c FULL OUTER JOIN uncompressed_results u ON c.query = u.query AND c.id = u.id WHERE c.id IS NULL OR u.id IS NULL OR abs(c.score - u.score) > 0.0001 LIMIT 20; -- Summary SELECT 'VALIDATION SUMMARY:' as status; SELECT (SELECT count(*) FROM compressed_results) as compressed_count, (SELECT count(*) FROM uncompressed_results) as uncompressed_count, (SELECT count(*) FROM compressed_results c JOIN uncompressed_results u ON c.query = u.query AND c.id = u.id WHERE abs(c.score - u.score) < 0.0001) as matching_count; -- Final verdict SELECT CASE WHEN (SELECT count(*) FROM compressed_results) = (SELECT count(*) FROM uncompressed_results) AND (SELECT count(*) FROM compressed_results c JOIN uncompressed_results u ON c.query = u.query AND c.id = u.id WHERE abs(c.score - u.score) < 0.0001) = (SELECT count(*) FROM compressed_results) THEN 'VALIDATION PASSED: Compressed and uncompressed results are identical' ELSE 'VALIDATION FAILED: Results differ between compressed and uncompressed' END as result; EOSQL - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_validate/data || true rm -rf tmp_validate timescale-pg_textsearch-b63fe58/.github/workflows/chinese-ci.yml000066400000000000000000000106611525762047700251140ustar00rootroot00000000000000name: Chinese CI # End-to-end Chinese full-text search test. pg_textsearch tokenizes through # the index's text_config, so this job pairs it with the third-party # zhparser extension (an SCWS dictionary-based Chinese word segmenter) and # runs the `chinese` regression test, which is intentionally kept out of the # default REGRESS schedule. on: push: branches: [ main, 'release/*' ] paths: - 'src/**' - 'sql/**' - 'Makefile' - 'pg_textsearch.control' - 'test/sql/chinese.sql' - 'test/expected/chinese.out' - '.github/workflows/chinese-ci.yml' pull_request: branches: [ main, 'release/*' ] paths: - 'src/**' - 'sql/**' - 'Makefile' - 'pg_textsearch.control' - 'test/sql/chinese.sql' - 'test/expected/chinese.out' - '.github/workflows/chinese-ci.yml' workflow_dispatch: permissions: contents: read env: # Pinned third-party commit SHAs for reproducible CI. Pinning zhparser to # its exact v2.3 commit pins the bundled dict.utf8.xdb — and thus the # segmentation dictionary and the expected test output. A commit SHA (not # a tag, which can be re-pointed upstream) is what makes this immutable. SCWS_REF: a04ef5e655213eb6b07fb57761d0eac72c281ad4 # SCWS 1.2.3 ZHPARSER_REF: dd292fd591edbcb7ebee79d3c3cdc969e9cddced # zhparser v2.3 jobs: chinese: runs-on: ubuntu-latest strategy: fail-fast: false matrix: pg_version: [17, 18] steps: - uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} - name: Install build dependencies run: | # autoconf/automake/libtool bootstrap the SCWS build from its git tag. sudo apt-get install -y build-essential autoconf automake libtool - name: Build and install pg_textsearch run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make clean make sudo env "PATH=$PATH" make install - name: Build and install SCWS (pinned) run: | git clone https://github.com/hightman/scws.git /tmp/scws cd /tmp/scws git checkout "$SCWS_REF" autoreconf -fi ./configure --prefix=/usr/local make sudo make install sudo ldconfig - name: Build and install zhparser (pinned) run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" git clone https://github.com/amutu/zhparser.git /tmp/zhparser cd /tmp/zhparser git checkout "$ZHPARSER_REF" SCWS_HOME=/usr/local make sudo env "PATH=$PATH" SCWS_HOME=/usr/local make install - name: Run Chinese regression test timeout-minutes: 10 run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" rm -rf tmp_check_chinese mkdir -p tmp_check_chinese initdb -D tmp_check_chinese/data --auth-local=trust --auth-host=trust -E UTF8 echo "port = 55440" >> tmp_check_chinese/data/postgresql.conf echo "shared_preload_libraries = 'pg_textsearch'" >> tmp_check_chinese/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_check_chinese'" >> tmp_check_chinese/data/postgresql.conf if ! pg_ctl start -D tmp_check_chinese/data -l tmp_check_chinese/data/logfile -w; then echo "PostgreSQL failed to start. Log contents:" cat tmp_check_chinese/data/logfile exit 1 fi if ! make test-chinese PGHOST=$PWD/tmp_check_chinese PGPORT=55440; then echo "Chinese test failed. Diff:" cat test/regression.diffs 2>/dev/null || true echo "PostgreSQL server log (last 100 lines):" tail -100 tmp_check_chinese/data/logfile || true pg_ctl stop -D tmp_check_chinese/data -l tmp_check_chinese/data/logfile || true rm -rf tmp_check_chinese || true exit 1 fi pg_ctl stop -D tmp_check_chinese/data -l tmp_check_chinese/data/logfile rm -rf tmp_check_chinese timescale-pg_textsearch-b63fe58/.github/workflows/ci.yml000066400000000000000000001546371525762047700235140ustar00rootroot00000000000000name: CI on: push: branches: [ main, 'release/*' ] paths: - 'src/**' - 'test/**' - 'sql/**' - 'Makefile' - 'pg_textsearch.control' - '.clang-format' - '.github/workflows/ci.yml' - '.github/scripts/build-postgres-injection.sh' - '.github/scripts/build-postgres-sanitizer.sh' - 'scripts/msvc-build.ps1' pull_request: branches: [ main, 'release/*' ] paths: - 'src/**' - 'test/**' - 'sql/**' - 'Makefile' - 'pg_textsearch.control' - '.clang-format' - '.github/workflows/ci.yml' - '.github/scripts/build-postgres-injection.sh' - '.github/scripts/build-postgres-sanitizer.sh' - 'scripts/msvc-build.ps1' workflow_dispatch: jobs: test: runs-on: ubuntu-latest strategy: matrix: pg_version: [17, 18] steps: - uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} - name: Install TimescaleDB run: | # Add TimescaleDB apt repository echo "deb https://packagecloud.io/timescale/timescaledb/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/timescaledb.list wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | sudo apt-key add - sudo apt-get update # Install TimescaleDB for the specific PostgreSQL version sudo apt-get install -y timescaledb-2-postgresql-${{ matrix.pg_version }} || echo "TimescaleDB installation failed, hypertable tests will be skipped" - name: Install build dependencies run: | sudo apt-get install -y build-essential bc - name: Start PostgreSQL run: | sudo systemctl start postgresql sudo -u postgres createuser -s $USER sudo -u postgres createdb $USER - name: Build extension run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make clean # Build with -Werror via PG_CFLAGS (CFLAGS is overridden by pgxs) make PG_CFLAGS="-Werror -Wno-error=unused-parameter" sudo env "PATH=$PATH" make install - name: Run cross-database registry regression timeout-minutes: 5 run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make test-cross-database-registry - name: Run multi-backend REINDEX regression timeout-minutes: 10 run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make test-reindex - name: Run DROP rollback regression timeout-minutes: 5 run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make test-drop-rollback - name: Run regression tests timeout-minutes: 10 run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" # Verify extension is installed echo "Checking if pg_textsearch.so is installed..." ls -la /usr/lib/postgresql/${{ matrix.pg_version }}/lib/pg_textsearch.so # Run the test setup manually since extension is already installed rm -rf tmp_check_shared mkdir -p tmp_check_shared initdb -D tmp_check_shared/data --auth-local=trust --auth-host=trust LIB_NAME="pg_textsearch" echo "port = 55433" >> tmp_check_shared/data/postgresql.conf echo "log_statement = 'all'" >> tmp_check_shared/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_check_shared/data/postgresql.conf echo "max_connections = 20" >> tmp_check_shared/data/postgresql.conf echo "shared_preload_libraries = 'timescaledb,${LIB_NAME}'" >> tmp_check_shared/data/postgresql.conf || true echo "unix_socket_directories = '$PWD/tmp_check_shared'" >> tmp_check_shared/data/postgresql.conf # Try to start PostgreSQL and show logs if it fails if ! pg_ctl start -D tmp_check_shared/data -l tmp_check_shared/data/logfile -w; then echo "PostgreSQL failed to start. Log contents:" cat tmp_check_shared/data/logfile exit 1 fi createdb -h $PWD/tmp_check_shared -p 55433 contrib_regression if ! make installcheck PGHOST=$PWD/tmp_check_shared PGPORT=55433; then echo "Tests failed. Showing regression diffs:" if [ -f test/regression.diffs ]; then cat test/regression.diffs fi echo "PostgreSQL server log (last 200 lines):" tail -200 tmp_check_shared/data/logfile || true pg_ctl stop -D tmp_check_shared/data -l tmp_check_shared/data/logfile || true rm -rf tmp_check_shared || true exit 1 fi pg_ctl stop -D tmp_check_shared/data -l tmp_check_shared/data/logfile rm -rf tmp_check_shared - name: Run shell-based tests timeout-minutes: 35 run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make test-concurrency echo "Running segment tests..." if make test-segment; then echo "✓ Segment tests passed" else echo "✗ Segment tests failed" exit 1 fi echo "Running crash recovery tests..." if (cd test/scripts && ./recovery.sh); then echo "✓ Crash recovery tests passed" else echo "✗ Crash recovery tests failed" exit 1 fi echo "Running deferred-reclaim crash recovery tests (#380)..." if (cd test/scripts && ./standby_reclaim.sh); then echo "✓ Deferred-reclaim crash recovery tests passed" else echo "✗ Deferred-reclaim crash recovery tests failed" exit 1 fi echo "Running per-index compaction recovery-guard tests..." if (cd test/scripts && ./compaction_recovery.sh); then echo "✓ Compaction recovery-guard tests passed" else echo "✗ Compaction recovery-guard tests failed" exit 1 fi echo "Running shutdown spill tests..." if (cd test/scripts && ./shutdown_spill.sh); then echo "✓ Shutdown spill tests passed" else echo "✗ Shutdown spill tests failed" exit 1 fi echo "Running CREATE INDEX CONCURRENTLY tests..." if (cd test/scripts && ./cic.sh); then echo "✓ CIC tests passed" else echo "✗ CIC tests failed" exit 1 fi echo "Running multi-index / multi-user / multi-schema tests..." if (cd test/scripts && ./multi_index.sh); then echo "✓ Multi-index tests passed" else echo "✗ Multi-index tests failed" exit 1 fi echo "Running replication tests..." if (cd test/scripts && ./replication.sh); then echo "✓ Replication tests passed" else echo "✗ Replication tests failed" exit 1 fi echo "Running spill-paths replication tests..." if (cd test/scripts && ./replication_spill_paths.sh); then echo "✓ Spill-paths replication tests passed" else echo "✗ Spill-paths replication tests failed" exit 1 fi echo "Running standby segment-reclaim acceptance tests (#380)..." if (cd test/scripts && ./replication_segment_reclaim.sh); then echo "✓ Standby segment-reclaim tests passed" else echo "✗ Standby segment-reclaim tests failed" exit 1 fi echo "Running WAL audit tests..." if (cd test/scripts && ./wal_audit.sh); then echo "✓ WAL audit tests passed" else echo "✗ WAL audit tests failed" exit 1 fi # hypertable.sh needs a PostgreSQL instance with TimescaleDB preloaded echo "Running hypertable tests..." LIB_NAME="pg_textsearch" rm -rf tmp_hypertable_test mkdir -p tmp_hypertable_test initdb -D tmp_hypertable_test/data --auth-local=trust --auth-host=trust echo "port = 55435" >> tmp_hypertable_test/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_hypertable_test/data/postgresql.conf echo "shared_preload_libraries = 'timescaledb,${LIB_NAME}'" >> tmp_hypertable_test/data/postgresql.conf || true echo "unix_socket_directories = '$PWD/tmp_hypertable_test'" >> tmp_hypertable_test/data/postgresql.conf pg_ctl start -D tmp_hypertable_test/data -l tmp_hypertable_test/data/logfile -w export PGHOST=$PWD/tmp_hypertable_test export PGPORT=55435 if (cd test/scripts && ./hypertable.sh); then echo "✓ Hypertable tests passed" else echo "✗ Hypertable tests failed" pg_ctl stop -D tmp_hypertable_test/data || true rm -rf tmp_hypertable_test exit 1 fi pg_ctl stop -D tmp_hypertable_test/data rm -rf tmp_hypertable_test - name: Upload test results on failure if: failure() uses: actions/upload-artifact@v4 with: name: test-results-pg${{ matrix.pg_version }} path: | test/regression.diffs test/regression.out test/results/ retention-days: 7 injection-points: name: PG${{ matrix.pg }} Injection Points runs-on: ubuntu-22.04 timeout-minutes: 45 strategy: fail-fast: false matrix: pg: ["17.10", "18.4"] env: PGPREFIX: /tmp/postgresql # Instrument one leg only; both run the same tests. COVERAGE_PG: "17.10" steps: - uses: actions/checkout@v4 - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y bison build-essential curl flex lcov - name: Cache injection-enabled PostgreSQL id: pg-cache uses: actions/cache@v4 with: path: ${{ env.PGPREFIX }} key: postgresql-injection-${{ matrix.pg }}-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.github/scripts/build-postgres-injection.sh') }} - name: Build PostgreSQL ${{ matrix.pg }} if: steps.pg-cache.outputs.cache-hit != 'true' run: | .github/scripts/build-postgres-injection.sh \ "${{ matrix.pg }}" "$PGPREFIX" - name: Verify injection-point build run: | "$PGPREFIX/bin/pg_config" --configure | grep -q -- '--enable-injection-points' test -f \ "$PGPREFIX/share/extension/injection_points.control" test -f "$PGPREFIX/share/extension/dblink.control" - name: Build and install pg_textsearch run: | export PATH="$PGPREFIX/bin:$PATH" make clean warnflags="-Werror -Wno-error=unused-parameter" if [ "${{ matrix.pg }}" = "$COVERAGE_PG" ]; then make PG_CFLAGS="$warnflags --coverage -O0 -g" \ SHLIB_LINK="--coverage" else make PG_CFLAGS="$warnflags" fi make install - name: Run injection-point tests timeout-minutes: 35 run: | export PATH="$PGPREFIX/bin:$PATH" cluster="$RUNNER_TEMP/pg_textsearch-injection" rm -rf "$cluster" initdb -D "$cluster" --auth-local=trust --auth-host=trust echo "port = 55439" >> "$cluster/postgresql.conf" echo "unix_socket_directories = '$cluster'" \ >> "$cluster/postgresql.conf" echo "shared_preload_libraries = 'pg_textsearch'" \ >> "$cluster/postgresql.conf" trap 'pg_ctl stop -D "$cluster" -m immediate >/dev/null 2>&1 || true' \ EXIT pg_ctl start -D "$cluster" -l "$cluster/server.log" -w createdb -h "$cluster" -p 55439 contrib_regression make test-injection-sql \ PGHOST="$cluster" PGPORT=55439 pg_ctl stop -D "$cluster" -m fast -w trap - EXIT make test-injection-shell - name: Capture coverage data if: matrix.pg == '17.10' run: | # ubuntu-22.04 ships lcov 1.x, so only its error classes are # available here; the main coverage job runs lcov 2.x. lcov --capture \ --directory . \ --output-file coverage.info \ --base-directory "$PWD" \ --no-external \ --ignore-errors gcov,source,graph lcov --remove coverage.info '*/test/*' \ --output-file coverage.info lcov --list coverage.info - name: Upload coverage to Codecov if: matrix.pg == '17.10' uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.info flags: injection fail_ci_if_error: true verbose: true build-test-multiple-configs: runs-on: ubuntu-latest strategy: matrix: config: - { cc: gcc, pg_version: 17 } - { cc: clang, pg_version: 17 } - { cc: gcc, pg_version: 18 } - { cc: clang, pg_version: 18 } steps: - uses: actions/checkout@v4 - name: Install PostgreSQL and build tools run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.config.pg_version }} postgresql-server-dev-${{ matrix.config.pg_version }} - name: Build with ${{ matrix.config.cc }} run: | export PATH="/usr/lib/postgresql/${{ matrix.config.pg_version }}/bin:$PATH" export CC=${{ matrix.config.cc }} make clean # Build with -Werror via PG_CFLAGS (CFLAGS is overridden by pgxs) make PG_CFLAGS="-Werror -Wno-error=unused-parameter" # PG19 is in beta and has no PGDG apt package yet, so it is built from # source here and exercised with both gcc and clang plus the full SQL # regression suite. This job is experimental: beta breakage must not # block merges (continue-on-error), and it can be dropped in favor of # the apt matrices once PG19 reaches GA and PGDG ships packages. test-pg19-beta: runs-on: ubuntu-latest continue-on-error: true strategy: fail-fast: false matrix: cc: [gcc, clang] env: PG19_REF: REL_19_BETA3 PGPREFIX: ${{ github.workspace }}/pg19-install steps: - uses: actions/checkout@v4 - name: Install build dependencies run: | sudo apt-get update sudo apt-get install -y build-essential clang bison flex \ libreadline-dev zlib1g-dev libicu-dev pkg-config bc - name: Cache PostgreSQL ${{ env.PG19_REF }} build id: pgcache uses: actions/cache@v4 with: path: ${{ env.PGPREFIX }} key: pg19-${{ env.PG19_REF }}-injection-v2-${{ matrix.cc }}-${{ runner.os }} - name: Build PostgreSQL ${{ env.PG19_REF }} from source if: steps.pgcache.outputs.cache-hit != 'true' run: | git clone --depth 1 --branch "$PG19_REF" \ https://git.postgresql.org/git/postgresql.git /tmp/pgsrc cd /tmp/pgsrc CC=${{ matrix.cc }} ./configure --prefix="$PGPREFIX" \ --with-icu --enable-debug --enable-cassert \ --enable-injection-points make -j"$(nproc)" make install make -C src/test/modules/injection_points install make -C contrib/dblink install - name: Verify injection-point build run: | "$PGPREFIX/bin/pg_config" --configure | grep -q -- '--enable-injection-points' test -f \ "$("$PGPREFIX/bin/pg_config" --sharedir)/extension/injection_points.control" test -f \ "$("$PGPREFIX/bin/pg_config" --sharedir)/extension/dblink.control" - name: Build extension with ${{ matrix.cc }} run: | export PATH="$PGPREFIX/bin:$PATH" export CC=${{ matrix.cc }} make clean make PG_CFLAGS="-Werror -Wno-error=unused-parameter" make install - name: Run regression tests timeout-minutes: 25 run: | export PATH="$PGPREFIX/bin:$PATH" rm -rf tmp_check_shared mkdir -p tmp_check_shared initdb -D tmp_check_shared/data --auth-local=trust --auth-host=trust echo "port = 55433" >> tmp_check_shared/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_check_shared/data/postgresql.conf echo "max_connections = 20" >> tmp_check_shared/data/postgresql.conf echo "shared_preload_libraries = 'pg_textsearch'" >> tmp_check_shared/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_check_shared'" >> tmp_check_shared/data/postgresql.conf if ! pg_ctl start -D tmp_check_shared/data -l tmp_check_shared/data/logfile -w; then echo "PostgreSQL failed to start. Log contents:" cat tmp_check_shared/data/logfile exit 1 fi createdb -h $PWD/tmp_check_shared -p 55433 contrib_regression if ! make installcheck PGHOST=$PWD/tmp_check_shared PGPORT=55433; then echo "Tests failed. Showing regression diffs:" [ -f test/regression.diffs ] && cat test/regression.diffs echo "PostgreSQL server log (last 200 lines):" tail -200 tmp_check_shared/data/logfile || true pg_ctl stop -D tmp_check_shared/data || true exit 1 fi pg_ctl stop -D tmp_check_shared/data make test-memtable-stale-tail make test-injection-shell build-msvc: name: PG${{ matrix.pg_version }} MSVC Build runs-on: windows-2022 timeout-minutes: 20 strategy: # Do not cancel the remaining leg when one fails: comparing both logs # shows whether a /WX failure comes from the toolset or from headers. fail-fast: false matrix: include: - pg_version: 17 url: 'https://get.enterprisedb.com/postgresql/postgresql-17.11-1-windows-x64-binaries.zip' sha256: '6eabdf00d2893713b75db4336a23c3fdf505f056e217ec6e2e95d901750cfea3' - pg_version: 18 url: 'https://get.enterprisedb.com/postgresql/postgresql-18.6-1-windows-x64-binaries.zip' sha256: 'fbe23da234ee31547bf8a36d29dfd81e82b849df2d2b78d2eecb43d360252f8c' steps: - uses: actions/checkout@v4 - name: Download PostgreSQL ${{ matrix.pg_version }} binaries shell: pwsh run: | $ErrorActionPreference = 'Stop' $ProgressPreference = 'SilentlyContinue' $zip = Join-Path $env:RUNNER_TEMP 'pgsql.zip' $dest = Join-Path $env:RUNNER_TEMP 'pg' $expected = '${{ matrix.sha256 }}' # The retry covers transport errors and hash mismatches alike: a # truncated download of a ~340 MB archive gives a wrong hash. This # job is a required check. $verified = $false for ($attempt = 1; $attempt -le 2; $attempt++) { if (Test-Path -LiteralPath $zip) { Remove-Item -LiteralPath $zip -Force } try { Invoke-WebRequest -Uri '${{ matrix.url }}' -OutFile $zip -UseBasicParsing -TimeoutSec 300 $actual = (Get-FileHash -LiteralPath $zip -Algorithm SHA256).Hash.ToLowerInvariant() if ($actual -eq $expected) { $verified = $true; break } Write-Host "attempt ${attempt}: SHA256 is $actual, expected $expected" } catch { Write-Host "attempt ${attempt}: download failed: $($_.Exception.Message)" } if ($attempt -lt 2) { Start-Sleep -Seconds 15 } } if (-not $verified) { throw "Could not fetch ${{ matrix.url }} with SHA256 $expected in 2 attempts." } Write-Host "SHA256 verified: $expected" if (Test-Path -LiteralPath $dest) { Remove-Item -LiteralPath $dest -Recurse -Force } New-Item -ItemType Directory -Path $dest -Force | Out-Null # tar is bsdtar here and unpacks this zip far faster than Expand-Archive. tar -xf $zip -C $dest if ($LASTEXITCODE -ne 0) { throw "tar -xf failed with exit code $LASTEXITCODE" } # EDB zips unpack into a single pgsql/ directory. A different root # means the archive changed and the pin must be updated. $pgRoot = Join-Path $dest 'pgsql' foreach ($required in @('include\server', 'lib\postgres.lib')) { $path = Join-Path $pgRoot $required if (-not (Test-Path -LiteralPath $path)) { throw "Extracted tree is missing $path" } } # A plain $env: assignment would not survive this step. "PGROOT=$pgRoot" >> $env:GITHUB_ENV - name: Build with MSVC shell: pwsh run: | $ErrorActionPreference = 'Stop' ./scripts/msvc-build.ps1 -PgRoot $env:PGROOT -ExpectedMajor ${{ matrix.pg_version }} performance-test: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - name: Install PostgreSQL 17 run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Start PostgreSQL run: | sudo systemctl start postgresql sudo -u postgres createuser -s $USER sudo -u postgres createdb $USER - name: Build and install extension run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean && make && sudo make install - name: Run performance benchmarks timeout-minutes: 20 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" # Run larger concurrency test for performance validation echo "Running performance benchmarks..." cd test/scripts && TEST_SIZE_MULTIPLIER=5.0 ./concurrency.sh - name: Upload performance results if: always() uses: actions/upload-artifact@v4 with: name: performance-results path: | test/tmp_concurrent_test/postgres.log retention-days: 7 format-check: name: Check C Code Formatting runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Install clang-format run: pip install clang-format==21.1.8 - name: Check formatting run: | # Find all C source and header files find src -name "*.c" -o -name "*.h" | while read file; do echo "Checking $file" if ! clang-format --dry-run --Werror "$file" > /dev/null 2>&1; then echo "❌ $file is not properly formatted" echo "Expected format:" clang-format "$file" exit 1 else echo "✅ $file is properly formatted" fi done - name: Suggest formatting command if: failure() run: | echo "" echo "To fix formatting issues, run:" echo " make format" echo "" echo "Or format individual files:" echo " clang-format -i src/filename.c" sanitizer: name: PG${{ matrix.pg }} Sanitizer runs-on: ubuntu-22.04 timeout-minutes: 45 strategy: fail-fast: false matrix: pg: ["17.2", "18.1"] env: PG_SRC_DIR: "pgbuild" PG_INSTALL_DIR: "postgresql" extra_packages: "clang-15 llvm-15" CC: "clang-15" CXX: "clang-15" CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" LDFLAGS: "-fsanitize=address,undefined" ASAN_OPTIONS: "detect_leaks=0 abort_on_error=1 exitcode=27 log_path=${{ github.workspace }}/sanitizer_logs/asan print_stacktrace=1 check_initialization_order=1" UBSAN_OPTIONS: "halt_on_error=1 exitcode=27 print_stacktrace=1 log_path=${{ github.workspace }}/sanitizer_logs/ubsan print_summary=1" steps: - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y flex bison build-essential libssl-dev \ libreadline-dev systemd-coredump gdb ${{ env.extra_packages }} - name: Checkout uses: actions/checkout@v4 - name: Create sanitizer logs directory run: mkdir -p ${{ github.workspace }}/sanitizer_logs - name: Cache PostgreSQL ${{ matrix.pg }} id: cache-postgresql uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: sanitizer-postgresql-${{ matrix.pg }}-${{ env.CC }}-standard-memory-injection-v1-${{ hashFiles('.github/scripts/build-postgres-sanitizer.sh', 'test/postgres-asan-instrumentation.patch', 'test/postgres-asan-instrumentation-PG18GE.patch') }} - name: Build PostgreSQL ${{ matrix.pg }} with sanitizers if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | .github/scripts/build-postgres-sanitizer.sh \ "${{ matrix.pg }}" "$HOME/$PG_SRC_DIR" "$HOME/$PG_INSTALL_DIR" - name: Install PostgreSQL ${{ matrix.pg }} run: | cd ~/$PG_SRC_DIR make install make -C src/test/modules/injection_points install make -C contrib/dblink install test -f "$HOME/$PG_INSTALL_DIR/share/extension/dblink.control" ~/$PG_INSTALL_DIR/bin/pg_config --version - name: Verify sanitizers are enabled run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" echo "Verifying PostgreSQL was built with sanitizers..." # Check if postgres binary contains sanitizer symbols echo "Checking for AddressSanitizer symbols..." if nm ~/$PG_INSTALL_DIR/bin/postgres | grep -i asan > /dev/null; then echo "AddressSanitizer symbols found in postgres binary" else echo "AddressSanitizer symbols NOT found in postgres binary" echo "This suggests PostgreSQL was not built with -fsanitize=address" exit 1 fi # Test that sanitizers are actually working by triggering a known issue echo "Testing sanitizer detection with intentional buffer overflow..." cat > test_sanitizer.c << 'EOF' #include #include int main() { char *buf = malloc(10); buf[10] = 'x'; // Buffer overflow - should be caught by AddressSanitizer free(buf); return 0; } EOF # Compile and run test - should fail with sanitizer error $CC $CFLAGS test_sanitizer.c -o test_sanitizer if ./test_sanitizer; then echo "WARNING: Sanitizer test did not catch buffer overflow!" else echo "Sanitizer correctly caught buffer overflow (exit code $?)" fi rm -f test_sanitizer test_sanitizer.c - name: Build and install pg_textsearch run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" make clean make make install - name: Run regression tests run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" # Set up test database rm -rf tmp_sanitizer_test mkdir -p tmp_sanitizer_test initdb -D tmp_sanitizer_test/data --auth-local=trust # Configure PostgreSQL LIB_NAME="pg_textsearch" echo "port = 55433" >> tmp_sanitizer_test/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_sanitizer_test'" >> tmp_sanitizer_test/data/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> tmp_sanitizer_test/data/postgresql.conf # Start PostgreSQL pg_ctl start -D tmp_sanitizer_test/data -l tmp_sanitizer_test/data/logfile -w # Create test database and run tests createdb -h $PWD/tmp_sanitizer_test -p 55433 contrib_regression echo "Running SQL regression tests under sanitizer..." make installcheck PGHOST=$PWD/tmp_sanitizer_test PGPORT=55433 echo "Running stale memtable-tail tests under sanitizer..." timeout 120s make test-memtable-stale-tail # Run shell-based tests under sanitizer (more likely to find memory issues) echo "Running spill-time inline compaction locking tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./inline_compaction_locking.sh " echo "Running concurrency tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./concurrency.sh " echo "Running concurrent duplicate-read tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./concurrent_duplicate_read.sh " # Covers the VACUUM-vs-merge races (#411, #465). This is the # only sanitized run of it on a PR, where the timing windows # are wide enough to hit. echo "Running VACUUM concurrency tests under sanitizer..." timeout 900s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./vacuum_concurrent_merge.sh " echo "Running crash recovery tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./recovery.sh " # Deferred-reclaim crash recovery (issue #380): parks displaced # segment pages, crashes (immediate stop), asserts WAL replay # reconstructs the tombstone chain. Spins up its own instance; # under ASan a use-after-recycle of a displaced page trips here. echo "Running deferred-reclaim crash recovery tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./standby_reclaim.sh " echo "Running shutdown spill tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./shutdown_spill.sh " echo "Running multi-backend segment tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 make test-segment " echo "Running CREATE INDEX CONCURRENTLY tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./cic.sh " echo "Running multi-index / multi-user / multi-schema tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./multi_index.sh " # Standby segment-reclaim acceptance (issue #380): a live standby # snapshot with hot_standby_feedback=on holds the primary's # reclaim horizon so a merge+VACUUM keeps displaced pages parked, # and the standby reads cleanly across merge replay. Spins up its # own primary + standby; under ASan, recycling a page out from # under the standby read surfaces as a use-after-recycle. echo "Running standby segment-reclaim acceptance tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./replication_segment_reclaim.sh " # Clean up pg_ctl stop -D tmp_sanitizer_test/data -l tmp_sanitizer_test/data/logfile - name: Collect sanitizer reports if: always() run: | echo "Collecting all sanitizer logs..." find ${{ github.workspace }}/sanitizer_logs -type f 2>/dev/null | \ head -20 | while read file; do echo "=== $file ===" head -100 "$file" 2>/dev/null || echo "(file not readable or empty)" echo done - name: Upload sanitizer logs and test artifacts if: always() uses: actions/upload-artifact@v4 with: name: sanitizer-logs-pg${{ matrix.pg }} path: | ${{ github.workspace }}/sanitizer_logs/ test/regression.diffs test/regression.out test/results/ tmp_sanitizer_test/data/logfile retention-days: 7 - name: Upload failure artifacts if: failure() uses: actions/upload-artifact@v4 with: name: sanitizer-failure-debug-pg${{ matrix.pg }} path: | tmp_sanitizer_test/ test/scripts/tmp_* test/tmp_* retention-days: 7 # The coverage job exercises managed compaction on PostgreSQL 17. Run the # same end-to-end lifecycle suite separately on the other required version. managed-compaction-pg18: runs-on: ubuntu-latest env: PG_DURABLE_VERSION: 0.2.8 PG_DURABLE_PACKAGE: pg-durable-postgresql-18_0.2.8-1_amd64.deb PG_DURABLE_SHA256: 090028f59b003613b150e61ccd5cdbebe70e69e65361913ad29b1f0b6af9e2bb steps: - uses: actions/checkout@v4 - name: Install PostgreSQL 18 run: | sudo apt-get update sudo apt-get install -y wget ca-certificates gnupg wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-18 postgresql-server-dev-18 \ build-essential bc - name: Build and install extension run: | export PATH="/usr/lib/postgresql/18/bin:$PATH" make sudo env "PATH=$PATH" make install - name: Download pg_durable release package run: | curl -fL --retry 3 \ -o "$PG_DURABLE_PACKAGE" \ "https://github.com/microsoft/pg_durable/releases/download/v${PG_DURABLE_VERSION}/${PG_DURABLE_PACKAGE}" echo "${PG_DURABLE_SHA256} ${PG_DURABLE_PACKAGE}" | sha256sum -c - mkdir pg_durable_package dpkg-deb -x "$PG_DURABLE_PACKAGE" pg_durable_package - name: Run managed pg_durable compaction test timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/18/bin:$PATH" sudo chown -R "$(id -u):$(id -g)" \ /usr/share/postgresql/18/extension export PG_DURABLE_PACKAGE_DIR="$PWD/pg_durable_package" make test-durable \ PG_CONFIG=/usr/lib/postgresql/18/bin/pg_config coverage: runs-on: ubuntu-latest env: # Managed background compaction is only exercisable against a live # pg_durable; src/index/compaction_job.c is otherwise unreachable. PG_DURABLE_VERSION: 0.2.8 PG_DURABLE_PACKAGE: pg-durable-postgresql-17_0.2.8-1_amd64.deb PG_DURABLE_SHA256: 927b091e57be42c63079fc50a548af5f491b223534eaca112c5d1f18299dc29f steps: - uses: actions/checkout@v4 - name: Install PostgreSQL 17 run: | sudo apt-get update sudo apt-get install -y wget ca-certificates gnupg wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo gpg --dearmor -o /usr/share/keyrings/postgresql-archive-keyring.gpg echo "deb [signed-by=/usr/share/keyrings/postgresql-archive-keyring.gpg] http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Install TimescaleDB run: | echo "deb https://packagecloud.io/timescale/timescaledb/ubuntu/ $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/timescaledb.list wget --quiet -O - https://packagecloud.io/timescale/timescaledb/gpgkey | sudo apt-key add - sudo apt-get update sudo apt-get install -y timescaledb-2-postgresql-17 || echo "TimescaleDB installation failed, hypertable tests will be skipped" - name: Install coverage tools run: | sudo apt-get install -y lcov build-essential bc - name: Build extension with coverage instrumentation run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean # Build with coverage flags - disable optimization for accurate line counts make PG_CFLAGS="--coverage -O0 -g" SHLIB_LINK="--coverage" sudo env "PATH=$PATH" make install - name: Setup test database run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" LIB_NAME="pg_textsearch" rm -rf tmp_check_shared mkdir -p tmp_check_shared initdb -D tmp_check_shared/data --auth-local=trust --auth-host=trust echo "port = 55433" >> tmp_check_shared/data/postgresql.conf echo "log_statement = 'all'" >> tmp_check_shared/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_check_shared/data/postgresql.conf echo "max_connections = 20" >> tmp_check_shared/data/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> tmp_check_shared/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_check_shared'" >> tmp_check_shared/data/postgresql.conf pg_ctl start -D tmp_check_shared/data -l tmp_check_shared/data/logfile -w createdb -h $PWD/tmp_check_shared -p 55433 contrib_regression - name: Zero coverage counters run: | lcov --zerocounters --directory . - name: Run SQL regression tests run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make test PGHOST=$PWD/tmp_check_shared PGPORT=55433 || true # Stop and restart Postgres to flush coverage data from backend pg_ctl stop -D tmp_check_shared/data -l tmp_check_shared/data/logfile pg_ctl start -D tmp_check_shared/data -l tmp_check_shared/data/logfile -w - name: Run shell-based tests timeout-minutes: 20 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" # Run all shell-based tests for coverage. # Each spawns its own Postgres instance; the coverage- # instrumented .so writes gcda data to the build dir. # Scripts that require an injection-point server are covered by # the PG17 leg of the injection-points job instead. (cd test/scripts && ./concurrency.sh) || true make test-segment || true (cd test/scripts && ./recovery.sh) || true (cd test/scripts && ./standby_reclaim.sh) || true (cd test/scripts && ./compaction_recovery.sh) || true (cd test/scripts && ./shutdown_spill.sh) || true (cd test/scripts && ./cic.sh) || true (cd test/scripts && ./multi_index.sh) || true (cd test/scripts && ./parallel_vacuum.sh) || true (cd test/scripts && ./vacuum_concurrent_merge.sh) || true (cd test/scripts && ./replication.sh) || true (cd test/scripts && ./replication_spill_paths.sh) || true (cd test/scripts && ./replication_segment_reclaim.sh) || true (cd test/scripts && ./wal_audit.sh) || true - name: Stop PostgreSQL and flush coverage run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_check_shared/data -l tmp_check_shared/data/logfile || true - name: Download pg_durable release package run: | curl -fL --retry 3 \ -o "$PG_DURABLE_PACKAGE" \ "https://github.com/microsoft/pg_durable/releases/download/v${PG_DURABLE_VERSION}/${PG_DURABLE_PACKAGE}" echo "${PG_DURABLE_SHA256} ${PG_DURABLE_PACKAGE}" | sha256sum -c - mkdir pg_durable_package dpkg-deb -x "$PG_DURABLE_PACKAGE" pg_durable_package - name: Run managed pg_durable compaction test timeout-minutes: 15 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" # durable_compaction.sh stages the pg_durable package into the # PostgreSQL sharedir, which is root-owned on the runner image. sudo chown -R "$(id -u):$(id -g)" \ /usr/share/postgresql/17/extension export PG_DURABLE_PACKAGE_DIR="$PWD/pg_durable_package" make test-durable \ PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config - name: Upload managed compaction failure diagnostics if: failure() uses: actions/upload-artifact@v4 with: name: coverage-managed-compaction-logs path: test/tmp_durable_compaction_logs/ if-no-files-found: ignore - name: Run hypertable tests (exercises CustomScan code paths) timeout-minutes: 5 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" LIB_NAME="pg_textsearch" rm -rf tmp_hypertable_test mkdir -p tmp_hypertable_test initdb -D tmp_hypertable_test/data --auth-local=trust --auth-host=trust echo "port = 55434" >> tmp_hypertable_test/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_hypertable_test/data/postgresql.conf echo "shared_preload_libraries = 'timescaledb,${LIB_NAME}'" >> tmp_hypertable_test/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_hypertable_test'" >> tmp_hypertable_test/data/postgresql.conf pg_ctl start -D tmp_hypertable_test/data -l tmp_hypertable_test/data/logfile -w export PGHOST=$PWD/tmp_hypertable_test export PGPORT=55434 (cd test/scripts && ./hypertable.sh) || true pg_ctl stop -D tmp_hypertable_test/data || true rm -rf tmp_hypertable_test - name: Run upgrade compat test (v6 metapage → in-place upgrade path) timeout-minutes: 5 run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PG_CONFIG="/usr/lib/postgresql/17/bin/pg_config" LIB_NAME="pg_textsearch" # Download and build v1.0.0 (V4 segment + v6 metapage) WITHOUT # coverage flags. Used to seed an index whose on-disk format is # the pre-1.3.0 layout; we then upgrade the binary and exercise # the v6→v7 metapage in-place upgrade path plus REINDEX-from- # heap as a fallback. curl -fsSL -o /tmp/old.tar.gz \ "https://github.com/timescale/pg_textsearch/releases/download/v1.0.0/pg_textsearch-1.0.0.tar.gz" tar -xzf /tmp/old.tar.gz -C /tmp (cd /tmp/pg_textsearch-1.0.0 && make clean && make && sudo make install) # Start PG with old binary and create a v6-metapage index with # V4 segments. rm -rf tmp_upgrade_cov mkdir -p tmp_upgrade_cov initdb -D tmp_upgrade_cov/data --auth-local=trust --auth-host=trust echo "port = 55435" >> tmp_upgrade_cov/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_upgrade_cov/data/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> tmp_upgrade_cov/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_upgrade_cov'" >> tmp_upgrade_cov/data/postgresql.conf pg_ctl start -D tmp_upgrade_cov/data -l tmp_upgrade_cov/data/logfile -w createdb -h $PWD/tmp_upgrade_cov -p 55435 upgrade_cov psql -h $PWD/tmp_upgrade_cov -p 55435 -d upgrade_cov -c \ "CREATE EXTENSION pg_textsearch VERSION '1.0.0';" psql -h $PWD/tmp_upgrade_cov -p 55435 -d upgrade_cov << 'SQL' CREATE TABLE docs (id SERIAL PRIMARY KEY, content TEXT NOT NULL); INSERT INTO docs (content) SELECT CASE (g % 4) WHEN 0 THEN 'database query optimization' WHEN 1 THEN 'text search ranking algorithm' WHEN 2 THEN 'machine learning neural network' WHEN 3 THEN 'distributed systems consensus' END || ' doc ' || g FROM generate_series(1, 100) g; CREATE INDEX idx ON docs USING bm25 (content) WITH (text_config = 'english'); INSERT INTO docs (content) SELECT 'extra terms for spill segment ' || g FROM generate_series(1, 50) g; SELECT bm25_spill_index('idx'); SQL # Stop PG, install coverage-instrumented current version pg_ctl stop -D tmp_upgrade_cov/data -w SHAREDIR=$($PG_CONFIG --sharedir) sudo rm -f "$SHAREDIR/extension/pg_textsearch--"*.sql sudo env "PATH=$PATH" make install pg_ctl start -D tmp_upgrade_cov/data -l tmp_upgrade_cov/data/logfile -w # Exercise v6-metapage in-place upgrade + REINDEX path. # Issue #383: the 1.3.0 binary now reads v6 metapages in place # (byte-identical prefix), normalizes the new chain head/tail # fields, and lazily upgrades to the current version on the # first metapage write. This step exercises the read path, the # ALTER EXTENSION UPDATE, an INSERT that triggers the in-place # v6-to-v9 upgrade via memtable_bootstrap_and_append, and finally # a REINDEX (still supported as the fallback recovery path) # plus VACUUM and force_merge. psql -h $PWD/tmp_upgrade_cov -p 55435 -d upgrade_cov << 'SQL' \set ON_ERROR_STOP on -- Pre-ALTER: queries against the v6 index should succeed -- (in-place compat read; no REINDEX required). DO $$ DECLARE n bigint; BEGIN SELECT count(*) INTO n FROM ( SELECT 1 FROM docs ORDER BY content <@> to_bm25query('database', 'idx') LIMIT 10) sub; IF n = 0 THEN RAISE EXCEPTION 'FAIL: pre-ALTER v6 query returned 0 rows'; END IF; RAISE NOTICE 'PASS: pre-ALTER v6 index queryable, n=%', n; END $$; -- Read-only queries MUST NOT upgrade the on-disk metapage -- (issue #383 contract: tp_get_metapage normalizes a v6 -- page in its palloc'd copy but does not mutate the -- buffer). Regression here would mean the read path -- accidentally emits a write. DO $$ DECLARE dump_text text; BEGIN dump_text := bm25_dump_index('idx'); IF dump_text NOT LIKE '%version: 6' || E'\n' || '%' THEN RAISE EXCEPTION 'FAIL: metapage upgraded by read-only queries; ' 'expected version: 6, dump excerpt: %', substr(dump_text, 1, 400); END IF; RAISE NOTICE 'PASS: read-only queries left v6 metapage intact'; END $$; ALTER EXTENSION pg_textsearch UPDATE; -- INSERT triggers memtable_bootstrap_and_append, which -- runs tp_metapage_upgrade_to_current() inside its -- GenericXLog record. Validates that the v6-to-v9 in-place -- upgrade landed on disk. INSERT INTO docs (content) VALUES ('compat upgrade insert one'), ('compat upgrade insert two'); DO $$ DECLARE dump_text text; BEGIN dump_text := bm25_dump_index('idx'); IF dump_text NOT LIKE '%version: 9' || E'\n' || '%' THEN RAISE EXCEPTION 'FAIL: metapage not upgraded to v9 after INSERT; ' 'dump excerpt: %', substr(dump_text, 1, 400); END IF; RAISE NOTICE 'PASS: in-place v6-to-v9 upgrade after first write'; END $$; -- REINDEX is still supported as the operator fallback; -- exercise the build-from-heap codepath for coverage. REINDEX INDEX idx; -- V7/V5 reading post-REINDEX SELECT count(*) FROM ( SELECT 1 FROM docs ORDER BY content <@> to_bm25query('search ranking', 'idx') LIMIT 10) sub; -- VACUUM with alive bitset on V5 segments DELETE FROM docs WHERE id <= 20; VACUUM docs; SELECT count(*) FROM ( SELECT 1 FROM docs ORDER BY content <@> to_bm25query('database', 'idx') LIMIT 10) sub; -- Merge V5 segments SELECT bm25_force_merge('idx'); SELECT count(*) FROM ( SELECT 1 FROM docs ORDER BY content <@> to_bm25query('optimization', 'idx') LIMIT 10) sub; SQL pg_ctl stop -D tmp_upgrade_cov/data || true rm -rf tmp_upgrade_cov /tmp/pg_textsearch-1.0.0 /tmp/old.tar.gz - name: Capture coverage data run: | # Capture coverage from all .gcda files lcov --capture \ --directory . \ --output-file coverage.info \ --base-directory $PWD \ --no-external \ --ignore-errors mismatch # Remove test files from coverage report (we only care about src/) lcov --remove coverage.info \ '*/test/*' \ '*/tmp_check_shared/*' \ --output-file coverage.info \ --ignore-errors unused # Show summary lcov --list coverage.info - name: Restore coverage baseline if: github.event_name == 'pull_request' uses: actions/cache/restore@v4 with: path: coverage-baseline.txt key: coverage-baseline-never-matches restore-keys: | coverage-baseline- - name: Coverage gate id: coverage-gate run: | # Read old baseline if available (before we overwrite it) BASELINE="" if [ -f coverage-baseline.txt ]; then BASELINE=$(cat coverage-baseline.txt) fi # Extract current line coverage percentage CURRENT=$(lcov --summary coverage.info 2>&1 \ | grep "lines" | sed 's/.*: \([0-9.]*\)%.*/\1/') echo "Current line coverage: ${CURRENT}%" echo "line_coverage=$CURRENT" >> "$GITHUB_OUTPUT" # Save current value as baseline for future runs echo "$CURRENT" > coverage-baseline.txt # Enforce minimum coverage threshold MIN_COVERAGE=85 if (( $(echo "$CURRENT < $MIN_COVERAGE" | bc -l) )); then echo "::error::Line coverage ${CURRENT}% is below" \ "minimum ${MIN_COVERAGE}%" exit 1 fi echo "Coverage ${CURRENT}% meets minimum ${MIN_COVERAGE}%" # Check for coverage reduction on PRs if [ -n "$BASELINE" ]; then DIFF=$(echo "$CURRENT - $BASELINE" | bc -l) echo "Baseline: ${BASELINE}%, Change: ${DIFF}%" # Tolerance relaxed to -3% to absorb the one-time drop from # the v6 metapage REINDEX-rejection design (issue #374, # Batch 1): the V4 segment reader is no longer reachable # post-1.3.0 because v6 indexes ERROR before any segment # data is touched, requiring REINDEX -> V5 segments. # Follow-up: physically delete V4 reader (then tighten # this back to -1%). if (( $(echo "$DIFF < -3.0" | bc -l) )); then echo "::error::Coverage decreased by more than 3%:" \ "${BASELINE}% -> ${CURRENT}% (change: ${DIFF}%)" exit 1 fi echo "Coverage change (${DIFF}%) within threshold" else echo "No baseline found, skipping reduction check" fi - name: Save coverage baseline if: github.ref == 'refs/heads/main' uses: actions/cache/save@v4 with: path: coverage-baseline.txt key: coverage-baseline-${{ github.sha }} - name: Generate HTML report run: | genhtml coverage.info \ --output-directory coverage-html \ --title "pg_textsearch Coverage" \ --legend \ --show-details - name: Upload coverage to Codecov uses: codecov/codecov-action@v5 with: token: ${{ secrets.CODECOV_TOKEN }} files: ./coverage.info fail_ci_if_error: true verbose: true - name: Upload HTML coverage report uses: actions/upload-artifact@v4 with: name: coverage-report path: coverage-html/ retention-days: 30 - name: Upload lcov info file uses: actions/upload-artifact@v4 with: name: coverage-lcov path: coverage.info retention-days: 30 - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_check_shared/data -l tmp_check_shared/data/logfile || true rm -rf tmp_check_shared - name: Coverage summary run: | echo "## Coverage Summary" >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY echo "Download the HTML report from the artifacts to see detailed line-by-line coverage." >> $GITHUB_STEP_SUMMARY echo "" >> $GITHUB_STEP_SUMMARY # Extract and display summary stats if [ -f coverage.info ]; then echo '```' >> $GITHUB_STEP_SUMMARY lcov --summary coverage.info 2>&1 | tail -10 >> $GITHUB_STEP_SUMMARY echo '```' >> $GITHUB_STEP_SUMMARY fi pgspot: name: Check Extension SQL runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install pgspot run: pip install pgspot - name: Run pgspot run: | # Check the current version's base install SQL file VERSION=$(awk -F"'" '/default_version/ {print $2}' pg_textsearch.control) echo "Checking sql/pg_textsearch--${VERSION}.sql" pgspot "sql/pg_textsearch--${VERSION}.sql" # Umbrella job for branch protection rules all-tests-passed: runs-on: ubuntu-latest needs: [test, injection-points, build-test-multiple-configs, build-msvc, performance-test, format-check, sanitizer, managed-compaction-pg18, coverage, pgspot] if: always() steps: - name: Check all jobs succeeded run: | if [[ "${{ contains(needs.*.result, 'failure') }}" == "true" ]]; then echo "One or more jobs failed" exit 1 fi if [[ "${{ contains(needs.*.result, 'cancelled') }}" == "true" ]]; then echo "One or more jobs were cancelled" exit 1 fi echo "All checks passed!" timescale-pg_textsearch-b63fe58/.github/workflows/claude-code-review.yml000066400000000000000000000036741525762047700265570ustar00rootroot00000000000000name: Claude Code Review # Disabled - use @claude mentions instead for on-demand reviews # on: # pull_request: # types: [opened, synchronize] # # Optional: Only run on specific file changes # # paths: # # - "src/**/*.ts" # # - "src/**/*.tsx" # # - "src/**/*.js" # # - "src/**/*.jsx" on: workflow_dispatch: # Manual trigger only jobs: claude-review: # Optional: Filter by PR author # if: | # github.event.pull_request.user.login == 'external-contributor' || # github.event.pull_request.user.login == 'new-developer' || # github.event.pull_request.author_association == 'FIRST_TIME_CONTRIBUTOR' runs-on: ubuntu-latest permissions: contents: read pull-requests: read issues: read id-token: write steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 1 - name: Run Claude Code Review id: claude-review uses: anthropics/claude-code-action@v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} prompt: | Please review this pull request and provide feedback on: - Code quality and best practices - Potential bugs or issues - Performance considerations - Security concerns - Test coverage Use the repository's CLAUDE.md for guidance on style and conventions. Be constructive and helpful in your feedback. Use `gh pr comment` with your Bash tool to leave your review as a comment on the PR. # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options claude_args: '--allowed-tools "Bash(gh issue view:*),Bash(gh search:*),Bash(gh issue list:*),Bash(gh pr comment:*),Bash(gh pr diff:*),Bash(gh pr view:*),Bash(gh pr list:*)"' timescale-pg_textsearch-b63fe58/.github/workflows/claude.yml000066400000000000000000000036031525762047700243400ustar00rootroot00000000000000name: Claude Code on: issue_comment: types: [created] pull_request_review_comment: types: [created] issues: types: [opened, assigned] pull_request_review: types: [submitted] jobs: claude: if: | (github.event_name == 'issue_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review_comment' && contains(github.event.comment.body, '@claude')) || (github.event_name == 'pull_request_review' && contains(github.event.review.body, '@claude')) || (github.event_name == 'issues' && (contains(github.event.issue.body, '@claude') || contains(github.event.issue.title, '@claude'))) runs-on: ubuntu-latest permissions: contents: read pull-requests: read issues: read id-token: write actions: read # Required for Claude to read CI results on PRs steps: - name: Checkout repository uses: actions/checkout@v4 with: fetch-depth: 1 - name: Run Claude Code id: claude uses: anthropics/claude-code-action@v1 with: anthropic_api_key: ${{ secrets.ANTHROPIC_API_KEY }} # This is an optional setting that allows Claude to read CI results on PRs additional_permissions: | actions: read # Optional: Give a custom prompt to Claude. If this is not specified, Claude will perform the instructions specified in the comment that tagged it. # prompt: 'Update the pull request description to include a summary of changes.' # Optional: Add claude_args to customize behavior and configuration # See https://github.com/anthropics/claude-code-action/blob/main/docs/usage.md # or https://docs.anthropic.com/en/docs/claude-code/sdk#command-line for available options # claude_args: '--model claude-opus-4-1-20250805 --allowed-tools Bash(gh pr:*)' timescale-pg_textsearch-b63fe58/.github/workflows/coverity.yml000066400000000000000000000041161525762047700247470ustar00rootroot00000000000000name: Coverity on: schedule: # Run weekly on Saturday at 22:00 UTC - cron: '0 22 * * SAT' push: branches: - coverity_scan - trigger/coverity pull_request: paths: - .github/workflows/coverity.yml workflow_dispatch: jobs: coverity: name: Coverity PG${{ matrix.pg }} runs-on: ubuntu-latest strategy: fail-fast: false matrix: pg: [17, 18] steps: - name: Checkout uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg }} postgresql-server-dev-${{ matrix.pg }} - name: Download Coverity tools run: | wget https://scan.coverity.com/download/linux64 \ --post-data "token=${{ secrets.COVERITY_TOKEN }}&project=pg_textsearch" \ -O coverity_tool.tgz -q tar xf coverity_tool.tgz mv cov-analysis-linux64-* coverity - name: Build with Coverity run: | export PATH="$GITHUB_WORKSPACE/coverity/bin:/usr/lib/postgresql/${{ matrix.pg }}/bin:$PATH" make clean || true cov-build --dir cov-int make - name: Upload to Coverity env: FORM_TOKEN: --form token=${{ secrets.COVERITY_TOKEN }} FORM_EMAIL: --form email=tj@tigerdata.com FORM_FILE: --form file=@pg_textsearch.tgz FORM_DESC: --form description="GitHub Actions CI" COVERITY_URL: https://scan.coverity.com/builds?project=pg_textsearch run: | tar czf pg_textsearch.tgz cov-int VERSION=$(grep '^default_version' pg_textsearch.control | sed "s/.*'\(.*\)'/\1/") curl $FORM_TOKEN $FORM_EMAIL $FORM_DESC $FORM_FILE \ --form version="${VERSION}-pg${{ matrix.pg }}" $COVERITY_URL timescale-pg_textsearch-b63fe58/.github/workflows/formatting.yml000066400000000000000000000023301525762047700252510ustar00rootroot00000000000000name: Code Formatting Check # NOTE: PR checks are consolidated in ci.yml. This workflow runs on main only. on: push: branches: [ main ] paths: - 'src/**' - '.clang-format' - '.github/workflows/formatting.yml' jobs: format-check: name: Check C Code Formatting runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Install clang-format run: pip install clang-format==21.1.8 - name: Check formatting run: | # Find all C source and header files find src -name "*.c" -o -name "*.h" | while read file; do echo "Checking $file" if ! clang-format --dry-run --Werror "$file" > /dev/null 2>&1; then echo "❌ $file is not properly formatted" echo "Expected format:" clang-format "$file" exit 1 else echo "✅ $file is properly formatted" fi done - name: Suggest formatting command if: failure() run: | echo "" echo "To fix formatting issues, run:" echo " make format" echo "" echo "Or format individual files:" echo " clang-format -i src/filename.c" timescale-pg_textsearch-b63fe58/.github/workflows/nightly-stress.yml000066400000000000000000000262151525762047700261060ustar00rootroot00000000000000# Nightly stress tests with memory leak detection # # Runs long-running tests that: # - Exercise core functionality over extended time # - Create many segments via low spill thresholds # - Check for memory leaks and resource leaks # - Validate continued correctness over time # name: Nightly Stress Tests on: schedule: # Run at 5 AM UTC daily (9 PM Pacific) - cron: '0 5 * * *' workflow_dispatch: inputs: duration_minutes: description: 'Test duration in minutes' required: false default: '30' pg_version: description: 'PostgreSQL version' required: false default: '17' type: choice options: - '17' - '18' env: PG_SRC_DIR: "pgbuild" PG_INSTALL_DIR: "postgresql" CC: "clang-15" CXX: "clang-15" # AddressSanitizer with leak detection ENABLED CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" LDFLAGS: "-fsanitize=address,undefined" # Enable leak detection for stress tests ASAN_OPTIONS: "detect_leaks=1 abort_on_error=0 exitcode=27 log_path=${{ github.workspace }}/sanitizer_logs/asan print_stacktrace=1 check_initialization_order=1" UBSAN_OPTIONS: "halt_on_error=0 exitcode=27 print_stacktrace=1 log_path=${{ github.workspace }}/sanitizer_logs/ubsan" # Leak sanitizer options LSAN_OPTIONS: "suppressions=${{ github.workspace }}/test/lsan.supp exitcode=27 print_suppressions=0" permissions: contents: read issues: write jobs: stress-test: name: Stress Test PG${{ matrix.pg }} runs-on: ubuntu-22.04 timeout-minutes: 120 strategy: fail-fast: false matrix: pg: ${{ github.event_name == 'workflow_dispatch' && fromJSON(format('["{0}"]', github.event.inputs.pg_version || '17')) || fromJSON('["17", "18"]') }} steps: - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y flex bison build-essential libssl-dev \ libreadline-dev clang-15 llvm-15 bc - name: Checkout uses: actions/checkout@v4 - name: Create directories run: | mkdir -p ${{ github.workspace }}/sanitizer_logs mkdir -p ${{ github.workspace }}/test-results - name: Create leak sanitizer suppressions file run: | # Suppress known PostgreSQL internal leaks that are not our concern cat > ${{ github.workspace }}/test/lsan.supp << 'EOF' # PostgreSQL allocates some memory at startup that lives for the process lifetime # These are not actual leaks, just long-lived allocations leak:AllocSetContextCreate leak:MemoryContextAlloc leak:palloc leak:repalloc # PostgreSQL frontend utilities (pg_config, etc.) use pg_strdup and don't free # memory before exit - this is normal for short-lived command-line tools leak:pg_strdup leak:get_configdata # OpenSSL has some one-time initialization allocations leak:libssl leak:libcrypto # PostgreSQL process title (argv) saving - intentionally never freed leak:save_ps_display_args # PostgreSQL initdb - short-lived tool doesn't free memory before exit leak:escape_single_quotes_ascii leak:escape_quotes_bki leak:pretty_wal_size leak:pg_malloc_internal leak:setup_config leak:bootstrap_template1 # System library allocations in short-lived tools leak:getaddrinfo # PostgreSQL server startup allocations - live for process lifetime leak:PostmasterMain EOF - name: Get date for caching id: get-date run: echo "date=$(date +\"%Y%m%d\")" >> $GITHUB_OUTPUT - name: Cache PostgreSQL ${{ matrix.pg }} id: cache-postgresql uses: actions/cache@v4 with: path: | ~/${{ env.PG_SRC_DIR }} ~/${{ env.PG_INSTALL_DIR }} key: "nightly-postgresql-${{ matrix.pg }}-${{ env.CC }}-${{ steps.get-date.outputs.date }}" - name: Determine PG version id: pg-version run: | # Map major version to full version case "${{ matrix.pg }}" in 17) echo "full_version=17.2" >> $GITHUB_OUTPUT ;; 18) echo "full_version=18.1" >> $GITHUB_OUTPUT ;; *) echo "full_version=${{ matrix.pg }}.0" >> $GITHUB_OUTPUT ;; esac - name: Build PostgreSQL ${{ matrix.pg }} with sanitizers if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | wget -q -O postgresql.tar.bz2 \ https://ftp.postgresql.org/pub/source/v${{ steps.pg-version.outputs.full_version }}/postgresql-${{ steps.pg-version.outputs.full_version }}.tar.bz2 mkdir -p ~/$PG_SRC_DIR tar --extract --file postgresql.tar.bz2 --directory ~/$PG_SRC_DIR --strip-components 1 # Apply ASAN instrumentation patch PG_MAJOR=${{ matrix.pg }} if [ ${PG_MAJOR} -lt 18 ]; then patch -F5 -p1 -d ~/$PG_SRC_DIR < test/postgres-asan-instrumentation.patch else patch -F5 -p1 -d ~/$PG_SRC_DIR < test/postgres-asan-instrumentation-PG18GE.patch fi cd ~/$PG_SRC_DIR ./configure --prefix=$HOME/$PG_INSTALL_DIR --enable-debug --enable-cassert \ --with-openssl --without-readline --without-zlib --without-libxml make -j$(nproc) - name: Install PostgreSQL if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | cd ~/$PG_SRC_DIR make install - name: Build and install pg_textsearch run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" make clean make make install - name: Run stress tests timeout-minutes: 90 run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" # Determine test duration if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then DURATION="${{ github.event.inputs.duration_minutes }}" else # Default 30 minutes for scheduled nightly runs DURATION=30 fi echo "Running stress tests for ${DURATION} minutes..." # Run stress tests with low spill threshold to create many segments cd test/scripts STRESS_DURATION_MINUTES=${DURATION} \ SPILL_THRESHOLD=500 \ DOCS_PER_BATCH=50 \ CONCURRENT_READERS=8 \ CONCURRENT_WRITERS=4 \ ./stress.sh 2>&1 | tee ${{ github.workspace }}/test-results/stress-test.log - name: Run regression tests under sanitizer run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" rm -rf tmp_nightly_test mkdir -p tmp_nightly_test initdb -D tmp_nightly_test/data --auth-local=trust LIB_NAME="pg_textsearch" echo "port = 55437" >> tmp_nightly_test/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_nightly_test'" >> tmp_nightly_test/data/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> tmp_nightly_test/data/postgresql.conf pg_ctl start -D tmp_nightly_test/data -l tmp_nightly_test/data/logfile -w createdb -h $PWD/tmp_nightly_test -p 55437 contrib_regression echo "Running SQL regression tests..." if ! make test PGHOST=$PWD/tmp_nightly_test PGPORT=55437; then echo "Regression tests failed" cat test/regression.diffs || true fi pg_ctl stop -D tmp_nightly_test/data - name: Run concurrency tests timeout-minutes: 30 run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" cd test/scripts ./concurrency.sh 2>&1 | tee ${{ github.workspace }}/test-results/concurrency.log - name: Run segment tests timeout-minutes: 20 run: | set -o pipefail export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" make test-segment 2>&1 | tee ${{ github.workspace }}/test-results/segment.log - name: Check for sanitizer errors if: always() run: | echo "Checking for sanitizer errors..." # Count sanitizer log files error_count=0 if [ -d "${{ github.workspace }}/sanitizer_logs" ]; then for log in ${{ github.workspace }}/sanitizer_logs/*; do if [ -f "$log" ] && [ -s "$log" ]; then echo "=== Sanitizer error in $log ===" cat "$log" echo "" error_count=$((error_count + 1)) fi done fi if [ $error_count -gt 0 ]; then echo "Found $error_count sanitizer error logs" # Don't fail here - let the upload happen first else echo "No sanitizer errors detected" fi - name: Upload test results if: always() uses: actions/upload-artifact@v4 with: name: nightly-stress-results-pg${{ matrix.pg }} path: | ${{ github.workspace }}/test-results/ ${{ github.workspace }}/sanitizer_logs/ test/regression.diffs test/regression.out test/results/ retention-days: 14 - name: Upload failure artifacts if: failure() uses: actions/upload-artifact@v4 with: name: nightly-failure-debug-pg${{ matrix.pg }} path: | test/scripts/tmp_*/ test/tmp_*/ tmp_nightly_test/ retention-days: 7 notify-on-failure: name: Notify on Failure runs-on: ubuntu-latest needs: stress-test if: failure() steps: - name: Create failure issue uses: actions/github-script@v7 with: script: | const title = `Nightly stress tests failed on ${new Date().toISOString().split('T')[0]}`; const body = `## Nightly Stress Test Failure The nightly stress tests have failed. This may indicate: - Memory leaks in the extension - Resource leaks - Correctness issues under extended operation - Segment handling problems **Workflow run:** ${context.serverUrl}/${context.repo.owner}/${context.repo.repo}/actions/runs/${context.runId} Please investigate the uploaded artifacts for details. `; // Check if there's already an open issue with this title prefix const issues = await github.rest.issues.listForRepo({ owner: context.repo.owner, repo: context.repo.repo, state: 'open', labels: 'nightly-failure' }); const existingIssue = issues.data.find(i => i.title.startsWith('Nightly stress tests failed') ); if (existingIssue) { // Add a comment to existing issue await github.rest.issues.createComment({ owner: context.repo.owner, repo: context.repo.repo, issue_number: existingIssue.number, body: `Another failure on ${new Date().toISOString().split('T')[0]}\n\n${body}` }); } else { // Create new issue await github.rest.issues.create({ owner: context.repo.owner, repo: context.repo.repo, title: title, body: body, labels: ['nightly-failure', 'bug'] }); } timescale-pg_textsearch-b63fe58/.github/workflows/package-release.yml000066400000000000000000000263641525762047700261250ustar00rootroot00000000000000name: Package Release on: workflow_dispatch: inputs: tag: description: 'Tag/Version (e.g. v0.0.2)' required: true type: string push: tags: - 'v*' permissions: contents: write # Needed to upload assets to releases jobs: test-upgrades: runs-on: ubuntu-latest strategy: matrix: pg_version: [17, 18] steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.tag || github.ref }} - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} sudo systemctl stop postgresql - name: Test upgrade from previous stable release run: | export PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config STRICT_UNSPILLED=1 bash test/scripts/upgrade_matrix.sh 1.5.0 build-packages: needs: test-upgrades runs-on: ubuntu-latest strategy: fail-fast: false matrix: pg_version: [17, 18] platform: - type: amd64 docker_platform: linux/amd64 - type: arm64 docker_platform: linux/arm64 steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.tag || github.ref }} - name: Set version run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "VERSION=${{ github.event.inputs.tag }}" >> $GITHUB_ENV else echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV fi echo "Building version: $VERSION" - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: platforms: arm64 - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 - name: Build extension in Docker run: | # Create a build script cat > build.sh << 'BUILDSCRIPT' #!/bin/bash set -ex PG_VERSION="$3" echo "Building for PostgreSQL version: $PG_VERSION" # Installing packages under QEMU emulation occasionally dies with # a segfault in a maintainer script (see issue #466), so retry # before giving up on the whole build. install_deps() { local attempt for attempt in 1 2 3; do if apt-get update -o Acquire::Retries=3 && apt-get install -y -o Acquire::Retries=3 \ build-essential \ "postgresql-server-dev-$PG_VERSION"; then return 0 fi echo "apt-get attempt $attempt failed; retrying in 15s" dpkg --configure -a || true sleep 15 done echo "ERROR: could not install build dependencies" return 1 } install_deps # Use the correct pg_config for the target PostgreSQL version # The postgresql-server-dev-XX package installs pg_config here export PG_CONFIG="/usr/lib/postgresql/$PG_VERSION/bin/pg_config" if [ ! -f "$PG_CONFIG" ]; then echo "ERROR: pg_config not found at $PG_CONFIG" echo "Make sure postgresql-server-dev-$PG_VERSION is installed" exit 1 fi echo "Using pg_config: $PG_CONFIG" $PG_CONFIG --version # Build the extension cd /build make clean CFLAGS="-O3 -DNDEBUG" make # Package the extension bash scripts/package-deb.sh "$1" "$PWD" "$2" "$3" BUILDSCRIPT chmod +x build.sh # Run build in TimescaleDB container with proper architecture # Run as root to install dependencies. arm64 runs under QEMU # emulation, which is prone to spurious crashes, so retry the # whole container run a few times before failing the job. for attempt in 1 2 3; do if docker run --rm \ --platform ${{ matrix.platform.docker_platform }} \ -v "$PWD:/build" \ -w /build \ --user root \ timescale/timescaledb-ha:pg${{ matrix.pg_version }}-all \ /build/build.sh "$VERSION" '${{ matrix.platform.type }}' '${{ matrix.pg_version }}'; then echo "Build succeeded on attempt $attempt" exit 0 fi echo "::warning::Build attempt $attempt failed" # Leftovers from the failed attempt are owned by root. sudo rm -rf dist debian-build sleep 30 done echo "::error::Build failed after 3 attempts" exit 1 - name: Verify package architecture run: | # Fix permissions after Docker build (Docker runs as root) sudo chown -R $USER:$USER dist/ # Extract and check the actual binary architecture mkdir -p check-arch dpkg-deb -x dist/*.deb check-arch/ file check-arch/usr/lib/postgresql/*/lib/pg_textsearch.so | grep -E "(x86-64|aarch64)" || true rm -rf check-arch - name: Verify PostgreSQL version in package run: | DEB_FILE=$(ls dist/*.deb) EXPECTED_PG_VERSION="${{ matrix.pg_version }}" echo "Checking package: $DEB_FILE" echo "Expected PostgreSQL version: $EXPECTED_PG_VERSION" # List package contents and extract the PostgreSQL version directory ACTUAL_PG_DIR=$(dpkg -c "$DEB_FILE" | grep -oP 'usr/lib/postgresql/\K\d+' | head -n1) echo "Actual PostgreSQL version in package: $ACTUAL_PG_DIR" # Verify the PostgreSQL version matches if [ "$ACTUAL_PG_DIR" != "$EXPECTED_PG_VERSION" ]; then echo "❌ ERROR: Package contains PostgreSQL $ACTUAL_PG_DIR binaries but should contain PostgreSQL $EXPECTED_PG_VERSION" echo "This indicates the build used the wrong pg_config" exit 1 fi # Also verify the .deb filename contains the correct version if ! echo "$DEB_FILE" | grep -q "postgresql-${EXPECTED_PG_VERSION}"; then echo "❌ ERROR: .deb filename does not contain postgresql-${EXPECTED_PG_VERSION}" echo "Found: $DEB_FILE" exit 1 fi echo "✅ Package verification passed: PostgreSQL version is correct" - name: Create zip archive run: | cd dist # Create zip with the expected naming convention for timescaledb-docker-cloud zip pg-textsearch-${{ env.VERSION }}-pg${{ matrix.pg_version }}-${{ matrix.platform.type }}.zip *.deb cd .. - name: Upload package artifacts uses: actions/upload-artifact@v4 with: name: pg-textsearch-${{ env.VERSION }}-pg${{ matrix.pg_version }}-${{ matrix.platform.type }} path: | dist/*.deb dist/*.zip retention-days: 30 build-source: needs: test-upgrades runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 with: ref: ${{ github.event.inputs.tag || github.ref }} - name: Set version run: | if [ "${{ github.event_name }}" = "workflow_dispatch" ]; then echo "VERSION=${{ github.event.inputs.tag }}" >> $GITHUB_ENV else echo "VERSION=${GITHUB_REF#refs/tags/}" >> $GITHUB_ENV fi echo "Building version: $VERSION" - name: Create source tarball run: | # Clean version without 'v' prefix CLEAN_VERSION=${VERSION#v} # Create directory with version mkdir -p pg_textsearch-${CLEAN_VERSION} # Copy source files cp -r src sql test benchmarks Makefile pg_textsearch.control \ README.md ARCHITECTURE.md CONTRIBUTING.md LICENSE \ pg_textsearch-${CLEAN_VERSION}/ # Create tarball tar -czf pg_textsearch-${CLEAN_VERSION}.tar.gz pg_textsearch-${CLEAN_VERSION}/ # Also create a .tar.bz2 tar -cjf pg_textsearch-${CLEAN_VERSION}.tar.bz2 pg_textsearch-${CLEAN_VERSION}/ mkdir -p dist mv pg_textsearch-${CLEAN_VERSION}.tar.* dist/ - name: Upload source artifacts uses: actions/upload-artifact@v4 with: name: pg-textsearch-source-${{ env.VERSION }} path: dist/pg_textsearch-*.tar.* retention-days: 30 release: needs: [build-packages, build-source] runs-on: ubuntu-latest if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - uses: actions/checkout@v4 - name: Download all artifacts uses: actions/download-artifact@v4 with: path: artifacts - name: Verify all platform packages were built run: | MISSING="" for pg in 17 18; do for arch in amd64 arm64; do if ! ls artifacts/*/pg-textsearch-*-pg${pg}-${arch}.zip \ >/dev/null 2>&1; then MISSING="$MISSING pg${pg}-${arch}" fi done done if [ -n "$MISSING" ]; then echo "::error::Missing platform packages:$MISSING" echo "Refusing to publish an incomplete set of release assets." exit 1 fi echo "✅ All platform packages present" - name: Upload Release Assets env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | TAG_NAME="${GITHUB_REF#refs/tags/}" # Check if release exists, create if it doesn't if ! gh release view "$TAG_NAME" &>/dev/null; then echo "Creating release for tag $TAG_NAME" gh release create "$TAG_NAME" \ --title "Release $TAG_NAME" \ --notes "Release $TAG_NAME" \ --draft fi # Upload .zip and source packages for file in artifacts/*/*.zip artifacts/*/*.tar.*; do if [ -f "$file" ]; then echo "Uploading $(basename "$file") to release $TAG_NAME" gh release upload "$TAG_NAME" "$file" --clobber fi done # A failed matrix leg skips the release job entirely, which silently # leaves the release without .deb packages (issue #466). Surface that # as an explicit, named failure in the run summary. release-status: needs: [build-packages, build-source, release] runs-on: ubuntu-latest if: always() && github.event_name == 'push' && startsWith(github.ref, 'refs/tags/') steps: - name: Check release assets were published run: | RESULT="${{ needs.release.result }}" if [ "$RESULT" = "success" ]; then echo "✅ Release assets published" >> $GITHUB_STEP_SUMMARY exit 0 fi { echo "## ❌ Release assets were NOT published" echo echo "- \`build-packages\`: ${{ needs.build-packages.result }}" echo "- \`build-source\`: ${{ needs.build-source.result }}" echo "- \`release\`: $RESULT" echo echo "The tagged release is missing its \`.deb\` packages." echo "Re-run the failed jobs with \`gh run rerun --failed\`." } >> $GITHUB_STEP_SUMMARY echo "::error::Release assets were not published for this tag" exit 1 timescale-pg_textsearch-b63fe58/.github/workflows/pgspot.yml000066400000000000000000000014461525762047700244220ustar00rootroot00000000000000name: pgspot Security Check # NOTE: PR checks are consolidated in ci.yml. This workflow runs on main only. on: push: branches: [ main ] paths: - 'sql/**' - '.github/workflows/pgspot.yml' jobs: pgspot: name: Check Extension SQL runs-on: ubuntu-latest steps: - name: Checkout code uses: actions/checkout@v4 - name: Set up Python uses: actions/setup-python@v5 with: python-version: '3.12' - name: Install pgspot run: pip install pgspot - name: Run pgspot run: | # Check the current version's base install SQL file VERSION=$(awk -F"'" '/default_version/ {print $2}' pg_textsearch.control) echo "Checking sql/pg_textsearch--${VERSION}.sql" pgspot "sql/pg_textsearch--${VERSION}.sql" timescale-pg_textsearch-b63fe58/.github/workflows/release.yml000066400000000000000000000220021525762047700245150ustar00rootroot00000000000000name: Release on: push: tags: - 'v*' workflow_dispatch: # Allow manual testing permissions: contents: write # Needed to upload assets to releases deployments: write # Needed for benchmark tracking jobs: # Performance regression check before release benchmark-gate: runs-on: ubuntu-latest timeout-minutes: 30 steps: - uses: actions/checkout@v4 - name: Install PostgreSQL 17 run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc jq wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | \ sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | \ sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-17 postgresql-server-dev-17 - name: Build extension (optimized) run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" make clean CFLAGS="-O3 -DNDEBUG" make sudo make install - name: Start PostgreSQL run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" rm -rf tmp_bench mkdir -p tmp_bench/socket initdb -D tmp_bench/data --auth-local=trust --auth-host=trust echo "shared_buffers = 256MB" >> tmp_bench/data/postgresql.conf echo "max_connections = 20" >> tmp_bench/data/postgresql.conf echo "shared_preload_libraries = 'pg_textsearch'" >> tmp_bench/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_bench/socket'" >> tmp_bench/data/postgresql.conf pg_ctl start -D tmp_bench/data -l tmp_bench/postgres.log -w export PGHOST="$PWD/tmp_bench/socket" createdb bench_test psql -d bench_test -c "CREATE EXTENSION pg_textsearch" echo "PGHOST=$PWD/tmp_bench/socket" >> $GITHUB_ENV - name: Run Cranfield benchmark run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" export PGDATABASE=bench_test cd benchmarks/sql/cranfield psql -f 01-load.sql 2>&1 | tee "$GITHUB_WORKSPACE/benchmark_results.txt" psql -f 02-queries.sql 2>&1 | tee -a "$GITHUB_WORKSPACE/benchmark_results.txt" - name: Extract and format metrics run: | chmod +x benchmarks/runner/extract_metrics.sh chmod +x benchmarks/runner/format_for_action.sh ./benchmarks/runner/extract_metrics.sh benchmark_results.txt benchmark_metrics.json jq '.dataset = "cranfield"' benchmark_metrics.json > tmp.json && mv tmp.json benchmark_metrics.json ./benchmarks/runner/format_for_action.sh benchmark_metrics.json benchmark_action.json - name: Download benchmark baseline uses: actions/cache@v4 with: path: ./cache key: benchmark-baseline-cranfield restore-keys: | benchmark-baseline-cranfield - name: Check for performance regression if: hashFiles('benchmark_action.json') != '' uses: benchmark-action/github-action-benchmark@v1 with: name: 'Cranfield Benchmarks' tool: 'customSmallerIsBetter' output-file-path: benchmark_action.json external-data-json-path: ./cache/benchmark-data.json github-token: ${{ secrets.GITHUB_TOKEN }} alert-threshold: '120%' fail-on-alert: true comment-on-alert: true save-data-file: false - name: Cleanup if: always() run: | export PATH="/usr/lib/postgresql/17/bin:$PATH" pg_ctl stop -D tmp_bench/data || true rm -rf tmp_bench build-and-test: needs: benchmark-gate runs-on: ubuntu-latest strategy: matrix: pg_version: [17, 18] steps: - uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates build-essential bc wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - echo "deb http://apt.postgresql.org/pub/repos/apt/ $(lsb_release -cs)-pgdg main" | sudo tee /etc/apt/sources.list.d/pgdg.list sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} - name: Build extension (optimized) run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" make clean # Build optimized version for release CFLAGS="-O3 -DNDEBUG" make sudo make install - name: Start PostgreSQL run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" # Create fresh PostgreSQL instance to avoid DSA segment accumulation rm -rf tmp_release_check mkdir -p tmp_release_check initdb -D tmp_release_check/data --auth-local=trust --auth-host=trust echo "port = 55433" >> tmp_release_check/data/postgresql.conf echo "shared_buffers = 256MB" >> tmp_release_check/data/postgresql.conf echo "max_connections = 20" >> tmp_release_check/data/postgresql.conf echo "shared_preload_libraries = 'pg_textsearch'" >> tmp_release_check/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_release_check'" >> tmp_release_check/data/postgresql.conf pg_ctl start -D tmp_release_check/data -l tmp_release_check/data/logfile -w createdb -h $PWD/tmp_release_check -p 55433 contrib_regression - name: Run full test suite run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" export PGHOST=$PWD/tmp_release_check export PGPORT=55433 # Run regression tests using pg_regress directly /usr/lib/postgresql/${{ matrix.pg_version }}/lib/pgxs/src/makefiles/../../src/test/regress/pg_regress \ --inputdir=test --outputdir=test --dbname=contrib_regression \ aerodocs basic deletion vacuum dropped empty implicit index inheritance limits lock \ manyterms memory merge mixed partitioned queries schema scoring1 scoring2 scoring3 \ scoring4 scoring5 scoring6 segment strings unsupported updates vector # Run concurrency tests directly cd test/scripts && TEST_SIZE_MULTIPLIER=3.0 ./concurrency.sh - name: Package extension run: | export PATH="/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH" mkdir -p dist/pg${{ matrix.pg_version }} # Copy extension files cp pg_textsearch.control dist/pg${{ matrix.pg_version }}/ cp sql/pg_textsearch--*.sql dist/pg${{ matrix.pg_version }}/ # Copy built library with version info PG_LIB_DIR=$(pg_config --pkglibdir) if [ -f "$PG_LIB_DIR/pg_textsearch.so" ]; then cp "$PG_LIB_DIR/pg_textsearch.so" dist/pg${{ matrix.pg_version }}/ elif [ -f "pg_textsearch.dylib" ]; then cp pg_textsearch.dylib dist/pg${{ matrix.pg_version }}/ fi # Create README for this version cat > dist/pg${{ matrix.pg_version }}/README.txt << 'READMEEOF' pg_textsearch PostgreSQL Extension PostgreSQL Version: ${{ matrix.pg_version }} Built: $(date) Commit: ${{ github.sha }} Installation: 1. Copy pg_textsearch.control and pg_textsearch--*.sql to your PostgreSQL extension directory 2. Copy the library file to your PostgreSQL library directory 3. Run: CREATE EXTENSION pg_textsearch; For detailed documentation, see: https://github.com/timescale/pg_textsearch READMEEOF # Create tarball cd dist tar -czf ../pg_textsearch-pg${{ matrix.pg_version }}-${{ github.ref_name }}.tar.gz pg${{ matrix.pg_version }}/ cd .. - name: Upload build artifacts uses: actions/upload-artifact@v4 with: name: pg_textsearch-pg${{ matrix.pg_version }}-${{ github.ref_name }} path: | dist/pg${{ matrix.pg_version }}/ pg_textsearch-pg${{ matrix.pg_version }}-${{ github.ref_name }}.tar.gz retention-days: 30 - name: Upload release assets if: github.event_name == 'release' || (github.event_name == 'push' && startsWith(github.ref, 'refs/tags/')) env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} run: | # For tag pushes, create a release first if it doesn't exist if [ "${{ github.event_name }}" = "push" ]; then # Check if release exists if ! gh release view "${{ github.ref_name }}" &>/dev/null; then echo "Creating release for tag ${{ github.ref_name }}" gh release create "${{ github.ref_name }}" \ --title "Release ${{ github.ref_name }}" \ --notes "Release ${{ github.ref_name }}" \ --draft fi fi # Upload the asset echo "Uploading pg_textsearch-pg${{ matrix.pg_version }}-${{ github.ref_name }}.tar.gz to release ${{ github.ref_name }}" gh release upload "${{ github.ref_name }}" \ "./pg_textsearch-pg${{ matrix.pg_version }}-${{ github.ref_name }}.tar.gz" \ --clobber timescale-pg_textsearch-b63fe58/.github/workflows/sanitizer-build-and-test.yml000066400000000000000000000301741525762047700277300ustar00rootroot00000000000000# Run regression tests under memory sanitizer # NOTE: PR checks are consolidated in ci.yml. This workflow runs on main only. name: Sanitizer test on: push: branches: [ main ] paths: - 'src/**' - 'test/**' - 'sql/**' - 'Makefile' - 'pg_textsearch.control' - '.github/workflows/sanitizer-build-and-test.yml' - '.github/scripts/build-postgres-sanitizer.sh' workflow_dispatch: env: name: "Sanitizer" PG_SRC_DIR: "pgbuild" PG_INSTALL_DIR: "postgresql" extra_packages: "clang-15 llvm-15" CC: "clang-15" CXX: "clang-15" CFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" CXXFLAGS: "-g -fsanitize=address,undefined -fno-omit-frame-pointer -Og -fno-inline-functions" LDFLAGS: "-fsanitize=address,undefined" # Sanitizer options - create logs directory for better debugging ASAN_OPTIONS: "detect_leaks=0 abort_on_error=1 exitcode=27 log_path=${{ github.workspace }}/sanitizer_logs/asan print_stacktrace=1 check_initialization_order=1" UBSAN_OPTIONS: "halt_on_error=1 exitcode=27 print_stacktrace=1 log_path=${{ github.workspace }}/sanitizer_logs/ubsan print_summary=1" jobs: sanitizer: name: PG${{ matrix.pg }} Sanitizer runs-on: ubuntu-22.04 timeout-minutes: 45 strategy: fail-fast: false matrix: pg: ["17.2", "18.1"] steps: - name: Install dependencies run: | sudo apt-get update sudo apt-get install -y flex bison build-essential libssl-dev \ libreadline-dev systemd-coredump gdb ${{ env.extra_packages }} - name: Checkout uses: actions/checkout@v4 - name: Create sanitizer logs directory run: mkdir -p ${{ github.workspace }}/sanitizer_logs - name: Cache PostgreSQL ${{ matrix.pg }} id: cache-postgresql uses: actions/cache@v4 with: path: ~/${{ env.PG_SRC_DIR }} key: sanitizer-postgresql-${{ matrix.pg }}-${{ env.CC }}-randomized-memory-injection-v1-${{ hashFiles('.github/scripts/build-postgres-sanitizer.sh', 'test/postgres-asan-instrumentation.patch', 'test/postgres-asan-instrumentation-PG18GE.patch') }} - name: Build PostgreSQL ${{ matrix.pg }} with sanitizers if: steps.cache-postgresql.outputs.cache-hit != 'true' run: | .github/scripts/build-postgres-sanitizer.sh \ "${{ matrix.pg }}" "$HOME/$PG_SRC_DIR" "$HOME/$PG_INSTALL_DIR" \ true true - name: Install PostgreSQL ${{ matrix.pg }} run: | cd ~/$PG_SRC_DIR make install make -C src/test/modules/injection_points install make -C contrib/dblink install test -f "$HOME/$PG_INSTALL_DIR/share/extension/dblink.control" # On cache hits the build step is skipped, so re-run the # pg_walinspect build here (no-op when already built) and # always install it. make -C contrib/pg_walinspect -j$(nproc) make -C contrib/pg_walinspect install # Fail fast if the contrib install regressed — clearer # error than a CREATE EXTENSION failure 8 minutes into # the test phase. test -f "$HOME/$PG_INSTALL_DIR/share/extension/pg_walinspect.control" ~/$PG_INSTALL_DIR/bin/pg_config --version - name: Verify sanitizers are enabled run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" echo "Verifying PostgreSQL was built with sanitizers..." # Check if postgres binary contains sanitizer symbols echo "Checking for AddressSanitizer symbols..." if nm ~/$PG_INSTALL_DIR/bin/postgres | grep -i asan > /dev/null; then echo "AddressSanitizer symbols found in postgres binary" else echo "AddressSanitizer symbols NOT found in postgres binary" echo "This suggests PostgreSQL was not built with -fsanitize=address" exit 1 fi # Test that sanitizers are actually working by triggering a known issue echo "Testing sanitizer detection with intentional buffer overflow..." cat > test_sanitizer.c << 'EOF' #include #include int main() { char *buf = malloc(10); buf[10] = 'x'; // Buffer overflow - should be caught by AddressSanitizer free(buf); return 0; } EOF # Compile and run test - should fail with sanitizer error $CC $CFLAGS test_sanitizer.c -o test_sanitizer if ./test_sanitizer; then echo "WARNING: Sanitizer test did not catch buffer overflow!" else echo "Sanitizer correctly caught buffer overflow (exit code $?)" fi rm -f test_sanitizer test_sanitizer.c - name: Build and install pg_textsearch run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" make clean make make install - name: Run regression tests run: | export PATH="$HOME/$PG_INSTALL_DIR/bin:$PATH" # Set up test database rm -rf tmp_sanitizer_test mkdir -p tmp_sanitizer_test initdb -D tmp_sanitizer_test/data --auth-local=trust # Configure PostgreSQL LIB_NAME="pg_textsearch" echo "port = 55433" >> tmp_sanitizer_test/data/postgresql.conf echo "unix_socket_directories = '$PWD/tmp_sanitizer_test'" >> tmp_sanitizer_test/data/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> tmp_sanitizer_test/data/postgresql.conf # Start PostgreSQL pg_ctl start -D tmp_sanitizer_test/data -l tmp_sanitizer_test/data/logfile -w # Create test database and run tests createdb -h $PWD/tmp_sanitizer_test -p 55433 contrib_regression echo "Running SQL regression tests under sanitizer..." make installcheck PGHOST=$PWD/tmp_sanitizer_test PGPORT=55433 echo "Running stale memtable-tail tests under sanitizer..." timeout 120s make test-memtable-stale-tail # Run shell-based tests under sanitizer (more likely to find memory issues) echo "Running spill-time inline compaction locking tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./inline_compaction_locking.sh " echo "Running concurrency tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./concurrency.sh " echo "Running VACUUM concurrency tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./vacuum_concurrent_merge.sh " echo "Running crash recovery tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./recovery.sh " # Deferred-reclaim crash recovery (issue #380): parks displaced # segment pages in the tombstone chain, crashes (immediate stop), # and asserts WAL replay reconstructs the chain. Under ASan a # use-after-recycle of a displaced page would trip immediately. echo "Running deferred-reclaim crash recovery tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./standby_reclaim.sh " echo "Running shutdown spill tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./shutdown_spill.sh " echo "Running multi-backend segment tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 make test-segment " echo "Running CREATE INDEX CONCURRENTLY tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./cic.sh " echo "Running multi-index / multi-user / multi-schema tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" export PGHOST=$PWD/tmp_sanitizer_test export PGPORT=55433 cd test/scripts && ./multi_index.sh " # Multi-backend reindex test (issue #390): exercises the # backend cache invalidation path after ALTER TABLE ADD COLUMN # ... GENERATED STORED rewrites the heap and reindexes BM25. # Spins up its own PG instance on a dedicated port; uses the # sanitizer-built pg_textsearch.so via $PATH. Sanitizers are # especially valuable here because the original bug was a # stale-pointer use-after-realloc class. echo "Running multi-backend reindex tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./multi_backend_reindex.sh " # WAL-audit asserts no custom rmgr records on stream # (issue #374) and that crash recovery reads the on-disk # memtable chain. Spins up its own PG instance on a # dedicated port; uses the sanitizer-built pg_textsearch.so # via $PATH. echo "Running WAL audit tests under sanitizer..." timeout 600s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./wal_audit.sh " # Replication spill paths exercises bulk-load PRE_COMMIT # spill and VACUUM bulkdelete spill on a primary + standby # pair, asserting both paths replicate cleanly. Spins up # its own primary + standby on dedicated ports. echo "Running replication spill-paths tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./replication_spill_paths.sh " # Standby segment-reclaim acceptance (issue #380): a live # standby snapshot with hot_standby_feedback=on must hold the # primary's reclaim horizon so a merge+VACUUM keeps displaced # pages parked, and the standby reads cleanly across merge # replay. Spins up its own primary + standby on dedicated # ports. Under ASan, recycling a page out from under the # standby read would surface as a use-after-recycle. echo "Running standby segment-reclaim acceptance tests under sanitizer..." timeout 1200s bash -c " export PATH=\"$HOME/$PG_INSTALL_DIR/bin:\$PATH\" cd test/scripts && ./replication_segment_reclaim.sh " # Clean up pg_ctl stop -D tmp_sanitizer_test/data -l tmp_sanitizer_test/data/logfile - name: Collect sanitizer reports if: always() run: | echo "Collecting all sanitizer logs..." find ${{ github.workspace }}/sanitizer_logs -type f 2>/dev/null | \ head -20 | while read file; do echo "=== $file ===" head -100 "$file" 2>/dev/null || echo "(file not readable or empty)" echo done - name: Upload sanitizer logs and test artifacts if: always() uses: actions/upload-artifact@v4 with: name: sanitizer-logs-pg${{ matrix.pg }} path: | ${{ github.workspace }}/sanitizer_logs/ test/regression.diffs test/regression.out test/results/ tmp_sanitizer_test/data/logfile retention-days: 7 - name: Upload failure artifacts if: failure() uses: actions/upload-artifact@v4 with: name: sanitizer-failure-debug-pg${{ matrix.pg }} path: | tmp_sanitizer_test/ test/scripts/tmp_* test/tmp_* retention-days: 7 timescale-pg_textsearch-b63fe58/.github/workflows/upgrade-tests.yml000066400000000000000000001117001525762047700256700ustar00rootroot00000000000000name: Upgrade Tests on: workflow_dispatch: inputs: target_version: description: 'Target version to test upgrades to (leave empty for current branch)' required: false type: string pull_request: paths: - 'sql/**' - 'src/segment/**' - 'src/index/**' - 'test/scripts/upgrade_matrix.sh' - 'pg_textsearch.control' - '.github/workflows/upgrade-tests.yml' push: tags: - 'v*' schedule: # Run weekly on Sundays at 2 AM UTC - cron: '0 2 * * 0' jobs: test-upgrades: runs-on: ubuntu-latest strategy: fail-fast: false matrix: pg_version: [17, 18] old_version: ['0.2.0', '0.3.0', '0.4.0', '0.4.1', '0.4.2', '0.5.0', '0.5.1', '0.6.0', '0.6.1', '1.0.0', '1.1.0', '1.2.0', '1.3.0', '1.3.1', '1.4.0', '1.5.0'] steps: - name: Checkout current code uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates gnupg sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} sudo systemctl stop postgresql - name: Download old release v${{ matrix.old_version }} run: | # Download the source tarball from GitHub releases DOWNLOAD_URL="https://github.com/timescale/pg_textsearch/releases/download/v${{ matrix.old_version }}/pg_textsearch-${{ matrix.old_version }}.tar.gz" echo "Downloading from: $DOWNLOAD_URL" if curl -fsSL -o pg_textsearch-old.tar.gz "$DOWNLOAD_URL"; then tar -xzf pg_textsearch-old.tar.gz ls -la else echo "::error::Failed to download v${{ matrix.old_version }}" exit 1 fi - name: Build and install old version v${{ matrix.old_version }} run: | cd pg_textsearch-${{ matrix.old_version }} export PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config make clean || true make sudo make install - name: Start PostgreSQL with old binary and create test data run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH export PGDATA=/tmp/upgrade_test_data LIB_NAME="pg_textsearch" initdb -D $PGDATA echo "shared_buffers = 256MB" >> $PGDATA/postgresql.conf echo "port = 5432" >> $PGDATA/postgresql.conf echo "unix_socket_directories = '/tmp'" >> $PGDATA/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> $PGDATA/postgresql.conf pg_ctl -D $PGDATA start -w -l $PGDATA/logfile createdb -h /tmp upgrade_test - name: Install old extension and create test data run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH # CREATE EXTENSION needs GHA variable; use -c to avoid # shell-expanding $$ in the heredoc SQL that follows. psql -h /tmp -d upgrade_test -c \ "CREATE EXTENSION pg_textsearch VERSION '${{ matrix.old_version }}';" psql -h /tmp -d upgrade_test << 'SQL' \set ON_ERROR_STOP on SELECT extversion AS old_version FROM pg_extension WHERE extname = 'pg_textsearch'; CREATE TABLE upgrade_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); -- Batch 1: 120 rows across 8 topic clusters INSERT INTO upgrade_docs (content) SELECT CASE (g % 8) WHEN 0 THEN 'PostgreSQL database management and query optimization' WHEN 1 THEN 'full text search engine with BM25 relevance ranking' WHEN 2 THEN 'machine learning classification using neural networks' WHEN 3 THEN 'distributed computing consensus protocols replication' WHEN 4 THEN 'programming language compiler design type inference' WHEN 5 THEN 'cloud infrastructure container orchestration deploy' WHEN 6 THEN 'database indexing strategies for analytical workloads' WHEN 7 THEN 'search engine document scoring with term weighting' END || ' document ' || g FROM generate_series(1, 120) g; -- Build batch 1 into segment 0 CREATE INDEX idx_upgrade_bm25 ON upgrade_docs USING bm25 (content) WITH (text_config = 'english'); SQL # Compat tier (read-compatible with the new binary, no # REINDEX): metapage v6 (issue #383, v0.5.1–1.2.x) and # metapage v7 (issue #374, v1.3.0) both upgrade in place on # the first write; v1.3.1–1.4.x write v8 and v1.5.x writes v9. # ALTER EXTENSION UPDATE alone is enough. # Add batch 2, spill, capture baselines. Pre-v0.5.1 # versions have a v5 metapage which is still REINDEX-only # (legacy tier below). # # The source metapage version drives the Phase 1a "still old # version" assertion (v6 for ≤1.2.x, v7 for 1.3.0, v8 for # 1.3.1+); v6/v7 then lazily upgrade to v8 on first write # (issue #380). if [[ "${{ matrix.old_version }}" =~ ^(0\.5\.1|0\.6\.[01]|1\.[0-5]\.[0-9]+)$ ]]; then psql -h /tmp -d upgrade_test << 'SQL' \set ON_ERROR_STOP on -- Batch 2: 80 rows (will be spilled to segment 1) INSERT INTO upgrade_docs (content) SELECT CASE (g % 8) WHEN 0 THEN 'relational database storage engine buffer management' WHEN 1 THEN 'information retrieval term scoring document ranking' WHEN 2 THEN 'deep learning optimization gradient descent training' WHEN 3 THEN 'network protocol design for distributed messaging' WHEN 4 THEN 'functional programming immutable data composition' WHEN 5 THEN 'serverless computing event driven scaling patterns' WHEN 6 THEN 'query planner cost estimation and join ordering' WHEN 7 THEN 'lexical analysis tokenization for search indexing' END || ' entry ' || g FROM generate_series(1, 80) g; -- Spill batch 2 to create segment 1 SELECT bm25_spill_index('idx_upgrade_bm25'); -- Helper: exact comparison against a baseline table CREATE FUNCTION validate_baseline( query_terms text, baseline_tbl text, label text ) RETURNS void AS $$ DECLARE diff_count int; BEGIN EXECUTE format( 'CREATE TEMP TABLE _vb AS SELECT id, content <@> to_bm25query(%L, ''idx_upgrade_bm25'') AS score FROM upgrade_docs ORDER BY content <@> to_bm25query(%L, ''idx_upgrade_bm25''), id LIMIT 10', query_terms, query_terms ); EXECUTE format( 'SELECT count(*) FROM ( (SELECT id, score FROM %I EXCEPT ALL SELECT id, score FROM _vb) UNION ALL (SELECT id, score FROM _vb EXCEPT ALL SELECT id, score FROM %I) ) d', baseline_tbl, baseline_tbl ) INTO diff_count; DROP TABLE _vb; IF diff_count > 0 THEN RAISE EXCEPTION 'FAIL [%]: % differences vs baseline', label, diff_count; END IF; RAISE NOTICE 'PASS [%]: results match baseline', label; END; $$ LANGUAGE plpgsql; -- Helper: verify no deleted rows appear, all scores valid CREATE FUNCTION validate_post_delete( query_terms text, max_deleted_id int, label text ) RETURNS void AS $$ DECLARE bad_count int; result_count int; bad_score float8; BEGIN EXECUTE format( 'CREATE TEMP TABLE _vd AS SELECT id, content <@> to_bm25query(%L, ''idx_upgrade_bm25'') AS score FROM upgrade_docs ORDER BY content <@> to_bm25query(%L, ''idx_upgrade_bm25''), id LIMIT 10', query_terms, query_terms ); SELECT count(*) INTO bad_count FROM _vd WHERE id <= max_deleted_id; IF bad_count > 0 THEN DROP TABLE _vd; RAISE EXCEPTION 'FAIL [%]: % deleted rows in results', label, bad_count; END IF; SELECT count(*) INTO result_count FROM _vd; IF result_count = 0 THEN DROP TABLE _vd; RAISE EXCEPTION 'FAIL [%]: query returned no results', label; END IF; SELECT max(score) INTO bad_score FROM _vd WHERE score >= 0; IF bad_score IS NOT NULL THEN DROP TABLE _vd; RAISE EXCEPTION 'FAIL [%]: non-negative score: %', label, bad_score; END IF; DROP TABLE _vd; RAISE NOTICE 'PASS [%]: % results, no deleted rows, valid scores', label, result_count; END; $$ LANGUAGE plpgsql; SQL fi - name: Stop PostgreSQL, install current version, restart run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH export PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config export PGDATA=/tmp/upgrade_test_data # Stop Postgres (still running old binary) pg_ctl -D $PGDATA stop -w # Remove old extension SQL files before installing current version, # otherwise stale shortcut upgrade scripts (e.g., 0.5.0--1.0.0.sql) # from the old release remain and Postgres may pick them over the # linear chain, causing version-check failures. SHAREDIR=$($PG_CONFIG --sharedir) sudo rm -f "$SHAREDIR/extension/pg_textsearch--"*.sql # Build and install current version (overwrites old binary) cd $GITHUB_WORKSPACE make clean make sudo make install # Restart with new binary pg_ctl -D $PGDATA start -w -l $PGDATA/logfile - name: Verify queries before ALTER EXTENSION (compat tier) run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH if [[ "${{ matrix.old_version }}" =~ ^(0\.5\.1|0\.6\.[01]|1\.[0-5]\.[0-9]+)$ ]]; then echo "=== Testing new binary reads old segments (pre-ALTER EXTENSION) ===" psql -h /tmp -d upgrade_test << 'SQL' \set ON_ERROR_STOP on -- Extension version should still be the old one DO $$ DECLARE ver text; BEGIN SELECT extversion INTO ver FROM pg_extension WHERE extname = 'pg_textsearch'; RAISE NOTICE 'Extension version (pre-ALTER): %', ver; END $$; -- Verify queries work with new binary on old segments. -- Capture baselines here (post-restart) because recovery -- rebuilds the memtable which can change corpus statistics. CREATE TABLE baseline_q1 AS SELECT id, content <@> to_bm25query('database', 'idx_upgrade_bm25') AS score FROM upgrade_docs ORDER BY content <@> to_bm25query('database', 'idx_upgrade_bm25'), id LIMIT 10; CREATE TABLE baseline_q2 AS SELECT id, content <@> to_bm25query('search ranking', 'idx_upgrade_bm25') AS score FROM upgrade_docs ORDER BY content <@> to_bm25query('search ranking', 'idx_upgrade_bm25'), id LIMIT 10; CREATE TABLE baseline_q3 AS SELECT id, content <@> to_bm25query('optimization', 'idx_upgrade_bm25') AS score FROM upgrade_docs ORDER BY content <@> to_bm25query('optimization', 'idx_upgrade_bm25'), id LIMIT 10; -- Verify results are valid (non-empty, negative scores) DO $$ DECLARE cnt int; bad float8; BEGIN SELECT count(*) INTO cnt FROM baseline_q1; IF cnt = 0 THEN RAISE EXCEPTION 'FAIL: pre-alter q1 returned no results'; END IF; SELECT max(score) INTO bad FROM baseline_q1 WHERE score >= 0; IF bad IS NOT NULL THEN RAISE EXCEPTION 'FAIL: pre-alter q1 non-negative score: %', bad; END IF; RAISE NOTICE 'PASS: pre-alter baselines captured (% results)', cnt; END $$; -- Verify the new binary actually uses the BM25 index scan on -- the v6-format index (issue #383). A sequential scan that -- fell back to standalone scoring would silently pass the -- result-count / score-sign checks above, hiding a broken -- v6 read path. Grep the EXPLAIN plan for the index name. DO $$ DECLARE rec record; plan_txt text := ''; BEGIN SET LOCAL enable_seqscan = off; FOR rec IN EXECUTE 'EXPLAIN SELECT id FROM upgrade_docs ' 'ORDER BY content <@> to_bm25query(''database'', ' '''idx_upgrade_bm25'') LIMIT 5' LOOP plan_txt := plan_txt || rec."QUERY PLAN" || E'\n'; END LOOP; IF plan_txt NOT LIKE '%idx_upgrade_bm25%' THEN RAISE EXCEPTION 'FAIL: pre-ALTER plan did not use bm25 index; got: %', plan_txt; END IF; RAISE NOTICE 'PASS: pre-ALTER plan uses BM25 index scan'; END $$; SELECT 'baseline_q1' AS query, * FROM baseline_q1; SELECT 'baseline_q2' AS query, * FROM baseline_q2; SELECT 'baseline_q3' AS query, * FROM baseline_q3; SQL else echo "Legacy tier: skipping pre-ALTER verification" fi - name: Upgrade extension to current version run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH if [[ "${{ matrix.old_version }}" =~ ^(0\.5\.1|0\.6\.[01]|1\.[0-5]\.[0-9]+)$ ]]; then echo "=== Compat tier: ALTER EXTENSION UPDATE (no REINDEX) ===" psql -h /tmp -d upgrade_test << 'SQL' \set ON_ERROR_STOP on ALTER EXTENSION pg_textsearch UPDATE; SELECT extversion AS new_version FROM pg_extension WHERE extname = 'pg_textsearch'; SQL else echo "=== Legacy tier: ALTER EXTENSION UPDATE + REINDEX ===" psql -h /tmp -d upgrade_test << 'SQL' \set ON_ERROR_STOP on ALTER EXTENSION pg_textsearch UPDATE; SELECT extversion AS new_version FROM pg_extension WHERE extname = 'pg_textsearch'; REINDEX INDEX idx_upgrade_bm25; SQL fi - name: Verify upgrade succeeded run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH EXPECTED_VERSION=$(awk -F"'" '/^[\t ]*default_version/ {print $2}' \ pg_textsearch.control | cut -d. -f1,2) psql -h /tmp -d upgrade_test << 'SQL' \set ON_ERROR_STOP on DO $$ BEGIN IF NOT EXISTS ( SELECT FROM pg_amop AS member JOIN pg_opfamily AS family ON family.oid = member.amopfamily JOIN pg_am AS am ON am.oid = family.opfmethod WHERE family.opfname = 'text_bm25_ops' AND am.amname = 'bm25' AND member.amopstrategy = 1 AND member.amoplefttype = 'text'::regtype AND member.amoprighttype = 'tsquery'::regtype AND member.amopopr = 'pg_catalog.@@(text,tsquery)'::regoperator ) THEN RAISE EXCEPTION 'FAIL: text_bm25_ops does not contain text @@ tsquery'; END IF; END $$; SQL # Source metapage version for the Phase 1a "still old version" # assertion: v1.3.0 natively writes v7 (issue #374); v1.3.1+ # through v1.4.x write v8; v1.5.x writes v9. # v0.5.1–1.2.x write v6 (issue #383). if [[ "${{ matrix.old_version }}" == "1.3.0" ]]; then OLD_METAPAGE_VERSION=7 elif [[ "${{ matrix.old_version }}" =~ ^1\.5\. ]]; then OLD_METAPAGE_VERSION=9 elif [[ "${{ matrix.old_version }}" =~ ^1\.(3\.[1-9][0-9]*|4\.[0-9]+)$ ]]; then OLD_METAPAGE_VERSION=8 else OLD_METAPAGE_VERSION=6 fi if [[ "${{ matrix.old_version }}" =~ ^(0\.5\.1|0\.6\.[01]|1\.[0-5]\.[0-9]+)$ ]]; then echo "=== Compat tier: multi-phase segment validation ===" psql -h /tmp -d upgrade_test \ -v expected_version="$EXPECTED_VERSION" \ -v old_metapage_version="$OLD_METAPAGE_VERSION" << 'SQL' \set ON_ERROR_STOP on -- Phase 1: Post-ALTER EXTENSION — old segments, new SQL SELECT validate_baseline('database', 'baseline_q1', 'post-alter q1'); SELECT validate_baseline('search ranking', 'baseline_q2', 'post-alter q2'); SELECT validate_baseline('optimization', 'baseline_q3', 'post-alter q3'); -- Phase 1a: After read-only queries the on-disk metapage -- MUST still be the source version — v6 (issue #383) or v7 -- (issue #374): tp_get_metapage normalizes an old page in -- its palloc'd copy but does NOT mutate the buffer. This -- phase catches a regression where the read path -- accidentally emits a write. The expected source version -- is passed via a temp table (psql does not interpolate -- :'var' inside a dollar-quoted block). CREATE TEMP TABLE _old_meta(version text); INSERT INTO _old_meta VALUES (:'old_metapage_version'); DO $$ DECLARE dump_text text; expected_meta text; BEGIN SELECT version INTO expected_meta FROM _old_meta; dump_text := bm25_dump_index('idx_upgrade_bm25'); IF dump_text NOT LIKE '%version: ' || expected_meta || E'\n' || '%' THEN RAISE EXCEPTION 'FAIL: metapage upgraded after read-only queries; ' 'expected version: %, dump excerpt: %', expected_meta, substr(dump_text, 1, 400); END IF; RAISE NOTICE 'PASS: read-only queries did not upgrade ' 'the v% metapage', expected_meta; END $$; -- Phase 1b: A single INSERT MUST trigger the lazy -- v6/v7/v8 -> current (v9) upgrade via -- memtable_bootstrap_and_append's GenericXLog record -- (src/memtable/log.c calls tp_metapage_upgrade_to_current, -- which jumps a v6 or v7 page straight to the current -- version). We assert here, BEFORE Phase 2's VACUUM and -- Phase 3's force_merge — both of which also independently -- call the upgrade helper — so that this phase isolates the -- INSERT path. A regression that removes only the -- INSERT-path call would still be caught here even if -- VACUUM / force_merge still upgrade. INSERT INTO upgrade_docs (content) VALUES ('first post-upgrade write to trigger legacy -> v9'); DO $$ DECLARE dump_text text; BEGIN dump_text := bm25_dump_index('idx_upgrade_bm25'); IF dump_text NOT LIKE '%version: 9' || E'\n' || '%' THEN RAISE EXCEPTION 'FAIL: metapage not upgraded to v9 after first ' 'INSERT (issues #383, #374, #380); dump excerpt: %', substr(dump_text, 1, 400); END IF; RAISE NOTICE 'PASS: single INSERT triggered legacy -> v9 ' 'lazy upgrade'; END $$; -- Phase 2: DELETE + VACUUM — alive bitset on old-format segments -- Delete ~20% of rows (40 of 200, 5 per topic cluster in batch 1) DELETE FROM upgrade_docs WHERE id <= 40; VACUUM upgrade_docs; SELECT validate_post_delete('database', 40, 'post-vacuum q1'); SELECT validate_post_delete('search ranking', 40, 'post-vacuum q2'); SELECT validate_post_delete('optimization', 40, 'post-vacuum q3'); -- Phase 3: force_merge — merges old segments into V5 SELECT bm25_force_merge('idx_upgrade_bm25'); SELECT validate_post_delete('database', 40, 'post-merge q1'); SELECT validate_post_delete('search ranking', 40, 'post-merge q2'); SELECT validate_post_delete('optimization', 40, 'post-merge q3'); -- Phase 4: INSERT new data alongside merged segments INSERT INTO upgrade_docs (content) VALUES ('newly inserted document about database query testing'), ('another new document with search ranking verification'); DO $$ DECLARE rec record; BEGIN SELECT * INTO rec FROM upgrade_docs ORDER BY content <@> to_bm25query('newly inserted', 'idx_upgrade_bm25') LIMIT 1; IF rec.id IS NULL THEN RAISE EXCEPTION 'FAIL: new data not searchable after merge'; END IF; RAISE NOTICE 'PASS: new data searchable, id=%', rec.id; END $$; -- Note: The legacy -> v9 upgrade is asserted in Phase 1b above -- (in isolation from VACUUM / force_merge / additional -- INSERTs). We rely on that assertion rather than -- re-checking here so that the test pinpoints the -- regression (INSERT path vs. VACUUM path vs. merge path). -- Phase 4b: UPDATE rows that came from the v6-format segment -- chain. In Postgres, UPDATE on a non-HOT-able index (bm25 -- never HOT-updates) is delete-old-tuple + insert-new-tuple; -- the bm25 index sees both versions until VACUUM cleans up -- the dead ones. This proves that: -- 1. INSERTs land in the upgraded chain (issue #383). -- 2. MVCC visibility correctly hides the pre-UPDATE -- content from queries that target the new content. -- 3. The post-UPDATE VACUUM marks the dead ctids in the -- v6-built segments via the alive bitset and the -- new content stays queryable through the update. -- -- Pick rows from batch 1 (ids 41-120, post-DELETE) whose -- original content matched 'database'. After UPDATE their -- content is 'mongodb sentinel ' — a deliberately unique -- term so the post-UPDATE query is unambiguous. UPDATE upgrade_docs SET content = 'mongodb sentinel update ' || id WHERE id BETWEEN 41 AND 120 AND content LIKE '%database%'; DO $$ DECLARE updated_count int; topk_match int; top_score float8; BEGIN SELECT count(*) INTO updated_count FROM upgrade_docs WHERE content LIKE 'mongodb sentinel update %'; IF updated_count = 0 THEN RAISE EXCEPTION 'FAIL: UPDATE affected no rows'; END IF; -- Query for the sentinel term: top-(updated_count) results -- must all be the UPDATEd rows. Pre-UPDATE content did -- not contain 'sentinel' anywhere in the corpus, so any -- match comes from the post-UPDATE ctid. SET LOCAL enable_seqscan = off; SELECT count(*) INTO topk_match FROM ( SELECT id, content FROM upgrade_docs ORDER BY content <@> to_bm25query('sentinel', 'idx_upgrade_bm25') LIMIT updated_count ) s WHERE s.content LIKE 'mongodb sentinel update %'; IF topk_match <> updated_count THEN RAISE EXCEPTION 'FAIL: post-UPDATE top-% sentinel hits contained ' 'only % UPDATEd rows (expected all %); MVCC or ' 'index out of sync', updated_count, topk_match, updated_count; END IF; -- Top score must be a real BM25 hit (strictly negative). SELECT content <@> to_bm25query('sentinel', 'idx_upgrade_bm25') INTO top_score FROM upgrade_docs ORDER BY content <@> to_bm25query('sentinel', 'idx_upgrade_bm25') LIMIT 1; IF top_score IS NULL OR top_score >= 0 THEN RAISE EXCEPTION 'FAIL: post-UPDATE sentinel top score not negative: %', top_score; END IF; RAISE NOTICE 'PASS: UPDATE on v6-built segment indexed correctly ' '(% rows, all in top-% sentinel results, top score %)', updated_count, updated_count, top_score; END $$; -- VACUUM after UPDATE: the alive bitset must mark the -- pre-UPDATE ctids dead in the v6-built segments. The -- sentinel query keeps working (post-UPDATE ctids are in -- a newer segment / the memtable chain, unaffected by the -- bitset flip), and the original content of those rows is -- gone (no row both contains 'mongodb sentinel update' -- AND any of the original batch-1 terms). VACUUM upgrade_docs; DO $$ DECLARE updated_count int; topk_match int; BEGIN SELECT count(*) INTO updated_count FROM upgrade_docs WHERE content LIKE 'mongodb sentinel update %'; SET LOCAL enable_seqscan = off; SELECT count(*) INTO topk_match FROM ( SELECT id FROM upgrade_docs ORDER BY content <@> to_bm25query('sentinel', 'idx_upgrade_bm25') LIMIT updated_count ) s WHERE s.id IN ( SELECT id FROM upgrade_docs WHERE content LIKE 'mongodb sentinel update %' ); IF topk_match <> updated_count THEN RAISE EXCEPTION 'FAIL: post-VACUUM sentinel hits = %, expected % ' '(alive bitset interaction broke UPDATE-inserted ' 'ctids)', topk_match, updated_count; END IF; RAISE NOTICE 'PASS: post-VACUUM, UPDATE-inserted ctids still ' 'queryable through the upgraded chain (% rows)', updated_count; END $$; -- Phase 5: Version check CREATE TEMP TABLE _expected(version_prefix text); INSERT INTO _expected VALUES (:'expected_version'); DO $$ DECLARE ver text; expected text; BEGIN SELECT version_prefix INTO expected FROM _expected; SELECT extversion INTO ver FROM pg_extension WHERE extname = 'pg_textsearch'; IF ver NOT LIKE expected || '%' THEN RAISE EXCEPTION 'FAIL: version %, expected %*', ver, expected; END IF; RAISE NOTICE 'PASS: extension version %', ver; END $$; -- Phase 6: 1.1.0 features (text[] indexes). -- Catches upgrade scripts that forget new SQL objects added -- during a release cycle. bm25_memory_usage was dropped in -- 1.3.0-dev (issue #374, Phase 7B) so it is no longer -- exercised here. CREATE TABLE upgrade_textarr (id serial, tags text[]); INSERT INTO upgrade_textarr (tags) VALUES (ARRAY['hello', 'world']); CREATE INDEX idx_upgrade_textarr ON upgrade_textarr USING bm25 (tags) WITH (text_config = 'english'); DO $$ DECLARE rec record; BEGIN SET LOCAL enable_seqscan = off; SELECT * INTO rec FROM upgrade_textarr ORDER BY tags <@> to_bm25query('hello', 'idx_upgrade_textarr') LIMIT 1; IF rec.id IS NULL THEN RAISE EXCEPTION 'FAIL: text[] index returned no results'; END IF; RAISE NOTICE 'PASS: text[] BM25 index works, id=%', rec.id; END $$; SELECT 'All compat-tier upgrade tests passed!' AS status; SQL else echo "=== Legacy tier: standard verification ===" psql -h /tmp -d upgrade_test \ -v expected_version="$EXPECTED_VERSION" << 'SQL' \set ON_ERROR_STOP on CREATE TEMP TABLE _expected(version_prefix text); INSERT INTO _expected VALUES (:'expected_version'); -- Test 1: Version check DO $$ DECLARE ver text; expected text; BEGIN SELECT version_prefix INTO expected FROM _expected; SELECT extversion INTO ver FROM pg_extension WHERE extname = 'pg_textsearch'; IF ver NOT LIKE expected || '%' THEN RAISE EXCEPTION 'FAIL: version %, expected %*', ver, expected; END IF; RAISE NOTICE 'PASS: extension version %', ver; END $$; -- Test 2: BM25 index scan works DO $$ DECLARE rec record; BEGIN SELECT * INTO rec FROM upgrade_docs ORDER BY content <@> to_bm25query('database', 'idx_upgrade_bm25') LIMIT 1; IF rec.id IS NULL THEN RAISE EXCEPTION 'FAIL: query returned no results'; END IF; RAISE NOTICE 'PASS: BM25 query works, id=%', rec.id; END $$; -- Test 3: Scores are negative DO $$ DECLARE score_val float8; BEGIN SELECT content <@> to_bm25query('database', 'idx_upgrade_bm25') INTO score_val FROM upgrade_docs ORDER BY 1 LIMIT 1; IF score_val IS NULL OR score_val >= 0 THEN RAISE EXCEPTION 'FAIL: score should be negative, got %', score_val; END IF; RAISE NOTICE 'PASS: scoring correct, score=%', score_val; END $$; -- Test 4: Index scan is used EXPLAIN (COSTS OFF) SELECT * FROM upgrade_docs ORDER BY content <@> to_bm25query('database', 'idx_upgrade_bm25') LIMIT 5; -- Test 5: Insert new data INSERT INTO upgrade_docs (content) VALUES ('New document added after upgrade to test index updates'); DO $$ DECLARE rec record; BEGIN SELECT * INTO rec FROM upgrade_docs ORDER BY content <@> to_bm25query('upgrade', 'idx_upgrade_bm25') LIMIT 1; IF rec.id IS NULL THEN RAISE EXCEPTION 'FAIL: could not find newly inserted document'; END IF; RAISE NOTICE 'PASS: new document searchable, id=%', rec.id; END $$; -- Test 6: New index after upgrade CREATE TABLE post_upgrade_docs (content TEXT); INSERT INTO post_upgrade_docs VALUES ('Testing post-upgrade index creation'); CREATE INDEX idx_post_upgrade ON post_upgrade_docs USING bm25 (content) WITH (text_config = 'english'); DO $$ DECLARE score_val float8; BEGIN SELECT content <@> to_bm25query('testing', 'idx_post_upgrade') INTO score_val FROM post_upgrade_docs ORDER BY 1 LIMIT 1; IF score_val IS NULL OR score_val >= 0 THEN RAISE EXCEPTION 'FAIL: new index query failed, score=%', score_val; END IF; RAISE NOTICE 'PASS: new index works, score=%', score_val; END $$; -- Test 7: 1.1.0 features (text[] indexes). -- Catches upgrade scripts that forget new SQL objects added -- during a release cycle. bm25_memory_usage was dropped in -- 1.3.0-dev (issue #374, Phase 7B) so it is no longer -- exercised here. CREATE TABLE upgrade_textarr (id serial, tags text[]); INSERT INTO upgrade_textarr (tags) VALUES (ARRAY['hello', 'world']); CREATE INDEX idx_upgrade_textarr ON upgrade_textarr USING bm25 (tags) WITH (text_config = 'english'); DO $$ DECLARE rec record; BEGIN SET LOCAL enable_seqscan = off; SELECT * INTO rec FROM upgrade_textarr ORDER BY tags <@> to_bm25query('hello', 'idx_upgrade_textarr') LIMIT 1; IF rec.id IS NULL THEN RAISE EXCEPTION 'FAIL: text[] index returned no results'; END IF; RAISE NOTICE 'PASS: text[] BM25 index works, id=%', rec.id; END $$; SELECT 'All legacy-tier upgrade tests passed!' AS status; SQL fi - name: Cleanup if: always() run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH pg_ctl -D /tmp/upgrade_test_data stop -m immediate || true rm -rf /tmp/upgrade_test_data # Note: v0.1.0 incompatible upgrade test was removed because # shared_preload_libraries is now required. The v0.1.0 extension SQL # references functions (e.g., bm25_get_current_score) that no longer exist # in the current binary. Upgrades from v0.2.0+ are tested in test-upgrades. # Test clean install (no upgrade) on both PG versions test-clean-install: runs-on: ubuntu-latest strategy: matrix: pg_version: [17, 18] steps: - name: Checkout current code uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates gnupg sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} sudo systemctl stop postgresql - name: Build and install run: | export PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config make clean make sudo make install - name: Test clean installation run: | export PATH=/usr/lib/postgresql/${{ matrix.pg_version }}/bin:$PATH export PGDATA=/tmp/clean_test_data LIB_NAME="pg_textsearch" initdb -D $PGDATA echo "port = 5432" >> $PGDATA/postgresql.conf echo "unix_socket_directories = '/tmp'" >> $PGDATA/postgresql.conf echo "shared_preload_libraries = '${LIB_NAME}'" >> $PGDATA/postgresql.conf pg_ctl -D $PGDATA start -w -l $PGDATA/logfile createdb -h /tmp clean_test psql -h /tmp -d clean_test << 'SQL' CREATE EXTENSION pg_textsearch; -- Verify version SELECT extversion FROM pg_extension WHERE extname = 'pg_textsearch'; -- Basic functionality test CREATE TABLE docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO docs (content) VALUES ('PostgreSQL full text search'), ('BM25 ranking algorithm'), ('Document retrieval system'); CREATE INDEX idx_docs ON docs USING bm25 (content) WITH (text_config = 'english'); -- Test query SELECT id, content <@> 'search ranking'::text AS score FROM docs ORDER BY score LIMIT 3; RAISE NOTICE 'Clean installation test passed!'; SQL pg_ctl -D $PGDATA stop -m immediate rm -rf $PGDATA # Data-integrity upgrade matrix. Unlike test-upgrades above — which # exercises a single data shape and captures its baseline from the # already-upgraded binary — this job builds several distinct on-disk # states per old release and checks recall against heap ground truth # captured both before and after the binary swap. See # test/scripts/upgrade_matrix.sh. upgrade-matrix: runs-on: ubuntu-latest strategy: fail-fast: false matrix: pg_version: [17, 18] steps: - name: Checkout current code uses: actions/checkout@v4 - name: Install PostgreSQL ${{ matrix.pg_version }} run: | sudo apt-get update sudo apt-get install -y wget ca-certificates gnupg sudo sh -c 'echo "deb http://apt.postgresql.org/pub/repos/apt $(lsb_release -cs)-pgdg main" > /etc/apt/sources.list.d/pgdg.list' wget --quiet -O - https://www.postgresql.org/media/keys/ACCC4CF8.asc | sudo apt-key add - sudo apt-get update sudo apt-get install -y postgresql-${{ matrix.pg_version }} postgresql-server-dev-${{ matrix.pg_version }} sudo systemctl stop postgresql - name: Run upgrade data-integrity matrix run: | export PG_CONFIG=/usr/lib/postgresql/${{ matrix.pg_version }}/bin/pg_config # Representative set: one release per distinct metapage/segment # on-disk format combination (v5 legacy, v6+segv3/v4/v5, v7, # and v8 before zero-lexeme documents were indexed), plus the # previous stable release. export OLD_VERSIONS="0.5.0 0.5.1 1.0.0 1.2.0 1.3.0 1.4.0 1.5.0" bash test/scripts/upgrade_matrix.sh timescale-pg_textsearch-b63fe58/.gitignore000066400000000000000000000013331525762047700207510ustar00rootroot00000000000000# Build artifacts *.o *.obj *.bc *.so *.dylib *.dll *.exp *.lib # Test artifacts test/regression.diffs test/regression.out test/results/ tmp_check_shared/ # Test scratch dirs, at any level (root, test/, test/scripts/) tmp_*/ # PGXS dependency-tracking output .deps/ # Coverage artifacts coverage-html/ coverage.info *.gcda *.gcno *.gcov # IDE/Editor files .vscode/ .idea/ *.swp *.swo *~ # Editor configuration files .clang-format .clangd .editorconfig .cache/ # OS files .DS_Store Thumbs.db # Claude Code files .claude/ # Git worktrees .worktrees/ # Design docs docs/superpowers/ # VSCode files compile_commands.json target/ __pycache__/ *.pyc PR_BREAKDOWN.md # Python virtual environment venv/ test/sql/*_validated.sql timescale-pg_textsearch-b63fe58/.pre-commit-config.yaml000066400000000000000000000010251525762047700232400ustar00rootroot00000000000000repos: - repo: https://github.com/pre-commit/mirrors-clang-format rev: 'v21.1.8' hooks: - id: clang-format types: [c] files: ^src/.*\.(c|h)$ args: [--style=file] - repo: https://github.com/pre-commit/pre-commit-hooks rev: v4.5.0 hooks: - id: trailing-whitespace exclude: (\.patch$|test/expected/.*\.out$|test/sql/.*\.sql$) - id: end-of-file-fixer exclude: \.patch$ - id: check-yaml - id: check-added-large-files args: ['--maxkb=500'] timescale-pg_textsearch-b63fe58/AGENTS.md000077700000000000000000000000001525762047700215362CLAUDE.mdustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/ARCHITECTURE.md000066400000000000000000000763211525762047700211760ustar00rootroot00000000000000# Architecture pg_textsearch is a PostgreSQL index access method for BM25-ranked full-text search. It uses an LSM-like layout: an on-disk memtable receives writes and spills into immutable segments that are compacted across levels. ## Source Layout - `src/access/` — access-method build, scan, vacuum, and SQL maintenance entry points - `src/types/` — `bm25query`, `bm25vector`, and SQL operators - `src/planner/` — planner hooks and cost estimation - `src/scoring/` — BM25 and Block-Max WAND - `src/index/` — index state, metapage, registry, limits, and posting sources - `src/memtable/` — on-disk write buffer and derived query cache - `src/segment/` — immutable segments, compression, merge, and deferred reclaim - `src/debug/` — optional diagnostic functions These directories are an organizational model, not a strictly enforced dependency graph. Some storage coordination crosses layers; for example, segment compaction uses reclaim-horizon logic from the access layer. Project includes use paths relative to `src/`. ## Storage and WAL Block 0 is the metapage. It points to the on-disk memtable chain, immutable segment chains for each LSM level, and the deferred-free tombstone chain. Writes append document records to the memtable chain under buffer locks. In-place and multi-page publication mutations use `GenericXLog`. Newly written segment pages use `log_newpage_buffer()` when `RelationNeedsWAL()` is true. Before reclaimed segment pages enter the FSM, pg_textsearch emits PostgreSQL's stock btree page-reuse conflict record; its redo path only resolves old standby snapshots and does not inspect btree storage. pg_textsearch has no custom WAL resource manager. The on-disk chain is authoritative through crash recovery and physical replication. An append retries if another writer has extended its candidate tail. If rereading the metapage returns that same stale tail, the index is corrupt: extension publishes the old tail's link and the new tail pointer atomically. The write raises an index-corruption error with a `REINDEX` hint rather than spinning while holding the per-index lock. Queries compose postings from the memtable and all published segments. Each live heap TID occurs in at most one published segment. Segment-local numeric `doc_id` values may repeat across segments. This disjointness permits merge and scoring paths to avoid cross-segment document deduplication. Published-graph consumers first read and release a candidate memtable head/tail, lock the candidate tail in share mode, then lock the metapage in share mode and validate that the candidate is still current. On mismatch they release metapage then tail and retry. With tail then metapage held, they copy every segment root and the bounded memtable endpoint, including the tail free offset. Empty chains validate both pointers without a tail lock. Level counts bound and validate root discovery. Query, debug, and maintenance work then use the explicit root arrays rather than lazily following published `next_segment` links. On recovery, ranked, standalone, and Boolean scoring consume the bounded chain endpoint instead of rereading the metapage. Segment and memtable inputs therefore come from one complete old or new generation while Generic WAL replay changes publication. Ranked scoring pins recovery mode before acquiring the snapshot, so promotion during a pause cannot switch that generation to the primary cache path. Ranked scans and standalone score expressions share one captured generation per physical index and executor, including larger-batch retries and cursor fetches. Corpus totals and term frequencies therefore stay consistent within that execution; a later statement captures fresh statistics. These are physical index statistics, not MVCC-filtered corpus statistics. Snapshots also identify the physical relation file, so a same-backend `REINDEX` or `TRUNCATE` captures a new generation rather than reusing old block numbers. ## Parallel Index Build Parallel build workers scan disjoint heap ranges and write current-format segments as flat temporary `BufFile` streams. Their TID-range scans enable the table AM's bulk-read strategy, allowing large heap scans to reuse a buffer ring rather than fill shared buffers alongside the build batches. Synchronized scanning remains disabled to preserve the disjoint ranges. These private worker segments contain only live documents; an absent alive-bitset section represents that all-live state. Empty-token documents still occupy the fieldnorm and CTID sections and contribute to corpus totals even when a worker segment has no dictionary terms. Each worker appends its segment directory to its temporary stream and reports only the directory offset and count through shared memory. Small build budgets can therefore produce more than 64 segments per worker without exhausting a fixed shared-memory reporting array. After all workers finish, the leader publishes one final L0 segment. It merges source dictionaries one term at a time, using bounded windows for worker string offsets and temporary streams for string metadata, exact source references, skip entries, and dictionary-entry backpatch data. Posting blocks are read from worker segments and written directly to the normal WAL-logged segment writer; complete output postings are never spooled or reread. Worker inputs are ordered and disjoint, so the leader assigns each source a document-ID base and concatenates its fieldnorm and split CTID sections in the same order. It emits the all-live bitmap incrementally. This avoids vocabulary-sized merged-term state and corpus-sized document remapping arrays; leader working memory is bounded by active inputs, posting blocks, offset windows, and fixed-size copy buffers. The output keeps the ordinary segment layout, compression, page index, dictionary backpatch, and publication order. Worker batch flushes use the bytes allocated by the batch's dedicated memory context, including the arena, dynahash, and document arrays. The worker threshold also reserves conservative serialization scratch for sorted terms, string offsets, term and dictionary metadata, the geometrically grown skip array, temporary CTID arrays, and the alive bitmap. Parallel builds divide the total `maintenance_work_mem` batch budget by the workers actually launched. A worker budget that cannot hold the initial context, one 1 MiB arena page, and minimum serialization scratch is rejected explicitly rather than being raised by a hidden per-worker floor. Serial builds retain their established arena-payload and document-capacity flush estimate, preserving their batching and inline-compaction behavior. Correcting serial dynahash and serialization accounting is deferred until it can be paired with a serial merge strategy that does not regress build time or peak memory. The worker batch estimate is not a hard backend RSS limit. PostgreSQL executor and backend overhead, temporary-file buffers, and the resettable tokenization of one document are outside it. A single document can therefore overshoot the worker threshold before the following flush check. A primary reader uses the memtable cache only when its physical relation file and applied endpoint match the captured chain. It holds the cache apply lock in shared mode for that scoring call to prevent catch-up from changing the view. Otherwise it reads the bounded on-disk chain. Reads discard caches from rolled-back relation files even when the restored chain is empty. Readers of an already matching cache share that lock without exclusive catch-up admission. The first ranked source also supplies the snapshot's corpus totals, avoiding a separate memtable walk. No LWLock is held between rows or cursor fetches; the executor's heap snapshot protects retired pages from reclaim. Executor memory-context cleanup releases the captured generations, including on errors and nested execution. ## Memtable Cache The shared-memory memtable cache accelerates reads but is derived and disposable. Writes update only the on-disk chain. Readers lazily build or catch up the cache; generation mismatches, spills, eviction, or memory pressure can drop it without affecting correctness. Standbys read the on-disk chain. Registry entries are keyed by database and index OID. Rebuilds preserve the shared allocation and its locks so existing backends retain valid wrappers. A successful build clears the cache in place and advances its generation. Cache cursors also record the relation's physical file identity: REINDEX, TRUNCATE, and rollback must never resume a cursor in a replacement file. Chain-page counts are lazily recounted against that same file identity before spill threshold checks; shutdown spills skip a busy recount lock. Index drops free registry state only at commit, before relation locks are released. Transaction or savepoint rollback preserves allocations referenced by other backends; releasing a savepoint transfers pending cleanup to its parent. `PREPARE TRANSACTION` rejects pending index-drop cleanup, as it does initial index creation, because this ownership is backend-local. Concurrent drops retain that state across PostgreSQL's intermediate commits and reader waits. Only successful completion of the utility command schedules cleanup for its final transaction; cancellation preserves the surviving index's state. `pg_textsearch.memory_limit` has three budget tiers: - per-index per-record growth guard (`limit / 8`): reject a record whose estimated growth would cross the guard and fall back to the chain; - global soft cap (`limit / 2`): attempt best-effort eviction of the largest non-caller cache; eviction may find nothing or a busy target; - global `limit` is an approximate admission threshold: catch-up or cold build falls back when the entry-time estimate is already at the limit. Admitted or concurrent work may increase estimated usage past it. `0` disables the limit. The setting is applied on SIGHUP. Cache lock order is: 1. per-index LWLock; 2. cache apply lock; 3. cache lifetime lock; 4. dshash bucket locks; 5. posting-list lock. The global eviction mutex is acquired before another index's per-index lock. ## Spill and Compaction A spill builds an immutable L0 segment from the memtable, publishes it through the metapage, and marks the old chain dead for deferred reclaim. Compaction combines adjacent immutable segments within `pg_textsearch.max_segment_size`. ### Admission and lock order The per-index LWLock uses writer-preference admission. An exclusive acquirer increments `exclusive_waiters` before waiting for the LWLock. New shared acquirers sleep while that count is nonzero. The exclusive acquirer decrements the count immediately after acquisition and wakes shared waiters when the last exclusive waiter has acquired. A shared acquirer can pass the gate just before the count changes, but only that bounded set can barge; a continuing stream of new readers cannot starve the exclusive waiter. Memtable writers also use a fair per-index writer/spill gate. Appends take it in shared mode before the per-index LWLock; spill takes it exclusively so the published chain stays stable while its segment is built. Ranked readers do not take this gate. Compaction, force merge, and VACUUM segment mutation serialize with an exclusive per-index heavyweight object lock in pg_textsearch's private `pg_am` subobject namespace. Its discriminator is distinct from managed background-compaction admission, so a signaled worker can compact while the spilling transaction is still completing pre-commit dispatch. The lock does not exclude scans, memtable appends, or ordinary relation locks. Graph publication also uses a private per-index publication barrier. Runtime compaction and VACUUM replacement acquire maintenance, then the publication barrier in `ExclusiveLock`, then phase-specific per-index LWLocks. Spill publication does not use maintenance; it acquires the publication barrier in `ShareLock`, the writer/spill gate in exclusive mode, and then phase-specific per-index LWLocks. No path acquires the publication barrier, maintenance, or writer/spill gate while holding the per-index LWLock. Buffer ordering then follows the storage operation. Existing-tail memtable extension is tail -> new page -> metapage. The common read snapshot uses tail -> metapage after an unlocked candidate read and retry validation; it never nests metapage -> tail. Segment and tombstone publication retain their own validated buffer order under the per-index lock. Empty-chain bootstrap is the only memtable path with no existing tail and locks metapage before its new unpublished page. No path may request maintenance while holding the per-index lock. A spill therefore completes L0 publication and releases `LW_EXCLUSIVE` before applying the configured compaction policy. ### Spill phases A runtime spill holds the publication barrier and writer/spill gate across three phases: 1. **Freeze and extract.** Briefly take the per-index lock in `LW_EXCLUSIVE`, recheck the threshold, and extract the stable memtable chain. 2. **Build.** Release the per-index lock and write the complete WAL-logged but unreachable L0 segment. Readers continue to scan the unchanged chain; writers wait on the writer/spill gate. 3. **Publish.** Briefly retake `LW_EXCLUSIVE`, publish the segment, disconnect and stamp the old chain for deferred reclaim, and advance the cache spill generation. An error before publication discards the unreachable segment and leaves the old chain published. Ownership transfers at WAL publication, before cache cleanup or chain retirement; a later error must not discard the live segment. Shutdown's no-wait spill skips busy publication, writer-gate, and per-index locks. If the publication-phase index lock is busy, it discards the unpublished output. Buffer, WAL, and I/O work may still wait after admission. The `compaction` index option controls spill-time behavior: - `inline` compacts eligible debt during spills, serial VACUUM, and index builds. It never waits for maintenance admission or exclusive index access: when another session is using or maintaining the index -- a concurrent `REINDEX INDEX CONCURRENTLY`, `VACUUM`, or explicit compaction -- the pass is skipped rather than blocking the writer, and the debt is picked up by the next spill or by explicit maintenance; - `background` dispatches a pre-commit request when possible. Runtime no-dispatch contexts such as autovacuum and callback re-entry compact inline. Index builds leave compaction to the managed workflow after activation, and temporary indexes do not support this mode; - `manual` leaves debt for explicit maintenance. The legacy `off` value is accepted as an alias for `manual`. Prepared transactions do not flush queued background requests. Unconfigured, unresolvable, or failed callbacks do not fall back inline; the compaction debt remains for a later spill or explicit maintenance. A level 0 that is already at `pg_textsearch.segments_per_level` blocks the spill outright, so `inline` and `background` indexes both compact it in the spilling process; a background worker's pass would come too late. If that compaction is skipped because maintenance is busy, the records stay in the durable memtable chain for a later spill to drain. If it runs but cannot reduce level 0 -- every segment is over `max_segment_size` -- the spill fails closed with a segment-count error rather than growing the chain without bound. `manual` indexes take that error directly. Each visible automatic invocation performs at most one bounded pass. Each pass uses brief per-index `LW_SHARED` selection, no per-index lock during output build, and fair `LW_EXCLUSIVE` validation and publication. `bm25_compact()` drives reducible debt to completion, taking the per-index maintenance lock for each pass and releasing it in between so a queued VACUUM is not starved by a long cascade. Each pass uses brief per-index `LW_SHARED` selection, no per-index lock during output build, and fair `LW_EXCLUSIVE` validation and publication. `bm25_compact_step()` runs at most one pass. A private `CREATE INDEX` build may drain compaction debt before the index becomes visible. Both, like `bm25_force_merge()`, wait for index maintenance: the lock is private to this extension, so waiting cannot close a deadlock cycle with a concurrent `REINDEX INDEX CONCURRENTLY`. A dispatched background worker instead defers to its next scheduled run. `bm25_needs_compaction()` reports whether `bm25_compact_step()` would run a pass: it runs the same selection without publishing, so a level that is full of over-budget segments reports false and is safe to loop on. It tries the maintenance lock instead of waiting for it, and reports true when another session holds it. ### Consolidation policy Count-triggered compaction remains available at `segments_per_level`. Merged outputs use their estimated surviving size for placement, with L1 as the minimum destination; small outputs no longer climb a level on every merge. Capacity-driven singleton promotion remains an exception. Below the count threshold, a segment with at least 50% dead documents can be rewritten alone. This also works inside a level's chain, retaining its level so publication needs only one predecessor splice. Completely dead segments retain their existing cleanup path. Otherwise, compaction considers head prefixes across levels whose estimated surviving sizes are within a factor of two of the first source. A pass combines at most `segments_per_level` sources into one bounded output. Incompatible heads are not rewritten merely to reach smaller segments behind them. This prevents each tiny spill from rewriting a large output. Surviving-size estimates guide selection and placement only: the conservative document, dictionary, and posting estimates still enforce merge-size and format bounds. An existing over-budget singleton can be rewritten for deletion cleanup, but cannot join a multi-source merge. Explicit maintenance, inline policy, and managed workers use the same planner. Background spills enqueue a request without running selection in the writer; the worker determines whether any pass is eligible. Policy scans read only segment headers, loading page maps and dictionaries for selected candidates. Serial VACUUM invokes the configured policy after cleanup, so partial deletion needs no subsequent spill to become eligible. Manual mode leaves nonempty segments for explicit maintenance. A scheduled worker also discovers below-threshold debt without a new signal. ### Compaction phases Each runtime pass uses the same phase engine: 1. **Maintenance admission.** Acquire the per-index maintenance lock. A caller that waited rechecks compaction debt after admission. 2. **Select.** Briefly take the per-index lock in `LW_SHARED` and copy the metapage. Release it before the maintenance-protected source walk records exact source roots, contiguous runs, and retained remainders. 3. **Build.** Hold only the maintenance lock while reading immutable sources, constructing complete WAL-logged but unreachable output segments, collecting displaced source pages, and building a detached tombstone batch with a provisional invalid reclaim stamp. Segment data pages, page-index pages, completed output roots, and tombstone container pages have explicit ownership records for handled-error cleanup. Validate the complete prepared segment and tombstone chains while they remain unreachable. 4. **Stamp reclaim.** Assign the compactor's full transaction ID and restamp every detached tombstone container with it while the batch remains unreachable and no runtime per-index lock is held. The in-progress transaction pins primary and standby horizons through publication. 5. **Prepare and validate.** Acquire the publication barrier in `ExclusiveLock`, then under `LW_SHARED` validate the selected runs and prepare any L0 prefix added by spills that completed before barrier admission. Release the shared lock, then request fair `LW_EXCLUSIVE`. Because later spills cannot publish while the barrier is held, an identity mismatch fails closed as an internal invariant violation rather than retrying preparation. 6. **Publish.** In one `GenericXLog` action, splice around any accepted L0 prefix, replace the selected runs, rebase counts and corpus shrinkage from current metapage values, and attach the detached tombstone batch to the current pending-free head. Exclusive work is limited to constant-time identity, predecessor, and detached-tail checks plus publication. Published physical changes are not undone by transaction rollback. Runtime segment and tombstone allocation must use the same `ExtendBufferedRel(..., EB_LOCK_FIRST)` fallback as memtable growth when the FSM has no reusable page. Mixing it with `ReadBufferExtended(P_NEW)` would let unlocked compaction and concurrent memtable extension reserve the same block on PostgreSQL 17. Publication remains exclusive even though it is bounded. Primary scans that started before publication finish from their copied old roots; later scans copy the new roots. Standby discovery holds the metapage buffer share lock, so replay cannot expose a metapage/link mixture while roots are being copied. The existing `LW_SHARED` lifetime of ranked index scans was not shortened. Readers and inserts continue during the long build phase, but fair admission can briefly gate new shared acquirers while publication waits. Inline compaction is still foreground work: the write transaction that triggers it waits for one selection, build, validation, and publication pass to complete. Output and tombstone pages are WAL-logged before the later reachability record; WAL insertion order provides durability without `FlushRelationBuffers()`. ## Managed Background Compaction Background mode requires pg_durable 0.2.8 or newer. pg_durable must be preloaded, installed and initialized in the current database, and usable by the index owner. The owner must have `LOGIN`; a superuser owner also requires `pg_durable.enable_superuser_instances = on`. pg_textsearch discovers the SQL API through extension metadata and records a normal extension dependency after the first successful activation. The pg_durable extension and BM25 index must be in the same database. pg_durable's `database` argument routes SQL activities after submission; it does not expose `df.start`, `df.signal`, or workflow metadata in another database. pg_textsearch rejects background mode when `pg_durable.database` names a different database. Use `manual` mode there. Each physical background index has one owner-scoped workflow identified by its database, physical relation identity, owner, schedule, and protocol version. The workflow runs a stepped cascade immediately, then waits for a spill signal or its cron schedule. Every step calls the private physical-target helper in a separate SQL node and transaction, releasing PostgreSQL locks between published passes. The startup wrapper and scheduled loop continue after SQL activity failures, so the same workflow can retry or handle a later wake. Graph, protocol, runtime, and infrastructure failures remain terminal. A later spill recovers a terminal workflow for the current managed generation. The helper revalidates the captured physical identity while holding the relation lock. Dropped, replaced, reindexed, re-owned, or reconfigured targets return false without touching another relation. Utility hooks reconcile workflows after relevant `CREATE INDEX` and `ALTER INDEX` operations. Spill requests are transaction-local and deduplicated by index. At pre-commit, after the compaction lock is released, pg_textsearch revalidates the target, finds or recovers its workflow, and signals that exact instance. Ordinary signaling failures warn without aborting the writer; cancellation and shutdown errors retain PostgreSQL's normal behavior. The periodic schedule repairs a signal lost during a loop transition. ## Deferred Reclaim Spill and compaction unlink old pages before they can be safely reused. Dead memtable pages and displaced segment pages are parked with a transaction horizon and returned to the free-space map only after `GetOldestNonRemovableTransactionId` passes that horizon. Query-serving hot standbys require `hot_standby_feedback = on` so their oldest snapshots hold the primary's reclaim horizon back. Use `bm25_pending_free_pages()` to observe displaced segment pages awaiting reuse. As a safety fallback for a standby that disconnects while an old-generation query remains active, both tombstone drain and DEAD-memtable reclaim emit the stock `XLOG_BTREE_REUSE_PAGE` conflict-only WAL record before reuse. Memtable reclaim uses each page's `dead_fxid`; tombstone drain uses the batch horizon. Spill samples `dead_fxid` only after the WAL record that unpublishes the old chain, so every standby snapshot that can still discover that chain is covered. Replay cancels any conflicting standby snapshot before later WAL can reuse those pages. Feedback therefore preserves query continuity; the conflict record preserves storage correctness when feedback is temporarily unavailable. Compaction constructs its tombstone containers as a detached chain whose tail initially points to `InvalidBlockNumber`. Publication links that tail to the then-current `pending_free_head` in the same WAL record that replaces the segment graph. The detached pages are restamped after the long unlocked build but before requesting the runtime publication lock. The stamp is the compactor's assigned, still-in-progress full transaction ID, so primary and standby snapshots that start on the old graph before publication cannot advance the reclaim horizon past it. Restamping scales with the number of tombstone containers but does not extend runtime reader exclusion. Selected source pages are never returned directly to the FSM. Tombstone drain unlinks a reclaimable batch under `LW_EXCLUSIVE`, then releases the per-index lock before stamping its already-unreachable pages free and returning them to the FSM. Free-before-unlink is forbidden; unlink-before-free is safe, and an error during the unlocked free loop can only leak the unfinished remainder until `REINDEX`. Tombstone walks reject metapage and past-EOF links before reading them. The drain warns and detaches the corrupt tail, preserving any valid prefix; `REINDEX` reclaims the leaked pages. `bm25_pending_free_pages()` instead raises a corruption error without changing the chain or returning a partial count. Metapage V8 already contains `pending_free_head`; compaction preserves that existing chain when upgrading and publishing. Only older metapage versions synthesize an empty pending-free head. Serial VACUUM segment replacement assigns its current full transaction ID before building replacement tombstones. That transaction remains in progress through the replacement `GenericXLog` publication, preventing a later standby snapshot from observing the old graph with a reclaim stamp that is already in its past. PostgreSQL can invoke index bulk-delete in a parallel worker or in a leader that has already entered parallel mode, where assigning an XID is forbidden. In that context VACUUM still persists V5 alive-bit changes, including an all-zero bitmap, but leaves an empty segment physically linked for later serial compaction. A VACUUM-triggered spill remains published while its compaction policy is deferred. For affected legacy segments, parallel VACUUM builds detached tombstones with a provisional invalid stamp before publication. Under the publication barrier and `LW_EXCLUSIVE`, one `GenericXLog` action replaces the graph and attaches that provisional batch atomically. Only after the publication WAL record is inserted does VACUUM sample `ReadNextFullTransactionId()` and WAL-restamp the attached containers. WAL order protects standby readers that can still see the old graph. Invalid stamps are unreclaimable; after a crash, a later tombstone drain activates them with a new post-publication stamp before any reuse. A handled error before publication returns every explicitly tracked output and tombstone allocation to the FSM without freeing selected source pages. A backend crash can leave unreachable pre-publication output pages; they cannot affect queries or be mistaken for live pages and are reclaimed by `REINDEX`. Force-merge relation truncation does not reclaim those orphans by inference: it removes only a contiguous EOF suffix whose pages already carry `TP_FREE_PAGE_MAGIC`. An unfinished publication `GenericXLog` state is aborted on handled errors, and the still-unreachable prepared pages are discarded. Once `GenericXLogFinish()` succeeds, cleanup does not recycle pages whose ownership has transferred. Recovery exposes either the old graph with no attached batch or the complete new graph with displaced pages reachable from the deferred-free chain. ## VACUUM Coordination VACUUM uses separate maintenance critical sections for bulk deletion and cleanup. The bulk-delete section covers segment document identification, alive-bit mutation, legacy replacement, and corpus-statistic adjustment. The cleanup section covers segment unlink and the full-fork scan that reclaims DEAD memtable pages, excluding force-merge truncation during that scan. Other maintenance may run between the two callbacks. The per-index LWLock is held only for root snapshots and brief validated publication; document identification, bitmap work, and dead-memtable reclaim remain unlocked from readers, inserts, and spills. When legacy and current segments disagree with the metapage token total, VACUUM verifies each current segment by summing its posting frequencies before attributing the residual to legacy headers. A current-format mismatch fails closed with a REINDEX hint before any replacement is published. Dead-memtable reclaim first records the currently reachable chain, then inspects pages under their buffer locks. A racing spill can only make that reachable set conservative, retaining pages until a later VACUUM. Live and newly allocated pages are not marked DEAD, while `dead_fxid` prevents a retired page from entering the FSM before old primary or feedback-protected standby snapshots are safe. Before each reclaimed page is free-stamped, stock conflict-only WAL protects disconnected or no-feedback standby readers. `bm25_force_merge()` may shrink the physical relation only across consecutive EOF pages already stamped `TP_FREE_PAGE_MAGIC`. The stamp proves normal reclaim completed: the page is detached from every owning structure and any required standby conflict WAL was inserted before it entered the FSM. A DEAD memtable page, a valid structural page, an unknown page, or an unreachable orphan stops truncation even when no current graph edge references it. Truncation retains maintenance admission and takes the exclusive memtable writer gate before the exclusive per-index lock, holding both from suffix inspection through `RelationTruncate()`. The writer gate excludes spill construction, which allocates pages without the per-index lock; maintenance admission excludes compaction and VACUUM allocation. If VACUUM is admitted first, compaction waits and later builds from the updated alive bits. If compaction is admitted first, VACUUM waits and then discovers the published output before applying deaths. This prevents both resurrection of deleted documents and mutation through stale source document IDs. Parallel VACUUM can persist a zero-alive bitmap but cannot assign the reclaim XID needed to unlink it. A later serial VACUUM or ordinary compaction therefore prioritizes removal of zero-alive segments even below the normal compaction threshold, including maintenance rounds with no newly reported dead TIDs. timescale-pg_textsearch-b63fe58/CLAUDE.md000066400000000000000000000440741525762047700202510ustar00rootroot00000000000000# CLAUDE.md This file provides guidance to Claude Code (claude.ai/code) when working with code in this repository. ## Project Overview pg_textsearch is a Postgres extension providing full-text search with BM25 ranking. (Internal name: Tapir) It implements a memtable-based architecture similar to LSM trees, with in-memory structures that spill to disk segments for scalability. **Current Version**: 1.5.1 **Postgres Version Support**: 17, 18 (tested in CI). 19 (beta) is supported on a best-effort basis; its CI job is allowed to fail while 19 is in beta. **Schema Organization**: Currently uses public schema. Future versions may consider a dedicated `pg_textsearch` schema for cleaner namespace management. ## Important Notes for Development - **pg_textsearch requires shared_preload_libraries** - Add `pg_textsearch` to `shared_preload_libraries` in postgresql.conf and restart the server before CREATE EXTENSION. A server restart is also required after updating the binary (e.g., after `make install` during development). - **Every test failure matters.** We keep CI green on main. Do not dismiss failures as "pre-existing" — they almost never are. If a test fails, investigate and fix the root cause. If you believe a failure is unrelated to your changes, verify by checking out main and running the same test. Even if it does reproduce on main, it still needs to be investigated and fixed, not ignored. - **Physical replication**: every page mutation is WAL-logged with stock records (`GenericXLog`, `log_newpage_buffer()`, and the btree page-reuse conflict record). pg_textsearch registers no custom rmgr, so stock replay reconstructs every page without loading `pg_textsearch.so`. **Read [ARCHITECTURE.md](ARCHITECTURE.md#storage-and-wal) before changing the write/read/spill flow.** - **Standby-safe segment reclaim**: displaced segment pages are parked in a WAL-logged tombstone chain off the metapage and returned to the FSM only once a later VACUUM or merge observes that the reclaim horizon has passed their stamp. **`hot_standby_feedback = on` is required** on hot standbys serving queries. Observe the parked count with `bm25_pending_free_pages(index_name)`. See [ARCHITECTURE.md](ARCHITECTURE.md#deferred-reclaim). ## Core Architecture ### Storage Architecture The index uses an LSM-like layered storage approach: - **Memtable (L0)**: An on-disk paged structure stored in the index relation itself. Writes append doc records (ctid + length + packed bm25vector bytes) to a tail page chained off the metapage. Mutations are WAL-logged via `GenericXLog`. See [ARCHITECTURE.md](ARCHITECTURE.md#storage-and-wal). - **Segments**: Immutable disk-based structures using V2 block storage format with skip lists for efficient top-k queries ### Query Optimization - **Block-Max WAND (BMW)**: Top-k query optimization that uses block-level upper bounds to skip non-contributing blocks - **Skip Lists**: Segment format includes skip entries for fast block seeking ### Source Code Structure The `src/` directory is organized in layers (see CONTRIBUTING.md for details): - **Layer 1 (Postgres interface):** `access/` (AM handler, build, scan, vacuum), `types/` (bm25query, bm25vector), `planner/` (optimizer hooks, cost estimation) - **Layer 2 (Index coordination):** `scoring/` (BM25, Block-Max WAND), `index/` (state lifecycle, registry, metapage, posting source abstraction) - **Layer 3 (Storage):** `memtable/` (on-disk paged L0 — chain of doc-record pages mutated under buffer locks, WAL-logged via `GenericXLog`; see [ARCHITECTURE.md](ARCHITECTURE.md#storage-and-wal)), `segment/` (on-disk segments, merge, compression) - **Cross-cutting:** `debug/` (dump utilities), `mod.c` (init) ### Data Types - `bm25vector` - Stores term frequencies with index context for BM25 scoring - Format: `"index_name:{lexeme1:freq1,lexeme2:freq2,...}"` - `bm25query` - Represents queries for BM25 scoring with optional index context - Format: `"query terms"` or `"index_name:{query terms}"` (via to_bm25query) ## Development Commands ### Building and Testing ```bash make # build extension make install # install to Postgres make test # run SQL regression tests only make installcheck # run SQL regression tests make test-all # run SQL, default shell, and replication tests # Run a single test $(pg_config --pgxs | xargs dirname)/../../src/test/regress/pg_regress \ --inputdir=test --outputdir=test basic # runs basic.sql only # Individual test types make test-concurrency # multi-session concurrency tests make test-recovery # crash recovery tests make test-segment # multi-backend segment tests make test-stress # long-running stress tests make test-shell # run default shell tests (excludes replication) make test-local # run tests with dedicated Postgres instance make expected # generate expected output from test results ``` ### Debug Builds ```bash # Enable debug index dumps (add to Makefile or command line) make PG_CPPFLAGS="-I$(pwd)/src -g -O2 -DDEBUG_DUMP_INDEX" ``` ### Code Quality ```bash make format # auto-format C code with clang-format make format-check # check C code formatting (alias: lint-format) make format-diff # show formatting differences make format-single FILE=path/to/file.c # format specific file ``` ## Configuration ### GUC Parameters | Parameter | Description | Default | |-----------|-------------|---------| | `pg_textsearch.default_limit` | Default limit for queries without LIMIT | 1000 | | `pg_textsearch.log_scores` | Log BM25 scores during scans | false | | `pg_textsearch.log_bmw_stats` | Log BMW blocks scanned/skipped | false | | `pg_textsearch.bulk_load_threshold` | Terms/xact to trigger spill (0 = disable) | 100000 | | `pg_textsearch.memtable_pages_threshold` | Chain pages before auto-spill (0 = disable) | 64 | | `pg_textsearch.allow_rls` | Allow BM25 indexes on RLS-protected tables (superuser-only) | on | | `pg_textsearch.segments_per_level` | Segments before compaction | 8 | | `pg_textsearch.max_segment_size` | Conservative size budget for newly merged multi-source segments (1-4095MB) | 4095MB | | `pg_textsearch.background_compaction_schedule` | Default cron schedule captured by indexes entering managed background mode | `*/5 * * * *` | | `pg_textsearch.compress_segments` | Enable compression for new segment blocks | true | | `pg_textsearch.filtered_seed` | Seed the BM25 internal top-K from estimated filter selectivity. The existing planner hook attaches private query-value hints to direct `Limit -> IndexScan` pairs with constant query and LIMIT/OFFSET values. Each scan reads its own hint and applies the current GUCs; missing hints use the default and backoff. | true | | `pg_textsearch.filtered_seed_margin` | Seed = `ceil(margin * LIMIT / selectivity)`. Higher captures the true top-k in one scoring pass more often, at the cost of scoring deeper. Range [1, 1000] | 3.0 | | `pg_textsearch.memtable_cache_enabled` | Serve query reads from the in-memory memtable cache instead of the on-disk chain (chain remains source of truth; standbys always use the chain) | true | | `pg_textsearch.log_cache_state` | Log in-memory cache apply outcomes (OK / BUDGET_EXCEEDED / cold_build / RETRY / ABORT / fall back to chain) | false | | `pg_textsearch.memory_limit` | Approximate shared-memory budget (KB, `PGC_SIGHUP`) for the in-memory memtable cache. Three tiers: per-index per-record growth guard (`limit/8`) → BUDGET_EXCEEDED + chain fallback before a record crosses it; global soft cap (`limit/2`) → best-effort eviction of the largest non-caller cache; the global `limit` is an approximate admission threshold → catch-up or cold-build fallback when the entry-time estimate is already at the limit. Admitted or concurrent work may increase estimated usage past the limit. `0` = unlimited. | 2 GB | ### Index Options | Option | Description | Default | |--------|-------------|---------| | `text_config` | Postgres text search configuration | (required) | | `k1` | BM25 term frequency saturation | 1.2 | | `b` | BM25 length normalization | 0.75 | | `compaction` | Spill-time compaction: `inline`, managed pg_durable `background`, or `manual`. Alterable with `ALTER INDEX ... SET` | inline | | `compaction_schedule` | Optional per-index cron schedule for background mode | global GUC | ## Test Structure - **SQL tests** (`test/sql/`): Core functionality, BM25 scoring (scoring1-6), storage/segments, query optimization (bmw, wand), table features (partitioned, inheritance), and edge cases - **Shell tests** (`test/scripts/`): concurrency.sh, recovery.sh, segment.sh, stress.sh for multi-session, crash recovery, and load testing - **Expected output** (`test/expected/`): Reference outputs for regression tests ## BM25 Implementation ### Query Processing - `text <@> bm25query` - primary operator, works in index scans and standalone - `text <@> text` - implicit form, planner rewrites to `text <@> bm25query` - Returns negative BM25 scores for Postgres ASC ordering - bm25query with index name required for WHERE clause queries ### Query Style Guidelines **Do not use WHERE clauses that compare against BM25 scores** as a generic way to filter for matching documents. Numeric score comparisons like `WHERE content <@> query < 0` are an anti-pattern when used as a convenience filter in tests and examples: - BM25 scores are for ranking, not filtering; their numeric values are opaque and not meaningful to compare against thresholds - A WHERE clause with `<@>` triggers standalone scoring (sequential scan), not an index scan; index scans require ORDER BY - The correct pattern is `ORDER BY content <@> query LIMIT n` For counting matching documents, use a subquery with ORDER BY: ```sql -- WRONG: standalone scoring as a convenience filter SELECT count(*) FROM docs WHERE content <@> to_bm25query('term', 'idx') < 0; -- RIGHT: index scan via ORDER BY SELECT count(*) FROM ( SELECT 1 FROM docs ORDER BY content <@> to_bm25query('term', 'idx') ) sub; ``` **Exception: standalone scoring as an explicit test target.** Tests that deliberately exercise the standalone scoring code path (e.g., comparing index scan results against standalone results) should use WHERE with `<@>` and document the intent. See `validation.sql` (`validate_index_vs_standalone`), `queries.sql`, and `vector.sql` for examples of this pattern. ### Block-Max WAND Optimization Top-k queries use Block-Max WAND optimization: - Block-level upper bounds computed during index build - Early termination when blocks cannot contribute to results - Skip lists for fast block seeking in segments ### VACUUM and Alive Bitset VACUUM uses per-segment alive bitsets (1 bit per doc) to mark dead documents instead of rebuilding segments. This is O(dead_docs) instead of O(all_docs). Dead docs are filtered during BMW scoring and physically removed during segment merge. The serial cleanup pass holds the maintenance object lock, but not the per-index LWLock, while scanning the full index fork for reclaimable DEAD memtable pages. This serializes force-merge truncation without blocking spills or later readers behind an O(index-pages) shared lock. **Stale statistics after VACUUM**: After VACUUM marks docs dead, the segment's `total_docs`, `total_tokens`, and per-term `doc_freq` are not updated. This means BM25 IDF calculations use slightly stale corpus statistics until the next segment merge/compaction, which corrects them. For small numbers of deletes this is negligible; mass deletion can affect ranking quality until the next merge. ## Benchmarking The `benchmarks/` directory contains performance testing tools: ### Datasets - `sql/cranfield/` - Classic IR benchmark (Cranfield collection) - `datasets/msmarco/` - MS MARCO passage ranking (download required) - `datasets/wikipedia/` - Wikipedia articles (download required) ### Scripts - `run_cranfield.sh` - Classic IR benchmark using Cranfield collection - `run_memory_stress.sh` - Memory usage and capacity testing - `sql/memory_stress*.sql` - Memory stress test configurations - `runner/run_benchmark.sh` - Full benchmark runner with metrics extraction ### Comparative Benchmarks - `engines/tantivy/` - Tantivy comparison for validation ## Continuous Integration GitHub Actions workflows: - `ci.yml` - Main CI pipeline testing Postgres 17, 18, and 19 (beta, allow-failure) - `sanitizer-build-and-test.yml` - Address sanitizer builds - `formatting.yml` - Code formatting validation - `release.yml` - Automated release workflow - `package-release.yml` - Release packaging - `upgrade-tests.yml` - Version upgrade testing - `benchmark.yml` - Performance benchmarks - `nightly-stress.yml` - Nightly stress tests with memory leak detection - `claude-code-review.yml` - AI code review integration - `claude.yml` - Claude Code automation See [RELEASING.md](RELEASING.md) for release instructions. ## Debug Functions - `bm25_dump_index(index_name)` - Shows internal index structure with term dictionary, posting lists, and document entries - `bm25_dump_index(index_name, filepath)` - Dumps full index to file - `bm25_summarize_index(index_name)` - Shows high-level index statistics - `bm25_spill_index(index_name)` - Forces memtable spill to disk segment, returns number of entries spilled - `bm25_force_merge(index_name)` - Runs one bounded, copy-on-write compaction pass, combining the largest adjacent groups that fit `pg_textsearch.max_segment_size`. It does **not** guarantee a single segment: large indexes may intentionally retain several immutable segments, and an existing segment that already exceeds the budget remains an uncombinable singleton. Published sources stay immutable while replacements are built; displaced pages enter deferred reclaim (see #380) rather than becoming immediately reusable. Waits for index maintenance when another session holds it. - `bm25_level_counts(idx regclass)` - Segments held at each of the eight LSM levels - `bm25_needs_compaction(idx regclass)` - Whether `bm25_compact_step()` would run a pass. Runs the same selection without publishing, so a level that is full of over-budget segments reports false; safe as a loop condition. Never waits: reports true when another session holds index maintenance. - `bm25_compact(idx regclass)` - Run compaction passes to completion, releasing same-index maintenance admission between passes so a long cascade does not starve a queued VACUUM. Requires index ownership. A published pass is a physical change and is **not** undone by ROLLBACK. Waits for index maintenance when another session holds it; the maintenance lock is private to this extension, so waiting cannot deadlock with a concurrent `REINDEX INDEX CONCURRENTLY`. - `bm25_compact_step(idx regclass)` - Run at most one pass and report whether one ran, letting a caller spread a cascade over several transactions. Requires index ownership. Waits on the same terms as `bm25_compact()`, so a `false` return always means "no reducible debt", never "maintenance was busy". - `bm25_pending_free_pages(index_name)` - Count displaced segment pages currently parked in the deferred-free tombstone chain (issue #380), awaiting standby-safe FSM reclaim - `bm25_debug_pageviz(index_name, file_path)` - Generate page layout visualization - `bm25_debug_scoring_passes([reset])` - Session-local count of BM25 scoring passes. A scan seeded for its own filter costs exactly one pass; each executor backoff re-drive adds another. Scan depth is invisible in query results, so this is how the per-scan seeding of issue #435 is asserted. Pass `true` to return the count and reset it ## Parallel Index Build Parallel index build uses multiple workers to speed up CREATE INDEX on large tables. ### Requirements - Table must have >= 100,000 estimated rows (configurable via `TP_MIN_PARALLEL_TUPLES`) - `max_parallel_maintenance_workers` must be > 0 - Workers are automatically allocated based on table statistics ### How It Works 1. Leader launches parallel workers and assigns heap block ranges 2. Workers scan their assigned blocks, tokenize documents, and build local in-memory indexes 3. Workers write intermediate segment data to temporary BufFiles 4. Leader performs an N-way merge of all worker BufFiles, writing the final merged segment directly to index pages 5. Result is a single merged segment ### Configuration | GUC | Description | Default | |-----|-------------|---------| | `max_parallel_maintenance_workers` | Max workers for parallel operations | 2 | ### Limitations - Postgres may launch fewer workers than requested for smaller tables ## Important Notes ### Concurrency Safety - All shared memory structures protected by appropriate locks - String interning is thread-safe via hash table locks - Transaction isolation maintained through proper cleanup ### Testing Requirements - Always run `make installcheck` before committing changes - Concurrency tests verify thread safety under load - Shell tests verify crash recovery and multi-backend behavior ### Code Style - Uses clang-format for consistent C formatting - Postgres coding conventions followed - Memory allocation patterns follow Postgres standards - Wrap all lines at 79 characters - The L0 memtable is an on-disk chain of pages in the index relation (one chain per index). Mutated under standard buffer locks and WAL-logged via `GenericXLog`. Read [ARCHITECTURE.md](ARCHITECTURE.md#storage-and-wal) before changing the write/read/spill flow. ### Pre-Commit Checklist **ALWAYS complete these steps before committing changes:** 1. `make` - Compile the extension 2. `make installcheck` - Run SQL regression tests 3. **Check test/regression.diffs** - Examine diffs to ensure your changes didn't introduce failures 4. If you modified error messages, update corresponding test/expected/*.out 5. `make format-check` - Verify code formatting 6. Re-run `make installcheck` after fixing any SQL test expectation files ### Git Workflow - **After PRs are merged**: Always check if main has been updated, even if you didn't do the merge yourself. Run `git fetch && git rebase origin/main` on active branches to avoid merge conflicts. - **Before pushing**: Verify your branch is up-to-date with main to prevent conflicts from divergent histories. timescale-pg_textsearch-b63fe58/CONTRIBUTING.md000066400000000000000000000047271525762047700212240ustar00rootroot00000000000000# Contributing to pg_textsearch pg_textsearch was originally named Tapir (Textual Analysis for Postgres Information Retrieval), which remains the project mascot and appears in some source names. ## Development Setup Install PostgreSQL 17 or 18 and its development headers, then build and install the extension: ```bash make make install ``` Add `pg_textsearch` to `shared_preload_libraries`, restart PostgreSQL, and run the installed-extension regression suite: ```bash make installcheck ``` See [test/README.md](test/README.md) for a temporary-cluster target and the specialized test suites. ## Before Submitting Run: ```bash make make installcheck make format-check ``` Changes to storage, WAL, the memtable cache, compaction, or deferred reclaim also require the relevant shell and replication tests documented in [test/README.md](test/README.md). If output changes intentionally, review and update the corresponding file under `test/expected/`. ## Code Style Follow the [PostgreSQL coding conventions](https://www.postgresql.org/docs/current/source-format.html): - Wrap lines at 79 characters and indent with tabs. - Use Allman braces, with opening braces on a new line. - Put `postgres.h` first in C source files. - Write project includes as paths relative to `src/`, such as `#include "segment/segment.h"`. Use `make format` to apply formatting and `make format-check` to verify it. ## Architecture See [ARCHITECTURE.md](ARCHITECTURE.md) for the source layout, storage design, and invariants that changes must preserve. ## Pull Requests Open focused pull requests against `main`. Explain the problem and approach, list the tests run, note user-visible or compatibility effects, and link related issues. Keep commits clear and scoped to the change. Performance work should include relevant measurements; see [benchmarks/README.md](benchmarks/README.md) for the benchmark tooling. ## Reporting Issues Search existing issues first. Bug reports should include PostgreSQL and pg_textsearch versions, operating system, reproduction steps, expected and actual behavior, and relevant logs. Feature requests should describe the problem, proposed outcome, and alternatives considered. Use [GitHub Issues](https://github.com/timescale/pg_textsearch/issues) for bugs and feature requests, and [GitHub Discussions](https://github.com/timescale/pg_textsearch/discussions) for general questions. ## License By contributing, you agree that your contributions are licensed under the PostgreSQL License. timescale-pg_textsearch-b63fe58/LICENSE000066400000000000000000000016161525762047700177720ustar00rootroot00000000000000The PostgreSQL License Permission to use, copy, modify, and distribute this software and its documentation for any purpose, without fee, and without a written agreement is hereby granted, provided that the above copyright notice and this paragraph and the following two paragraphs appear in all copies. IN NO EVENT SHALL TIGER DATA BE LIABLE TO ANY PARTY FOR DIRECT, INDIRECT, SPECIAL, INCIDENTAL, OR CONSEQUENTIAL DAMAGES, INCLUDING LOST PROFITS, ARISING OUT OF THE USE OF THIS SOFTWARE AND ITS DOCUMENTATION, EVEN IF TIGER DATA HAS BEEN ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. TIGER DATA SPECIFICALLY DISCLAIMS ANY WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE. THE SOFTWARE PROVIDED HEREUNDER IS ON AN "AS IS" BASIS, AND TIGER DATA HAS NO OBLIGATIONS TO PROVIDE MAINTENANCE, SUPPORT, UPDATES, ENHANCEMENTS, OR MODIFICATIONS. timescale-pg_textsearch-b63fe58/Makefile000066400000000000000000000467431525762047700204370ustar00rootroot00000000000000EXTENSION = pg_textsearch EXTVERSION = $(shell awk -F"'" '/default_version/ {print $$2}' pg_textsearch.control) DATA = sql/pg_textsearch--1.5.1.sql \ sql/pg_textsearch--0.0.1--0.0.2.sql \ sql/pg_textsearch--0.0.2--0.0.3.sql \ sql/pg_textsearch--0.0.3--0.0.4.sql \ sql/pg_textsearch--0.0.4--0.0.5.sql \ sql/pg_textsearch--0.0.5--0.1.0.sql \ sql/pg_textsearch--0.1.0--0.2.0.sql \ sql/pg_textsearch--0.2.0--0.3.0.sql \ sql/pg_textsearch--0.3.0--0.4.0.sql \ sql/pg_textsearch--0.4.0--0.4.1.sql \ sql/pg_textsearch--0.4.1--0.4.2.sql \ sql/pg_textsearch--0.4.2--0.5.0.sql \ sql/pg_textsearch--0.5.0--0.5.1.sql \ sql/pg_textsearch--0.5.1--0.6.0.sql \ sql/pg_textsearch--0.6.0--0.6.1.sql \ sql/pg_textsearch--0.6.1--1.0.0.sql \ sql/pg_textsearch--1.0.0--1.1.0.sql \ sql/pg_textsearch--1.1.0--1.2.0.sql \ sql/pg_textsearch--1.2.0--1.3.0.sql \ sql/pg_textsearch--1.3.0--1.3.1.sql \ sql/pg_textsearch--1.3.1--1.4.0.sql \ sql/pg_textsearch--1.4.0--1.5.0.sql \ sql/pg_textsearch--1.5.0--1.5.1.sql # Source files organized by directory OBJS = \ src/mod.o \ src/layout_check.o \ src/access/handler.o \ src/access/build.o \ src/access/build_context.o \ src/access/build_merge.o \ src/access/build_parallel.o \ src/access/compaction_api.o \ src/access/boolean.o \ src/access/rls.o \ src/access/scan.o \ src/access/vacuum.o \ src/memtable/arena.o \ src/memtable/cache.o \ src/memtable/cache_source.o \ src/memtable/chain_source.o \ src/memtable/chain_walker.o \ src/memtable/expull.o \ src/memtable/log.o \ src/memtable/page.o \ src/memtable/posting.o \ src/memtable/scan.o \ src/memtable/stringtable.o \ src/segment/segment.o \ src/segment/graph_snapshot.o \ src/segment/dictionary.o \ src/segment/scan.o \ src/segment/merge.o \ src/segment/compaction.o \ src/segment/tombstone.o \ src/segment/docmap.o \ src/segment/alive_bitset.o \ src/segment/compression.o \ src/segment/fieldnorm.o \ src/scoring/bmw.o \ src/scoring/bm25.o \ src/scoring/snapshot.o \ src/types/array.o \ src/types/vector.o \ src/types/query.o \ src/index/compaction_job.o \ src/index/compaction_request.o \ src/index/state.o \ src/index/registry.o \ src/index/metapage.o \ src/index/freepage.o \ src/index/limit.o \ src/index/resolve.o \ src/index/source.o \ src/planner/hooks.o \ src/planner/cost.o \ src/debug/injection.o \ src/debug/dump.o # Shared library target MODULE_big = pg_textsearch # Include directories, debug flags, and warning flags for unused code PG_CPPFLAGS = -I$(srcdir)/src -g -O2 -Wall -Wextra -Wunused-function -Wunused-variable -Wunused-parameter -Wunused-but-set-variable -DPG_TEXTSEARCH_VERSION=\"$(EXTVERSION)\" # Suppress GCC-only false positives (clang silently ignores unknown -Wno-* # flags thanks to -Wno-unknown-warning-option): # -Wclobbered: PG_TRY uses setjmp/longjmp; GCC warns about locals across it # -Wpacked-not-aligned: intentionally packed on-disk structs PG_CPPFLAGS += -Wno-unknown-warning-option -Wno-clobbered -Wno-packed-not-aligned # Uncomment the following line to enable debug index dumps # PG_CPPFLAGS += -DDEBUG_DUMP_INDEX # Test configuration REGRESS = preload abort aerodocs basic binary_io bmw bmw_skip_advance boolean_queries build_progress bulk_load cache_apply cache_memory_cap cache_source cache_spill catalog_stats chain_source compaction compaction_request compression concurrent_build coverage deletion vacuum vacuum_bitmap vacuum_extended vacuum_rebuild dropped empty explicit_index expression_index filtered_seed force_merge implicit index inheritance large_documents limits lock manyterms memory memtable_append memtable_page memtable_spill memtable_spill_dead memtable_reclaim merge mixed parallel_build parallel_bmw partitioned partitioned_many partial_index pgstats queries quoted_identifiers rescan rls schema scoring1 scoring2 scoring3 scoring4 scoring5 scoring6 security security_acl segment segment_integrity segment_reclaim tombstone_reuse tombstone_recover strings temp_table text_array text_config unsupported updates vector vector_v1_rejected unlogged_index wand INJECTION_REGRESS = scoring_injection merge_injection compaction_injection \ compaction_error_injection \ force_merge_injection segment_reclaim_injection tombstone_bounds_injection \ vacuum_rebuild_injection query_hint_roundtrip memtable_stale_tail_injection REGRESS += compaction_policy REGRESS_OPTS = --inputdir=test --outputdir=test PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) # Makefile.global (included by PGXS) reports how the server was # configured. The injection tests and the helper module that drives # them only exist for a --enable-injection-points server. ifeq ($(enable_injection_points),yes) REGRESS += $(INJECTION_REGRESS) install: install-test-injection install-test-injection: @# PGXS does not encode PG_CONFIG in object dependencies, so stale @# objects from another server version would silently be reused. @$(MAKE) -C test/modules/pg_textsearch_test \ PG_CONFIG="$(PG_CONFIG)" clean @$(MAKE) -C test/modules/pg_textsearch_test PG_CONFIG="$(PG_CONFIG)" @$(MAKE) -C test/modules/pg_textsearch_test \ PG_CONFIG="$(PG_CONFIG)" install test-injection-sql: @$(pg_regress_installcheck) $(REGRESS_OPTS) $(INJECTION_REGRESS) test-injection-shell: @cd test/scripts && ./inline_compaction_locking.sh injection @cd test/scripts && ./crash_safety_spill.sh @cd test/scripts && ./nonblocking_compaction.sh @cd test/scripts && ./nonblocking_spill.sh @cd test/scripts && ./compaction_recovery.sh @cd test/scripts && ./parallel_vacuum.sh injection @cd test/scripts && ./standby_reclaim.sh else install-test-injection test-injection-sql test-injection-shell: @echo "PostgreSQL injection points are disabled; skipping $@" endif # SQL regression tests test: test-segment-io-limits test-compression \ test-mixed-update-query-benchmark test-filtered-seed-benchmark \ test-build-memory-benchmark @echo "Running SQL regression tests..." @$(pg_regress_installcheck) $(REGRESS_OPTS) $(REGRESS) test-segment-io-limits: @set -e; tmp_dir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmp_dir"' EXIT; \ $(CC) -std=gnu11 \ -Isrc \ -I"$$($(PG_CONFIG) --includedir-server)" \ -I"$$($(PG_CONFIG) --includedir)" \ test/scripts/segment_io_limits_test.c \ -o "$$tmp_dir/segment_io_limits_test"; \ "$$tmp_dir/segment_io_limits_test" test-compression: @set -e; tmp_dir="$$(mktemp -d)"; \ trap 'rm -rf "$$tmp_dir"' EXIT; \ for flags in "" "-DTP_TEST_NO_AVX2"; do \ $(CC) $(CFLAGS) $(CPPFLAGS) $$flags -std=gnu11 \ -Isrc \ -isystem "$$($(PG_CONFIG) --includedir-server)" \ -isystem "$$($(PG_CONFIG) --includedir)" \ test/scripts/compression_test.c \ -o "$$tmp_dir/compression_test"; \ "$$tmp_dir/compression_test"; \ done test-mixed-update-query-benchmark: @./test/scripts/mixed_update_query_benchmark_test.sh test-filtered-seed-benchmark: @bash test/scripts/filtered_seed_benchmark_test.sh test-build-memory-benchmark: @python3 test/scripts/build_memory_benchmark_test.py test-durable: @echo "Running managed pg_durable compaction tests..." @cd test/scripts && ./durable_compaction.sh installcheck: test-segment-io-limits test-compression \ test-mixed-update-query-benchmark test-filtered-seed-benchmark \ test-build-memory-benchmark test-local: test-segment-io-limits test-compression \ test-mixed-update-query-benchmark test-filtered-seed-benchmark \ test-build-memory-benchmark # Custom local test target with dedicated PostgreSQL instance test-local: install @echo "Setting up temporary PostgreSQL instance for local testing..." @rm -rf tmp_check_shared @mkdir -p tmp_check_shared @initdb -D tmp_check_shared/data --auth-local=trust --auth-host=trust @echo "port = 55433" >> tmp_check_shared/data/postgresql.conf @echo "log_statement = 'all'" >> tmp_check_shared/data/postgresql.conf @echo "shared_buffers = 256MB" >> tmp_check_shared/data/postgresql.conf @echo "max_connections = 20" >> tmp_check_shared/data/postgresql.conf @echo "shared_preload_libraries = '$(MODULE_big)'" >> tmp_check_shared/data/postgresql.conf @pg_ctl start -D tmp_check_shared/data -l tmp_check_shared/data/logfile -w @createdb -p 55433 contrib_regression @$(pg_regress_installcheck) --use-existing --port=55433 --inputdir=test --outputdir=test $(REGRESS) --dbname=contrib_regression @pg_ctl stop -D tmp_check_shared/data -l tmp_check_shared/data/logfile @rm -rf tmp_check_shared # Clean test directories clean: clean-test-dirs clean-test-dirs: @rm -rf tmp_check_shared coverage-html coverage.info @find . -name "*.gcda" -delete 2>/dev/null || true @find . -name "*.gcno" -delete 2>/dev/null || true @$(MAKE) -C test/modules/pg_textsearch_test \ PG_CONFIG="$(PG_CONFIG)" clean >/dev/null 2>&1 || true # Shell script test targets (assume extension is already installed) test-rls-locking: @echo "Running RLS DDL locking tests..." @cd test/scripts && ./rls_ddl_locking.sh test-nonblocking-compaction: @cd test/scripts && ./nonblocking_compaction.sh test-nonblocking-spill: @cd test/scripts && ./nonblocking_spill.sh test-standalone-snapshot: @cd test/scripts && ./standalone_snapshot.sh test-memtable-stale-tail: @cd test/scripts && ./memtable_stale_tail.sh test-concurrency: test-rls-locking test-memtable-stale-tail @echo "Running concurrency tests..." @cd test/scripts && ./standalone_snapshot.sh @cd test/scripts && ./inline_compaction_locking.sh @cd test/scripts && ./parallel_vacuum.sh @cd test/scripts && ./concurrency.sh @cd test/scripts && ./boolean_concurrent_merge.sh @cd test/scripts && ./partial_concurrent_read.sh @cd test/scripts && ./concurrent_duplicate_read.sh @cd test/scripts && ./vacuum_concurrent_merge.sh test-recovery: @echo "Running crash recovery tests..." @cd test/scripts && ./recovery.sh @cd test/scripts && ./shutdown_spill.sh @cd test/scripts && ./compaction_recovery.sh test-segment: @echo "Running multi-backend segment tests..." @cd test/scripts && ./segment.sh @cd test/scripts && ./parallel_build_bulkread.sh test-stress: @echo "Running stress tests..." @cd test/scripts && ./stress.sh test-cic: @echo "Running CREATE INDEX CONCURRENTLY tests..." @cd test/scripts && ./cic.sh # Chinese tokenization end-to-end test. Optional and NOT part of the # default REGRESS schedule: it requires the third-party zhparser extension # (https://github.com/amutu/zhparser) and its SCWS library. Run it # explicitly, e.g. from a CI job that installs zhparser first. test-chinese: @echo "Running Chinese tokenization test (requires zhparser)..." @$(pg_regress_installcheck) $(REGRESS_OPTS) chinese # Replication tests (not in test-shell: each spawns two Postgres instances) test-replication: @echo "Running physical replication tests..." @cd test/scripts && TMPDIR=.. REPL_HOST=127.0.0.1 \ REPL_SOCKET_DIR= ./replication.sh @cd test/scripts && ./standby_reclaim.sh test-logical-replication: @echo "Running logical replication tests..." @cd test/scripts && ./logical_replication.sh test-replication-extended: @echo "Running extended physical replication tests..." @scripts="\ replication.sh \ replication_issue_342.sh \ replication_parallel_build.sh \ replication_correctness.sh \ replication_concurrency.sh \ replication_failover.sh \ replication_compat.sh \ replication_cascading.sh \ replication_spill_paths.sh \ replication_memtable_dead_reclaim.sh \ replication_segment_reclaim.sh \ standby_reclaim.sh \ wal_audit.sh"; \ failed=""; \ for s in $$scripts; do \ echo ""; \ echo "==> $$s"; \ if ! (cd test/scripts && "./$$s"); then \ failed="$$failed $$s"; \ fi; \ done; \ if [ -n "$$failed" ]; then \ echo ""; \ echo "Failed scripts:$$failed"; \ exit 1; \ fi test-multi-index: @echo "Running multi-index / multi-user / multi-schema tests..." @cd test/scripts && ./multi_index.sh test-reindex: @echo "Running multi-backend reindex regression tests (issue #390)..." @cd test/scripts && ./multi_backend_reindex.sh test-cross-database-registry: @echo "Running cross-database registry regression tests (issue #464)..." @cd test/scripts && ./cross_database_registry.sh test-drop-rollback: @bash test/scripts/drop_index_rollback.sh test-shell: test-concurrency test-recovery test-segment test-cic test-multi-index test-reindex test-cross-database-registry test-drop-rollback @echo "All shell-based tests completed" test-all: test test-shell test-replication @echo "All tests (SQL regression + shell scripts) completed successfully" # Generate expected output files from current test results expected: @echo "Generating expected output files from current results..." @for test in $(REGRESS); do \ if [ -f test/results/$$test.out ]; then \ cp test/results/$$test.out test/expected/$$test.out; \ echo " Updated test/expected/$$test.out"; \ else \ echo " Warning: No results file for $$test"; \ fi; \ done @echo "Expected files updated. Review changes before committing." # Code formatting targets lint-format: @echo "Checking C code formatting with clang-format..." @if command -v clang-format >/dev/null 2>&1; then \ find src/ -name "*.c" -o -name "*.h" | xargs clang-format --dry-run --Werror --style=file; \ else \ echo "clang-format not found - install with: brew install clang-format (macOS) or apt install clang-format (Linux)"; \ exit 1; \ fi @echo "Code formatting check passed" format: @echo "Formatting C code with clang-format..." @if command -v clang-format >/dev/null 2>&1; then \ find src/ -name "*.c" -o -name "*.h" | xargs clang-format -i --style=file; \ else \ echo "clang-format not found - install with: brew install clang-format (macOS) or apt install clang-format (Linux)"; \ exit 1; \ fi @echo "Code formatting completed" format-diff: @echo "Showing formatting differences..." @if command -v clang-format >/dev/null 2>&1; then \ for file in `find src/ -name "*.c" -o -name "*.h"`; do \ echo "=== $$file ==="; \ clang-format --style=file "$$file" | diff -u "$$file" - || true; \ done; \ else \ echo "clang-format not found"; \ exit 1; \ fi format-single: @if [ -z "$(FILE)" ]; then \ echo "Usage: make format-single FILE=path/to/file.c"; \ exit 1; \ fi @echo "Formatting $(FILE)..." @clang-format -i --style=file $(FILE) @echo "$(FILE) formatted" format-check: lint-format # Code coverage targets (Linux only - requires lcov: apt install lcov) # Note: macOS is not supported due to gcov runtime crashes with Postgres extensions COVERAGE_DIR = coverage-html COVERAGE_INFO = coverage.info coverage-clean: @echo "Cleaning coverage data..." @rm -rf $(COVERAGE_DIR) $(COVERAGE_INFO) @find . -name "*.gcda" -delete @find . -name "*.gcno" -delete coverage-build: coverage-clean @echo "Building with coverage instrumentation..." @if ! command -v lcov >/dev/null 2>&1; then \ echo "lcov not found - install with: brew install lcov (macOS) or apt install lcov (Linux)"; \ exit 1; \ fi $(MAKE) clean ifeq ($(shell uname),Darwin) $(MAKE) PG_CFLAGS="-fprofile-arcs -ftest-coverage -O0 -g" else $(MAKE) PG_CFLAGS="--coverage -O0 -g" SHLIB_LINK="--coverage" endif $(MAKE) install coverage: coverage-build @echo "Running tests and collecting coverage..." @lcov --zerocounters --directory . -$(MAKE) test @echo "Capturing coverage data..." @lcov --capture \ --directory . \ --output-file $(COVERAGE_INFO) \ --base-directory $(shell pwd) \ --no-external \ --rc branch_coverage=1 \ --ignore-errors mismatch 2>/dev/null || \ lcov --capture \ --directory . \ --output-file $(COVERAGE_INFO) \ --base-directory $(shell pwd) \ --no-external @lcov --remove $(COVERAGE_INFO) '*/test/*' \ --output-file $(COVERAGE_INFO) \ --rc branch_coverage=1 \ --ignore-errors unused 2>/dev/null || \ lcov --remove $(COVERAGE_INFO) '*/test/*' \ --output-file $(COVERAGE_INFO) @echo "Generating HTML report..." @genhtml $(COVERAGE_INFO) \ --output-directory $(COVERAGE_DIR) \ --title "pg_textsearch Coverage" \ --legend \ --show-details \ --rc branch_coverage=1 @echo "" @echo "Coverage report generated in $(COVERAGE_DIR)/index.html" @lcov --summary $(COVERAGE_INFO) --rc branch_coverage=1 coverage-report: @if [ ! -f $(COVERAGE_INFO) ]; then \ echo "No coverage data found. Run 'make coverage' first."; \ exit 1; \ fi @lcov --summary $(COVERAGE_INFO) --rc branch_coverage=1 # Help target .PHONY: help help: @echo "pg_textsearch Makefile" @echo "" @echo "Build targets:" @echo " make - Build the extension" @echo " make install - Build and install the extension" @echo " make clean - Clean build artifacts and test directories" @echo "" @echo "Testing targets:" @echo " make test - Run SQL regression tests" @echo " make installcheck - Run SQL regression tests" @echo " make test-local - Run tests with dedicated PostgreSQL instance" @echo " make test-all - Run SQL, default shell, and replication tests" @echo " make test-shell - Run default shell tests (excludes replication)" @echo " make test-nonblocking-compaction - Run compaction overlap tests (needs injection points)" @echo " make test-concurrency - Run concurrency tests" @echo " make test-recovery - Run crash recovery tests" @echo " make test-segment - Run multi-backend segment tests" @echo " make test-stress - Run long-running stress tests" @echo " make test-cic - Run CREATE INDEX CONCURRENTLY tests" @echo " make test-chinese - Run Chinese tokenization test (needs zhparser)" @echo " make test-reindex - Run multi-backend reindex regression tests (issue #390)" @echo " make test-durable - Run managed pg_durable compaction tests" @echo " make test-cross-database-registry - Run issue #464 registry regression" @echo " make test-drop-rollback - Run multi-backend DROP rollback regression" @echo " make expected - Generate expected output files from test results" @echo "" @echo "Code formatting targets:" @echo " make format - Auto-format C code with clang-format" @echo " make format-check - Check C code formatting (alias: lint-format)" @echo " make format-diff - Show formatting differences" @echo " make format-single FILE=path/to/file.c - Format specific file" @echo "" @echo "Code coverage targets (Linux only, requires lcov):" @echo " make coverage - Build with coverage, run tests, generate HTML report" @echo " make coverage-build - Build with coverage instrumentation only" @echo " make coverage-clean - Remove coverage data and reports" @echo " make coverage-report - Show coverage summary (after running coverage)" @echo " Note: macOS not supported; coverage runs in GitHub Actions CI" @echo "" @echo "Configuration:" @echo " PG_CONFIG - Path to pg_config (default: pg_config)" @echo "" @echo "Examples:" @echo " make && make install" @echo " make test-all" @echo " make format" .PHONY: \ test test-segment-io-limits test-compression \ test-mixed-update-query-benchmark \ test-filtered-seed-benchmark test-build-memory-benchmark \ test-durable \ test-injection-sql test-injection-shell install-test-injection \ clean-test-dirs installcheck test-rls-locking test-concurrency \ test-standalone-snapshot test-nonblocking-compaction \ test-nonblocking-spill test-memtable-stale-tail \ test-recovery test-segment test-stress test-cic test-chinese \ test-replication test-replication-extended \ test-logical-replication test-multi-index test-reindex \ test-cross-database-registry test-drop-rollback \ test-shell test-all expected lint-format format format-check \ format-diff format-single coverage coverage-build coverage-clean \ coverage-report help timescale-pg_textsearch-b63fe58/Makefile.win000066400000000000000000000130721525762047700212200ustar00rootroot00000000000000EXTENSION = pg_textsearch EXTVERSION = 1.5.1 DATA = sql/pg_textsearch--1.5.1.sql \ sql/pg_textsearch--0.0.1--0.0.2.sql \ sql/pg_textsearch--0.0.2--0.0.3.sql \ sql/pg_textsearch--0.0.3--0.0.4.sql \ sql/pg_textsearch--0.0.4--0.0.5.sql \ sql/pg_textsearch--0.0.5--0.1.0.sql \ sql/pg_textsearch--0.1.0--0.2.0.sql \ sql/pg_textsearch--0.2.0--0.3.0.sql \ sql/pg_textsearch--0.3.0--0.4.0.sql \ sql/pg_textsearch--0.4.0--0.4.1.sql \ sql/pg_textsearch--0.4.1--0.4.2.sql \ sql/pg_textsearch--0.4.2--0.5.0.sql \ sql/pg_textsearch--0.5.0--0.5.1.sql \ sql/pg_textsearch--0.5.1--0.6.0.sql \ sql/pg_textsearch--0.6.0--0.6.1.sql \ sql/pg_textsearch--0.6.1--1.0.0.sql \ sql/pg_textsearch--1.0.0--1.1.0.sql \ sql/pg_textsearch--1.1.0--1.2.0.sql \ sql/pg_textsearch--1.2.0--1.3.0.sql \ sql/pg_textsearch--1.3.0--1.3.1.sql \ sql/pg_textsearch--1.3.1--1.4.0.sql \ sql/pg_textsearch--1.4.0--1.5.0.sql \ sql/pg_textsearch--1.5.0--1.5.1.sql # Source files organized by directory OBJS = \ src/mod.obj \ src/layout_check.obj \ src/access/handler.obj \ src/access/build.obj \ src/access/build_context.obj \ src/access/build_parallel.obj \ src/access/compaction_api.obj \ src/access/scan.obj \ src/access/vacuum.obj \ src/memtable/arena.obj \ src/memtable/cache.obj \ src/memtable/cache_source.obj \ src/memtable/chain_source.obj \ src/memtable/chain_walker.obj \ src/memtable/expull.obj \ src/memtable/log.obj \ src/memtable/page.obj \ src/memtable/posting.obj \ src/memtable/scan.obj \ src/memtable/stringtable.obj \ src/segment/segment.obj \ src/segment/dictionary.obj \ src/segment/scan.obj \ src/segment/merge.obj \ src/segment/compaction.obj \ src/segment/tombstone.obj \ src/segment/docmap.obj \ src/segment/alive_bitset.obj \ src/segment/compression.obj \ src/segment/fieldnorm.obj \ src/scoring/bmw.obj \ src/scoring/bm25.obj \ src/scoring/snapshot.obj \ src/types/array.obj \ src/types/vector.obj \ src/types/query.obj \ src/index/compaction_request.obj \ src/index/state.obj \ src/index/registry.obj \ src/index/metapage.obj \ src/index/freepage.obj \ src/index/limit.obj \ src/index/resolve.obj \ src/index/source.obj \ src/planner/hooks.obj \ src/planner/cost.obj \ src/debug/dump.obj # Shared library target MODULE_big = pg_textsearch # Include directories, debug flags, and warning flags for unused code PG_CFLAGS = /I"src" /O2 /fp:fast -DPG_TEXTSEARCH_VERSION=\"$(EXTVERSION)\" # Suppress GCC-only false positives (clang silently ignores unknown -Wno-* # flags thanks to -Wno-unknown-warning-option): # -Wclobbered: PG_TRY uses setjmp/longjmp; GCC warns about locals across it # -Wpacked-not-aligned: intentionally packed on-disk structs #PG_CFLAGS = $(PG_CFLAGS) -Wno-unknown-warning-option -Wno-clobbered -Wno-packed-not-aligned # Uncomment the following line to enable debug index dumps # PG_CPPFLAGS += -DDEBUG_DUMP_INDEX # Test configuration REGRESS = abort aerodocs basic binary_io bmw bmw_skip_advance bulk_load cache_apply cache_memory_cap cache_source cache_spill catalog_stats chain_source compression concurrent_build coverage deletion vacuum vacuum_bitmap vacuum_extended vacuum_rebuild dropped empty explicit_index expression_index force_merge implicit index inheritance large_documents limits lock manyterms memory memtable_append memtable_page memtable_spill memtable_spill_dead memtable_reclaim merge mixed parallel_build parallel_bmw partitioned partitioned_many partial_index pgstats queries quoted_identifiers rescan schema scoring1 scoring2 scoring3 scoring4 scoring5 scoring6 security segment segment_integrity segment_reclaim strings temp_table text_array text_config unsupported updates vector vector_v1_rejected unlogged_index wand REGRESS_OPTS = --inputdir=test --outputdir=test # TODO use pg_config !ifndef PGROOT !error PGROOT is not set !endif BINDIR = $(PGROOT)\bin INCLUDEDIR = $(PGROOT)\include INCLUDEDIR_SERVER = $(PGROOT)\include\server LIBDIR = $(PGROOT)\lib PKGLIBDIR = $(PGROOT)\lib SHAREDIR = $(PGROOT)\share # Use $(PGROOT)\bin\pg_regress for Postgres < 17 PG_REGRESS = $(LIBDIR)\pgxs\src\test\regress\pg_regress CFLAGS = /nologo /I"$(INCLUDEDIR_SERVER)\port\win32_msvc" /I"$(INCLUDEDIR_SERVER)\port\win32" /I"$(INCLUDEDIR_SERVER)" /I"$(INCLUDEDIR)" CFLAGS = $(CFLAGS) $(PG_CFLAGS) SHLIB = $(EXTENSION).dll LIBS = "$(LIBDIR)\postgres.lib" # PGXS = "C:\Users\fu0547\Downloads\postgresql\pgsql\lib\pgxs\src\makefiles\pgxs.mk" # !include $(PGXS) all: $(SHLIB) $(DATA_built) .c.obj: $(CC) $(CFLAGS) /c $< /Fo$@ $(SHLIB): $(OBJS) $(CC) $(CFLAGS) $(OBJS) $(LIBS) /link /DLL /OUT:$(SHLIB) sql\$(EXTENSION)--$(EXTVERSION).sql: sql\$(EXTENSION).sql copy sql\$(EXTENSION).sql $@ install: all copy $(SHLIB) "$(PKGLIBDIR)" copy $(EXTENSION).control "$(SHAREDIR)\extension" copy sql\$(EXTENSION)--*.sql "$(SHAREDIR)\extension" if not exist "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" mkdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" for %f in ($(HEADERS)) do copy %f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" installcheck: "$(PG_REGRESS)" --bindir="$(BINDIR)" $(REGRESS_OPTS) $(REGRESS) uninstall: del /f "$(PKGLIBDIR)\$(SHLIB)" del /f "$(SHAREDIR)\extension\$(EXTENSION).control" del /f "$(SHAREDIR)\extension\$(EXTENSION)--*.sql" del /f "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)\*.h" rmdir "$(INCLUDEDIR_SERVER)\extension\$(EXTENSION)" clean: del /f $(SHLIB) $(EXTENSION).lib $(EXTENSION).exp del /f $(DATA_built) del /f $(OBJS) del /f /s /q results regression.diffs regression.out tmp_check tmp_check_iso log output_isotimescale-pg_textsearch-b63fe58/NOTICE000066400000000000000000000012061525762047700176640ustar00rootroot00000000000000pg_textsearch by Tiger Data (TM) Copyright (c) 2025-2026 Timescale, Inc. d/b/a Tiger Data. All Rights Reserved. Licensed under the PostgreSQL License (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at https://github.com/timescale/pg_textsearch/blob/main/LICENSE Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License. timescale-pg_textsearch-b63fe58/README.md000066400000000000000000000622231525762047700202450ustar00rootroot00000000000000![pg_textsearch](images/banner.png) [![CI](https://github.com/timescale/pg_textsearch/actions/workflows/ci.yml/badge.svg)](https://github.com/timescale/pg_textsearch/actions/workflows/ci.yml) [![Benchmarks](https://github.com/timescale/pg_textsearch/actions/workflows/benchmark.yml/badge.svg)](https://timescale.github.io/pg_textsearch/benchmarks/) [![Coverity Scan](https://scan.coverity.com/projects/32822/badge.svg)](https://scan.coverity.com/projects/pg_textsearch) Modern ranked text search for Postgres. - Simple syntax: `ORDER BY content <@> 'search terms'` - BM25 ranking with configurable `k1` and `b` - PostgreSQL text search configurations - Expression, partial, and partitioned indexes - Fast top-k queries with Block-Max WAND - Parallel index builds for large tables ## PostgreSQL Version Compatibility pg_textsearch supports PostgreSQL 17 and 18. PostgreSQL 19 (beta) is supported on a best-effort basis while it is in beta; its CI is allowed to fail and prebuilt binaries are not published for it yet. ## Installation pg_textsearch supports PostgreSQL 17 and 18. ### Pre-built Packages Download pre-built binaries from the [Releases page](https://github.com/timescale/pg_textsearch/releases). Available for Linux on amd64 and arm64. ### Build from Source #### Linux and Mac ```sh cd /tmp git clone https://github.com/timescale/pg_textsearch cd pg_textsearch make make install # may need sudo ``` #### Windows Ensure you have a working MSVC environment on your path (`cl.exe` and `nmake`). ```cmd set "PGROOT=C:\Program Files\PostgreSQL\18" cd %TEMP% git clone https://github.com/timescale/pg_textsearch cd pgvector nmake /F Makefile.win nmake /F Makefile.win install ``` ## Getting Started Add pg_textsearch to `shared_preload_libraries` in `postgresql.conf`, then restart the server: ``` shared_preload_libraries = 'pg_textsearch' # add to existing list if needed ``` Enable the extension in each database: ```sql CREATE EXTENSION pg_textsearch; ``` Create a table with text content ```sql CREATE TABLE documents ( id bigserial PRIMARY KEY, content text, category_id integer ); INSERT INTO documents (content, category_id) VALUES ('PostgreSQL is a powerful database system', 1), ('BM25 is an effective ranking function', 1), ('Full text search with custom scoring', 2); ``` Create a pg_textsearch index on the text column ```sql CREATE INDEX docs_idx ON documents USING bm25(content) WITH (text_config='english'); ``` ## Querying Get the most relevant documents using the `<@>` operator ```sql SELECT * FROM documents ORDER BY content <@> 'database system' LIMIT 5; ``` `<@>` returns negative BM25 scores for ascending index scans, so lower scores rank first. `<@>` can also score rows outside a BM25 index scan. This is standalone scoring; it still uses corpus statistics from the selected BM25 index. The index is detected from the column. Specify it explicitly when needed: ```sql SELECT * FROM documents ORDER BY content <@> to_bm25query('database system', 'docs_idx') LIMIT 5; ``` Supported operations: - `text <@> 'query'` - Score text against a query (index auto-detected) - `text <@> bm25query` - Score text with explicit index specification ### Boolean Filtering Use PostgreSQL's `@@` operator and `tsquery` syntax to filter through a BM25 index: ```sql SELECT * FROM documents WHERE content @@ to_tsquery('english', 'postgres & (search | database) & !mysql'); ``` Supported `tsquery` features include `&` (AND), `|` (OR), `!` (NOT), phrase operators such as `<->`, prefix matching with `:*`, and weight restrictions. Phrase and weight checks may be rechecked against the table row after the index finds candidates. The `default_text_search_config` used to parse the left-hand `text` value must match the index configuration: ```sql SET default_text_search_config = 'english'; ``` If that setting changes after a Boolean prepared statement has switched to a generic plan, `DEALLOCATE` and prepare the statement again. A newly planned query can choose the correct sequential fallback, while the cached plan is rejected to avoid incorrect index results. Combining Boolean filtering with BM25 ranking is supported, but is not yet optimized as a single index scan. PostgreSQL currently evaluates the filter, calculates standalone scores for the matching rows, and then sorts them. This is most effective when the Boolean filter matches relatively few rows. ### Verifying Index Usage ```sql EXPLAIN SELECT * FROM documents ORDER BY content <@> 'database system' LIMIT 5; ``` PostgreSQL may prefer standalone scoring with a sequential scan for small tables. To test the index plan: ```sql SET enable_seqscan = off; ``` ### Pre-filtering and Post-filtering PostgreSQL can use a separate index to pre-filter rows before standalone scoring: ```sql CREATE INDEX ON documents (category_id); SELECT * FROM documents WHERE category_id = 1 ORDER BY content <@> 'search terms' LIMIT 10; ``` When PostgreSQL chooses the ordered BM25 index scan, other conditions are post-filters applied after BM25 scoring: ```sql SELECT * FROM documents WHERE length(content) > 100 ORDER BY content <@> 'search terms' LIMIT 10; ``` Post-filtered scans automatically grow their internal scoring batch until the `LIMIT` is filled or matches are exhausted. ## Indexing Create a BM25 index on a text column: ```sql CREATE INDEX ON documents USING bm25(content) WITH (text_config='english'); ``` ### Index Options Option | Default | Description --- | --- | --- [`text_config`](https://www.postgresql.org/docs/current/textsearch-configuration.html) | required | PostgreSQL text search configuration `k1` | 1.2 | Term frequency saturation (0.1-10.0) `b` | 0.75 | Length normalization (0.0-1.0) `compaction` | inline | Spill-time compaction: `inline`, `background`, or `manual`; see [Background Compaction](#background-compaction) `compaction_schedule` | `pg_textsearch.background_compaction_schedule` | Optional cron schedule captured when the index enters background mode ```sql CREATE INDEX ON documents USING bm25(content) WITH (text_config='english', k1=1.5, b=0.8); ``` ### Expression Indexes Index expressions for JSONB fields, multiple columns, or text transformations: ```sql -- JSONB field extraction CREATE INDEX events_expr_idx ON events USING bm25 ((data->>'description')) WITH (text_config='english'); SELECT * FROM events ORDER BY (data->>'description') <@> to_bm25query('network error', 'events_expr_idx') LIMIT 10; -- Text transformation CREATE INDEX ON documents USING bm25 ((lower(content))) WITH (text_config='simple'); -- Multi-column search CREATE INDEX ON articles USING bm25 ((coalesce(title, '') || ' ' || coalesce(body, ''))) WITH (text_config='english'); -- Array expression CREATE INDEX ON articles USING bm25 ((ARRAY[title, body])) WITH (text_config='english'); ``` The expression must evaluate to a text-like type or array (`text`, `varchar`, `char`, or arrays thereof) and use only IMMUTABLE functions. Queries must repeat the same expression in the `ORDER BY` clause. ### Partial Indexes Add a `WHERE` clause to index a subset of rows: ```sql CREATE INDEX documents_category_bm25_idx ON documents USING bm25 (content) WITH (text_config='english') WHERE category_id = 1; SELECT * FROM documents WHERE category_id = 1 ORDER BY content <@> to_bm25query('search terms', 'documents_category_bm25_idx') LIMIT 10; ``` Partial indexes require explicit index naming via `to_bm25query()` — the implicit `text <@> 'query'` syntax skips them. ### Multilingual Tables Create one partial index per language: ```sql ALTER TABLE documents ADD COLUMN lang CHAR(2) NOT NULL DEFAULT 'en'; CREATE INDEX docs_en_idx ON documents USING bm25 (content) WITH (text_config='english') WHERE lang = 'en'; CREATE INDEX docs_de_idx ON documents USING bm25 (content) WITH (text_config='german') WHERE lang = 'de'; CREATE INDEX docs_fr_idx ON documents USING bm25 (content) WITH (text_config='french') WHERE lang = 'fr'; ``` Query with the matching predicate and index name: ```sql SELECT * FROM documents WHERE lang = 'en' ORDER BY content <@> to_bm25query('databases', 'docs_en_idx') LIMIT 10; ``` #### Chinese Full-Text Search Use a Chinese-aware PostgreSQL text search configuration such as [zhparser](https://github.com/amutu/zhparser). The same `text_config` tokenizes documents and queries. ```sql CREATE EXTENSION zhparser; CREATE TEXT SEARCH CONFIGURATION public.chinese (PARSER = zhparser); ALTER TEXT SEARCH CONFIGURATION public.chinese ADD MAPPING FOR n, v, a, i, e, l WITH simple; CREATE TABLE chinese_documents (id bigserial PRIMARY KEY, content text); INSERT INTO chinese_documents (content) VALUES ('机器学习'); CREATE INDEX chinese_documents_bm25 ON chinese_documents USING bm25 (content) WITH (text_config='public.chinese'); SELECT id FROM chinese_documents ORDER BY content <@> to_bm25query('机器学习', 'chinese_documents_bm25') LIMIT 10; ``` ## Explicit Queries `bm25query` can carry an explicit index name: ```sql SELECT to_bm25query('search query text', 'docs_idx'); SELECT 'docs_idx:search query text'::bm25query; ``` Explicit index names are required when the planner cannot infer an index, including partial indexes, PL/pgSQL, prepared or dynamic query text, and ambiguous expressions. Standalone scoring requires `SELECT` on the indexed table or columns. ### Functions Function | Description --- | --- to_bm25query(text) → bm25query | Create bm25query without explicit index context to_bm25query(text, text) → bm25query | Create bm25query with query text and index name text <@> bm25query → double precision | BM25 scoring operator (returns negative scores) bm25query = bm25query → boolean | Equality comparison ## Performance For initial loads, create the index after loading data. ### Parallel Index Builds PostgreSQL uses parallel workers automatically for sufficiently large tables. ```sql SET max_parallel_maintenance_workers = 4; SET maintenance_work_mem = '256MB'; ``` `maintenance_work_mem` sets the total batch budget shared by parallel workers. Lower it to spill batches to disk earlier. Leave memory headroom beyond this budget and `shared_buffers` for other server activity; `pg_textsearch.memory_limit` controls the memtable cache, not index builds. Partitioned tables build each partition separately. See [build-memory benchmarking](benchmarks/README.md#build-memory) for details. ### Query Performance When PostgreSQL chooses a BM25 index scan for `ORDER BY`, scoring uses [Block-Max WAND](https://research.engineering.nyu.edu/~suel/papers/bmw.pdf). `LIMIT n` requests the top-k result count. For filtered queries, selectivity seeding may start with a deeper internal scoring batch. Without a pushed-down SQL `LIMIT`, `pg_textsearch.default_limit` sets the initial scoring batch, which can grow as more rows are requested. ```sql SELECT * FROM documents ORDER BY content <@> 'search terms' LIMIT 10; ``` Segment compression is enabled by default. Disable it only when decompression is a measured bottleneck: ```sql SET pg_textsearch.compress_segments = off; ``` Update-heavy workloads can fragment index pages. Use `REINDEX` during a low-traffic window if cold-cache latency degrades: ```sql REINDEX INDEX docs_idx; ``` ### Compaction Compaction consolidates comparable-sized segments across levels and rewrites segments with at least 50% dead documents, even below the segment-count threshold. With the default `inline` policy, spills and serial VACUUM provide an opportunity for one synchronous pass. Readers and other memtable writers can continue while merged output is built, because the long build holds no per-index LWLock. This is reader non-blocking, not foreground-writer non-blocking: the invoking writer still spends the time required to build and publish the merge. Compaction is skipped, and left to the next spill, when another session is reindexing, vacuuming, or compacting the index. These functions provide manual and scheduled control. They wait when another session holds index maintenance: ```sql SELECT bm25_force_merge('docs_idx'); SELECT bm25_compact('docs_idx'::regclass); SELECT bm25_compact_step('docs_idx'::regclass); ``` These report compaction state without waiting for maintenance: ```sql SELECT bm25_needs_compaction('docs_idx'::regclass); SELECT bm25_level_counts('docs_idx'::regclass); ``` `bm25_force_merge()` runs one bounded best-effort pass over eligible adjacent segments; oversized or otherwise uncombinable segments may remain. `bm25_compact()` processes all eligible levels, while `bm25_compact_step()` processes at most one pass. - Long merge work checks for cancellation, but published replacements remain physical and are not undone by `ROLLBACK`. - `bm25_needs_compaction()` reports whether `bm25_compact_step()` would run a pass, so either can drive a maintenance loop. - Mutating functions require index ownership and do not operate on partitioned parent indexes or during recovery. See [ARCHITECTURE.md](ARCHITECTURE.md#spill-and-compaction) for sizing, publication, locking, and page-reclaim details. Hot standbys serving queries must set `hot_standby_feedback = on` so active snapshots delay physical page reuse on the primary. If a standby disconnects while an old snapshot remains active, stock PostgreSQL recovery-conflict WAL cancels that snapshot before reclaimed segment pages can be reused on replay. ### Settings Setting | Default | Description --- | --- | --- `pg_textsearch.default_limit` | 1000 | Initial scoring batch when no SQL LIMIT is available `pg_textsearch.compress_segments` | on | Compress posting blocks in new segments `pg_textsearch.segments_per_level` | 8 | Count-triggered compaction threshold and maximum inputs per opportunistic merge (2-64) `pg_textsearch.max_segment_size` | 4095MB | Conservative size budget for newly merged multi-source segments (1-4095MB) `pg_textsearch.background_compaction_schedule` | `*/5 * * * *` | Default cron schedule captured by indexes entering managed background mode `pg_textsearch.bulk_load_threshold` | 100000 | Terms per transaction before auto-spill (0 = disable) `pg_textsearch.memtable_pages_threshold` | 64 | Chain pages before auto-spill (0 = disable) `pg_textsearch.allow_rls` | on | Allow BM25 indexes on RLS-protected tables; superuser-only `pg_textsearch.memtable_cache_enabled` | on | Cache memtable data in shared memory for faster queries `pg_textsearch.memory_limit` | 2GB | Approximate shared-memory budget for the memtable cache across all indexes; changes take effect after a configuration reload without a restart (0 = no limit) ### Memtable Architecture The L0 memtable is stored in the index as a WAL-logged chain of pages. It is the durable source of truth and can be restored by PostgreSQL without loading `pg_textsearch.so`. See [ARCHITECTURE.md](ARCHITECTURE.md#storage-and-wal). Queries use a shared-memory cache when enabled. An index falls back to the chain when the next record's estimated growth would cross `memory_limit / 8`; global pressure triggers best-effort eviction at `memory_limit / 2`. The global `memory_limit` is an approximate admission threshold: incremental catch-up and cold builds fall back when the entry-time estimate is already at the limit, but admitted or concurrent work may take estimated usage above it. The cache is rebuilt from the chain when missing or stale, while standbys always read the chain directly. Memtables spill automatically based on `memtable_pages_threshold` and `bulk_load_threshold`, and during VACUUM. ```sql -- Manual spill SELECT bm25_spill_index('docs_idx'); ``` ## Monitoring ```sql -- Check index usage SELECT s.schemaname, s.relname AS tablename, s.indexrelname AS indexname, s.idx_scan, s.idx_tup_read, s.idx_tup_fetch FROM pg_stat_user_indexes AS s JOIN pg_class AS i ON i.oid = s.indexrelid JOIN pg_am AS am ON am.oid = i.relam WHERE am.amname = 'bm25'; ``` ## Limitations ### Row-Level Security BM25 corpus statistics include all indexed rows, including rows hidden by RLS. A user who already knows a term can infer frequency information affected by inaccessible rows, though the index does not reveal unknown terms. This is analogous to [Elastic's security limitation](https://www.elastic.co/docs/deploy-manage/security/limitations). `pg_textsearch.allow_rls` defaults to `on`. Set it to `off` as a superuser to reject creating or rebuilding BM25 indexes on RLS-protected tables and enabling RLS where BM25 indexes already exist. This does not disable combinations that already exist when the setting is changed. ### Background Compaction The default `inline` policy compacts during memtable spills, skipping the pass when another session is using or maintaining the index. Managed `background` mode uses [pg_durable](https://github.com/microsoft/pg_durable) 0.2.8 or newer rather than a built-in worker. pg_durable must be preloaded, initialized in the current database, and granted to the index owner. The owner must have `LOGIN`; a superuser owner also requires `pg_durable.enable_superuser_instances = on`. Each physical index has one managed workflow scoped to its captured owner. The index owner, or a role PostgreSQL permits to act as that owner, may enable background mode. A separate insert-only writer may later trigger a spill, but pg_textsearch submits the workflow and calls `df.signal` under the index owner's identity. The compaction SQL nodes reached through either a spill signal or the cron backstop execute in pg_durable connections authenticated as the index owner, not as the DML writer; pg_durable's worker role provides only the orchestration infrastructure. pg_durable must be installed in the same database as the BM25 index. Use `manual` compaction for indexes outside `pg_durable.database`. This restriction is expected to be lifted in a future version of pg_durable. ```sql CREATE INDEX documents_bm25 ON documents USING bm25(content) WITH ( text_config = 'english', compaction = 'background' ); ``` Change modes with `ALTER INDEX`. Resetting `compaction_schedule` uses the current `pg_textsearch.background_compaction_schedule` default. Set the per-index option only when the default schedule is unsuitable. ```sql ALTER INDEX documents_bm25 SET (compaction = 'background'); ALTER INDEX documents_bm25 RESET (compaction_schedule); ALTER INDEX documents_bm25 SET (compaction = 'manual'); ``` Use `manual` with an external scheduler when pg_durable is unavailable or not desired and foreground compaction causes unacceptable write transaction stalls. The legacy `off` value remains accepted as an alias for `manual`. Temporary indexes do not support background mode. See [ARCHITECTURE.md](ARCHITECTURE.md#managed-background-compaction) for workflow lifecycle and safety details. ### Partitioned Tables BM25 statistics are local to each partition. Scores are comparable within a partition but may use different IDF scales across partitions. Query individual partitions when cross-row score comparability matters. ### Token and Document Limits PostgreSQL ignores lexemes longer than 2047 bytes. This mainly affects base64 data, long URLs, and concatenated identifiers. pg_textsearch splits raw inputs larger than 256KB at whitespace or multibyte-character boundaries before tokenization, then merges the term frequencies. Text arrays are flattened with spaces before this processing and do not preserve element boundaries. ### PL/pgSQL and Stored Procedures Planner hooks do not resolve the implicit query syntax inside PL/pgSQL. Use an explicit index name: ```sql SELECT * FROM documents ORDER BY content <@> to_bm25query('search terms', 'docs_idx') LIMIT 10; ``` ### Two-Phase Commit `PREPARE TRANSACTION` is not supported after creating a BM25 index or while BM25 index-drop cleanup is pending. Commit or roll back that DDL first. ## Troubleshooting List installed text search configurations: ```sql SELECT cfgname FROM pg_ts_config; ``` List BM25 indexes: ```sql SELECT indexname FROM pg_indexes WHERE indexdef LIKE '%USING bm25%'; ``` For multiple PostgreSQL installations, set `PG_CONFIG` before building: ```sh export PG_CONFIG=/Library/PostgreSQL/18/bin/pg_config # or 17 make clean && make && make install ``` Compilation requires PostgreSQL development files: ```sh sudo apt install postgresql-server-dev-18 # use 17 for PostgreSQL 17 ``` ## Reference ### Compaction Functions These functions take an index `regclass`. Mutating functions require index ownership. See [Compaction](#compaction) before scripting them. Function | Description --- | --- bm25_level_counts(index) → int4[] | Segments held at each of the eight LSM levels bm25_needs_compaction(index) → bool | Whether a compaction pass is eligible (advisory) bm25_compact(index) → void | Run eligible compaction passes bm25_compact_step(index) → bool | Run at most one pass ### Development Functions These interfaces may change without notice. Functions marked with † require superuser privileges. Function | Description --- | --- bm25_force_merge(index_name) → void | Run one bounded best-effort merge pass bm25_spill_index(index_name) → int4 | Force memtable spill to disk segment bm25_pending_free_pages(index_name) † → int8 | Count pages awaiting standby-safe reclaim bm25_dump_index(index_name) † → text | Dump internal index structure (truncated) bm25_summarize_index(index_name) † → text | Show index statistics without content ## Extension Compatibility pg_textsearch uses LWLock tranche IDs 1001-1012. Another extension using the same IDs can cause incorrect wait-event names in `pg_stat_activity`. If you encounter a conflict, [open an issue](https://github.com/timescale/pg_textsearch/issues). ## Version History Version | Highlights --- | --- [`v1.5.1`](https://github.com/timescale/pg_textsearch/releases/tag/v1.5.1) | Fix concurrent spill/merge corruption, expression indexes, and maintenance without the extension [`v1.5.0`](https://github.com/timescale/pg_textsearch/releases/tag/v1.5.0) | Background compaction, Boolean and phrase queries, and faster builds and scans [`v1.4.0`](https://github.com/timescale/pg_textsearch/releases/tag/v1.4.0) | Faster filtered top-k queries, Chinese search, and large-corpus improvements [`v1.3.1`](https://github.com/timescale/pg_textsearch/releases/tag/v1.3.1) | Standby-safe page reclaim and concurrency, VACUUM, and parallel-build fixes [`v1.3.0`](https://github.com/timescale/pg_textsearch/releases/tag/v1.3.0) | On-disk memtable with a shared-memory read cache and stateless WAL replay [`v1.2.0`](https://github.com/timescale/pg_textsearch/releases/tag/v1.2.0) | Physical replication and update-heavy workload correctness [`v1.1.0`](https://github.com/timescale/pg_textsearch/releases/tag/v1.1.0) | Concurrent writes, alive-bitset VACUUM, memory controls, arrays, expression indexes, and partial indexes [`v1.0.0`](https://github.com/timescale/pg_textsearch/releases/tag/v1.0.0) | First generally available release [`v0.6.1`](https://github.com/timescale/pg_textsearch/releases/tag/v0.6.1) | Transaction-abort, VACUUM, large-posting, and BMW stability [`v0.6.0`](https://github.com/timescale/pg_textsearch/releases/tag/v0.6.0) | Large-scale performance, lower memory use, and required shared preloading [`v0.5.1`](https://github.com/timescale/pg_textsearch/releases/tag/v0.5.1) | Concurrent index builds, iterative scans, progress reporting, and scan statistics [`v0.5.0`](https://github.com/timescale/pg_textsearch/releases/tag/v0.5.0) | Parallel index builds and improved hypertable support [`v0.4.2`](https://github.com/timescale/pg_textsearch/releases/tag/v0.4.2) | Hypertable scoring fix [`v0.4.1`](https://github.com/timescale/pg_textsearch/releases/tag/v0.4.1) | Hypertable scan locking fix [`v0.4.0`](https://github.com/timescale/pg_textsearch/releases/tag/v0.4.0) | Compressed posting lists, ordered-scan planner support, and partitioned-table fixes [`v0.3.0`](https://github.com/timescale/pg_textsearch/releases/tag/v0.3.0) | Block-Max WAND, bounded memtable growth, and expanded performance testing [`v0.2.0`](https://github.com/timescale/pg_textsearch/releases/tag/v0.2.0) | Public benchmarks, block-based segment storage, and a dynamic index registry [`v0.1.0`](https://github.com/timescale/pg_textsearch/releases/tag/v0.1.0) | First open-source release, simpler query syntax, and partitioned tables [`v0.0.5`](https://github.com/timescale/pg_textsearch/releases/tag/v0.0.5) | Pre-release [`v0.0.4`](https://github.com/timescale/pg_textsearch/releases/tag/v0.0.4) | BM25 indexing, PostgreSQL 17/18 support, crash recovery, and memory budgets [`v0.0.3`](https://github.com/timescale/pg_textsearch/releases/tag/v0.0.3) | PostgreSQL 18 support [`v0.0.2`](https://github.com/timescale/pg_textsearch/releases/tag/v0.0.2) | Schema insertion fixes and an inheritance safety check [`v0.0.1`](https://github.com/timescale/pg_textsearch/releases/tag/v0.0.1) | Initial release See [GitHub Releases](https://github.com/timescale/pg_textsearch/releases) for complete release notes. ## Contributing See [CONTRIBUTING.md](CONTRIBUTING.md) for development setup and contribution guidelines. See [ARCHITECTURE.md](ARCHITECTURE.md) for implementation details and storage invariants. - **Bug Reports**: [Create an issue](https://github.com/timescale/pg_textsearch/issues/new?labels=bug&template=bug_report.md) - **Feature Requests**: [Request a feature](https://github.com/timescale/pg_textsearch/issues/new?labels=enhancement&template=feature_request.md) - **General Discussion**: [Start a discussion](https://github.com/timescale/pg_textsearch/discussions) timescale-pg_textsearch-b63fe58/RELEASING.md000066400000000000000000000276371525762047700206330ustar00rootroot00000000000000# Releasing pg_textsearch This document describes the release process for pg_textsearch. ## Version Scheme We use semantic versioning: `MAJOR.MINOR.PATCH`. - Development versions use `-dev` suffix (e.g., `1.2.0-dev`). - Release versions drop the suffix (e.g., `1.2.0`). ## Cutting a Release ### Patch releases Maintain bugfix releases on `release/MAJOR.MINOR`, starting from the latest stable tag in that series, not from `main`. Cherry-pick reviewed fixes with `git cherry-pick -x`, including their regression tests. Do not backport new features or the next development-version bump. Open the release PR against that maintenance branch. Require the same CI and upgrade checks as a release from `main`, including upgrades from the previous patch release. Keep `main` on its existing development version; do not merge the maintenance branch's version bump back into it. For a patch with no development cycle, use the bump script to stage `PREV` to `CURRENT-dev`, then finalize the generated version references and SQL filenames to `CURRENT`. Check the script's reported stragglers, including `Makefile.win` and test scripts. Tag the merged maintenance-branch release commit, not `main`. Before publishing, prepare the forward upgrade chain on `main`: `PREV -> CURRENT -> NEXT-dev`. Once the release assets are published, add `CURRENT` to `main`'s upgrade-test matrices. Do not add an unpublished version to a matrix that downloads its release tarball. ### 1. Audit the upgrade SQL script This is the only piece of the release that cannot be automated, and the piece most likely to be wrong. Skim it carefully. The upgrade script `sql/pg_textsearch--PREV--CURRENT-dev.sql` must recreate, for an installation still on `PREV`, every catalog object that exists in the current main SQL file but not in the previous release's main SQL file. To enumerate what the upgrade script must cover, diff the two main SQL files: ```sh git show vPREV:sql/pg_textsearch--PREV.sql > /tmp/prev.sql diff /tmp/prev.sql sql/pg_textsearch--CURRENT-dev.sql ``` For every statement that is new in the current main file, verify the upgrade script has a matching statement: | New in main file | Required in upgrade script | |-------------------------------|-----------------------------------| | `CREATE FUNCTION` | `CREATE FUNCTION` | | `CREATE OPERATOR` | `CREATE OPERATOR` | | `CREATE OPERATOR CLASS` | `CREATE OPERATOR CLASS` | | `CREATE TYPE` | `CREATE TYPE` | | `CREATE CAST` | `CREATE CAST` | | `ALTER OPERATOR FAMILY` | `ALTER OPERATOR FAMILY` | | Catalog `UPDATE pg_catalog.*` | Same `UPDATE` | Renamed, signature-changed, or removed objects need matching `DROP` / `ALTER` statements in the upgrade script. The `upgrade-tests` CI workflow exercises the upgrade against every version in its `old_version` matrix and is the primary safety net for gaps — but it can only catch what the regression suite touches. A new operator that has no test coverage will pass CI and ship broken. The audit catches what the workflow can't. ### 2. Run the bump script ```sh ./scripts/bump-version.sh CURRENT-dev CURRENT ``` This renames the SQL files and updates every version reference across the tree. The script reports what's left for the human to do. ### 3. Add the previous release to the upgrade-tests matrix In `.github/workflows/upgrade-tests.yml`, add `PREV` to the `old_version` matrix so future releases are tested for upgrade compatibility from this version. ### 4. Update the README version history Add `CURRENT` to the Version History table in `README.md`, linking to the GitHub release and summarizing its main user-facing changes. ### 5. Open the PR ```sh git checkout -b release-CURRENT git add -A git commit -m "Release vCURRENT" gh pr create --draft --base main --title "Release vCURRENT" ``` For a patch release, use `--base release/MAJOR.MINOR` instead. CI runs the full test suite, including upgrade-tests against every matrix version. ### 6. After PR merges Merging the PR does not create a tag or publish a release. After the PR merges, tag its merged commit and push the tag: ```sh git fetch origin git tag -a vCURRENT -m "Release vCURRENT" git push origin vCURRENT ``` The tag push triggers `release.yml` and `package-release.yml` to build release artifacts and upload them to a draft GitHub release. Review the workflow results and assets before publishing the draft. Pushing the `v*` release tag also triggers `upgrade-tests.yml`, including the `upgrade-matrix` data-integrity job (`test/scripts/upgrade_matrix.sh`), so every release is checked for silent upgrade data loss against the representative old-version matrix on PG 17/18. ### 7. Follow up on Debian package publication Once PGDG apt onboarding is complete, follow the [Debian package process](#debian-package-publication) below. Apt publication may follow the GitHub release; it does not block the release or the next development-version bump. ## Debian Package Publication PGDG apt publication is being established in [#304](https://github.com/timescale/pg_textsearch/issues/304). Our existing release ZIPs contain `.deb` files, but do not provide an apt repository. The pg_textsearch project should own upstream packaging updates, working with PGDG maintainers on acceptance and repository publication. ### Initial onboarding Start before the 1.5.0 release, without making acceptance a release gate: 1. Open a request with the [PGDG apt maintainers](https://github.com/pgdg-packaging/apt.postgresql.org-issues). Note that PGDG already ships pg_textsearch RPMs. Agree on the packaging maintainer, repository, package names, supported targets, and submission procedure; record those details here once agreed. 2. Prepare Debian source packaging using `debhelper` and `pg_buildext`, initially for PostgreSQL 17/18. Target Debian bookworm/trixie and amd64/arm64, subject to agreement with PGDG. Include licensing, upstream release tracking, runtime dependencies, and the required extension SQL and upgrade scripts. Coordinate any package-name transition from our existing `pg-textsearch-postgresql-N` packages. 3. Exercise installed packages in clean environments, including regression tests and supported upgrades. Configure temporary clusters to preload `pg_textsearch` before startup. Document preload configuration and the restart required after replacing the library; do not silently change or restart users' clusters during package installation. Packaging can be prepared against the release candidate, but publication must use the final tagged source. If onboarding finishes after 1.5.0, package that release then; packaging alone does not require a new upstream patch release. ### Each release after onboarding 1. Update the upstream version and Debian changelog in the agreed packaging repository. Adjust build rules, dependencies, supported PostgreSQL versions, and file lists when the release requires it. 2. Submit the packaging update through the agreed PGDG process, using the final release source rather than our binary ZIPs. 3. Check package build and installed-package test results, then confirm apt can install the new version on the agreed targets. Track publication separately from the GitHub release until it is available. CloudNativePG extension-container integration is a separate downstream follow-up, not a release prerequisite. Once apt packages are available, revisit the previous [container submission](https://github.com/cloudnative-pg/postgres-extensions-containers/pull/84) with its contributor, updating repository installation, version tracking, preload configuration, and container tests. ## Bumping to the Next Dev Version After a release is published, open a follow-up PR: ```sh git checkout main && git pull git checkout -b bump-to-NEXT-dev ./scripts/bump-version.sh CURRENT NEXT-dev git add -A git commit -m "chore: bump version to NEXT-dev" gh pr create --draft --title "chore: bump version to NEXT-dev" ``` The script creates the new main SQL file (renamed from the previous release's), creates a stub upgrade file `sql/pg_textsearch--CURRENT--NEXT-dev.sql` containing only the library-version check, and updates every other version reference. Contributors then append to the upgrade file as features land in the dev cycle. ## SQL Upgrade Path Requirements **Upgrade scripts must form a single linear chain.** Every version connects to exactly one next version — no shortcuts that skip intermediate steps. This keeps the number of upgrade scripts minimal and the path predictable. ``` ... → 0.5.0 → 0.5.1 → 0.6.0 → 0.6.1 → 1.0.0 → 1.1.0 ``` Every release must have an upgrade path from the previous stable release (e.g., `1.0.0--1.1.0.sql`). Dev versions are not supported for direct upgrades; users on dev versions should reinstall the extension. ## Upgrade Compatibility Not all version upgrades are compatible with `ALTER EXTENSION UPDATE`. Breaking changes that require index recreation or server restart: | Change Type | Impact | User Action Required | |-------------|--------|---------------------| | Metapage version bump | Existing indexes incompatible | `REINDEX` or recreate index | | Shared memory structure change | Server crash on mixed versions | Restart Postgres after upgrade | | Segment format change | Old segments unreadable | Recreate index | **Current compatibility matrix:** | From Version | To Version | Compatible? | Notes | |--------------|------------|-------------|-------| | 0.5.x | 0.6.0 | ⚠️ REINDEX | Segment format v3→v4 (uint32→uint64 offsets) | | 0.3.0 | 0.4.0 | ❌ No | Segment format v2→v3 for compression | | 0.2.0 | 0.4.0 | ❌ No | Segment format v2→v3 | | 0.2.0 | 0.3.0 | ✅ Yes | Direct upgrade works | | 0.1.0 | 0.3.0 | ❌ No | Metapage v4→v5, shmem size change | | < 0.1.0 | 0.3.0 | ❌ No | Multiple breaking changes | When releasing a version with breaking changes: 1. Update `.github/workflows/upgrade-tests.yml` to exclude incompatible versions. 2. Document in release notes that users must recreate indexes. 3. Consider providing a migration guide for major version bumps. ## On-disk Format Versions If any on-disk format changed during a release cycle, bump the corresponding version constant: | Constant | Header | Purpose | |----------|--------|---------| | `TP_METAPAGE_VERSION` | `src/constants.h` | Index metapage format | | `TP_DOCID_PAGE_VERSION` | `src/constants.h` | Document ID page format | | `TP_PAGE_INDEX_VERSION` | `src/constants.h` | Page index format | | `TP_SEGMENT_FORMAT_VERSION` | `src/segment/segment.h` | Segment block storage format | | `TPQUERY_VERSION` | `src/types/query.h` | bm25query binary format | Version bumps break upgrade compatibility — exclude incompatible old versions from the upgrade-tests matrix and document the breaking change in release notes. ## Automated Workflows | Workflow | Trigger | Purpose | |----------|---------|---------| | `ci.yml` | PR, push to main | Build and test on PG17/18 | | `upgrade-tests.yml` | PR (sql/control changes), weekly | Test extension upgrades | | `release.yml` | `v*` tag push, manual | Build artifacts and create a draft release | | `package-release.yml` | `v*` tag push, manual | Build and upload release packages | | `benchmark.yml` | Weekly, manual | Performance regression testing | ## Troubleshooting ### Old SQL files in the Postgres share directory If tests fail with old version messages, check for stale files: ```sh ls ~/pg18/share/postgresql/extension/pg_textsearch* ``` Remove old dev versions that shouldn't be installed: ```sh rm ~/pg18/share/postgresql/extension/pg_textsearch--X.Y.Z-dev.sql ``` ### Extension won't upgrade If `ALTER EXTENSION pg_textsearch UPDATE` fails, verify: 1. The upgrade SQL file exists in the share directory. 2. The control file has the correct `default_version`. 3. The upgrade path exists (check `pg_extension_update_paths('pg_textsearch')`). timescale-pg_textsearch-b63fe58/SECURITY.md000066400000000000000000000015731525762047700205600ustar00rootroot00000000000000# Security Policy ## Reporting a Vulnerability If you discover a security vulnerability in pg_textsearch, please report it responsibly by emailing **security@tigerdata.com**. Please include: - A description of the vulnerability - Steps to reproduce - Potential impact - Any suggested fixes (optional) We will acknowledge receipt within 48 hours and aim to provide a fix or mitigation within 30 days for confirmed vulnerabilities. Please do not disclose the vulnerability publicly until we have had a chance to address it. ## Security Considerations pg_textsearch is a Postgres extension that runs with the same privileges as the Postgres server. When using this extension: - Follow Postgres security best practices - Limit `CREATE EXTENSION` privileges to trusted users - Review index configurations before deployment - Keep Postgres and pg_textsearch updated to the latest versions timescale-pg_textsearch-b63fe58/benchmarks/000077500000000000000000000000001525762047700210765ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/.gitignore000066400000000000000000000002751525762047700230720ustar00rootroot00000000000000# Benchmark results (local only) results/ # Downloaded datasets datasets/*/data/ # Generated ground truth (CI creates from versioned files) datasets/*/ground_truth.tsv # macOS .DS_Store timescale-pg_textsearch-b63fe58/benchmarks/README.md000066400000000000000000000240651525762047700223640ustar00rootroot00000000000000# pg_textsearch Benchmarks Performance benchmarks for the pg_textsearch BM25 full-text search extension. ## Quick Start ```bash # Run Cranfield benchmark (quick validation, ~1400 docs) ./runner/run_benchmark.sh cranfield # Run MS MARCO benchmark (8.8M passages) - requires download ./runner/run_benchmark.sh msmarco --download --load --query # Run Wikipedia benchmark ./runner/run_benchmark.sh wikipedia --download --load --query # Run all benchmarks ./runner/run_benchmark.sh all # Measure the filtered-seed top-K optimization (synthetic, no download) ./run_filtered_seed.sh # Measure index-build memory (synthetic, Linux, no download) python3 run_build_memory.py ``` ## Datasets ### Cranfield Collection (Quick Validation) - **Size:** 1,400 aerodynamics abstracts, 225 queries - **Purpose:** Quick validation of BM25 correctness and basic performance - **Time:** ~1-2 minutes total ### MS MARCO Passage Ranking (Full Scale) - **Size:** 8.8 million passages, 6,980 dev queries with relevance judgments - **Purpose:** Large-scale performance benchmarking, search quality evaluation - **Source:** [Microsoft MS MARCO](https://microsoft.github.io/msmarco/) - **Download:** ~2GB compressed - **Time:** Index build may take 30+ minutes depending on hardware ### Wikipedia (Real-World Content) - **Size:** Configurable (10K, 100K, 1M, or full ~6M articles) - **Purpose:** Real-world document lengths and vocabulary - **Source:** [Wikimedia Dumps](https://dumps.wikimedia.org/) - **Time:** Varies significantly by size ### Filtered-Seed Top-K (Synthetic, Self-Contained) - **Size:** Configurable, default 200K synthetic documents - **Purpose:** Measure `pg_textsearch.filtered_seed` on filtered top-k queries (`WHERE ORDER BY LIMIT k`), where an unseeded scan pays executor backoff re-drives - **Script:** `./run_filtered_seed.sh [ndocs]` - **Time:** ~10 seconds at the default size; no download needed - **Method:** Toggles the GUC over identical data and queries. Reports median latency and, where `bm25_debug_scoring_passes` exists, scoring passes. Drops cells unless every arm uses the BM25 index; result parity is covered separately by the `filtered_seed` regression - **Note:** Compare only the *seed on* columns across versions. The *seed off* baseline is not stable across the #435 change, because the `LIMIT` binding became per-scan even with seeding disabled ## Benchmark Runner The main runner script is `runner/run_benchmark.sh`: ```bash ./runner/run_benchmark.sh [dataset] [options] # Datasets: # msmarco - MS MARCO Passage Ranking (8.8M passages) # wikipedia - Wikipedia articles # cranfield - Cranfield collection (1,400 docs) # all - Run all benchmarks # Options: # --download - Download dataset if not present # --load - Load data and create index (drops existing) # --query - Run query benchmarks only # --report - Generate markdown report # --port PORT - Postgres port (default: 5433 for release build) ``` ## Build Memory `run_build_memory.py` reproduces vocabulary-driven build-memory growth with 100,000 synthetic documents. It starts a disposable local cluster using `PG_CONFIG` (default: `pg_config`), with the **installed** extension. It never installs binaries or connects to an existing server. Requires Linux, Python 3, and PostgreSQL server tools; run as a non-root user. ```bash PG_CONFIG=/path/to/pg_config python3 run_build_memory.py # Run only the one-million-term case, or change the corpus/workers python3 run_build_memory.py --case unique python3 run_build_memory.py --case unique --rows 200000 --workers 4 ``` The default matrix compares 1,000 repeated terms, 500K/1M/2M unique terms, and the 1M case with a lower memory budget and with parallelism disabled. Every document also contains one shared term for a result-count check. Defaults are two workers and `maintenance_work_mem=64MB`; the low-budget and serial controls use 16MB. Tables explicitly request the chosen worker count; builds verify the actual count and query results rather than silently comparing different degrees of parallelism. Allow about a minute and 1.5 GiB of available memory for the unfixed implementation. Parallel cases require at least 100K rows. The serial-only case accepts smaller corpora. Results go into a new `results/build_memory_/` directory: `summary.json` records versions, settings, timings, index sizes, sampled memory peaks by phase, and failures; per-case CSV files contain leader RSS/private memory, worker private memory, aggregate build PSS, and temporary-file sizes. Logs are retained; the cluster data is removed. PSS avoids counting shared pages multiple times. Sampling can miss brief peaks; `--interval` controls the delay between samples (default 0.1s). For an **intentional OOM experiment**, use a private cgroup v2 scope: ```bash systemd-run --user --scope -p MemoryMax=384M -p MemorySwapMax=0 \ -p OOMPolicy=continue \ python3 run_build_memory.py --case unique --cgroup ``` This requires a systemd user manager with memory-controller delegation. Use 512M for a higher-limit comparison. `--cgroup` records the current scope's memory and OOM counters; these include the runner and client processes, not just PostgreSQL. An OOM/failure returns nonzero and records the error rather than publishing a successful result. Keep results/data on a **disk-backed filesystem**, not tmpfs: `--output /disk/new-directory` selects another location. Tmpfs data would itself consume the memory cap. Benchmark failures are expected on unfixed builds under tight limits. ## Running Benchmarks ### Prerequisites 1. PostgreSQL with pg_textsearch installed 2. For Wikipedia: `pip install 'wikiextractor==3.0.6'` 3. For best results, use a release build of Postgres (port 5433) ### Download Data ```bash # MS MARCO cd datasets/msmarco && ./download.sh # Wikipedia (full) cd datasets/wikipedia && ./download.sh full # Wikipedia (subset for testing) cd datasets/wikipedia && ./download.sh 100K ``` ### Load and Index ```bash # Using psql directly psql -p 5433 -v data_dir="'$PWD/datasets/msmarco/data'" \ -f datasets/msmarco/load.sql # Or use the runner ./runner/run_benchmark.sh msmarco --load --port 5433 ``` ### Run Query Benchmarks ```bash psql -p 5433 -f datasets/msmarco/queries.sql ``` ### Run Concurrent Queries and Updates The mixed update/query benchmark reproduces a workload with continuous top-10 disjunction queries and one client targeting 1,000 indexed updates per second. It runs a read-only phase followed by an identical query phase with updates, records reader and writer throughput, and samples PostgreSQL wait events once per second. The defaults target the MS MARCO v2 schema and run each measured phase for ten minutes with 32 query clients: ```bash PGPORT=5433 \ ./datasets/msmarco/mixed-update-query/run.sh ``` To run against the original 8.8 million-passage MS MARCO schema: ```bash PGPORT=5433 \ TABLE=msmarco_passages \ INDEX=msmarco_bm25_idx \ QUERY_TABLE=msmarco_queries \ ./datasets/msmarco/mixed-update-query/run.sh ``` Relation settings can be schema-qualified, for example `TABLE=benchmarks.msmarco_passages` and `INDEX=benchmarks.msmarco_bm25_idx`. Use shorter phases for a smoke run: ```bash PGPORT=5433 DURATION=30 WARMUP=10 READ_CLIENTS=4 \ UPDATE_TARGETS=1000 \ ./datasets/msmarco/mixed-update-query/run.sh ``` The writer toggles a trailing `pgtsupdate` token in the selected source rows, so run against a disposable database or restore the corpus after benchmarking. Results are written below `benchmarks/results/mixed-update-query//`. `metadata.env` records the resolved non-secret connection and workload settings. `summary.tsv` contains read-only and mixed query QPS, average and p99 latency, completed updates, update TPS, and update latency. `waits.tsv` contains sampled reader and writer wait events, and `wait_summary.tsv` aggregates them. `warmup.log`, `read_only.out`, `mixed_reader.out`, and `mixed_writer.out` preserve pgbench output. The `read_only_pgbench.*`, `mixed_reader_pgbench.*`, and `mixed_writer_pgbench.*` files are per-transaction pgbench logs. ## Metrics Collected ### Index Build - Total time to build BM25 index - Memory usage during index build ### Query Performance - Single query latency (p50, p95, p99) - Batch query throughput (QPS) - Query latency by query type: - Single-word queries - Multi-word queries - Question-style queries - Rare term queries ### Search Quality (MS MARCO) - MRR@10 (Mean Reciprocal Rank) - Comparison with known relevance judgments ## CI Integration Benchmarks run automatically: - **On PR:** Cranfield only (quick validation) - **Nightly:** MS MARCO subset - **Weekly:** Full benchmark suite See `.github/workflows/benchmark.yml` for configuration. ## Results Historical results are stored in `results/` (gitignored). Each run produces: - `benchmark_[dataset]_[timestamp].md` - Markdown report - `[dataset]_load_[timestamp].log` - Load phase logs - `[dataset]_queries_[timestamp].log` - Query benchmark logs - `metrics_[timestamp].env` - Machine-readable metrics ## Interpreting Results ### Index Build Time | Dataset | Expected Time (Release Build) | |---------|------------------------------| | Cranfield | < 5 seconds | | MS MARCO | 15-60 minutes | | Wikipedia (100K) | 2-10 minutes | | Wikipedia (full) | 1-4 hours | ### Query Latency For a well-tuned system with sufficient memory: - Top-10 queries: < 100ms - Batch throughput: 10-100 QPS (depending on query complexity) ### Memory Requirements - Cranfield: < 64MB - MS MARCO: 1-4GB (depending on index memory limit) - Wikipedia: 512MB-8GB (depending on size) ## Adding New Benchmarks To add a new dataset: 1. Create directory: `datasets/[name]/` 2. Add scripts: - `download.sh` - Download and prepare data - `load.sql` - Load data and create index - `queries.sql` - Query benchmarks 3. Update this README ## Comparison with Other Systems For competitive benchmarks, you can run the same queries against: - Native Postgres `ts_rank` (built-in full-text search) - Other PostgreSQL full-text search extensions - External systems (Elasticsearch, Meilisearch) See `datasets/[name]/queries_native.sql` for native Postgres equivalents. timescale-pg_textsearch-b63fe58/benchmarks/datasets/000077500000000000000000000000001525762047700227065ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/000077500000000000000000000000001525762047700246745ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/benchmark_queries.tsv000066400000000000000000000775661525762047700311460ustar00rootroot00000000000000196232 government does do 1 549219 what a pathologist does 1 549235 what a pronoun 1 558263 what are endotoxins? 1 559009 what are foragers 1 561448 what are mandrakes 1 563347 what are plastids 1 563359 what are platitudes 1 568709 what are the dent 1 639545 what does increment of 1 645252 what does pbr322 do 1 680250 what is a cwe 1 687375 what is a hydromyelia 1 687615 what is a interstim 1 689885 what is a malanga 1 693469 what is a pacemaker for 1 694678 what is a photostat 1 695993 what is a primatologist 1 699510 what is a serigraph 1 699872 what is a shotgun 1 699873 what is a shoul 1 701663 what is a straddle 1 702790 what is a tanf 1 702792 what is a tanga 1 703383 what is a thoracentesis 1 704398 what is a tsp? 1 708517 what is advertisement? 1 708781 what is agave? 1 710297 what is amex ? 1 710914 what is an acl 1 711811 what is an anoscopy for 1 714636 what is an fcr 1 714672 what is an fmo 1 715588 what is an interlude 1 718782 what is anxiety 1 722413 what is barron 1 724121 what is bioflavonoids 1 724571 what is blackleg 1 725047 what is bobtail? 1 726076 what is broth 1 726098 what is brownfields 1 727707 what is carbohydrates? 1 728060 what is carpuject 1 729697 what is chattahoochee 1 731759 what is cobol 1 732618 what is compassion? 1 735343 what is counter 1 736347 what is csfb 1 738484 what is dendrochronology 1 739743 what is dihydroquercetin 1 740416 what is djgpp 1 741977 what is dynamics 1 742022 what is dysgraphia? 1 743708 what is emulation 1 744891 what is espresso 1 745559 what is excavatum 1 747937 what is floof 1 749267 what is fructose? 1 751778 what is gomorrah 1 753168 what is gwa 1 755465 what is hominid 1 756949 what is iboss 1 757275 what is ied 1 758901 what is inductance 1 758909 what is inductive 1 759503 what is instamed 1 760512 what is ip? 1 762111 what is javascript? 1 762761 what is kalel 1 763878 what is kyc? 1 764691 what is lavf 1 766272 what is lispro 1 767745 what is mag 1 768133 what is malmsey 1 771694 what is mhs? 1 772928 what is moca 1 773998 what is mrsi 1 778139 what is nrn 1 782696 what is perched 1 784961 what is polarised 1 786021 what is ppp 1 788035 what is ptb 1 788702 what is qa? 1 788851 what is qualfon 1 790059 what is rectocele? 1 791140 what is respite 1 792900 what is rylon 1 792977 what is sacrococcygeal 1 795540 what is shaft 1 796383 what is simplicheck 1 798945 what is spotting 1 800987 what is sunchoke 1 801478 what is sweetfx 1 804687 what is the altitude of be 1 814282 what is the ddc 1 841020 what is the principle? 1 856171 what is tipm 1 857943 what is trump 1 859669 what is usaid 1 860573 what is venator 1 10312 accordian definition 2 19552 anxiety: definition 2 27932 at what age can a kitten 2 39577 average of rn 2 49435 beard define 2 49802 belizean cuisine 2 53814 bindlestiff definition 2 54307 blackmail definition 2 54843 blueberries pint 2 57270 burning libraries 2 58234 calan medication 2 58801 calgary population 2 61531 campagnolo bikes 2 90941 chateau meaning 2 93311 cliff definition 2 96602 concomitant, definition 2 97295 consolidate function 2 97652 contacts meaning 2 100013 cortana what is the apocalypse 2 107077 cost of slate 2 111995 cours definition 2 112035 courtesy of meaning 2 114638 cva causes 2 116431 deed restrictions 2 117683 define anglicised 2 118365 define biobanks 2 118702 define calving 2 119089 define cleft 2 119168 define coherent 2 119761 define crusades 2 122440 define humidors 2 123710 define marmite 2 123975 define mib 2 124128 define mni 2 124787 define oppress 2 125545 define populace 2 126491 define resit 2 126525 define retrogressed 2 126821 define scooped out 2 127315 define spalting 2 127876 define tapestry 2 129517 define: entity's 2 129565 define: homologate 2 130825 definition for daring 2 130932 definition for naive 2 131768 definition of a pest 2 131873 definition of a surge 2 131925 definition of aberration 2 132104 definition of airbrush 2 132263 definition of an empath 2 132359 definition of anthropology 2 132639 definition of attorney 2 135079 definition of hamburger 2 135386 definition of imagination 2 135464 definition of indeterminism 2 135516 definition of infraction 2 135633 definition of introspective. 2 136209 definition of menses 2 137440 definition of prostration 2 137508 definition of purse 2 138223 definition of slouch 2 139090 definition of tumor 2 139239 definition of velvet 2 139767 definition quint 2 139897 definition sharing 2 139929 definition socorro 2 140161 definition wedgie 2 140804 demasculated definition 2 141472 derriere definition 2 156889 do i have to sponsor my husband for 2 160339 do you have to collect copays 2 175251 dove meaning 2 176065 drops definition 2 180693 enlarge definition 2 181301 eradicated definition 2 182393 exacting definition 2 185397 fastest lamborghini 2 187818 flash definition 2 193866 general izzam 2 194724 gesundheit meaning 2 195199 glioma meaning 2 195693 golf cost 2 197024 greenhorns meaning 2 197964 haemoptysis causes 2 200600 heels or heal 2 201376 here there be dragons comic 2 224548 how do you wrap a sari 2 234165 how fast is a saluki 2 236580 how is drive booster 2 236708 how is gasoline made 2 263889 how long is if your on disability 2 309402 how much does a engineer 2 321363 how much is a switch 2 334558 how old is canada 2 334916 how old is eminem? 2 342285 how titanic facts 2 344955 how to become a clnc? 2 355484 how to do a basic squat 2 358240 how to find a slit 2 377805 how to rig for crappie 2 2 Androgen receptor define 3 1576 60x40 slab cost 3 4947 Ludacris Net Worth 3 6217 The hormone that does the opposite of calcitonin is 3 6791 What Does Noel Mean in the Bible 3 10157 academic cheating definition 3 10205 accelerated cml symptoms 3 10264 access parallels cost 3 17586 an eating disorder is characterized by _____. 3 17848 anatomy origin definition 3 18840 another name for reaper 3 19457 antithesis definition and examples 3 21861 are corn snakes poisonous 3 21948 are cycling shoes necessary 3 22479 are florida mangroves protected 3 24979 are snails good for plants 3 25344 are the chemicals in instant coffee? 3 26079 are vets considered doctors 3 26485 argue the meaning of a debate 3 32642 average cost of a baby 3 35996 average developer salary 3 36214 average egg prices 3 38098 average kwh for tv 3 46081 average time for an appraisal 3 47270 average weight for a pug 3 47864 awareness of others definition 3 50891 benefits of hydrogenated water 3 53109 biggest german shepherds 3 53813 binding spell definition 3 53897 biological culture definition 3 54040 bipolar mood symptoms 3 56033 brandywine md population 3 57882 cabin fever meaning 3 59084 calling powershell functions 3 59204 calorie of granola bar 3 59426 calories crab meat 3 59654 calories in a flour wrap 3 60357 calories in ham sandwich 3 61452 cambridge wi in what county 3 66154 can crystals cause dizziness 3 68095 can hives be a sign of pregnancy 3 70787 can lice be transmitted to pets 3 72435 can pine floors be whitewashed 3 72613 can prednisone cause headaches 3 73106 can router slow internet 3 73119 can running strengthen back 3 73853 can stdev be higher than the mean 3 74356 can tetracaine be used in the ear 3 75342 can vinegar reduce fat 3 75801 can you absorb metals from plants 3 76283 can you catch a sinus infection 3 77424 can you find emphysema on a xray 3 78730 can you move a hyperlink in excel 3 79763 can you spray roundup after it rains 3 81649 cancer of the stomach symptoms 3 81993 cannot delete partition 3 82161 cannot uninstall windirstat 3 82842 carnelian color meaning 3 83458 cast of movie cowboys 3 83621 cast of the seal team 3 84778 causes for bowel leakage 3 86094 causes of drugs and alcohol 3 87892 causes of petechial hemorrhage 3 89786 central city definition 3 91345 chemistry amu definition 3 92437 cincinnati baseball teams 3 92542 circulation money definition 3 93649 clutch baseball definition 3 94865 coloring pages numbers 3 94953 columbus iron works 3 95286 commercial roaming definition 3 96250 complex protein definition 3 96379 compressive atelectasis definition 3 96749 concurrent disorders definition 3 97895 contract number piid 3 97972 control panel define control panel 3 98847 convert oz to gallons 3 99183 convert wh to joules 3 100250 cosmology what is the dark ages 3 100661 cost estimation technique 3 102695 cost of bamboo hr 3 106125 cost of pet snakes 3 107812 cost of us passport 3 108037 cost of wooden shutters 3 108622 cost root canal 3 109276 cost to dig a pond 3 112318 crate engine how heavy 3 112477 creative arts definition 3 113664 curious definition synonyms 3 113826 current fad diet 3 114037 current price of soybeans 3 115930 dcu routing number' 3 116939 define simplify math 3 117036 define a multichannel radio 3 121017 define engineering controls 3 123859 define memorandum of understanding 3 124534 define nfc technology 3 125627 define precocious puberty 3 127098 define sim card 3 127984 define tension stress 3 2235 Bethel University was founded in what year 4 2962 Explain what a bone scan is and what it is used for. 4 9454 a simple definition for the word bias. 4 10276 access, how to go to most recent record 4 11006 actress who plays alice on the magicians 4 12903 age discrimination in the workplace cases 4 14151 age requirements for name change 4 16860 american translators association how to study 4 19940 aplastic anemia causes and symptoms 4 20356 approximately how many moons does uranus have 4 21741 are cold sores and fever blisters the same 4 21765 are comic books considered a genre 4 22670 are giraffes the tallest mammals on earth 4 24115 are parents allowed to spank children 4 25603 are there bad side effects of htp 4 27743 asus phones price list 4 28352 at what age does a colt become fertile 4 28442 at what age does the average person retire 4 29097 at what temperature should cooling fans start 4 30188 avatar the last airbender game 4 31595 average car bill in florida 4 34015 average cost of maid service 4 36703 average gas prices in ga 4 38608 average loan payment for car 4 44072 average snowfall in chatsworth ga 4 44686 average temp in hana hi 4 46095 average time for hair to grow 4 47588 average wpm typing speed 4 50833 benefits of green tea and cinnamon 4 54531 blood coagulation is a component of this process 4 54544 blood diseases that are sexually transmitted 4 55682 botulism symptoms diagnosis and treatment 4 57258 burning feeling in stomach symptoms 4 57411 business identification number bin 4 58130 caffeine takes how long effect 4 58409 calculate days past ovulation 4 59392 calories burned using elliptical 4 60339 calories in ground beef stroganoff 4 60677 calories in one slice ham 4 61623 can Climbing Roses grow as bush roses? 4 61882 can a backache cause stomach pain 4 62648 can a lp gas furnace be serviced 4 64528 can anyone take prenatal vitamins? 4 65000 can blind chameleon change colors? 4 65583 can chewing gum cause ulcers 4 65584 can chewing gum prevent heartburn 4 74759 can tmj cause your jaw lock 4 77323 can you faint with eyes fixed open 4 78076 can you grow red onions in pot 4 82293 capital expenditure definition example 4 85904 causes of cramping pelvic pain 4 85954 causes of death in the world's history 4 86203 causes of elderly leg to swell 4 86264 causes of elevated serum bilirubin 4 88160 causes of roof of mouth soreness 4 88284 causes of severe itching all over body 4 88375 causes of slow blood flow 4 88577 causes of swelling in the feet and ankles 4 88831 causes of trigger finger in children 4 91722 child support legal definition of 4 92509 circadian rhythm disorders symptoms 4 99556 copperhead snake bite effects 4 100616 cost estimate glass shower 4 102506 cost of appendectomy surgery in usa 4 103125 cost of cbs all access on roku 4 106508 cost of redwood privacy fence 4 109647 cost to install a front door 4 109819 cost to install french drain 4 112718 crew fort worth application 4 116820 deferred action literal definition 4 118448 define body muscular endurance 4 128113 define the indirect function in excel 4 128633 define unadjusted trial balance 4 133037 definition of capias issued on a background 4 137919 definition of routine foot care 4 142579 devine paint color stores 4 144285 did jonah die in the belly of the whale 4 146244 difference between a spruce and a fir tree 4 147073 difference between discrete and process manufacturing 4 149221 difference between yellow jackets and wasps 4 149767 different foods of oaxaca mexico 4 149801 different kind of kidney stones 4 149853 different kinds of spruce trees 4 150087 different types led lights 4 155086 do azaleas like acidic soil 4 156052 do dry beans have a shelf life 4 156723 do hulled strawberries need to be refrigerated? 4 160255 do you give a toast at rehearsal dinner 4 160735 do you need a license to get a notary 4 160808 do you need a transit visa for iran 4 161828 does LOS ANGELES have buses or metro 4 164946 does comcast charge extra for hd 4 166043 does estrogen make me start my period 4 166625 does gastritis cause leaky gut 4 166748 does global entry include tsa 4 166784 does glycerin tincture need to be refrigerated 4 167156 does highster mobile really work 4 168069 does kurt cobain buried or cremated 4 168238 does lemon make water alkaline 4 169305 does motion sensor device have camera 4 1215 3 levels of government in canada and their responsibilities 5 9926 about how many fans does mexico soccer team 5 13397 age not to take off shoes at airline security 5 15607 all of the following were created by the first congress in 1789 5 22231 are eggs or grapes better to fight colds 5 24441 are premature roth ira distributions taxable 5 30860 average amount of money stna make 5 30956 average annual income for rn in illinois 5 34039 average cost of medicare drug plan 5 37952 average interest rate for saving account 5 39360 average new rn salary in mass. 5 42361 average salary for high school counselor 5 42568 average salary for public relations manager 5 47716 average yield per strawberry plant 5 51276 benefits of the master cleanse lemonade diet 5 54235 black death effects on europe culture 5 58571 calculate total days between two dates 5 59030 call delta reservations phone number 5 62411 can a grandparent open a 529 plan for a grandchild 5 63548 can a yeast infection cause swollen bumps 5 64960 can birth control cause hormonal imbalance 5 65488 can celiac disease cause excessive sweating 5 66908 can eating cabbage help you lose weight 5 69789 can i take the mortgage interest deduction and the standard deduction? 5 70340 can inheritance monies be taken by the trustee in bankruptcy 5 70709 can lease renewal date be changed by landlord 5 73257 can seizure meds cause low sodium? 5 75266 can vcenter standard manage essentials plus 5 77391 can you file bankruptcy separately if you are getting divorced 5 77878 can you get refunded on an amazon prime subscription 5 80590 can you use denture tablets to clean a brace 5 81945 cannot connect to citrix xenapp server 5 82100 cannot print to adobe pdf printer 5 84473 cause of painful swollen lymph nodes 5 89777 centra credit union routing number 5 95409 common causes for elevated liver enzymes 5 97964 contributory aged parent visa cost 5 98682 convert lumens to foot-candles 5 102627 cost of attending kennesaw state university 5 108507 cost planned parenthood tubal ligation 5 114573 customer service number for atmos energy 5 115365 danny phantom the ultimate enemy game 5 115833 days sales in cash ratio definition 5 125996 define pull the wool over someone's eyes 5 142382 determining marginal cost from total cost equation 5 142411 detox bath ingredients epsom salt 5 144254 did john lennon say he was more famous than jesus 5 144857 did princess margaret ever marry peter 5 147337 difference between ford supercab and crew cab 5 148761 difference between soy and whey protein powder 5 151547 dimensions of lazy susan base cabinet 5 161224 do you set up txt record like a cname? 5 162351 does a refrigerator normally use a standard outlet 5 163602 does bacterial infection cause joint pain 5 165002 does contraction of the ciliary muscles shorten the lens 5 170770 does quasi experimental use pretest and posttest 5 171527 does solar radiation cause global warming 5 173001 does the temperature affects the rate of buffer solution 5 176677 during pregnancy, when does the baby get in the head down position 5 178468 effects hot stone has on the circulatory system 5 185009 famous people who was born on pi day 5 185276 fastest car in the world motor tr 5 186446 ferrous sulfate how many elemental iron 5 188714 foods and supplements to lower blood sugar 5 190307 foods to help lower cholesterol naturally 5 191632 freedom home mortgage phone number 5 192579 function to count a certain character in cell 5 192894 furthest distance man has traveled in space 5 193422 gas price in mount gilead oh 5 194531 geranium rose essential oil benefits 5 198581 harland clarke customer service number 5 203039 highest state income tax rate 5 206549 house is settling and outside wall is cracked 5 230082 how far do bears homes range in miles? 5 231298 how far is it from colorado springs to la junta 5 231482 how far is lake como to lugano switzerland 5 239189 how long after pap smear will i receive results 5 239511 how long ago can exposure to hiv show up? 5 240504 how long before female dog shows pregnancy 5 242713 how long can u keep fresh eggs for 5 243139 how long can you keep frozen meat in freezer 5 247717 how long do you cook pork tenderloin in oven 5 248086 how long do you have to go college to become nurse 5 248385 how long do you keep credit card statements 5 249321 how long does a duck egg take to hatch 5 249792 how long does a mild sprain take to heal 5 249866 how long does a pa marriage license last 5 250228 how long does a stress test usually take 5 250636 how long does an average runner run 10kn 5 255633 how long does it take to become a us citizen 5 256052 how long does it take to cook dry beans 5 258337 how long does it take to walk a quarter mile 5 258485 how long does it tske for duck eggs to hatch 5 259239 how long does pink eye last bacterial 5 267187 how long should one hold bank statements? 5 267566 how long should you rinse your eyes in a safety eye wash 5 273014 how long until you see gains from test e 5 273449 how long will cut melon stay good 5 276979 how many calories in a medium smoothie king smoothie 5 277977 how many calories mcdonalds chicken nuggets 5 7968 When did the earthquake hit San Francisco during the World Series 6 32176 average cost for assisted living in syracuse ny 6 36473 average fifty year old retirement savings 6 37685 average income for north salem new york 6 59190 calorie intake for normal guy lifting weights 6 66161 can cuisinart electric pressure cooker be used as a canner 6 75698 can windows media player play amr files 6 79891 can you substitute chocolate chips for semi-sweet 6 80712 can you use merge things together in one table 6 86701 causes of headaches retina eye stops working 6 93823 co-curricular vs extracurricular definition 6 128200 define the term globalization as it applies in agribusiness management 6 153794 distance in miles pittsburgh, pa to new york 6 159078 do sports medicine physicians have to go to medical school 6 160671 do you need a certification to become a mindfulness meditation practitioner 6 166111 does excessive spinach consumption cause kidney stones 6 176015 driving distance littleton co to ft. collins co 6 231109 how far is grand hyatt denver to the convention center? 6 253678 how long does it take for collections to come off credit report 6 256783 how long does it take to get a va refinance loan? 6 257335 how long does it take to get your lottery jackpot check? 6 259763 how long does tap water have to sit before i put my fish in it 6 261521 how long for weed to get out of your system urine test 6 273481 how long will food last in the refrigerator without power 6 277632 how many calories in one large baked potato 6 278658 how many carbs in one cup of cooked quinoa 6 279718 how many coats of watco butcher block finish on butcher block 6 281002 how many days do have in advance of a election to register to vote 6 286160 how many hours of light do indoor plants need 6 290779 how many oscars has clint eastwood won?pdrijgheposrgijapeoikgjpesoar 6 290830 how many oscars has peter jackson wolllnlll?l 6 292094 how many people die from snake bites a year? 6 298444 how many times has amnesty been given to illegal aliens? 6 298940 how many types of files are on a single video dvd 6 300312 how many words one page double spaced 6 303777 how much coffee is safe per day pregnancy 6 303790 how much college education is recommended for being a computer engineer 6 320320 how much how much per hour does a private housekeeper make 6 323798 how much jail time is a class d felony 6 328611 how much time it take to get transit visa 6 329515 how much to replace honda civic ignition switch 6 330560 how much wider is a california king vs regular king 6 335711 how old is lil niqo when he made television love 6 351820 how to convert voice recording to mp4 samsung galaxy 6 353623 how to create one pivot table from multiple tabs 6 359499 how to find threads per inch on a metric bolt 6 368728 how to make dd mango pineapple ice tea 6 399617 is a claim alleging tcpa violation a d&o claim? 6 400311 is a lower ph level better in drinking water 6 406923 is commemorative air force contribution tax deductible? 6 407131 is copper one of the metals needed by human body 6 408945 is dylan o'brien related to adam brody 6 414714 is it still dangerous to go to the place of where the chernobyl happened 6 435794 largest airport near lake clear ny 6 437752 lasik surgery how long to wear goggles at night 6 496244 should comp time for overtime for government employees be accrued 6 497132 should you use lead or lag time in project schedules 6 507434 symptoms of depersonalization/derealization disorder include all of the following except 6 520627 tila-respa integrated disclosure definition 6 537301 virginia corporate income tax due date 6 552868 what airport to use to get to tuscany region of italy 6 589586 what causes low pressure on a pump for well water 6 595236 what chemical process are used to make synthetic rubber 6 670476 what impact did world war ii have on the american economy? 6 760817 what is irs individual taxpayer identification number (itin) 6 805900 what is the average cost of a home in santa rosa california 6 806688 what is the average pay per line for medical transcription 6 841165 what is the process called for exchanges gases between body and air 6 859274 what is unreserved designated fund balance in government accounting 6 885986 what percent workers get martin luther king off 6 900924 what temp do you cook pork chops on in the oven? and for how long? 6 904389 what time does chick-fil-a breakfast close 6 907127 what to eat and drink to help lose belly fat 6 909547 what tv series theme song was woke up this morning? 6 928753 what year was why havent i heard from you reba mcentire released 6 932495 what's the movie with meryl streep that a dingo ate her baby 6 987671 who said never let a tragedy go to waste 6 988653 who is the president of the republic of texas right now mark smith 6 992605 which variety of rosemary grows well in north carolina? 6 992660 which type of tissue has abundant nonliving extracellular matrix 6 993041 which state is legal cultivation medical marijuana for business 6 994133 which kind of biologist would most likely study whales? 6 995380 which climate prevails in the northwest coast of the united states 6 997648 where is the bronx located within new york city 6 997713 where is the animal foundation located in las vegas, nv 6 1003277 when to plant bermuda grass in piedmont region of nc 6 1007473 _______ sediments originate from elements in seawater that precipitate out of solution. 6 1009109 which environment would be likely to produce a black shale? 6 1010524 what year did the song vehicle by the ides of march come out 6 1011721 which is more important for weight loss the distance or the speed of walking? 6 1015347 what type of government does a small group of people rule 6 1016790 which pair of chromosomes would result in the birth of a male 6 1021907 which type of governmental power does a unitary system hold? 6 1023025 what part of florida is patrick air force base 6 1024528 what muscles could be involved in a rotator cuff injury 6 1025801 what law schools did justice anthony kennedy graduate from 6 1033927 what is the purpose of a conceptual model for advanced practice nursing 6 1039195 who is the actress that played gretchen schwartz on breaking bad 6 1042364 what is the cost of hot air ballooning in temecula, ca 6 1043914 what is the best way to treat a cold sore that is scabbed over 6 20671 are a category of sugars that contain either one or two molecules 7 42555 average salary for primary care sports medicine doctors 7 62439 can a health care proxy give consent for hiv testing 7 153037 distance between cities amarillo tx and mora new mexico 7 157149 do i really need to separate baby clothes by color before washing? 7 159667 do us citizens need passports to enter and exit canada 7 172062 does tarrant county assessor figure your taxes on assessed or appraised 7 172608 does the independent or dependent variable go on the x-axis 7 244011 how long do I bake skinless seared chicken breast at 350 7 250367 how long does a us passport need to be valid to visit germany 7 257309 how long does it take to get your bsrn if you already have a bachelors degree 7 257885 how long does it take to process your request for ssa retirement benefits? 7 270140 how long to cook sauerkraut and fresh kielbasa in slow cooker 7 278429 how many carbs can a type 2 diabetic have per day 7 283141 how many episodes in Better call saul season 2 7 285537 how many horsepower needed to plane a flat bottom boat 7 288884 how many miles in altitude is the united states satellite constellation 7 301061 how many years is the term for mi supreme court judges 7 324159 how much milk is the average to bring home after every pump? 7 328629 how much time must you give your tenants for an notice of inspection 7 628056 what does a 2005 pro crappie 175 bass tracker weigh 7 662524 what formula in excel returns the number of the current business day 7 808528 what is the best size grout lines to use for tiling shower 7 881582 what network connection transmits data quickly via light? 7 914771 what type of substance minimizes the changes in h+ and oh- in a solution. 7 918324 what was first tv show to show married couple sleeping in same bed 7 926019 what year did california girls-beach boys come out 7 926064 what year did college basketball start a three point shot 7 988787 who is the judge michael day district court of south dakota 7 1000574 where did josiah henson create his afro-canadian community 7 1001810 where are located the central chemoreceptors and the peripheral chemoreceptors for the breathing control process? 7 1005163 which amendment prohibited states from violating citizens' civil rights? 7 1007628 when applying for a new passport do i get to keep my old one 7 1009959 which function automatically counts cells that meet multiple conditions 7 1014697 which of the following events must cause equilibrium price to rise? 7 1014911 which of the following is a genetic condition in which the body is unable to make melanin? 7 1018918 what testing phase ensures that the code meets customer requirements 7 1021554 which two inventors worked to improve the light bulb and electricity 7 1078198 wind power requires how many acres per megawatt 7 1081595 what federal statute gives the epa authority to regulate pesticides 7 1082263 what energy source is earth using primarily for its internal heat 7 1084906 what diseases are associated with low white blood cell count 7 1089964 texas roadhouse glen mills pa phone number 7 1092161 northampton county tax department pa phone number 7 1094271 is a complex molecule in the cell's chromosomes that carries genetic information. 7 1096476 how many years must a person be a u.s. citizen before they can be senator 7 1096509 how many years back to the clinton tax returns go 7 1096628 how many students at mount vernon middle school fortville 7 1096827 how many people died from the flu who didn't take the vaccine 7 1096870 how many ounces in dunkin donuts iced coffee medium 7 1097469 how many calories in a salad containing chicken apples and walnuts 7 1098222 how long does it take for the moon to do a full circle around earth 7 1099452 how do duty based ethical standards differ from outcome-based ethical standards? 7 8714 _____ is the name used to refer to the era of legalized segregation in the united states 8 8854 ________ disparity refers to the slightly different view of the world that each eye receives.cyclopeanbinocularmonoculartrichromatic 8 17635 an increase of adh causes the kidneys to ________ the reabsorption of water, causing them to produce _______ urine. 8 517386 the medical term meaning the separation of the retina from the choroid in the back of the eye 8 659230 what famous hollywood composer wrote the film music to king kong 8 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 8 665009 what happened in europe as a result of the cooling in climate that occurred in the early fourteenth century? 8 840532 what is the prevalence of overweight among children and adolescents in the u.s, and what are the contributing factors? 8 846806 what is the simplest whole-number ratio of atoms in a molecule or formula unit called? 8 914637 what type of solution is made when mixing table salt to water until it dissolves 8 925571 what would happen if you start car without maf sensor plug connected 8 992132 who created an engraving of the boston massacre to increase support for the patriot cause? 8 993255 which rotator cuff muscle originates on the subscapular fossa of the scapula and inserts on the lesser tubercle of the humerus? 8 993834 which of the following structures form cytoplasmic channels that connect adjacent plant cells through the cell walls? 8 994085 which law says elastic limits strain produced is proportional to the stress called 8 994479 which health care system provides all citizens or residents with equal access to health care services 8 1007691 when allocating service department costs, the method which ignores serviced provided to other service departments is called 8 1009994 _______ ratios measure an organization's ability to meet its short term obligations. 8 1010277 which greek deity was honored with a towering gold-and-ivory statue inside the parthenon? 8 1015766 which of the following steps occurs last in the initiation phase of translation?hints 8 1016406 which one of the following organelles is involved in the production of proteins that are exported from the cell? 8 1016703 which organisms utilize a one-way air flow through their lungs to maximize air flow efficiency and oxygen consumption? 8 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 8 1022198 what president of the united states was born in lamar missouri and farmed for several years 8 1030722 what is the underlying reason a governmental unit uses separate funds to account for its transactions? 8 1042752 what is the colorless, gaseous element that makes up four-fifths of the air 8 1062457 why did franklin d roosevelt launch a new programs during his first hundred days in office 8 1068924 why hemorrhagic disease tests due to vitamin k deficiency in infancy 8 1080253 what fraction of the light falling on a piece of photographic film is typically wasted 8 1085760 what contains inner membranes arranged in stacks of membranous sacs called grana 8 1085764 what constitutional clause prohibits a state from drawing unreasonable distinctions between its own residents and those of persons living in other states 8 1086430 what book was on new york times list for patti callahan henry 8 1086701 what are the two species that are considered mountain yellow-legged frogs? 8 1093322 longest-tenured cast member of saturday night live 8 1095469 how much square foot per child in a mn child care center 8 1095699 how much money did the nra give to ted cruz of texas senate 8 1096454 how much apple cider vinegar is needed to help clear sinus 8 1097066 how many homes in heritage oak park port charlotte fl 8 timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/download.sh000077500000000000000000000064601525762047700270500ustar00rootroot00000000000000#!/bin/bash # Download MS MARCO v2 Passage Ranking dataset # Full dataset: ~138M passages, ~3,903 dev queries # Source: https://microsoft.github.io/msmarco/TREC-Deep-Learning # # Usage: ./download.sh # # This downloads the full 138M passage collection (no subsetting). # The tar file contains 70 gzipped JSONL files which are streamed # to a single TSV, deleting each .gz after processing to save disk. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATA_DIR="$SCRIPT_DIR/data" mkdir -p "$DATA_DIR" cd "$DATA_DIR" echo "=== MS MARCO v2 Passage Ranking Dataset Download ===" echo "Full dataset: ~138M passages" echo "" # Collection (passages) - 20.3GB tar of 70 gzipped JSONL files if [ ! -f "collection.tsv" ]; then if [ ! -d "msmarco_v2_passage" ]; then if [ ! -f "msmarco_v2_passage.tar" ]; then echo "Downloading passage collection (~20GB)..." wget -q --show-progress \ https://msmarco.z22.web.core.windows.net/msmarcoranking/msmarco_v2_passage.tar fi echo "Extracting tar archive..." tar xf msmarco_v2_passage.tar rm msmarco_v2_passage.tar echo "Tar extracted, converting JSONL files to TSV..." fi # Convert 70 gzipped JSONL files to single TSV # Each JSONL line: {"pid": "msmarco_passage_XX_NNNNN", "passage": "..."} # Delete each .gz after processing to save disk space echo "Converting JSONL files to collection.tsv..." > collection.tsv file_count=0 total_files=$(ls -1 msmarco_v2_passage/*.gz 2>/dev/null | wc -l) for f in msmarco_v2_passage/*.gz; do file_count=$((file_count + 1)) echo " Processing $f ($file_count/$total_files)..." zcat "$f" | python3 -c " import sys, json for line in sys.stdin: obj = json.loads(line) pid = obj['pid'] text = obj['passage'].replace('\t', ' ').replace('\n', ' ') print(f'{pid}\t{text}') " >> collection.tsv rm "$f" done # Clean up empty directory rmdir msmarco_v2_passage 2>/dev/null || true echo "Collection converted: $(wc -l < collection.tsv) passages" else echo "Collection already exists: $(wc -l < collection.tsv) passages" fi # Dev1 queries (3,903 queries) if [ ! -f "passv2_dev_queries.tsv" ]; then echo "Downloading dev queries..." wget -q --show-progress \ https://msmarco.z22.web.core.windows.net/msmarcoranking/passv2_dev_queries.tsv echo "Dev queries downloaded: $(wc -l < passv2_dev_queries.tsv) queries" else echo "Dev queries already exist: $(wc -l < passv2_dev_queries.tsv) queries" fi # Dev1 relevance judgments (qrels) if [ ! -f "passv2_dev_qrels.tsv" ]; then echo "Downloading relevance judgments..." wget -q --show-progress \ https://msmarco.z22.web.core.windows.net/msmarcoranking/passv2_dev_qrels.tsv echo "Relevance judgments downloaded: $(wc -l < passv2_dev_qrels.tsv) qrels" else echo "Relevance judgments already exist: $(wc -l < passv2_dev_qrels.tsv) qrels" fi echo "" echo "=== Download Complete ===" echo "Files in $DATA_DIR:" ls -lh "$DATA_DIR"/*.tsv 2>/dev/null | head -10 echo "" echo "Dataset statistics:" echo " Passages: $(wc -l < collection.tsv)" echo " Dev queries: $(wc -l < passv2_dev_queries.tsv 2>/dev/null || echo 'N/A')" echo " Relevance judgments: $(wc -l < passv2_dev_qrels.tsv 2>/dev/null || echo 'N/A')" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/generate_benchmark_queries.sh000077500000000000000000000052101525762047700325720ustar00rootroot00000000000000#!/bin/bash # Generate benchmark query samples bucketed by token count # # This script creates benchmark_queries.tsv with deterministically-sampled # queries from MS-MARCO v2 Dev1, bucketed by the number of search tokens # after stop word removal and stemming (via to_tsvector). # # Output format: query_idquery_texttoken_count # # Usage: ./generate_benchmark_queries.sh # # Requires: PostgreSQL running locally set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATA_DIR="$SCRIPT_DIR/data" OUTPUT_FILE="$SCRIPT_DIR/benchmark_queries.tsv" INPUT_FILE="$DATA_DIR/passv2_dev_queries.tsv" TEMP_FILE=$(mktemp) trap "rm -f $TEMP_FILE" EXIT # Check if queries file exists if [ ! -f "$INPUT_FILE" ]; then echo "Error: $INPUT_FILE not found" echo "Run ./download.sh first to get the MS-MARCO v2 dataset" exit 1 fi echo "Generating benchmark queries with 100 queries per token bucket..." echo "Loading and tokenizing $(wc -l < "$INPUT_FILE") queries..." # Preprocess: convert to CSV format to handle special characters echo "Preprocessing queries..." awk -F'\t' 'BEGIN {OFS=","} {gsub(/"/, "\"\"", $2); print $1, "\"" $2 "\""}' "$INPUT_FILE" > "$TEMP_FILE" # Run query in psql - output to stdout, redirect to file # Use -q to suppress notices, and filter out COPY/CREATE lines echo "Running tokenization and bucketing..." psql -d postgres -p 5432 -t -A -F $' ' -q -v ON_ERROR_STOP=1 << EOSQL | grep -v "^CREATE\|^COPY" > "$OUTPUT_FILE" CREATE TEMP TABLE all_queries (query_id int, query_text text); \copy all_queries FROM '$TEMP_FILE' WITH (FORMAT csv) WITH tokenized AS ( SELECT query_id, query_text, array_length(string_to_array(to_tsvector('english', query_text)::text, ' '), 1) as token_count FROM all_queries ), bucketed AS ( SELECT query_id, query_text, CASE WHEN token_count >= 8 THEN 8 ELSE token_count END as token_bucket, row_number() OVER ( PARTITION BY CASE WHEN token_count >= 8 THEN 8 ELSE token_count END ORDER BY query_id ) as rn FROM tokenized WHERE token_count IS NOT NULL ) SELECT query_id, query_text, token_bucket FROM bucketed WHERE rn <= 100 ORDER BY token_bucket, query_id; EOSQL # Count results echo "" echo "Generated $OUTPUT_FILE" TOTAL=$(wc -l < "$OUTPUT_FILE") echo "Total queries: $TOTAL" echo "" echo "Distribution by token bucket:" cut -f3 "$OUTPUT_FILE" | sort | uniq -c | sort -k2 -n # Show sample queries from each bucket echo "" echo "Sample queries per bucket:" for bucket in 1 2 3 4 5 6 7 8; do echo " Bucket $bucket: $(awk -F'\t' -v b="$bucket" '$3==b {print $2; exit}' "$OUTPUT_FILE" | head -c 60)..." done timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/ground_truth_pg17.tsv000066400000000000000000012676741525762047700310430ustar00rootroot00000000000000query_id query_text rank doc_id score 118702 define calving 1 msmarco_passage_12_462325159 21.156060005285305 118702 define calving 2 msmarco_passage_46_391039997 20.087673257722116 118702 define calving 3 msmarco_passage_12_462327965 19.674024622745918 118702 define calving 4 msmarco_passage_57_594118640 19.120872939415502 118702 define calving 5 msmarco_passage_22_286324488 18.78238889978915 118702 define calving 6 msmarco_passage_35_302695585 18.433308059217726 118702 define calving 7 msmarco_passage_12_462327528 18.26410357133054 118702 define calving 8 msmarco_passage_11_631671316 18.25816068981713 118702 define calving 9 msmarco_passage_04_790681077 18.00489020248718 118702 define calving 10 msmarco_passage_14_671675529 18.00489020248718 812190 what is the constitution act 1867 1 msmarco_passage_10_39419827 31.768198601468875 812190 what is the constitution act 1867 2 msmarco_passage_62_774231050 31.251893512161956 812190 what is the constitution act 1867 3 msmarco_passage_10_39411476 31.251893512161956 812190 what is the constitution act 1867 4 msmarco_passage_18_217847424 30.984274880476693 812190 what is the constitution act 1867 5 msmarco_passage_26_457337817 29.988721812559625 812190 what is the constitution act 1867 6 msmarco_passage_00_478176063 29.81011350763636 812190 what is the constitution act 1867 7 msmarco_passage_64_130667460 29.81011350763636 812190 what is the constitution act 1867 8 msmarco_passage_18_449195213 29.645349997802818 812190 what is the constitution act 1867 9 msmarco_passage_18_273167843 29.645349997802818 812190 what is the constitution act 1867 10 msmarco_passage_53_792234001 29.615637775409382 1056437 what is does un stands for 1 msmarco_passage_43_474891851 20.077948774304588 1056437 what is does un stands for 2 msmarco_passage_63_261204313 18.471342002129 1056437 what is does un stands for 3 msmarco_passage_63_261235009 18.471342002129 1056437 what is does un stands for 4 msmarco_passage_10_155983102 18.471342002129 1056437 what is does un stands for 5 msmarco_passage_05_637865473 17.573666131132164 1056437 what is does un stands for 6 msmarco_passage_43_479617166 17.573666131132164 1056437 what is does un stands for 7 msmarco_passage_63_261212538 17.33391154593536 1056437 what is does un stands for 8 msmarco_passage_42_85996281 16.887530007714343 1056437 what is does un stands for 9 msmarco_passage_10_149198387 16.69564235466817 1056437 what is does un stands for 10 msmarco_passage_02_225420365 16.66143682184463 1090107 taxes what does donation go under 1 msmarco_passage_42_17240560 21.92669354995355 1090107 taxes what does donation go under 2 msmarco_passage_35_186542722 21.671500383359284 1090107 taxes what does donation go under 3 msmarco_passage_35_193350532 21.671500383359284 1090107 taxes what does donation go under 4 msmarco_passage_35_186541133 21.13029573862813 1090107 taxes what does donation go under 5 msmarco_passage_23_508231884 20.69448286108243 1090107 taxes what does donation go under 6 msmarco_passage_35_380787753 20.497465044013524 1090107 taxes what does donation go under 7 msmarco_passage_45_30542053 20.44865389583035 1090107 taxes what does donation go under 8 msmarco_passage_30_113714635 20.339084692745722 1090107 taxes what does donation go under 9 msmarco_passage_13_583358626 20.15571665330843 1090107 taxes what does donation go under 10 msmarco_passage_30_97156356 20.12072336141278 1079868 world's largest bass caught 1 msmarco_passage_23_195152757 30.270983663299415 1079868 world's largest bass caught 2 msmarco_passage_04_170131051 30.014095658098213 1079868 world's largest bass caught 3 msmarco_passage_04_170138716 28.673862178378958 1079868 world's largest bass caught 4 msmarco_passage_29_41124008 28.32253645741502 1079868 world's largest bass caught 5 msmarco_passage_04_839018138 28.093841977715968 1079868 world's largest bass caught 6 msmarco_passage_19_664827951 27.99825127113843 1079868 world's largest bass caught 7 msmarco_passage_33_844034898 26.78126060491236 1079868 world's largest bass caught 8 msmarco_passage_09_715579301 26.648514854370937 1079868 world's largest bass caught 9 msmarco_passage_33_92677522 26.648514854370937 1079868 world's largest bass caught 10 msmarco_passage_11_123187018 26.426796925624092 345453 how to become a teacher assistant 1 msmarco_passage_03_194964276 22.641878503361806 345453 how to become a teacher assistant 2 msmarco_passage_62_194513385 22.196458105825762 345453 how to become a teacher assistant 3 msmarco_passage_14_551945778 22.17403751286661 345453 how to become a teacher assistant 4 msmarco_passage_33_132480860 21.82445049621744 345453 how to become a teacher assistant 5 msmarco_passage_36_453230548 21.792857430383478 345453 how to become a teacher assistant 6 msmarco_passage_51_427164106 21.494453371242784 345453 how to become a teacher assistant 7 msmarco_passage_62_194519137 21.44602308463927 345453 how to become a teacher assistant 8 msmarco_passage_15_252718153 21.40803414368849 345453 how to become a teacher assistant 9 msmarco_passage_62_704219918 21.271127516288892 345453 how to become a teacher assistant 10 msmarco_passage_63_825666354 21.183805480432603 1090542 sorcerer definition 1 msmarco_passage_53_836831070 21.124917065575595 1090542 sorcerer definition 2 msmarco_passage_43_753592872 20.204112266375745 1090542 sorcerer definition 3 msmarco_passage_32_473619025 20.174609215759332 1090542 sorcerer definition 4 msmarco_passage_43_563090250 19.098155864812803 1090542 sorcerer definition 5 msmarco_passage_39_116220842 18.782218897779245 1090542 sorcerer definition 6 msmarco_passage_53_786733162 18.74948092450789 1090542 sorcerer definition 7 msmarco_passage_03_222205090 18.59379893293606 1090542 sorcerer definition 8 msmarco_passage_33_515160539 18.59379893293606 1090542 sorcerer definition 9 msmarco_passage_33_514975617 18.59379893293606 1090542 sorcerer definition 10 msmarco_passage_05_514693573 17.728279023917164 1034845 what is the opposite of profile 1 msmarco_passage_00_376934018 16.449366772526073 1034845 what is the opposite of profile 2 msmarco_passage_57_407302329 16.198885182560186 1034845 what is the opposite of profile 3 msmarco_passage_59_298700209 16.113660226723894 1034845 what is the opposite of profile 4 msmarco_passage_00_376934548 16.059193419945576 1034845 what is the opposite of profile 5 msmarco_passage_61_69873428 15.799480958690609 1034845 what is the opposite of profile 6 msmarco_passage_59_416588476 15.727962220461635 1034845 what is the opposite of profile 7 msmarco_passage_10_772307275 15.64785292233761 1034845 what is the opposite of profile 8 msmarco_passage_34_756630086 15.64785292233761 1034845 what is the opposite of profile 9 msmarco_passage_16_472347154 15.586526115520602 1034845 what is the opposite of profile 10 msmarco_passage_14_152897977 15.460923282544398 1009668 which folate is known as mthfr 1 msmarco_passage_48_751042493 33.3242477578635 1009668 which folate is known as mthfr 2 msmarco_passage_06_765247346 33.3242477578635 1009668 which folate is known as mthfr 3 msmarco_passage_04_370902875 31.535193595847506 1009668 which folate is known as mthfr 4 msmarco_passage_38_598898478 31.535193595847506 1009668 which folate is known as mthfr 5 msmarco_passage_28_205248638 31.309169866919934 1009668 which folate is known as mthfr 6 msmarco_passage_31_854703227 31.275369371597854 1009668 which folate is known as mthfr 7 msmarco_passage_29_458276822 30.77968585925054 1009668 which folate is known as mthfr 8 msmarco_passage_16_693796099 30.637027116492924 1009668 which folate is known as mthfr 9 msmarco_passage_00_830180123 30.60723576972524 1009668 which folate is known as mthfr 10 msmarco_passage_42_289429961 30.533329254509134 1048359 what is php in mental health care 1 msmarco_passage_38_761497514 30.397908954411534 1048359 what is php in mental health care 2 msmarco_passage_38_761452094 29.97105566851598 1048359 what is php in mental health care 3 msmarco_passage_38_275290623 27.927814894365383 1048359 what is php in mental health care 4 msmarco_passage_38_275288998 27.11166978881753 1048359 what is php in mental health care 5 msmarco_passage_38_761297904 27.091692187757104 1048359 what is php in mental health care 6 msmarco_passage_38_761349140 26.744737740040247 1048359 what is php in mental health care 7 msmarco_passage_39_814466354 26.509220925830974 1048359 what is php in mental health care 8 msmarco_passage_59_99180936 26.31612837002757 1048359 what is php in mental health care 9 msmarco_passage_53_7511749 25.855786370442154 1048359 what is php in mental health care 10 msmarco_passage_42_151440864 25.729488207574285 195199 glioma meaning 1 msmarco_passage_55_244786744 22.742684782689384 195199 glioma meaning 2 msmarco_passage_36_117403702 20.818574410328832 195199 glioma meaning 3 msmarco_passage_03_567576027 20.818574410328832 195199 glioma meaning 4 msmarco_passage_62_754663651 20.140407354851952 195199 glioma meaning 5 msmarco_passage_62_754584500 20.140407354851952 195199 glioma meaning 6 msmarco_passage_36_835459260 20.129536510364606 195199 glioma meaning 7 msmarco_passage_55_244517143 19.97056220785215 195199 glioma meaning 8 msmarco_passage_55_451186135 19.86479844067509 195199 glioma meaning 9 msmarco_passage_61_570963016 19.803802802695603 195199 glioma meaning 10 msmarco_passage_36_117486867 19.69512399181008 504335 stress effects on the body 1 msmarco_passage_31_251880975 20.370259266872374 504335 stress effects on the body 2 msmarco_passage_38_84217308 20.154901194552792 504335 stress effects on the body 3 msmarco_passage_38_84134735 20.154901194552792 504335 stress effects on the body 4 msmarco_passage_38_84175100 20.154901194552792 504335 stress effects on the body 5 msmarco_passage_12_302239183 20.154901194552792 504335 stress effects on the body 6 msmarco_passage_38_84123991 20.154901194552792 504335 stress effects on the body 7 msmarco_passage_38_84206564 20.154901194552792 504335 stress effects on the body 8 msmarco_passage_04_239936880 20.154901194552792 504335 stress effects on the body 9 msmarco_passage_38_88449999 20.154901194552792 504335 stress effects on the body 10 msmarco_passage_38_88470235 20.154901194552792 744092 what is enteritis symptoms 1 msmarco_passage_38_24153190 14.316793815436554 744092 what is enteritis symptoms 2 msmarco_passage_06_749594759 14.21850720252759 744092 what is enteritis symptoms 3 msmarco_passage_48_650863429 14.21850720252759 744092 what is enteritis symptoms 4 msmarco_passage_08_790944006 14.06000807712215 744092 what is enteritis symptoms 5 msmarco_passage_41_149236487 13.794352659519596 744092 what is enteritis symptoms 6 msmarco_passage_29_540869271 13.784293076051693 744092 what is enteritis symptoms 7 msmarco_passage_53_492845979 13.464044376711948 744092 what is enteritis symptoms 8 msmarco_passage_24_115300926 13.367582963129362 744092 what is enteritis symptoms 9 msmarco_passage_24_115559832 13.367582963129362 744092 what is enteritis symptoms 10 msmarco_passage_24_167045672 13.367582963129362 109276 cost to dig a pond 1 msmarco_passage_46_413129339 30.20481017614391 109276 cost to dig a pond 2 msmarco_passage_02_602403940 29.467644151668306 109276 cost to dig a pond 3 msmarco_passage_49_606039662 27.687752108555614 109276 cost to dig a pond 4 msmarco_passage_07_18949637 26.428542881605168 109276 cost to dig a pond 5 msmarco_passage_49_298337294 26.428542881605168 109276 cost to dig a pond 6 msmarco_passage_06_81542713 25.856340011880135 109276 cost to dig a pond 7 msmarco_passage_09_4767290 25.62777749048214 109276 cost to dig a pond 8 msmarco_passage_22_518073078 25.42353149494891 109276 cost to dig a pond 9 msmarco_passage_30_195021926 25.324551339352865 109276 cost to dig a pond 10 msmarco_passage_28_121949606 25.016213370118173 985173 definition of product strategy in marketing 1 msmarco_passage_57_558402064 27.64225988211777 985173 definition of product strategy in marketing 2 msmarco_passage_32_337698358 27.571468967275237 985173 definition of product strategy in marketing 3 msmarco_passage_09_235072067 25.99127520472439 985173 definition of product strategy in marketing 4 msmarco_passage_03_163385443 25.67374118740109 985173 definition of product strategy in marketing 5 msmarco_passage_14_344491856 25.54123731278591 985173 definition of product strategy in marketing 6 msmarco_passage_56_149965637 25.32527513614238 985173 definition of product strategy in marketing 7 msmarco_passage_03_40449793 25.202012080248515 985173 definition of product strategy in marketing 8 msmarco_passage_06_561198812 25.050332900008755 985173 definition of product strategy in marketing 9 msmarco_passage_63_820220755 24.812000837942733 985173 definition of product strategy in marketing 10 msmarco_passage_27_263966241 24.265489962933728 783098 what is persuasive essay 1 msmarco_passage_61_800285522 28.116455179601125 783098 what is persuasive essay 2 msmarco_passage_17_220707332 28.095503197621216 783098 what is persuasive essay 3 msmarco_passage_36_700865357 27.974765688150335 783098 what is persuasive essay 4 msmarco_passage_47_11278653 27.974765688150335 783098 what is persuasive essay 5 msmarco_passage_29_429175682 27.925269371931748 783098 what is persuasive essay 6 msmarco_passage_42_85581259 27.61492229615056 783098 what is persuasive essay 7 msmarco_passage_54_726643363 27.61208955093528 783098 what is persuasive essay 8 msmarco_passage_47_11288939 27.434255180361802 783098 what is persuasive essay 9 msmarco_passage_29_429181670 27.411828325374714 783098 what is persuasive essay 10 msmarco_passage_47_11282723 27.3403081477456 1095807 how much is sales tax in rancho cordova 1 msmarco_passage_09_309145709 47.874867694418135 1095807 how much is sales tax in rancho cordova 2 msmarco_passage_59_631539710 47.874867694418135 1095807 how much is sales tax in rancho cordova 3 msmarco_passage_59_631539328 45.03730293175758 1095807 how much is sales tax in rancho cordova 4 msmarco_passage_59_631540597 44.988783950930554 1095807 how much is sales tax in rancho cordova 5 msmarco_passage_04_823766697 42.728293311914385 1095807 how much is sales tax in rancho cordova 6 msmarco_passage_59_631539157 41.32733480039404 1095807 how much is sales tax in rancho cordova 7 msmarco_passage_09_309146495 39.74915112702317 1095807 how much is sales tax in rancho cordova 8 msmarco_passage_59_631539935 39.57732015785722 1095807 how much is sales tax in rancho cordova 9 msmarco_passage_04_823770534 39.516319296881434 1095807 how much is sales tax in rancho cordova 10 msmarco_passage_38_197787774 38.94253240780304 1088938 cadillac meaning 1 msmarco_passage_23_457118322 18.645605481337608 1088938 cadillac meaning 2 msmarco_passage_34_381087505 18.2715612197705 1088938 cadillac meaning 3 msmarco_passage_03_333442639 18.2715612197705 1088938 cadillac meaning 4 msmarco_passage_04_770036192 18.12710906613629 1088938 cadillac meaning 5 msmarco_passage_40_804748749 18.12710906613629 1088938 cadillac meaning 6 msmarco_passage_14_20954800 18.12710906613629 1088938 cadillac meaning 7 msmarco_passage_14_20941385 18.12710906613629 1088938 cadillac meaning 8 msmarco_passage_36_705948652 17.519136811336345 1088938 cadillac meaning 9 msmarco_passage_38_492411979 17.0097272089895 1088938 cadillac meaning 10 msmarco_passage_38_475426250 16.859089199757058 1094215 is blighted ovum the same as molar pregnancy 1 msmarco_passage_06_197743458 44.34171470876626 1094215 is blighted ovum the same as molar pregnancy 2 msmarco_passage_30_690205173 43.93447388092566 1094215 is blighted ovum the same as molar pregnancy 3 msmarco_passage_57_445933778 41.01075215602471 1094215 is blighted ovum the same as molar pregnancy 4 msmarco_passage_32_143985260 40.547075399928524 1094215 is blighted ovum the same as molar pregnancy 5 msmarco_passage_67_36454463 39.08561871734858 1094215 is blighted ovum the same as molar pregnancy 6 msmarco_passage_66_23215746 38.59715890503327 1094215 is blighted ovum the same as molar pregnancy 7 msmarco_passage_42_238952531 38.44214031822369 1094215 is blighted ovum the same as molar pregnancy 8 msmarco_passage_22_593368157 38.40895145598683 1094215 is blighted ovum the same as molar pregnancy 9 msmarco_passage_66_12230059 38.40101004036155 1094215 is blighted ovum the same as molar pregnancy 10 msmarco_passage_07_154263023 38.40101004036155 690705 what is a merchant's mark 1 msmarco_passage_57_18967090 19.423145673707246 690705 what is a merchant's mark 2 msmarco_passage_17_458766444 18.854231670829527 690705 what is a merchant's mark 3 msmarco_passage_20_35309467 18.052110202845313 690705 what is a merchant's mark 4 msmarco_passage_20_35307145 17.886423037189815 690705 what is a merchant's mark 5 msmarco_passage_57_18966002 17.618602790919027 690705 what is a merchant's mark 6 msmarco_passage_22_43283320 17.530389312903985 690705 what is a merchant's mark 7 msmarco_passage_55_129894186 17.521275235652116 690705 what is a merchant's mark 8 msmarco_passage_15_233047497 17.408504217618976 690705 what is a merchant's mark 9 msmarco_passage_54_536372408 17.399993275625143 690705 what is a merchant's mark 10 msmarco_passage_20_35306263 17.384244640310108 601128 what convinces employees want 1 msmarco_passage_21_461900689 18.950736581881355 601128 what convinces employees want 2 msmarco_passage_06_43543071 18.773590291901588 601128 what convinces employees want 3 msmarco_passage_47_39852061 18.251758340687584 601128 what convinces employees want 4 msmarco_passage_34_858347252 18.108843793069923 601128 what convinces employees want 5 msmarco_passage_06_137551297 17.95223193744602 601128 what convinces employees want 6 msmarco_passage_45_205640426 17.95223193744602 601128 what convinces employees want 7 msmarco_passage_40_731249641 17.719362884132586 601128 what convinces employees want 8 msmarco_passage_40_367219668 17.49264989764407 601128 what convinces employees want 9 msmarco_passage_31_342665405 17.394954391414753 601128 what convinces employees want 10 msmarco_passage_30_778355810 17.271845774328455 1063777 what is an achilles heel injury 1 msmarco_passage_54_61815689 30.74978494796595 1063777 what is an achilles heel injury 2 msmarco_passage_61_355749659 30.334215969472677 1063777 what is an achilles heel injury 3 msmarco_passage_09_654009267 30.334215969472677 1063777 what is an achilles heel injury 4 msmarco_passage_48_600477514 30.015940458397747 1063777 what is an achilles heel injury 5 msmarco_passage_61_356512297 29.968710349789188 1063777 what is an achilles heel injury 6 msmarco_passage_61_356660232 29.968710349789188 1063777 what is an achilles heel injury 7 msmarco_passage_59_568246878 29.893355843990975 1063777 what is an achilles heel injury 8 msmarco_passage_31_515116256 29.88038554010223 1063777 what is an achilles heel injury 9 msmarco_passage_66_685619983 29.88038554010223 1063777 what is an achilles heel injury 10 msmarco_passage_57_451266147 29.789469033716514 1101861 windstream troubleshooting phone number 1 msmarco_passage_23_338157978 30.17124690944207 1101861 windstream troubleshooting phone number 2 msmarco_passage_48_503465558 29.292760378296254 1101861 windstream troubleshooting phone number 3 msmarco_passage_06_714204345 29.292760378296254 1101861 windstream troubleshooting phone number 4 msmarco_passage_29_225040051 29.126528868621985 1101861 windstream troubleshooting phone number 5 msmarco_passage_23_338162112 28.3468777347346 1101861 windstream troubleshooting phone number 6 msmarco_passage_23_338160739 27.950338311819745 1101861 windstream troubleshooting phone number 7 msmarco_passage_23_338164507 27.2555238808009 1101861 windstream troubleshooting phone number 8 msmarco_passage_29_225033306 27.188101744368865 1101861 windstream troubleshooting phone number 9 msmarco_passage_23_338160083 26.92247933738504 1101861 windstream troubleshooting phone number 10 msmarco_passage_23_338163257 26.84515581047486 1000574 where did josiah henson create his afro-canadian community 1 msmarco_passage_05_773443600 47.226498646968025 1000574 where did josiah henson create his afro-canadian community 2 msmarco_passage_27_291603935 45.188213890354135 1000574 where did josiah henson create his afro-canadian community 3 msmarco_passage_12_88920226 38.934578556546384 1000574 where did josiah henson create his afro-canadian community 4 msmarco_passage_04_469784512 34.92797501981724 1000574 where did josiah henson create his afro-canadian community 5 msmarco_passage_13_504461646 34.454647310851215 1000574 where did josiah henson create his afro-canadian community 6 msmarco_passage_59_12368480 34.03140581518552 1000574 where did josiah henson create his afro-canadian community 7 msmarco_passage_64_34466761 33.546166855902854 1000574 where did josiah henson create his afro-canadian community 8 msmarco_passage_21_216149522 33.542585446519155 1000574 where did josiah henson create his afro-canadian community 9 msmarco_passage_13_504463450 32.68103558487067 1000574 where did josiah henson create his afro-canadian community 10 msmarco_passage_59_12365938 31.99175261656437 855968 what is ti allowance lease 1 msmarco_passage_12_415068553 24.564679434276666 855968 what is ti allowance lease 2 msmarco_passage_05_15481340 24.192995814742382 855968 what is ti allowance lease 3 msmarco_passage_25_219367746 23.5922598369963 855968 what is ti allowance lease 4 msmarco_passage_25_219368810 22.81516592983273 855968 what is ti allowance lease 5 msmarco_passage_05_15486713 22.605876412641837 855968 what is ti allowance lease 6 msmarco_passage_35_718908070 22.208591192747093 855968 what is ti allowance lease 7 msmarco_passage_66_865783435 21.697762195570498 855968 what is ti allowance lease 8 msmarco_passage_30_350528265 21.697762195570498 855968 what is ti allowance lease 9 msmarco_passage_38_425001331 21.171280456337094 855968 what is ti allowance lease 10 msmarco_passage_38_424922867 20.806827670760295 513061 temple university student population 1 msmarco_passage_05_220456087 26.98149000236105 513061 temple university student population 2 msmarco_passage_04_268935256 25.925225720910507 513061 temple university student population 3 msmarco_passage_38_197581143 25.925225720910507 513061 temple university student population 4 msmarco_passage_42_105074706 23.819675425186755 513061 temple university student population 5 msmarco_passage_21_132956777 22.929670050217624 513061 temple university student population 6 msmarco_passage_65_613426633 22.929670050217624 513061 temple university student population 7 msmarco_passage_00_598253945 22.929670050217624 513061 temple university student population 8 msmarco_passage_65_611264095 22.850380734160915 513061 temple university student population 9 msmarco_passage_03_194195607 22.82036366245984 513061 temple university student population 10 msmarco_passage_14_324653956 22.723948285324546 1022359 what political party did abraham lincoln run 1 msmarco_passage_04_129078831 39.29625740350767 1022359 what political party did abraham lincoln run 2 msmarco_passage_04_129079260 38.28218062293112 1022359 what political party did abraham lincoln run 3 msmarco_passage_04_222263009 36.946560684099005 1022359 what political party did abraham lincoln run 4 msmarco_passage_04_129079674 36.12161470324362 1022359 what political party did abraham lincoln run 5 msmarco_passage_04_129080583 35.97702584184281 1022359 what political party did abraham lincoln run 6 msmarco_passage_11_348347200 34.60685731700801 1022359 what political party did abraham lincoln run 7 msmarco_passage_18_563387041 33.98590166788714 1022359 what political party did abraham lincoln run 8 msmarco_passage_04_129077827 33.57921832216138 1022359 what political party did abraham lincoln run 9 msmarco_passage_04_129080990 33.2444089442441 1022359 what political party did abraham lincoln run 10 msmarco_passage_04_220407463 33.199608534268265 1097242 abid cardiology 1 msmarco_passage_48_577564198 24.465822254213947 1097242 abid cardiology 2 msmarco_passage_48_615885650 24.44029185074499 1097242 abid cardiology 3 msmarco_passage_48_615881886 21.336028925962406 1097242 abid cardiology 4 msmarco_passage_48_577562111 17.73010377071855 1097242 abid cardiology 5 msmarco_passage_61_779754134 17.0155024310025 1097242 abid cardiology 6 msmarco_passage_48_615884646 16.77710088509664 1097242 abid cardiology 7 msmarco_passage_60_642092051 16.24817513534648 1097242 abid cardiology 8 msmarco_passage_54_748469754 16.226087794980867 1097242 abid cardiology 9 msmarco_passage_48_608949673 16.173800781976322 1097242 abid cardiology 10 msmarco_passage_53_404507756 15.963553201786514 1049200 who recorded loving you 1 msmarco_passage_19_329008764 13.67195237912328 1049200 who recorded loving you 2 msmarco_passage_19_403231147 13.575830779817744 1049200 who recorded loving you 3 msmarco_passage_25_505883250 13.371142549445233 1049200 who recorded loving you 4 msmarco_passage_11_189579821 13.240221174387429 1049200 who recorded loving you 5 msmarco_passage_38_584232780 13.162572518089561 1049200 who recorded loving you 6 msmarco_passage_01_706251346 13.139871596981003 1049200 who recorded loving you 7 msmarco_passage_57_357641706 13.122206304480898 1049200 who recorded loving you 8 msmarco_passage_18_5956397 12.964014483832447 1049200 who recorded loving you 9 msmarco_passage_20_286480593 12.964014483832447 1049200 who recorded loving you 10 msmarco_passage_36_206174502 12.945021822592517 68095 can hives be a sign of pregnancy 1 msmarco_passage_54_267662213 24.79846693050642 68095 can hives be a sign of pregnancy 2 msmarco_passage_08_214612920 24.79846693050642 68095 can hives be a sign of pregnancy 3 msmarco_passage_08_214611123 22.53239891973933 68095 can hives be a sign of pregnancy 4 msmarco_passage_54_267660901 22.53239891973933 68095 can hives be a sign of pregnancy 5 msmarco_passage_37_36866111 21.135732506383427 68095 can hives be a sign of pregnancy 6 msmarco_passage_03_769569287 21.135732506383427 68095 can hives be a sign of pregnancy 7 msmarco_passage_37_36875227 21.135732506383427 68095 can hives be a sign of pregnancy 8 msmarco_passage_11_824635085 21.06238192949255 68095 can hives be a sign of pregnancy 9 msmarco_passage_00_138619522 21.06238192949255 68095 can hives be a sign of pregnancy 10 msmarco_passage_06_861562235 20.90126483611332 1053901 what is hydroxyz pam used for 1 msmarco_passage_03_69190689 26.647643686724333 1053901 what is hydroxyz pam used for 2 msmarco_passage_09_692940499 25.91843726230439 1053901 what is hydroxyz pam used for 3 msmarco_passage_61_532981532 25.91843726230439 1053901 what is hydroxyz pam used for 4 msmarco_passage_62_422123564 18.590927133968254 1053901 what is hydroxyz pam used for 5 msmarco_passage_09_692939675 18.331404274860866 1053901 what is hydroxyz pam used for 6 msmarco_passage_61_532980708 18.331404274860866 1053901 what is hydroxyz pam used for 7 msmarco_passage_16_182550058 17.54206078125158 1053901 what is hydroxyz pam used for 8 msmarco_passage_17_58681891 17.47505475913346 1053901 what is hydroxyz pam used for 9 msmarco_passage_26_742944977 17.445346538930142 1053901 what is hydroxyz pam used for 10 msmarco_passage_36_233362489 17.420420844332227 1087634 what are examples of accruals 1 msmarco_passage_42_114595810 22.035750493348875 1087634 what are examples of accruals 2 msmarco_passage_66_367772315 20.926185696031723 1087634 what are examples of accruals 3 msmarco_passage_37_1499492 20.41949427620657 1087634 what are examples of accruals 4 msmarco_passage_37_1655573 20.41949427620657 1087634 what are examples of accruals 5 msmarco_passage_44_148486938 20.204444507706683 1087634 what are examples of accruals 6 msmarco_passage_52_352550967 20.138681774459386 1087634 what are examples of accruals 7 msmarco_passage_16_61160318 20.133960004792126 1087634 what are examples of accruals 8 msmarco_passage_30_381788745 20.08849806787898 1087634 what are examples of accruals 9 msmarco_passage_53_778766940 20.08849806787898 1087634 what are examples of accruals 10 msmarco_passage_36_864910139 19.81756789511409 709559 what is all considered common stock for accounting 1 msmarco_passage_30_711453584 23.865087112626544 709559 what is all considered common stock for accounting 2 msmarco_passage_65_406068594 23.095768000358156 709559 what is all considered common stock for accounting 3 msmarco_passage_69_3613661 22.993837484330406 709559 what is all considered common stock for accounting 4 msmarco_passage_30_122141512 22.136804637169387 709559 what is all considered common stock for accounting 5 msmarco_passage_30_99040753 21.670120553029125 709559 what is all considered common stock for accounting 6 msmarco_passage_58_190172969 21.612807983000643 709559 what is all considered common stock for accounting 7 msmarco_passage_01_244012674 21.561894829505533 709559 what is all considered common stock for accounting 8 msmarco_passage_44_148734406 20.920637513737656 709559 what is all considered common stock for accounting 9 msmarco_passage_37_2087543 20.856345072437875 709559 what is all considered common stock for accounting 10 msmarco_passage_65_402013795 20.544573957713613 1083345 what does numericals mean 1 msmarco_passage_65_106878241 15.7276294932069 1083345 what does numericals mean 2 msmarco_passage_12_240162591 15.59008884692037 1083345 what does numericals mean 3 msmarco_passage_65_106879008 15.428723219072563 1083345 what does numericals mean 4 msmarco_passage_15_479205522 14.90012864808919 1083345 what does numericals mean 5 msmarco_passage_53_284432109 14.833972651991816 1083345 what does numericals mean 6 msmarco_passage_21_714990361 14.666123503449802 1083345 what does numericals mean 7 msmarco_passage_07_158177244 14.63503813812291 1083345 what does numericals mean 8 msmarco_passage_07_498603009 14.63503813812291 1083345 what does numericals mean 9 msmarco_passage_44_806071322 14.634057187593662 1083345 what does numericals mean 10 msmarco_passage_58_378267636 14.559031658627397 731902 what is coffee flour 1 msmarco_passage_01_209384665 21.038268113542262 731902 what is coffee flour 2 msmarco_passage_01_209387710 20.698294417737692 731902 what is coffee flour 3 msmarco_passage_67_83265684 20.671657924550665 731902 what is coffee flour 4 msmarco_passage_01_209382088 20.580951937541727 731902 what is coffee flour 5 msmarco_passage_66_44540149 20.43917338235672 731902 what is coffee flour 6 msmarco_passage_01_209385101 20.124707592589704 731902 what is coffee flour 7 msmarco_passage_46_333568049 20.09255808277599 731902 what is coffee flour 8 msmarco_passage_01_209387251 19.983761272994062 731902 what is coffee flour 9 msmarco_passage_10_180694092 19.951997057593086 731902 what is coffee flour 10 msmarco_passage_22_586580 19.835252644567944 316803 how much does it cost to rewire electricity in a home? 1 msmarco_passage_07_18340400 33.34984913434725 316803 how much does it cost to rewire electricity in a home? 2 msmarco_passage_49_296412230 33.34984913434725 316803 how much does it cost to rewire electricity in a home? 3 msmarco_passage_01_655815568 32.20282567733592 316803 how much does it cost to rewire electricity in a home? 4 msmarco_passage_25_93070543 32.20282567733592 316803 how much does it cost to rewire electricity in a home? 5 msmarco_passage_01_813098146 31.423823440984574 316803 how much does it cost to rewire electricity in a home? 6 msmarco_passage_64_269625418 31.227239571735673 316803 how much does it cost to rewire electricity in a home? 7 msmarco_passage_23_827949825 30.466366561096965 316803 how much does it cost to rewire electricity in a home? 8 msmarco_passage_07_616392656 29.942933893276656 316803 how much does it cost to rewire electricity in a home? 9 msmarco_passage_46_414627451 29.561799809093813 316803 how much does it cost to rewire electricity in a home? 10 msmarco_passage_64_269625845 29.47662266299053 1078752 withdrawal symptoms of amitriptyline 1 msmarco_passage_30_141519916 34.328021727447435 1078752 withdrawal symptoms of amitriptyline 2 msmarco_passage_30_141465929 34.328021727447435 1078752 withdrawal symptoms of amitriptyline 3 msmarco_passage_08_860028009 34.328021727447435 1078752 withdrawal symptoms of amitriptyline 4 msmarco_passage_27_668353191 33.84880953374912 1078752 withdrawal symptoms of amitriptyline 5 msmarco_passage_02_253188192 33.84880953374912 1078752 withdrawal symptoms of amitriptyline 6 msmarco_passage_63_589559424 32.802497273329784 1078752 withdrawal symptoms of amitriptyline 7 msmarco_passage_36_684991053 32.61101453992511 1078752 withdrawal symptoms of amitriptyline 8 msmarco_passage_04_230175761 32.43458006311106 1078752 withdrawal symptoms of amitriptyline 9 msmarco_passage_36_684998712 31.135714978527282 1078752 withdrawal symptoms of amitriptyline 10 msmarco_passage_11_542887160 29.541343576829192 781877 what is pataday drops used for 1 msmarco_passage_48_60767063 30.521024481568364 781877 what is pataday drops used for 2 msmarco_passage_48_60765074 29.981568410803135 781877 what is pataday drops used for 3 msmarco_passage_29_490302255 29.379304609423002 781877 what is pataday drops used for 4 msmarco_passage_37_552508398 28.746758470658968 781877 what is pataday drops used for 5 msmarco_passage_22_695553581 28.708144262973693 781877 what is pataday drops used for 6 msmarco_passage_25_653293746 27.754522402877132 781877 what is pataday drops used for 7 msmarco_passage_25_653134829 27.754522402877132 781877 what is pataday drops used for 8 msmarco_passage_37_552508683 27.743688709210357 781877 what is pataday drops used for 9 msmarco_passage_29_490297299 27.503209162300816 781877 what is pataday drops used for 10 msmarco_passage_00_99218216 27.412941109173083 1038184 what is the ideal woman in our society 1 msmarco_passage_07_145247961 22.935113124953936 1038184 what is the ideal woman in our society 2 msmarco_passage_52_384010162 22.468546714005463 1038184 what is the ideal woman in our society 3 msmarco_passage_03_691869029 21.91745441381307 1038184 what is the ideal woman in our society 4 msmarco_passage_23_603241525 21.41172817697154 1038184 what is the ideal woman in our society 5 msmarco_passage_03_693394369 21.38846205058639 1038184 what is the ideal woman in our society 6 msmarco_passage_46_838870775 21.294467127429233 1038184 what is the ideal woman in our society 7 msmarco_passage_31_587702267 20.237577537937092 1038184 what is the ideal woman in our society 8 msmarco_passage_33_140466929 20.219590500571016 1038184 what is the ideal woman in our society 9 msmarco_passage_21_585019337 20.181128257263275 1038184 what is the ideal woman in our society 10 msmarco_passage_03_509135094 20.180333982812996 928753 what year was why havent i heard from you reba mcentire released 1 msmarco_passage_21_555612570 42.565912878786264 928753 what year was why havent i heard from you reba mcentire released 2 msmarco_passage_21_555609954 41.07649098543089 928753 what year was why havent i heard from you reba mcentire released 3 msmarco_passage_20_660298623 40.723232834951475 928753 what year was why havent i heard from you reba mcentire released 4 msmarco_passage_66_576112889 40.36527642267973 928753 what year was why havent i heard from you reba mcentire released 5 msmarco_passage_20_764755165 40.11934716511497 928753 what year was why havent i heard from you reba mcentire released 6 msmarco_passage_20_147331657 40.03258819796791 928753 what year was why havent i heard from you reba mcentire released 7 msmarco_passage_21_48283615 39.04268065621323 928753 what year was why havent i heard from you reba mcentire released 8 msmarco_passage_42_446266773 39.00090123880349 928753 what year was why havent i heard from you reba mcentire released 9 msmarco_passage_04_140530701 38.60321442519677 928753 what year was why havent i heard from you reba mcentire released 10 msmarco_passage_23_272186996 38.492925925665 637234 what does exempt or non-exempt employee mean 1 msmarco_passage_69_55445434 60.184731049838135 637234 what does exempt or non-exempt employee mean 2 msmarco_passage_13_468345245 55.93865962562081 637234 what does exempt or non-exempt employee mean 3 msmarco_passage_42_211344624 55.53999349048462 637234 what does exempt or non-exempt employee mean 4 msmarco_passage_27_852180688 55.519903696212054 637234 what does exempt or non-exempt employee mean 5 msmarco_passage_11_570060269 55.519903696212054 637234 what does exempt or non-exempt employee mean 6 msmarco_passage_13_468440744 55.5129666586629 637234 what does exempt or non-exempt employee mean 7 msmarco_passage_36_467703292 55.430832112746586 637234 what does exempt or non-exempt employee mean 8 msmarco_passage_03_648552702 55.430832112746586 637234 what does exempt or non-exempt employee mean 9 msmarco_passage_01_64138669 55.08809069498304 637234 what does exempt or non-exempt employee mean 10 msmarco_passage_06_137000301 55.08809069498304 1086439 what body parts does pku affect 1 msmarco_passage_59_560866962 23.783108124007935 1086439 what body parts does pku affect 2 msmarco_passage_17_324047742 23.681718830010478 1086439 what body parts does pku affect 3 msmarco_passage_25_311234718 23.339414603201615 1086439 what body parts does pku affect 4 msmarco_passage_39_276691232 22.816364041496087 1086439 what body parts does pku affect 5 msmarco_passage_39_276690811 22.777527245903965 1086439 what body parts does pku affect 6 msmarco_passage_08_841338110 22.66016299951899 1086439 what body parts does pku affect 7 msmarco_passage_08_841342416 22.66016299951899 1086439 what body parts does pku affect 8 msmarco_passage_57_158679629 22.66016299951899 1086439 what body parts does pku affect 9 msmarco_passage_08_841333804 22.66016299951899 1086439 what body parts does pku affect 10 msmarco_passage_53_342468555 22.64404510549094 983438 where is glenfinnan 1 msmarco_passage_19_102478946 23.09537528262875 983438 where is glenfinnan 2 msmarco_passage_00_36789201 22.38235129778906 983438 where is glenfinnan 3 msmarco_passage_19_102473282 22.210921739577596 983438 where is glenfinnan 4 msmarco_passage_59_156924133 22.074752020772262 983438 where is glenfinnan 5 msmarco_passage_60_685533789 21.391711203513882 983438 where is glenfinnan 6 msmarco_passage_14_216784446 20.92856339262324 983438 where is glenfinnan 7 msmarco_passage_03_604248804 20.92856339262324 983438 where is glenfinnan 8 msmarco_passage_34_585120916 20.795111093050533 983438 where is glenfinnan 9 msmarco_passage_21_207861354 20.778605533174876 983438 where is glenfinnan 10 msmarco_passage_34_585121655 20.630781348982126 890890 what region was rome in 1 msmarco_passage_68_33077533 21.246243708964407 890890 what region was rome in 2 msmarco_passage_09_245665091 20.718318955971085 890890 what region was rome in 3 msmarco_passage_09_245664554 20.702898295009824 890890 what region was rome in 4 msmarco_passage_20_638387096 19.495151909812616 890890 what region was rome in 5 msmarco_passage_03_27296020 18.673696585473497 890890 what region was rome in 6 msmarco_passage_09_245665492 18.3239400573571 890890 what region was rome in 7 msmarco_passage_66_365501745 18.196320908808385 890890 what region was rome in 8 msmarco_passage_38_288724843 18.09084165416768 890890 what region was rome in 9 msmarco_passage_09_245377452 18.09084165416768 890890 what region was rome in 10 msmarco_passage_21_731740743 18.09084165416768 102695 cost of bamboo hr 1 msmarco_passage_11_743003806 24.612657762780472 102695 cost of bamboo hr 2 msmarco_passage_61_108563933 23.474671780943417 102695 cost of bamboo hr 3 msmarco_passage_35_542169726 22.321626861010085 102695 cost of bamboo hr 4 msmarco_passage_61_107844180 21.717828230151817 102695 cost of bamboo hr 5 msmarco_passage_26_268399133 21.569803803357622 102695 cost of bamboo hr 6 msmarco_passage_26_268230465 21.569803803357622 102695 cost of bamboo hr 7 msmarco_passage_11_743003377 21.53900394659228 102695 cost of bamboo hr 8 msmarco_passage_22_543730846 21.370686938203185 102695 cost of bamboo hr 9 msmarco_passage_31_530618296 19.805552607769027 102695 cost of bamboo hr 10 msmarco_passage_49_446974816 19.393878077539295 1051422 what is mean streaming 1 msmarco_passage_47_461722438 14.571179728856524 1051422 what is mean streaming 2 msmarco_passage_54_2060893 14.571179728856524 1051422 what is mean streaming 3 msmarco_passage_06_544204772 14.571179728856524 1051422 what is mean streaming 4 msmarco_passage_25_368890859 14.528384060735702 1051422 what is mean streaming 5 msmarco_passage_66_741189052 14.528040521876367 1051422 what is mean streaming 6 msmarco_passage_32_765441860 14.338322586624892 1051422 what is mean streaming 7 msmarco_passage_61_683110230 14.258466674198086 1051422 what is mean streaming 8 msmarco_passage_43_324570994 14.234812807840836 1051422 what is mean streaming 9 msmarco_passage_16_139957680 14.155783149440579 1051422 what is mean streaming 10 msmarco_passage_42_863788885 14.140133482472242 636949 what does eobr means 1 msmarco_passage_37_54397802 23.773511122178803 636949 what does eobr means 2 msmarco_passage_25_67723731 23.658814180707243 636949 what does eobr means 3 msmarco_passage_25_67722980 23.602818985221337 636949 what does eobr means 4 msmarco_passage_25_67725095 23.424769112087876 636949 what does eobr means 5 msmarco_passage_37_54397306 23.01749276631507 636949 what does eobr means 6 msmarco_passage_25_67727573 22.98175486671041 636949 what does eobr means 7 msmarco_passage_25_67724665 22.906380423320726 636949 what does eobr means 8 msmarco_passage_18_715009006 22.09157189502948 636949 what does eobr means 9 msmarco_passage_25_67728511 21.781651685453156 636949 what does eobr means 10 msmarco_passage_25_67722539 21.629929743908974 985167 why are insulators used to cover the conducting wires 1 msmarco_passage_26_128740712 32.751404601870014 985167 why are insulators used to cover the conducting wires 2 msmarco_passage_58_271829368 32.22358682366002 985167 why are insulators used to cover the conducting wires 3 msmarco_passage_40_497664685 30.024967533474424 985167 why are insulators used to cover the conducting wires 4 msmarco_passage_63_742006794 29.958875063775118 985167 why are insulators used to cover the conducting wires 5 msmarco_passage_01_22639048 29.958875063775118 985167 why are insulators used to cover the conducting wires 6 msmarco_passage_15_80486693 29.430156528693644 985167 why are insulators used to cover the conducting wires 7 msmarco_passage_49_815093403 29.166368933775374 985167 why are insulators used to cover the conducting wires 8 msmarco_passage_63_714755468 28.375824091934007 985167 why are insulators used to cover the conducting wires 9 msmarco_passage_52_213824530 28.225037577433742 985167 why are insulators used to cover the conducting wires 10 msmarco_passage_54_247275406 28.179658963532418 993419 where is st martin 1 msmarco_passage_64_700215663 22.89763771563078 993419 where is st martin 2 msmarco_passage_64_700203817 22.89763771563078 993419 where is st martin 3 msmarco_passage_64_790997218 22.799137185877413 993419 where is st martin 4 msmarco_passage_64_790993211 22.799137185877413 993419 where is st martin 5 msmarco_passage_64_790988987 22.799137185877413 993419 where is st martin 6 msmarco_passage_64_700192336 22.77410407349263 993419 where is st martin 7 msmarco_passage_64_700210166 22.700286205350622 993419 where is st martin 8 msmarco_passage_64_700198369 22.700286205350622 993419 where is st martin 9 msmarco_passage_64_700221211 22.700286205350622 993419 where is st martin 10 msmarco_passage_64_777600030 22.6688743468983 786477 what is presidential reserve with wyndham resorts 1 msmarco_passage_15_168236389 39.3899909858777 786477 what is presidential reserve with wyndham resorts 2 msmarco_passage_15_168235564 39.01863294118039 786477 what is presidential reserve with wyndham resorts 3 msmarco_passage_10_355170196 37.918688138542116 786477 what is presidential reserve with wyndham resorts 4 msmarco_passage_35_514187962 36.692847869443796 786477 what is presidential reserve with wyndham resorts 5 msmarco_passage_15_168237054 35.87991799219499 786477 what is presidential reserve with wyndham resorts 6 msmarco_passage_15_168235984 35.86814065534766 786477 what is presidential reserve with wyndham resorts 7 msmarco_passage_13_620509191 35.24356608417197 786477 what is presidential reserve with wyndham resorts 8 msmarco_passage_09_420838622 33.260345814364314 786477 what is presidential reserve with wyndham resorts 9 msmarco_passage_15_168238193 32.510684920161495 786477 what is presidential reserve with wyndham resorts 10 msmarco_passage_15_168238662 32.00282796117394 851490 what is the top selling pampered chef item 1 msmarco_passage_13_485768784 36.18888597639362 851490 what is the top selling pampered chef item 2 msmarco_passage_41_375717704 34.06460365478429 851490 what is the top selling pampered chef item 3 msmarco_passage_41_375708307 34.06460365478429 851490 what is the top selling pampered chef item 4 msmarco_passage_41_375690475 33.651982994417594 851490 what is the top selling pampered chef item 5 msmarco_passage_41_375745978 33.41593001869032 851490 what is the top selling pampered chef item 6 msmarco_passage_32_798646568 31.870826717348 851490 what is the top selling pampered chef item 7 msmarco_passage_54_733542825 31.32503568684793 851490 what is the top selling pampered chef item 8 msmarco_passage_29_285248038 31.29688446033141 851490 what is the top selling pampered chef item 9 msmarco_passage_32_862257164 31.024386311552547 851490 what is the top selling pampered chef item 10 msmarco_passage_01_231318423 30.64779956345319 1090399 stents are placed by what kind of md 1 msmarco_passage_05_165382533 25.199244151496316 1090399 stents are placed by what kind of md 2 msmarco_passage_31_536394002 24.745548267688662 1090399 stents are placed by what kind of md 3 msmarco_passage_06_735879517 23.406923581680154 1090399 stents are placed by what kind of md 4 msmarco_passage_27_609040738 23.37357758470371 1090399 stents are placed by what kind of md 5 msmarco_passage_24_146821526 22.136581006949196 1090399 stents are placed by what kind of md 6 msmarco_passage_51_724075793 21.747454120062763 1090399 stents are placed by what kind of md 7 msmarco_passage_07_504359649 21.747454120062763 1090399 stents are placed by what kind of md 8 msmarco_passage_48_713417446 21.099905526213934 1090399 stents are placed by what kind of md 9 msmarco_passage_09_224280564 21.06237400429127 1090399 stents are placed by what kind of md 10 msmarco_passage_22_78779285 21.06237400429127 1094999 accounting definition of office supplies 1 msmarco_passage_04_685304482 23.522104993138846 1094999 accounting definition of office supplies 2 msmarco_passage_04_174780262 22.755819811434833 1094999 accounting definition of office supplies 3 msmarco_passage_36_865467879 22.626803145951563 1094999 accounting definition of office supplies 4 msmarco_passage_32_342733989 20.674744667474307 1094999 accounting definition of office supplies 5 msmarco_passage_68_526155848 20.583068235834823 1094999 accounting definition of office supplies 6 msmarco_passage_11_347242442 20.575785769190826 1094999 accounting definition of office supplies 7 msmarco_passage_04_198514763 20.575785769190826 1094999 accounting definition of office supplies 8 msmarco_passage_28_524176883 20.575785769190826 1094999 accounting definition of office supplies 9 msmarco_passage_04_681592209 20.53052679420134 1094999 accounting definition of office supplies 10 msmarco_passage_36_865468193 20.480400356047674 1080419 wow stonetalon mountains 1 msmarco_passage_35_396364436 32.60488126347428 1080419 wow stonetalon mountains 2 msmarco_passage_03_674773859 32.38513589856476 1080419 wow stonetalon mountains 3 msmarco_passage_36_540802537 28.44318097947105 1080419 wow stonetalon mountains 4 msmarco_passage_12_823456113 28.330784170442346 1080419 wow stonetalon mountains 5 msmarco_passage_66_435543226 28.330784170442346 1080419 wow stonetalon mountains 6 msmarco_passage_36_535362044 27.74645228442361 1080419 wow stonetalon mountains 7 msmarco_passage_36_535355428 27.74645228442361 1080419 wow stonetalon mountains 8 msmarco_passage_15_73425209 27.74645228442361 1080419 wow stonetalon mountains 9 msmarco_passage_03_674774745 27.46323310664105 1080419 wow stonetalon mountains 10 msmarco_passage_36_553274910 27.46323310664105 632923 what does bbb rating mean for a bond 1 msmarco_passage_30_124935738 30.754104338938212 632923 what does bbb rating mean for a bond 2 msmarco_passage_30_124937617 28.655843378069303 632923 what does bbb rating mean for a bond 3 msmarco_passage_50_477731823 27.420153965462195 632923 what does bbb rating mean for a bond 4 msmarco_passage_39_150382637 27.414648391568488 632923 what does bbb rating mean for a bond 5 msmarco_passage_51_63035365 27.361172096119752 632923 what does bbb rating mean for a bond 6 msmarco_passage_46_602965728 27.099681860589186 632923 what does bbb rating mean for a bond 7 msmarco_passage_08_789194932 27.057957560288166 632923 what does bbb rating mean for a bond 8 msmarco_passage_63_821365265 26.93876577928662 632923 what does bbb rating mean for a bond 9 msmarco_passage_40_354276199 26.70240488119861 632923 what does bbb rating mean for a bond 10 msmarco_passage_27_277746950 26.70240488119861 1096620 how many subscribers does the ny times have? 1 msmarco_passage_33_730877512 22.71167377699015 1096620 how many subscribers does the ny times have? 2 msmarco_passage_60_391415652 22.649615308072377 1096620 how many subscribers does the ny times have? 3 msmarco_passage_33_730878434 20.71773288995494 1096620 how many subscribers does the ny times have? 4 msmarco_passage_60_391416117 19.74586585875132 1096620 how many subscribers does the ny times have? 5 msmarco_passage_01_774879055 18.862917151685856 1096620 how many subscribers does the ny times have? 6 msmarco_passage_25_537214746 18.646732437689664 1096620 how many subscribers does the ny times have? 7 msmarco_passage_01_719866264 18.646732437689664 1096620 how many subscribers does the ny times have? 8 msmarco_passage_05_451946088 18.646732437689664 1096620 how many subscribers does the ny times have? 9 msmarco_passage_50_428999418 18.146280066015894 1096620 how many subscribers does the ny times have? 10 msmarco_passage_35_183402894 18.087152107674733 137411 definition of products photosynthesis 1 msmarco_passage_64_460797829 22.43051639482564 137411 definition of products photosynthesis 2 msmarco_passage_64_462353121 22.43051639482564 137411 definition of products photosynthesis 3 msmarco_passage_03_162416673 21.79773158423091 137411 definition of products photosynthesis 4 msmarco_passage_30_582317376 21.63725178985408 137411 definition of products photosynthesis 5 msmarco_passage_53_781181929 21.590001717152923 137411 definition of products photosynthesis 6 msmarco_passage_64_460798041 21.40433069857153 137411 definition of products photosynthesis 7 msmarco_passage_64_462353333 21.40433069857153 137411 definition of products photosynthesis 8 msmarco_passage_51_704621503 21.357512146602023 137411 definition of products photosynthesis 9 msmarco_passage_45_165171515 21.16083585333205 137411 definition of products photosynthesis 10 msmarco_passage_04_135280756 21.060194732167435 831601 what is the meaning of the name Brandon 1 msmarco_passage_23_810098089 26.65453532898963 831601 what is the meaning of the name Brandon 2 msmarco_passage_08_43151325 26.572596551701828 831601 what is the meaning of the name Brandon 3 msmarco_passage_28_419985181 26.392082814078968 831601 what is the meaning of the name Brandon 4 msmarco_passage_04_385404460 25.258981239552472 831601 what is the meaning of the name Brandon 5 msmarco_passage_51_680706508 24.98198280667619 831601 what is the meaning of the name Brandon 6 msmarco_passage_04_122407589 24.779142733470756 831601 what is the meaning of the name Brandon 7 msmarco_passage_63_503632074 24.330344253794475 831601 what is the meaning of the name Brandon 8 msmarco_passage_63_503630414 24.239969132033572 831601 what is the meaning of the name Brandon 9 msmarco_passage_28_419988467 24.1966992786366 831601 what is the meaning of the name Brandon 10 msmarco_passage_63_503631444 24.189746814368807 186727 fifth season vikings 1 msmarco_passage_02_842022515 28.181516189413955 186727 fifth season vikings 2 msmarco_passage_58_799864416 27.67423432498443 186727 fifth season vikings 3 msmarco_passage_10_415254376 27.307910965560847 186727 fifth season vikings 4 msmarco_passage_37_104586236 26.35869805228625 186727 fifth season vikings 5 msmarco_passage_54_337395376 26.119414441876955 186727 fifth season vikings 6 msmarco_passage_54_337395095 25.746203929168455 186727 fifth season vikings 7 msmarco_passage_17_578795129 25.257748650014854 186727 fifth season vikings 8 msmarco_passage_21_478289553 24.940362842157334 186727 fifth season vikings 9 msmarco_passage_54_342078481 24.940362842157334 186727 fifth season vikings 10 msmarco_passage_16_66244937 24.83203209206816 703268 what is a the accrual method of accounting 1 msmarco_passage_13_468908853 30.918072356680575 703268 what is a the accrual method of accounting 2 msmarco_passage_52_352550967 30.741605603296215 703268 what is a the accrual method of accounting 3 msmarco_passage_15_66416098 30.15444035460381 703268 what is a the accrual method of accounting 4 msmarco_passage_51_176576856 29.457839005029484 703268 what is a the accrual method of accounting 5 msmarco_passage_15_514126225 29.26843110620236 703268 what is a the accrual method of accounting 6 msmarco_passage_07_347411407 29.01966750973181 703268 what is a the accrual method of accounting 7 msmarco_passage_51_37714997 29.01966750973181 703268 what is a the accrual method of accounting 8 msmarco_passage_66_367775990 28.99926269673476 703268 what is a the accrual method of accounting 9 msmarco_passage_04_579731770 28.991167655126805 703268 what is a the accrual method of accounting 10 msmarco_passage_46_628835768 28.55014146923448 827791 what is the lcd math 1 msmarco_passage_04_119647472 20.017154902975317 827791 what is the lcd math 2 msmarco_passage_04_119647811 19.756461963453454 827791 what is the lcd math 3 msmarco_passage_53_293924851 18.764822964477894 827791 what is the lcd math 4 msmarco_passage_31_601970252 18.648000324774415 827791 what is the lcd math 5 msmarco_passage_04_723653433 17.90081929052116 827791 what is the lcd math 6 msmarco_passage_40_231867811 17.90081929052116 827791 what is the lcd math 7 msmarco_passage_40_516894407 17.90081929052116 827791 what is the lcd math 8 msmarco_passage_04_669389616 17.90081929052116 827791 what is the lcd math 9 msmarco_passage_04_119649056 17.618093385215886 827791 what is the lcd math 10 msmarco_passage_53_293925314 17.49007655585233 489858 ron hiw net worth 1 msmarco_passage_41_210463113 31.837924490267575 489858 ron hiw net worth 2 msmarco_passage_41_218399091 31.032950862564213 489858 ron hiw net worth 3 msmarco_passage_41_207705187 30.608328332002642 489858 ron hiw net worth 4 msmarco_passage_35_93968806 30.531465898875897 489858 ron hiw net worth 5 msmarco_passage_04_814130939 30.15734470200939 489858 ron hiw net worth 6 msmarco_passage_59_302018765 30.09088278088307 489858 ron hiw net worth 7 msmarco_passage_41_218401465 30.03057198459514 489858 ron hiw net worth 8 msmarco_passage_04_816107036 30.03057198459514 489858 ron hiw net worth 9 msmarco_passage_41_218400624 29.992977058238473 489858 ron hiw net worth 10 msmarco_passage_41_206994312 29.798610308009923 1077002 what is (prospectus 1 msmarco_passage_51_16706263 18.809219193678874 1077002 what is (prospectus 2 msmarco_passage_47_79332439 18.5129262690214 1077002 what is (prospectus 3 msmarco_passage_06_479615048 18.5129262690214 1077002 what is (prospectus 4 msmarco_passage_43_687220071 18.43324851019177 1077002 what is (prospectus 5 msmarco_passage_68_82561870 18.43324851019177 1077002 what is (prospectus 6 msmarco_passage_62_662510692 18.42090512020801 1077002 what is (prospectus 7 msmarco_passage_38_201977173 18.215425305167713 1077002 what is (prospectus 8 msmarco_passage_52_132916772 18.192569241044758 1077002 what is (prospectus 9 msmarco_passage_60_329892870 18.121279900026757 1077002 what is (prospectus 10 msmarco_passage_04_131314168 18.05335656972382 1093181 martinica meaning 1 msmarco_passage_00_760753967 25.503593867922593 1093181 martinica meaning 2 msmarco_passage_05_510670354 24.52847867250536 1093181 martinica meaning 3 msmarco_passage_00_760754347 18.476620588237154 1093181 martinica meaning 4 msmarco_passage_05_510671140 17.363123223854913 1093181 martinica meaning 5 msmarco_passage_40_53501442 16.376207678273865 1093181 martinica meaning 6 msmarco_passage_19_638684287 16.376207678273865 1093181 martinica meaning 7 msmarco_passage_00_760753091 16.146763191259332 1093181 martinica meaning 8 msmarco_passage_17_512751959 15.923659289016467 1093181 martinica meaning 9 msmarco_passage_17_815407274 15.706636725236725 1093181 martinica meaning 10 msmarco_passage_45_719144542 15.28986741780093 1057996 what is considered the love apple 1 msmarco_passage_23_624221420 19.29185863952688 1057996 what is considered the love apple 2 msmarco_passage_32_176723932 18.025132462760254 1057996 what is considered the love apple 3 msmarco_passage_07_444089144 17.942157415831417 1057996 what is considered the love apple 4 msmarco_passage_35_396049832 17.93703657098283 1057996 what is considered the love apple 5 msmarco_passage_22_289500310 17.59883816734302 1057996 what is considered the love apple 6 msmarco_passage_09_745607873 17.419094454332214 1057996 what is considered the love apple 7 msmarco_passage_48_338810192 17.20043633798772 1057996 what is considered the love apple 8 msmarco_passage_16_262237862 17.186427439093546 1057996 what is considered the love apple 9 msmarco_passage_32_168210582 17.10951097734971 1057996 what is considered the love apple 10 msmarco_passage_60_609675762 17.090490601248984 453856 military salary pay average 1 msmarco_passage_03_641799511 27.275479765797837 453856 military salary pay average 2 msmarco_passage_36_448072532 27.275479765797837 453856 military salary pay average 3 msmarco_passage_56_592084469 26.861285190399066 453856 military salary pay average 4 msmarco_passage_27_772199816 25.53504443399857 453856 military salary pay average 5 msmarco_passage_08_176409612 25.53504443399857 453856 military salary pay average 6 msmarco_passage_36_124375222 25.390940510304706 453856 military salary pay average 7 msmarco_passage_40_734259928 25.217953585805365 453856 military salary pay average 8 msmarco_passage_59_623422158 24.677091291861366 453856 military salary pay average 9 msmarco_passage_03_653365741 24.658388991910762 453856 military salary pay average 10 msmarco_passage_03_657630985 24.615174225084402 831380 what is the meaning of pound 1 msmarco_passage_12_166002200 15.719918194531537 831380 what is the meaning of pound 2 msmarco_passage_61_513888661 15.610131772503355 831380 what is the meaning of pound 3 msmarco_passage_61_514218241 15.610131772503355 831380 what is the meaning of pound 4 msmarco_passage_61_512327615 15.610131772503355 831380 what is the meaning of pound 5 msmarco_passage_09_685385500 15.610131772503355 831380 what is the meaning of pound 6 msmarco_passage_43_698518038 15.058873162561184 831380 what is the meaning of pound 7 msmarco_passage_42_847193528 15.001790980693507 831380 what is the meaning of pound 8 msmarco_passage_11_178821189 14.985472875105621 831380 what is the meaning of pound 9 msmarco_passage_11_353158200 14.859583466172152 831380 what is the meaning of pound 10 msmarco_passage_00_714174194 14.667544578826309 1029058 what is used to help with urinary leakage surgeries 1 msmarco_passage_28_274793935 28.690156373010172 1029058 what is used to help with urinary leakage surgeries 2 msmarco_passage_28_274800387 28.690156373010172 1029058 what is used to help with urinary leakage surgeries 3 msmarco_passage_02_345423304 28.690156373010172 1029058 what is used to help with urinary leakage surgeries 4 msmarco_passage_04_747512622 28.10462676518388 1029058 what is used to help with urinary leakage surgeries 5 msmarco_passage_01_516801409 26.915193210510104 1029058 what is used to help with urinary leakage surgeries 6 msmarco_passage_24_92811114 26.915193210510104 1029058 what is used to help with urinary leakage surgeries 7 msmarco_passage_48_553647303 26.915193210510104 1029058 what is used to help with urinary leakage surgeries 8 msmarco_passage_27_616210619 26.753124118178015 1029058 what is used to help with urinary leakage surgeries 9 msmarco_passage_53_615381253 26.727804719643594 1029058 what is used to help with urinary leakage surgeries 10 msmarco_passage_09_760577517 26.706769051075256 1000004 where does name nora come from 1 msmarco_passage_34_202945969 24.334702277900966 1000004 where does name nora come from 2 msmarco_passage_34_202943391 23.446678593136692 1000004 where does name nora come from 3 msmarco_passage_38_638092986 23.23455757056811 1000004 where does name nora come from 4 msmarco_passage_04_376045326 23.23455757056811 1000004 where does name nora come from 5 msmarco_passage_20_235233318 23.014562378941896 1000004 where does name nora come from 6 msmarco_passage_34_202942995 22.900211148562335 1000004 where does name nora come from 7 msmarco_passage_67_18648577 22.846934884668567 1000004 where does name nora come from 8 msmarco_passage_38_628888873 22.611406630987567 1000004 where does name nora come from 9 msmarco_passage_67_18646060 22.33396900039004 1000004 where does name nora come from 10 msmarco_passage_67_18645327 22.304172067499024 597395 what color birthstone is for december 1 msmarco_passage_26_135434741 32.77192724414006 597395 what color birthstone is for december 2 msmarco_passage_01_808703543 32.77192724414006 597395 what color birthstone is for december 3 msmarco_passage_23_233511691 31.479847030006887 597395 what color birthstone is for december 4 msmarco_passage_26_2313420 31.10287668948927 597395 what color birthstone is for december 5 msmarco_passage_67_64018392 30.931001914395477 597395 what color birthstone is for december 6 msmarco_passage_49_89642552 30.504460256783105 597395 what color birthstone is for december 7 msmarco_passage_26_2314643 29.841403083258946 597395 what color birthstone is for december 8 msmarco_passage_13_91545577 29.805245237484442 597395 what color birthstone is for december 9 msmarco_passage_14_247601976 29.19086930929017 597395 what color birthstone is for december 10 msmarco_passage_26_135435218 29.090383500235355 1092484 most nigeria read newspaper 1 msmarco_passage_55_335405602 29.643327686801857 1092484 most nigeria read newspaper 2 msmarco_passage_55_335405138 27.571090594795464 1092484 most nigeria read newspaper 3 msmarco_passage_55_335403277 26.629592431944054 1092484 most nigeria read newspaper 4 msmarco_passage_54_778995488 26.098142636212625 1092484 most nigeria read newspaper 5 msmarco_passage_52_524822648 25.52938949398484 1092484 most nigeria read newspaper 6 msmarco_passage_08_831236264 25.293356591890014 1092484 most nigeria read newspaper 7 msmarco_passage_52_524819819 25.097316955739927 1092484 most nigeria read newspaper 8 msmarco_passage_12_283413228 24.396632140458344 1092484 most nigeria read newspaper 9 msmarco_passage_28_661335797 24.298528996751227 1092484 most nigeria read newspaper 10 msmarco_passage_55_335404650 24.091526958350588 323096 how much is the average dj for a wedding 1 msmarco_passage_09_419271917 32.79078426553045 323096 how much is the average dj for a wedding 2 msmarco_passage_16_435456847 32.66097303903924 323096 how much is the average dj for a wedding 3 msmarco_passage_63_434376630 30.600957416881982 323096 how much is the average dj for a wedding 4 msmarco_passage_64_272108371 30.0377272765977 323096 how much is the average dj for a wedding 5 msmarco_passage_66_776974540 29.430016087347084 323096 how much is the average dj for a wedding 6 msmarco_passage_10_695028600 29.23012400971516 323096 how much is the average dj for a wedding 7 msmarco_passage_63_434381316 28.907880313534488 323096 how much is the average dj for a wedding 8 msmarco_passage_16_435458967 28.708165793642344 323096 how much is the average dj for a wedding 9 msmarco_passage_66_776958442 27.643776450630668 323096 how much is the average dj for a wedding 10 msmarco_passage_41_136150775 26.683635192858922 1086309 what car class is the mclaren 1 msmarco_passage_27_445852959 25.68376946761596 1086309 what car class is the mclaren 2 msmarco_passage_54_411830708 25.39370282230416 1086309 what car class is the mclaren 3 msmarco_passage_58_533812207 25.34326373694605 1086309 what car class is the mclaren 4 msmarco_passage_40_814175796 24.596257822550783 1086309 what car class is the mclaren 5 msmarco_passage_17_516086715 24.495495405825213 1086309 what car class is the mclaren 6 msmarco_passage_61_771755135 23.96098866646939 1086309 what car class is the mclaren 7 msmarco_passage_20_14974346 23.921148566410118 1086309 what car class is the mclaren 8 msmarco_passage_03_106654057 23.903032173747313 1086309 what car class is the mclaren 9 msmarco_passage_44_759779863 23.731382662533807 1086309 what car class is the mclaren 10 msmarco_passage_40_675465450 23.51065689602487 1009724 what's a tpo roof 1 msmarco_passage_07_248965852 29.76780393096788 1009724 what's a tpo roof 2 msmarco_passage_50_434264011 29.76780393096788 1009724 what's a tpo roof 3 msmarco_passage_06_386850994 29.4629612916271 1009724 what's a tpo roof 4 msmarco_passage_51_490546598 29.31881092337182 1009724 what's a tpo roof 5 msmarco_passage_51_490553891 29.31881092337182 1009724 what's a tpo roof 6 msmarco_passage_07_248960387 28.671703677602334 1009724 what's a tpo roof 7 msmarco_passage_50_434257256 28.671703677602334 1009724 what's a tpo roof 8 msmarco_passage_07_248959112 28.671703677602334 1009724 what's a tpo roof 9 msmarco_passage_50_434258534 28.671703677602334 1009724 what's a tpo roof 10 msmarco_passage_23_548041723 28.664059757232163 1096866 how many oz in a lb? 1 msmarco_passage_58_719045977 28.479840743504866 1096866 how many oz in a lb? 2 msmarco_passage_09_137005394 28.479840743504866 1096866 how many oz in a lb? 3 msmarco_passage_05_270533752 26.01762409235783 1096866 how many oz in a lb? 4 msmarco_passage_05_271085411 25.957050005473015 1096866 how many oz in a lb? 5 msmarco_passage_42_349291746 25.957050005473015 1096866 how many oz in a lb? 6 msmarco_passage_37_145674738 25.590185566181102 1096866 how many oz in a lb? 7 msmarco_passage_37_145653243 25.0004344335797 1096866 how many oz in a lb? 8 msmarco_passage_37_145653772 25.0004344335797 1096866 how many oz in a lb? 9 msmarco_passage_42_350172291 24.993787791846742 1096866 how many oz in a lb? 10 msmarco_passage_42_349290368 24.973254819836065 199837 health benefits of neem tree 1 msmarco_passage_05_733232168 35.07667424178338 199837 health benefits of neem tree 2 msmarco_passage_34_533714128 34.44779625102436 199837 health benefits of neem tree 3 msmarco_passage_66_519322629 32.10041900920059 199837 health benefits of neem tree 4 msmarco_passage_62_616385067 31.912697344584387 199837 health benefits of neem tree 5 msmarco_passage_62_590946634 31.65268125583255 199837 health benefits of neem tree 6 msmarco_passage_34_662998987 31.541302280113737 199837 health benefits of neem tree 7 msmarco_passage_08_400803667 31.351778661540756 199837 health benefits of neem tree 8 msmarco_passage_34_294868293 31.327190852302266 199837 health benefits of neem tree 9 msmarco_passage_05_751342294 30.84282879048253 199837 health benefits of neem tree 10 msmarco_passage_38_563250459 29.689788187387187 1083095 causes of shaking and blurred vision 1 msmarco_passage_09_665606580 28.566139000338083 1083095 causes of shaking and blurred vision 2 msmarco_passage_06_290005250 28.323763838441433 1083095 causes of shaking and blurred vision 3 msmarco_passage_48_624213827 28.01839686146772 1083095 causes of shaking and blurred vision 4 msmarco_passage_06_741178388 28.01839686146772 1083095 causes of shaking and blurred vision 5 msmarco_passage_48_619620907 27.644231405805865 1083095 causes of shaking and blurred vision 6 msmarco_passage_48_624221571 27.644231405805865 1083095 causes of shaking and blurred vision 7 msmarco_passage_06_741186150 27.644231405805865 1083095 causes of shaking and blurred vision 8 msmarco_passage_66_128849433 27.551896302715374 1083095 causes of shaking and blurred vision 9 msmarco_passage_65_793528061 27.551896302715374 1083095 causes of shaking and blurred vision 10 msmarco_passage_33_520194581 27.261810882498317 1082653 what does the rellis campus name mean 1 msmarco_passage_35_175236076 33.2818706204015 1082653 what does the rellis campus name mean 2 msmarco_passage_17_448706457 30.69548328182591 1082653 what does the rellis campus name mean 3 msmarco_passage_17_448705995 27.118531853704617 1082653 what does the rellis campus name mean 4 msmarco_passage_62_109389590 27.090654035779288 1082653 what does the rellis campus name mean 5 msmarco_passage_31_160939697 27.00038359855533 1082653 what does the rellis campus name mean 6 msmarco_passage_31_160940486 26.11638797317036 1082653 what does the rellis campus name mean 7 msmarco_passage_13_525064191 25.979686109493926 1082653 what does the rellis campus name mean 8 msmarco_passage_31_160937958 25.6454678387607 1082653 what does the rellis campus name mean 9 msmarco_passage_18_168602123 24.309613796997454 1082653 what does the rellis campus name mean 10 msmarco_passage_31_160939216 22.446142191264986 993748 which organ is supplied with blood by the coronary arteries 1 msmarco_passage_54_423949599 38.95263648372421 993748 which organ is supplied with blood by the coronary arteries 2 msmarco_passage_66_118096238 38.92742459153999 993748 which organ is supplied with blood by the coronary arteries 3 msmarco_passage_65_864520174 38.92742459153999 993748 which organ is supplied with blood by the coronary arteries 4 msmarco_passage_39_275312275 37.46420263949845 993748 which organ is supplied with blood by the coronary arteries 5 msmarco_passage_60_485500018 36.962387721509245 993748 which organ is supplied with blood by the coronary arteries 6 msmarco_passage_14_31808014 36.88100268742504 993748 which organ is supplied with blood by the coronary arteries 7 msmarco_passage_27_615799778 36.789826725193045 993748 which organ is supplied with blood by the coronary arteries 8 msmarco_passage_45_863759852 36.63045997946052 993748 which organ is supplied with blood by the coronary arteries 9 msmarco_passage_05_383272988 36.31931819731567 993748 which organ is supplied with blood by the coronary arteries 10 msmarco_passage_00_683124501 36.31931819731567 524166 transportation to us bank stadium 1 msmarco_passage_13_790189776 26.811974140930293 524166 transportation to us bank stadium 2 msmarco_passage_02_708653590 26.1492800122127 524166 transportation to us bank stadium 3 msmarco_passage_18_82494089 24.237523499001647 524166 transportation to us bank stadium 4 msmarco_passage_02_708653073 24.235089519745774 524166 transportation to us bank stadium 5 msmarco_passage_10_550760077 23.561995475190198 524166 transportation to us bank stadium 6 msmarco_passage_64_280267968 23.422261512310325 524166 transportation to us bank stadium 7 msmarco_passage_09_83743190 22.862813724490575 524166 transportation to us bank stadium 8 msmarco_passage_09_83746237 22.55604809930975 524166 transportation to us bank stadium 9 msmarco_passage_10_575197244 22.336989553439903 524166 transportation to us bank stadium 10 msmarco_passage_65_551567826 22.20381402330902 1092996 meaning of muuaahh in text 1 msmarco_passage_04_149686649 15.778016841586593 1092996 meaning of muuaahh in text 2 msmarco_passage_51_680946788 14.796293861816785 1092996 meaning of muuaahh in text 3 msmarco_passage_12_698277632 14.591678869930966 1092996 meaning of muuaahh in text 4 msmarco_passage_05_343658744 14.530633054084468 1092996 meaning of muuaahh in text 5 msmarco_passage_65_361431312 14.497137836706539 1092996 meaning of muuaahh in text 6 msmarco_passage_04_19690575 14.438245933017793 1092996 meaning of muuaahh in text 7 msmarco_passage_44_780917496 14.434227953158246 1092996 meaning of muuaahh in text 8 msmarco_passage_55_599019250 14.4252311969593 1092996 meaning of muuaahh in text 9 msmarco_passage_21_543472338 14.213178439396449 1092996 meaning of muuaahh in text 10 msmarco_passage_31_201541498 14.198486279610393 863623 what is wrong if your ear keeps popping 1 msmarco_passage_35_45850192 21.82342356772384 863623 what is wrong if your ear keeps popping 2 msmarco_passage_37_730280020 21.69469770131591 863623 what is wrong if your ear keeps popping 3 msmarco_passage_49_18724004 21.412117669218382 863623 what is wrong if your ear keeps popping 4 msmarco_passage_08_351983218 21.30640410191386 863623 what is wrong if your ear keeps popping 5 msmarco_passage_00_798096342 21.250712047109005 863623 what is wrong if your ear keeps popping 6 msmarco_passage_48_502645015 21.047935142258837 863623 what is wrong if your ear keeps popping 7 msmarco_passage_49_6080312 20.606373083503627 863623 what is wrong if your ear keeps popping 8 msmarco_passage_38_64576395 20.420939228744693 863623 what is wrong if your ear keeps popping 9 msmarco_passage_62_94725093 20.247273789275496 863623 what is wrong if your ear keeps popping 10 msmarco_passage_68_802857509 20.19276712207467 991241 where is quincy market in boston? 1 msmarco_passage_56_283310747 30.614081136377145 991241 where is quincy market in boston? 2 msmarco_passage_64_697898205 30.43630608420078 991241 where is quincy market in boston? 3 msmarco_passage_56_283317126 30.353040441460635 991241 where is quincy market in boston? 4 msmarco_passage_56_283308476 30.181565735525965 991241 where is quincy market in boston? 5 msmarco_passage_56_283311608 29.969858818006884 991241 where is quincy market in boston? 6 msmarco_passage_56_283308020 29.307526795027627 991241 where is quincy market in boston? 7 msmarco_passage_27_41304820 27.763937659852196 991241 where is quincy market in boston? 8 msmarco_passage_31_499218410 27.537291393368314 991241 where is quincy market in boston? 9 msmarco_passage_20_533264360 27.120066436828566 991241 where is quincy market in boston? 10 msmarco_passage_56_283317574 26.62844249238114 58234 calan medication 1 msmarco_passage_44_349331346 22.383379542082988 58234 calan medication 2 msmarco_passage_44_349324114 21.84554770299053 58234 calan medication 3 msmarco_passage_59_522106082 21.334769657856924 58234 calan medication 4 msmarco_passage_09_271297256 21.334769657856924 58234 calan medication 5 msmarco_passage_01_554999894 21.252748986570403 58234 calan medication 6 msmarco_passage_44_275960286 21.025523996329856 58234 calan medication 7 msmarco_passage_45_177713273 21.0066914719147 58234 calan medication 8 msmarco_passage_44_332829098 20.84745978607284 58234 calan medication 9 msmarco_passage_44_332827539 20.599508640973227 58234 calan medication 10 msmarco_passage_59_527477664 20.585486542913447 991938 where is royal swim located 1 msmarco_passage_01_562943977 20.751992595605067 991938 where is royal swim located 2 msmarco_passage_24_412861934 20.751992595605067 991938 where is royal swim located 3 msmarco_passage_36_299992703 20.625959914953043 991938 where is royal swim located 4 msmarco_passage_04_337181572 20.599585700593806 991938 where is royal swim located 5 msmarco_passage_40_621974929 20.41861068263843 991938 where is royal swim located 6 msmarco_passage_06_795195981 20.36078328280195 991938 where is royal swim located 7 msmarco_passage_40_621975253 19.995565557139486 991938 where is royal swim located 8 msmarco_passage_59_204415153 19.93191844335689 991938 where is royal swim located 9 msmarco_passage_59_204380589 19.93191844335689 991938 where is royal swim located 10 msmarco_passage_59_204403726 19.93191844335689 783277 what is ph of bleach 1 msmarco_passage_22_467619342 23.682211873889774 783277 what is ph of bleach 2 msmarco_passage_22_467620386 22.819609700363344 783277 what is ph of bleach 3 msmarco_passage_22_467618945 22.819609700363344 783277 what is ph of bleach 4 msmarco_passage_38_6707890 22.23647577323917 783277 what is ph of bleach 5 msmarco_passage_65_14591358 22.195607895454664 783277 what is ph of bleach 6 msmarco_passage_45_565870192 21.62500395235157 783277 what is ph of bleach 7 msmarco_passage_46_341964749 21.541165451975388 783277 what is ph of bleach 8 msmarco_passage_14_192278603 21.380795384578402 783277 what is ph of bleach 9 msmarco_passage_09_5910291 21.317312604891562 783277 what is ph of bleach 10 msmarco_passage_01_11005453 21.2584469362858 830551 what is the mayor of torrington ct salary 1 msmarco_passage_09_307114396 31.45100529535906 830551 what is the mayor of torrington ct salary 2 msmarco_passage_09_307115358 30.97488750742037 830551 what is the mayor of torrington ct salary 3 msmarco_passage_27_20571615 30.063821082553282 830551 what is the mayor of torrington ct salary 4 msmarco_passage_05_95531153 29.605404985324235 830551 what is the mayor of torrington ct salary 5 msmarco_passage_61_607798858 29.39549948477746 830551 what is the mayor of torrington ct salary 6 msmarco_passage_24_77158557 29.341025591203064 830551 what is the mayor of torrington ct salary 7 msmarco_passage_42_519473172 28.765880013606463 830551 what is the mayor of torrington ct salary 8 msmarco_passage_68_507962817 28.75624541819851 830551 what is the mayor of torrington ct salary 9 msmarco_passage_48_133473192 28.441989599101433 830551 what is the mayor of torrington ct salary 10 msmarco_passage_57_268886284 28.119695652358637 1090242 symptoms of ptsd in vietnam veterans 1 msmarco_passage_49_29444345 39.24582644586695 1090242 symptoms of ptsd in vietnam veterans 2 msmarco_passage_49_29447333 38.71782943088329 1090242 symptoms of ptsd in vietnam veterans 3 msmarco_passage_49_169429089 35.557678523834056 1090242 symptoms of ptsd in vietnam veterans 4 msmarco_passage_57_742566497 35.5398220767687 1090242 symptoms of ptsd in vietnam veterans 5 msmarco_passage_53_672509228 35.15316067351529 1090242 symptoms of ptsd in vietnam veterans 6 msmarco_passage_08_119729757 35.15316067351529 1090242 symptoms of ptsd in vietnam veterans 7 msmarco_passage_27_525108031 34.975188458096824 1090242 symptoms of ptsd in vietnam veterans 8 msmarco_passage_35_543749646 34.30699727999466 1090242 symptoms of ptsd in vietnam veterans 9 msmarco_passage_24_620671113 34.283003544990045 1090242 symptoms of ptsd in vietnam veterans 10 msmarco_passage_27_72027421 34.2610336069235 1093698 ketamine withdrawal symptoms 1 msmarco_passage_01_817689895 34.95631511352915 1093698 ketamine withdrawal symptoms 2 msmarco_passage_36_208013769 33.23511187237523 1093698 ketamine withdrawal symptoms 3 msmarco_passage_44_260318161 32.586410513520285 1093698 ketamine withdrawal symptoms 4 msmarco_passage_37_97656781 31.992184514277152 1093698 ketamine withdrawal symptoms 5 msmarco_passage_01_817693316 31.68094202751761 1093698 ketamine withdrawal symptoms 6 msmarco_passage_53_730893500 31.408699448545658 1093698 ketamine withdrawal symptoms 7 msmarco_passage_53_730896044 31.18039065110551 1093698 ketamine withdrawal symptoms 8 msmarco_passage_11_540828324 31.161192131264322 1093698 ketamine withdrawal symptoms 9 msmarco_passage_26_182450929 31.137551166471006 1093698 ketamine withdrawal symptoms 10 msmarco_passage_01_817633691 31.137551166471006 1090742 sherwin williams phone number 1 msmarco_passage_64_247532130 35.491403950368536 1090742 sherwin williams phone number 2 msmarco_passage_06_434526158 33.383495326798894 1090742 sherwin williams phone number 3 msmarco_passage_67_68189726 31.952731659967952 1090742 sherwin williams phone number 4 msmarco_passage_06_434527584 31.507079466350902 1090742 sherwin williams phone number 5 msmarco_passage_67_68190173 30.139279921844015 1090742 sherwin williams phone number 6 msmarco_passage_14_742034292 28.92801108804177 1090742 sherwin williams phone number 7 msmarco_passage_67_68189195 28.53555165273327 1090742 sherwin williams phone number 8 msmarco_passage_57_132826612 28.391452657738775 1090742 sherwin williams phone number 9 msmarco_passage_67_68190568 27.965349749807366 1090742 sherwin williams phone number 10 msmarco_passage_06_434527330 27.853981217717067 412340 is holland the netherlands 1 msmarco_passage_10_34519160 30.32597384645806 412340 is holland the netherlands 2 msmarco_passage_10_34517774 29.711588561622882 412340 is holland the netherlands 3 msmarco_passage_66_431308680 28.38684898809872 412340 is holland the netherlands 4 msmarco_passage_04_142018306 28.333891373353666 412340 is holland the netherlands 5 msmarco_passage_07_7472863 28.201846511589935 412340 is holland the netherlands 6 msmarco_passage_04_142019191 28.155867481273106 412340 is holland the netherlands 7 msmarco_passage_43_549029738 27.60092519080002 412340 is holland the netherlands 8 msmarco_passage_43_509588961 27.35649251667583 412340 is holland the netherlands 9 msmarco_passage_05_659320894 27.35649251667583 412340 is holland the netherlands 10 msmarco_passage_43_549030816 27.298282062482656 1082948 what does starling mean 1 msmarco_passage_59_299408258 20.827875770285665 1082948 what does starling mean 2 msmarco_passage_12_251093783 20.687050119938974 1082948 what does starling mean 3 msmarco_passage_27_456958274 20.672353392212848 1082948 what does starling mean 4 msmarco_passage_27_465466120 20.672353392212848 1082948 what does starling mean 5 msmarco_passage_27_47174818 20.372945933951637 1082948 what does starling mean 6 msmarco_passage_22_144902712 19.319208289550023 1082948 what does starling mean 7 msmarco_passage_12_62032691 18.903182168076256 1082948 what does starling mean 8 msmarco_passage_21_734046981 18.699144008040268 1082948 what does starling mean 9 msmarco_passage_08_784157764 18.585939898704297 1082948 what does starling mean 10 msmarco_passage_43_810821598 18.534932829323047 1028711 what is wells fargo go far reward 1 msmarco_passage_49_856855581 38.716453038471826 1028711 what is wells fargo go far reward 2 msmarco_passage_55_212937521 38.17003450169006 1028711 what is wells fargo go far reward 3 msmarco_passage_55_195860000 38.17003450169006 1028711 what is wells fargo go far reward 4 msmarco_passage_38_740390578 37.26341531462018 1028711 what is wells fargo go far reward 5 msmarco_passage_27_787648272 36.7502553830305 1028711 what is wells fargo go far reward 6 msmarco_passage_66_843361778 36.36221854036163 1028711 what is wells fargo go far reward 7 msmarco_passage_10_702868506 36.36221854036163 1028711 what is wells fargo go far reward 8 msmarco_passage_66_843357900 36.36221854036163 1028711 what is wells fargo go far reward 9 msmarco_passage_42_181460007 36.344469242123594 1028711 what is wells fargo go far reward 10 msmarco_passage_38_740389281 36.22888867032299 558263 what are endotoxins? 1 msmarco_passage_05_664331342 19.377984133567544 558263 what are endotoxins? 2 msmarco_passage_43_535316037 19.377984133567544 558263 what are endotoxins? 3 msmarco_passage_11_305101811 19.06427123280652 558263 what are endotoxins? 4 msmarco_passage_30_723066375 19.011529451999134 558263 what are endotoxins? 5 msmarco_passage_00_46759260 19.011529451999134 558263 what are endotoxins? 6 msmarco_passage_13_712636577 18.917297577370125 558263 what are endotoxins? 7 msmarco_passage_60_135902089 18.917297577370125 558263 what are endotoxins? 8 msmarco_passage_07_754671945 18.823995228243323 558263 what are endotoxins? 9 msmarco_passage_04_28773913 18.71373443355519 558263 what are endotoxins? 10 msmarco_passage_43_535319423 18.640124629513572 108622 cost root canal 1 msmarco_passage_27_15074224 27.860582065226996 108622 cost root canal 2 msmarco_passage_55_303101374 27.762529065276205 108622 cost root canal 3 msmarco_passage_27_15072896 27.762529065276205 108622 cost root canal 4 msmarco_passage_27_15073731 27.511140520715507 108622 cost root canal 5 msmarco_passage_52_803153538 27.502077681389878 108622 cost root canal 6 msmarco_passage_08_817305281 27.077574095559275 108622 cost root canal 7 msmarco_passage_08_250159178 26.64506940886637 108622 cost root canal 8 msmarco_passage_08_250148896 26.397930405298148 108622 cost root canal 9 msmarco_passage_48_460851387 26.39044846615711 108622 cost root canal 10 msmarco_passage_43_253833295 26.11241121939633 394021 in what county is punxsutawney pa 1 msmarco_passage_42_502277510 31.351207480602227 394021 in what county is punxsutawney pa 2 msmarco_passage_62_99376412 30.58217342621556 394021 in what county is punxsutawney pa 3 msmarco_passage_01_868993713 29.915828731980515 394021 in what county is punxsutawney pa 4 msmarco_passage_02_530652700 29.732845542215372 394021 in what county is punxsutawney pa 5 msmarco_passage_42_515774538 29.527685149838575 394021 in what county is punxsutawney pa 6 msmarco_passage_42_456810132 29.509190691249906 394021 in what county is punxsutawney pa 7 msmarco_passage_42_502294907 29.10795003950073 394021 in what county is punxsutawney pa 8 msmarco_passage_42_481206295 29.082637852788153 394021 in what county is punxsutawney pa 9 msmarco_passage_42_457745142 29.0084467418214 394021 in what county is punxsutawney pa 10 msmarco_passage_42_481075007 28.900354665438993 909048 what training methods is shuttle runs using 1 msmarco_passage_66_30613940 24.45578238381195 909048 what training methods is shuttle runs using 2 msmarco_passage_46_395386236 23.551736893331068 909048 what training methods is shuttle runs using 3 msmarco_passage_46_395396390 23.551736893331068 909048 what training methods is shuttle runs using 4 msmarco_passage_65_857486343 23.28440333143053 909048 what training methods is shuttle runs using 5 msmarco_passage_66_43401730 23.28440333143053 909048 what training methods is shuttle runs using 6 msmarco_passage_50_580866350 23.082401147642496 909048 what training methods is shuttle runs using 7 msmarco_passage_65_785343557 22.329981110514428 909048 what training methods is shuttle runs using 8 msmarco_passage_06_374160594 22.303546452568128 909048 what training methods is shuttle runs using 9 msmarco_passage_54_120882286 21.76199557715912 909048 what training methods is shuttle runs using 10 msmarco_passage_00_289992096 21.625202010282422 566335 what are strengths in writing 1 msmarco_passage_03_692659978 16.208678856828328 566335 what are strengths in writing 2 msmarco_passage_15_150504406 16.05157239223477 566335 what are strengths in writing 3 msmarco_passage_38_802352906 16.05157239223477 566335 what are strengths in writing 4 msmarco_passage_26_37660339 15.76014713827336 566335 what are strengths in writing 5 msmarco_passage_50_509859899 15.660741467795189 566335 what are strengths in writing 6 msmarco_passage_38_802351570 15.660741467795189 566335 what are strengths in writing 7 msmarco_passage_38_802353756 15.660741467795189 566335 what are strengths in writing 8 msmarco_passage_61_438586204 15.62810707460784 566335 what are strengths in writing 9 msmarco_passage_62_698145107 15.342732772793244 566335 what are strengths in writing 10 msmarco_passage_29_625364851 15.329142422131987 783822 what is pictorial drawing 1 msmarco_passage_30_780564205 24.07863190471933 783822 what is pictorial drawing 2 msmarco_passage_62_112755877 23.912084070788783 783822 what is pictorial drawing 3 msmarco_passage_09_794871048 23.912084070788783 783822 what is pictorial drawing 4 msmarco_passage_03_500616986 23.144886231564033 783822 what is pictorial drawing 5 msmarco_passage_26_527576284 22.965447151045375 783822 what is pictorial drawing 6 msmarco_passage_37_260946755 22.927924822623087 783822 what is pictorial drawing 7 msmarco_passage_17_279475036 22.84579823926333 783822 what is pictorial drawing 8 msmarco_passage_09_794876737 22.615509651247724 783822 what is pictorial drawing 9 msmarco_passage_62_112761566 22.615509651247724 783822 what is pictorial drawing 10 msmarco_passage_27_572563181 22.497159481432842 724571 what is blackleg 1 msmarco_passage_56_804873174 20.038159072455645 724571 what is blackleg 2 msmarco_passage_40_574792872 19.831980340526368 724571 what is blackleg 3 msmarco_passage_04_751952034 19.486390769109228 724571 what is blackleg 4 msmarco_passage_47_314751209 19.270783761555794 724571 what is blackleg 5 msmarco_passage_04_751950813 19.12430801242421 724571 what is blackleg 6 msmarco_passage_48_547081835 18.980042167843976 724571 what is blackleg 7 msmarco_passage_28_680794567 18.69794311838971 724571 what is blackleg 8 msmarco_passage_64_277493831 18.69794311838971 724571 what is blackleg 9 msmarco_passage_26_105373574 18.621087549423386 724571 what is blackleg 10 msmarco_passage_30_350357070 18.621087549423386 1098905 how is delaware sales tax calculated 1 msmarco_passage_09_801791212 35.79396939373757 1098905 how is delaware sales tax calculated 2 msmarco_passage_09_310227255 34.839707876419304 1098905 how is delaware sales tax calculated 3 msmarco_passage_09_310225865 33.08975109767623 1098905 how is delaware sales tax calculated 4 msmarco_passage_09_310228704 31.823805863030845 1098905 how is delaware sales tax calculated 5 msmarco_passage_09_810014132 31.22989695484315 1098905 how is delaware sales tax calculated 6 msmarco_passage_09_310227597 30.462039398222224 1098905 how is delaware sales tax calculated 7 msmarco_passage_09_801806345 29.94459994105007 1098905 how is delaware sales tax calculated 8 msmarco_passage_58_612731441 28.572999023288013 1098905 how is delaware sales tax calculated 9 msmarco_passage_09_308852669 28.43716918105239 1098905 how is delaware sales tax calculated 10 msmarco_passage_09_801805639 28.414093024734857 1006751 when did hunger games book come out 1 msmarco_passage_04_205092662 31.039306067505144 1006751 when did hunger games book come out 2 msmarco_passage_04_103754807 29.42003062994751 1006751 when did hunger games book come out 3 msmarco_passage_26_816224953 28.170256387386033 1006751 when did hunger games book come out 4 msmarco_passage_22_471690396 26.678766081765147 1006751 when did hunger games book come out 5 msmarco_passage_04_206153741 26.45492847145698 1006751 when did hunger games book come out 6 msmarco_passage_66_407122089 26.320837905539143 1006751 when did hunger games book come out 7 msmarco_passage_59_794123271 25.972399013287372 1006751 when did hunger games book come out 8 msmarco_passage_25_845683406 25.333769723184968 1006751 when did hunger games book come out 9 msmarco_passage_31_698612795 25.015016224662133 1006751 when did hunger games book come out 10 msmarco_passage_25_845684841 25.015016224662133 107077 cost of slate 1 msmarco_passage_50_433820316 21.139729641068335 107077 cost of slate 2 msmarco_passage_49_297054490 20.543121838953745 107077 cost of slate 3 msmarco_passage_46_418226165 20.06523673153704 107077 cost of slate 4 msmarco_passage_64_271514276 19.440853320215716 107077 cost of slate 5 msmarco_passage_39_853671202 19.26057766222507 107077 cost of slate 6 msmarco_passage_31_864396431 19.20930326963827 107077 cost of slate 7 msmarco_passage_31_864385340 19.20930326963827 107077 cost of slate 8 msmarco_passage_09_191099228 19.19999713272163 107077 cost of slate 9 msmarco_passage_39_853636878 19.19958016429033 107077 cost of slate 10 msmarco_passage_46_417717889 19.075024784972634 931726 what's the difference between jambalaya and paella 1 msmarco_passage_64_453525727 37.33185078216687 931726 what's the difference between jambalaya and paella 2 msmarco_passage_04_175028799 35.92767520614809 931726 what's the difference between jambalaya and paella 3 msmarco_passage_37_381554693 33.034703178331945 931726 what's the difference between jambalaya and paella 4 msmarco_passage_01_16478206 32.706064987913564 931726 what's the difference between jambalaya and paella 5 msmarco_passage_33_580977760 30.759186630319412 931726 what's the difference between jambalaya and paella 6 msmarco_passage_33_581494975 30.581373691333177 931726 what's the difference between jambalaya and paella 7 msmarco_passage_37_199832852 30.53530424421666 931726 what's the difference between jambalaya and paella 8 msmarco_passage_57_120682704 30.089356360962434 931726 what's the difference between jambalaya and paella 9 msmarco_passage_64_453526148 30.089356360962434 931726 what's the difference between jambalaya and paella 10 msmarco_passage_35_325708376 29.751099360893328 184916 family roots definition 1 msmarco_passage_53_788113972 19.054312972487132 184916 family roots definition 2 msmarco_passage_23_441599580 17.530102206356588 184916 family roots definition 3 msmarco_passage_23_441564775 17.49388353555866 184916 family roots definition 4 msmarco_passage_23_441545105 17.432895294209516 184916 family roots definition 5 msmarco_passage_23_441554898 17.432895294209516 184916 family roots definition 6 msmarco_passage_23_441555315 17.432895294209516 184916 family roots definition 7 msmarco_passage_23_441601627 17.38666289813093 184916 family roots definition 8 msmarco_passage_02_491604445 17.352172663799102 184916 family roots definition 9 msmarco_passage_23_441570761 17.34869986540047 184916 family roots definition 10 msmarco_passage_23_441558581 17.34869986540047 1087911 calories in fresh vegetable juice 1 msmarco_passage_07_687611430 34.60192118941726 1087911 calories in fresh vegetable juice 2 msmarco_passage_33_164225617 30.732138023330368 1087911 calories in fresh vegetable juice 3 msmarco_passage_05_865758261 30.305051366246452 1087911 calories in fresh vegetable juice 4 msmarco_passage_48_787873978 29.535941480295833 1087911 calories in fresh vegetable juice 5 msmarco_passage_55_725840588 29.069265484793714 1087911 calories in fresh vegetable juice 6 msmarco_passage_04_16102457 28.93799441946114 1087911 calories in fresh vegetable juice 7 msmarco_passage_63_791690295 28.556254622790824 1087911 calories in fresh vegetable juice 8 msmarco_passage_00_121522401 28.39339512075554 1087911 calories in fresh vegetable juice 9 msmarco_passage_37_469634589 28.39339512075554 1087911 calories in fresh vegetable juice 10 msmarco_passage_40_545215297 28.320495013437753 699510 what is a serigraph 1 msmarco_passage_38_254645853 23.82512915117656 699510 what is a serigraph 2 msmarco_passage_36_582586783 23.16657168789242 699510 what is a serigraph 3 msmarco_passage_14_701494160 22.621981397920194 699510 what is a serigraph 4 msmarco_passage_04_809772457 22.621981397920194 699510 what is a serigraph 5 msmarco_passage_29_303457539 21.94371332745477 699510 what is a serigraph 6 msmarco_passage_56_513240006 20.98660775340644 699510 what is a serigraph 7 msmarco_passage_05_513788015 20.92739502112417 699510 what is a serigraph 8 msmarco_passage_56_513058800 20.835148755315405 699510 what is a serigraph 9 msmarco_passage_53_280568515 20.538695861749318 699510 what is a serigraph 10 msmarco_passage_45_36772422 20.48198026693673 568895 what are the different kinds of arthritis 1 msmarco_passage_08_669586957 23.08384420784327 568895 what are the different kinds of arthritis 2 msmarco_passage_15_657662454 21.712428414728087 568895 what are the different kinds of arthritis 3 msmarco_passage_05_729410828 21.59922791128301 568895 what are the different kinds of arthritis 4 msmarco_passage_66_819633003 20.94938892364772 568895 what are the different kinds of arthritis 5 msmarco_passage_48_565707351 20.81232137393654 568895 what are the different kinds of arthritis 6 msmarco_passage_01_824895933 20.6360728725902 568895 what are the different kinds of arthritis 7 msmarco_passage_26_229773875 20.6360728725902 568895 what are the different kinds of arthritis 8 msmarco_passage_29_532865804 20.634820596962985 568895 what are the different kinds of arthritis 9 msmarco_passage_29_510313670 20.634820596962985 568895 what are the different kinds of arthritis 10 msmarco_passage_38_113866644 20.539819448173372 1096944 how many microns thick is house paint? 1 msmarco_passage_14_504783662 31.55117542302453 1096944 how many microns thick is house paint? 2 msmarco_passage_43_165478811 28.88603311564861 1096944 how many microns thick is house paint? 3 msmarco_passage_43_151520223 28.70946537180614 1096944 how many microns thick is house paint? 4 msmarco_passage_43_318976849 28.075732542024674 1096944 how many microns thick is house paint? 5 msmarco_passage_28_581955184 27.64026642037396 1096944 how many microns thick is house paint? 6 msmarco_passage_57_816149889 27.588738021714175 1096944 how many microns thick is house paint? 7 msmarco_passage_51_694271492 27.558265869490455 1096944 how many microns thick is house paint? 8 msmarco_passage_57_816148707 27.163537071846967 1096944 how many microns thick is house paint? 9 msmarco_passage_37_29843027 26.413646562850406 1096944 how many microns thick is house paint? 10 msmarco_passage_43_318975163 26.289226167307987 786937 what is process improvement (lean six sigma) 1 msmarco_passage_09_111460680 43.647566344711755 786937 what is process improvement (lean six sigma) 2 msmarco_passage_57_824000887 43.647566344711755 786937 what is process improvement (lean six sigma) 3 msmarco_passage_52_779294525 43.554159059675065 786937 what is process improvement (lean six sigma) 4 msmarco_passage_06_183928336 42.645540051982834 786937 what is process improvement (lean six sigma) 5 msmarco_passage_42_378547624 42.08124863740554 786937 what is process improvement (lean six sigma) 6 msmarco_passage_48_254595553 41.84374190702603 786937 what is process improvement (lean six sigma) 7 msmarco_passage_44_803793685 41.30579109458633 786937 what is process improvement (lean six sigma) 8 msmarco_passage_05_666431494 40.92057070888533 786937 what is process improvement (lean six sigma) 9 msmarco_passage_43_544370001 40.92057070888533 786937 what is process improvement (lean six sigma) 10 msmarco_passage_45_445847686 40.59054403197698 103125 cost of cbs all access on roku 1 msmarco_passage_40_332817832 32.14642813482742 103125 cost of cbs all access on roku 2 msmarco_passage_40_362287237 32.14642813482742 103125 cost of cbs all access on roku 3 msmarco_passage_54_565611459 32.14642813482742 103125 cost of cbs all access on roku 4 msmarco_passage_68_205067293 31.925912476066774 103125 cost of cbs all access on roku 5 msmarco_passage_40_332815980 31.628017773362796 103125 cost of cbs all access on roku 6 msmarco_passage_40_362285385 31.628017773362796 103125 cost of cbs all access on roku 7 msmarco_passage_54_565609641 30.704638728518965 103125 cost of cbs all access on roku 8 msmarco_passage_43_596210177 30.650411646164187 103125 cost of cbs all access on roku 9 msmarco_passage_61_84673199 30.617120246968646 103125 cost of cbs all access on roku 10 msmarco_passage_67_41119893 30.093824231006742 992191 definition of dongle 1 msmarco_passage_52_827897862 23.624644980250643 992191 definition of dongle 2 msmarco_passage_17_595664890 23.58152313583098 992191 definition of dongle 3 msmarco_passage_60_297662564 20.99958851817935 992191 definition of dongle 4 msmarco_passage_52_827898293 20.056657992472545 992191 definition of dongle 5 msmarco_passage_13_30330077 19.613729948926977 992191 definition of dongle 6 msmarco_passage_42_119227534 19.436302228095983 992191 definition of dongle 7 msmarco_passage_33_725766216 19.091860683537835 992191 definition of dongle 8 msmarco_passage_41_810355406 18.234845924408397 992191 definition of dongle 9 msmarco_passage_41_813969055 18.234845924408397 992191 definition of dongle 10 msmarco_passage_61_172846684 18.134269539963785 891565 what river is near scotts seadoos in sacramento 1 msmarco_passage_35_518460428 31.59250952502379 891565 what river is near scotts seadoos in sacramento 2 msmarco_passage_35_56411737 27.075909394813806 891565 what river is near scotts seadoos in sacramento 3 msmarco_passage_65_585452255 26.44371535367926 891565 what river is near scotts seadoos in sacramento 4 msmarco_passage_07_674471435 26.193453298016763 891565 what river is near scotts seadoos in sacramento 5 msmarco_passage_23_510817775 26.193453298016763 891565 what river is near scotts seadoos in sacramento 6 msmarco_passage_13_104810608 25.926002757564568 891565 what river is near scotts seadoos in sacramento 7 msmarco_passage_35_8357104 25.85157296266551 891565 what river is near scotts seadoos in sacramento 8 msmarco_passage_00_472193918 25.451612015426043 891565 what river is near scotts seadoos in sacramento 9 msmarco_passage_14_429098544 25.451612015426043 891565 what river is near scotts seadoos in sacramento 10 msmarco_passage_65_585456850 25.38874198478657 1028796 what is vulnerable animals 1 msmarco_passage_09_504682984 19.32619922133016 1028796 what is vulnerable animals 2 msmarco_passage_11_372327458 18.57292508120322 1028796 what is vulnerable animals 3 msmarco_passage_29_841860047 17.941135557953366 1028796 what is vulnerable animals 4 msmarco_passage_29_411468883 17.277676275332738 1028796 what is vulnerable animals 5 msmarco_passage_28_662245993 17.277676275332738 1028796 what is vulnerable animals 6 msmarco_passage_39_582405922 17.035214245554066 1028796 what is vulnerable animals 7 msmarco_passage_21_496482680 16.822678017656557 1028796 what is vulnerable animals 8 msmarco_passage_25_362943918 16.733422089407217 1028796 what is vulnerable animals 9 msmarco_passage_68_646646180 16.587464576982214 1028796 what is vulnerable animals 10 msmarco_passage_57_266644076 16.56496058746135 391125 ie import bookmarks 1 msmarco_passage_33_688906759 30.076611792917024 391125 ie import bookmarks 2 msmarco_passage_12_97299388 27.17974077637864 391125 ie import bookmarks 3 msmarco_passage_33_688904858 26.659255801779242 391125 ie import bookmarks 4 msmarco_passage_02_657409243 26.361149240097987 391125 ie import bookmarks 5 msmarco_passage_33_688908701 26.13568930484405 391125 ie import bookmarks 6 msmarco_passage_23_665906778 26.119827255299953 391125 ie import bookmarks 7 msmarco_passage_33_688907656 25.94516221131076 391125 ie import bookmarks 8 msmarco_passage_31_640265396 25.815341604964313 391125 ie import bookmarks 9 msmarco_passage_48_266906303 25.619246124164373 391125 ie import bookmarks 10 msmarco_passage_23_665907715 25.458506483904635 1034703 what is the peg ratio and what does it do 1 msmarco_passage_66_380445396 24.89987356173024 1034703 what is the peg ratio and what does it do 2 msmarco_passage_62_670506300 24.748406360482857 1034703 what is the peg ratio and what does it do 3 msmarco_passage_25_659664118 24.565844716223324 1034703 what is the peg ratio and what does it do 4 msmarco_passage_25_659220798 24.565844716223324 1034703 what is the peg ratio and what does it do 5 msmarco_passage_15_131323941 24.454261469061862 1034703 what is the peg ratio and what does it do 6 msmarco_passage_28_65035907 24.130296967522696 1034703 what is the peg ratio and what does it do 7 msmarco_passage_68_700615483 24.085633343026615 1034703 what is the peg ratio and what does it do 8 msmarco_passage_15_530580877 24.06826721566045 1034703 what is the peg ratio and what does it do 9 msmarco_passage_25_659220528 23.853601038994917 1034703 what is the peg ratio and what does it do 10 msmarco_passage_25_659663848 23.853601038994917 1045374 what is targeted victory 1 msmarco_passage_37_518106659 16.27690379322936 1045374 what is targeted victory 2 msmarco_passage_38_825175433 16.204410670064444 1045374 what is targeted victory 3 msmarco_passage_33_480172390 16.016342521187376 1045374 what is targeted victory 4 msmarco_passage_21_431196745 15.959548387199689 1045374 what is targeted victory 5 msmarco_passage_17_677567666 15.959548387199689 1045374 what is targeted victory 6 msmarco_passage_04_261926922 15.94489843783635 1045374 what is targeted victory 7 msmarco_passage_31_684648640 15.94489843783635 1045374 what is targeted victory 8 msmarco_passage_03_201402145 15.760258031111572 1045374 what is targeted victory 9 msmarco_passage_37_520851308 15.368653304971271 1045374 what is targeted victory 10 msmarco_passage_04_11006582 15.368653304971271 786674 what is prime rate in canada 1 msmarco_passage_58_729319081 27.351305182867367 786674 what is prime rate in canada 2 msmarco_passage_58_730610003 26.75993862316711 786674 what is prime rate in canada 3 msmarco_passage_59_753765037 26.414570753259486 786674 what is prime rate in canada 4 msmarco_passage_58_730612232 24.913791524920242 786674 what is prime rate in canada 5 msmarco_passage_58_728548776 24.891461136225306 786674 what is prime rate in canada 6 msmarco_passage_58_728535495 24.891461136225306 786674 what is prime rate in canada 7 msmarco_passage_58_730591402 24.767200556451307 786674 what is prime rate in canada 8 msmarco_passage_58_730611036 24.57087892170604 786674 what is prime rate in canada 9 msmarco_passage_36_546997992 24.567266821146426 786674 what is prime rate in canada 10 msmarco_passage_59_753768868 24.293711927972794 81945 cannot connect to citrix xenapp server 1 msmarco_passage_03_210226824 49.121472997271155 81945 cannot connect to citrix xenapp server 2 msmarco_passage_26_307450785 48.15588076908652 81945 cannot connect to citrix xenapp server 3 msmarco_passage_15_343818383 47.996069160829414 81945 cannot connect to citrix xenapp server 4 msmarco_passage_16_366349651 47.908667927382325 81945 cannot connect to citrix xenapp server 5 msmarco_passage_45_759837043 47.62834520577481 81945 cannot connect to citrix xenapp server 6 msmarco_passage_45_759595381 47.538590151006844 81945 cannot connect to citrix xenapp server 7 msmarco_passage_16_366351330 47.01467626615935 81945 cannot connect to citrix xenapp server 8 msmarco_passage_16_366349204 46.004003964012455 81945 cannot connect to citrix xenapp server 9 msmarco_passage_04_263017124 46.00102003050529 81945 cannot connect to citrix xenapp server 10 msmarco_passage_16_366353569 45.04860337036121 562821 what are overpacks used for in hazardous material 1 msmarco_passage_48_181670360 37.511316307830484 562821 what are overpacks used for in hazardous material 2 msmarco_passage_48_181668662 37.20741338284415 562821 what are overpacks used for in hazardous material 3 msmarco_passage_57_216149299 31.378088420305566 562821 what are overpacks used for in hazardous material 4 msmarco_passage_55_9293739 31.356445437730567 562821 what are overpacks used for in hazardous material 5 msmarco_passage_15_839400609 30.304835292669345 562821 what are overpacks used for in hazardous material 6 msmarco_passage_37_271908308 29.421828639332364 562821 what are overpacks used for in hazardous material 7 msmarco_passage_37_272020899 29.1736503515199 562821 what are overpacks used for in hazardous material 8 msmarco_passage_52_510581091 29.13758366020332 562821 what are overpacks used for in hazardous material 9 msmarco_passage_52_510554148 29.13758366020332 562821 what are overpacks used for in hazardous material 10 msmarco_passage_62_785378763 29.121088892347696 782696 what is perched 1 msmarco_passage_36_30836249 16.438507357924177 782696 what is perched 2 msmarco_passage_42_349466125 16.40444538144293 782696 what is perched 3 msmarco_passage_17_169807547 16.250583824136807 782696 what is perched 4 msmarco_passage_22_519160401 16.102645518159576 782696 what is perched 5 msmarco_passage_46_383119730 16.102645518159576 782696 what is perched 6 msmarco_passage_31_460329063 15.997473707614501 782696 what is perched 7 msmarco_passage_07_587576925 15.872081247260976 782696 what is perched 8 msmarco_passage_20_379148576 15.81944401466243 782696 what is perched 9 msmarco_passage_17_584316491 15.81944401466243 782696 what is perched 10 msmarco_passage_62_36153291 15.81944401466243 822937 what is the function of the platelets 1 msmarco_passage_01_708728948 21.69586630703876 822937 what is the function of the platelets 2 msmarco_passage_04_177819498 21.363070063922898 822937 what is the function of the platelets 3 msmarco_passage_17_326836183 21.22585177289472 822937 what is the function of the platelets 4 msmarco_passage_17_353972876 21.161080382700856 822937 what is the function of the platelets 5 msmarco_passage_01_45950545 21.161080382700856 822937 what is the function of the platelets 6 msmarco_passage_01_850039840 21.153915896514306 822937 what is the function of the platelets 7 msmarco_passage_45_120394155 21.153915896514306 822937 what is the function of the platelets 8 msmarco_passage_45_120384001 21.153915896514306 822937 what is the function of the platelets 9 msmarco_passage_01_708731004 21.117535697376827 822937 what is the function of the platelets 10 msmarco_passage_27_624129506 21.08498326591757 132104 definition of airbrush 1 msmarco_passage_53_779983351 24.63282986455941 132104 definition of airbrush 2 msmarco_passage_53_779982658 23.708307949445846 132104 definition of airbrush 3 msmarco_passage_28_340043491 20.99644138462415 132104 definition of airbrush 4 msmarco_passage_37_256599941 20.841697212500133 132104 definition of airbrush 5 msmarco_passage_03_821675289 20.841697212500133 132104 definition of airbrush 6 msmarco_passage_22_847092822 20.50090316608405 132104 definition of airbrush 7 msmarco_passage_11_517640786 19.834882424987924 132104 definition of airbrush 8 msmarco_passage_43_373710348 19.008420431527444 132104 definition of airbrush 9 msmarco_passage_05_556744337 19.008420431527444 132104 definition of airbrush 10 msmarco_passage_59_461625990 18.743306124055735 521018 time in crested butte co 1 msmarco_passage_64_629003948 37.83104843396672 521018 time in crested butte co 2 msmarco_passage_53_101965577 34.29676550797814 521018 time in crested butte co 3 msmarco_passage_64_629003507 33.75193169496766 521018 time in crested butte co 4 msmarco_passage_53_101965085 33.43454215877744 521018 time in crested butte co 5 msmarco_passage_41_346080826 31.94808480343527 521018 time in crested butte co 6 msmarco_passage_45_836362303 31.608792037254553 521018 time in crested butte co 7 msmarco_passage_43_664370786 31.470966575823454 521018 time in crested butte co 8 msmarco_passage_41_346071785 31.017940279418987 521018 time in crested butte co 9 msmarco_passage_64_629007834 30.745968233264787 521018 time in crested butte co 10 msmarco_passage_34_126338685 30.70062333541344 999637 where is volbeat from? 1 msmarco_passage_14_310684621 22.25641895230291 999637 where is volbeat from? 2 msmarco_passage_23_140485304 22.08463823500094 999637 where is volbeat from? 3 msmarco_passage_14_294291638 21.906195897773543 999637 where is volbeat from? 4 msmarco_passage_30_160128003 21.748910952727645 999637 where is volbeat from? 5 msmarco_passage_67_212257355 21.423238172730493 999637 where is volbeat from? 6 msmarco_passage_14_310690427 20.969624571631115 999637 where is volbeat from? 7 msmarco_passage_14_310688526 20.952614861185126 999637 where is volbeat from? 8 msmarco_passage_14_310685899 20.65018751403587 999637 where is volbeat from? 9 msmarco_passage_14_294298172 20.65018751403587 999637 where is volbeat from? 10 msmarco_passage_14_310686802 20.65018751403587 741392 what is drop in baseball bat selection 1 msmarco_passage_30_331840312 31.901323490969823 741392 what is drop in baseball bat selection 2 msmarco_passage_12_843064412 30.74398446626037 741392 what is drop in baseball bat selection 3 msmarco_passage_38_832733821 30.435375736565394 741392 what is drop in baseball bat selection 4 msmarco_passage_31_308554567 29.61014309271104 741392 what is drop in baseball bat selection 5 msmarco_passage_34_15412402 29.318767499595964 741392 what is drop in baseball bat selection 6 msmarco_passage_30_331841158 29.212504744450893 741392 what is drop in baseball bat selection 7 msmarco_passage_38_849046543 29.113930935809094 741392 what is drop in baseball bat selection 8 msmarco_passage_31_308554145 29.10033409889803 741392 what is drop in baseball bat selection 9 msmarco_passage_10_31852025 29.09026461539497 741392 what is drop in baseball bat selection 10 msmarco_passage_38_832731604 29.00511768027667 691004 what is a migrant worker 1 msmarco_passage_61_498194270 25.404213887470547 691004 what is a migrant worker 2 msmarco_passage_27_713160699 24.730149751347 691004 what is a migrant worker 3 msmarco_passage_61_509353560 24.387015264486458 691004 what is a migrant worker 4 msmarco_passage_03_814624875 24.057079304056657 691004 what is a migrant worker 5 msmarco_passage_11_628492830 24.057079304056657 691004 what is a migrant worker 6 msmarco_passage_20_64833207 24.045536741679342 691004 what is a migrant worker 7 msmarco_passage_17_693474150 23.858130386186815 691004 what is a migrant worker 8 msmarco_passage_10_25051988 23.682833432780235 691004 what is a migrant worker 9 msmarco_passage_14_167056574 23.579486559875093 691004 what is a migrant worker 10 msmarco_passage_20_64849012 23.50004328475397 405985 is chatham ct incorporated 1 msmarco_passage_42_59443408 25.770280694035993 405985 is chatham ct incorporated 2 msmarco_passage_49_747402087 20.85550067966757 405985 is chatham ct incorporated 3 msmarco_passage_09_141931978 20.484648363704153 405985 is chatham ct incorporated 4 msmarco_passage_14_782652345 20.029808698266493 405985 is chatham ct incorporated 5 msmarco_passage_58_656216253 20.029808698266493 405985 is chatham ct incorporated 6 msmarco_passage_64_723097724 20.007399606191 405985 is chatham ct incorporated 7 msmarco_passage_05_60339491 19.932597775402982 405985 is chatham ct incorporated 8 msmarco_passage_49_747404196 19.816775432807013 405985 is chatham ct incorporated 9 msmarco_passage_64_723099239 19.624773560485334 405985 is chatham ct incorporated 10 msmarco_passage_64_723103111 19.537322181537927 1084233 what does android sdk tools do 1 msmarco_passage_62_362208786 33.151389318167475 1084233 what does android sdk tools do 2 msmarco_passage_16_478168380 32.859244197359004 1084233 what does android sdk tools do 3 msmarco_passage_15_192397595 32.642364613109194 1084233 what does android sdk tools do 4 msmarco_passage_27_554705966 32.55319497719138 1084233 what does android sdk tools do 5 msmarco_passage_16_681037735 32.52190205329032 1084233 what does android sdk tools do 6 msmarco_passage_12_552887976 32.487893682643225 1084233 what does android sdk tools do 7 msmarco_passage_15_192396323 32.44423226626468 1084233 what does android sdk tools do 8 msmarco_passage_23_697302622 32.34230839788043 1084233 what does android sdk tools do 9 msmarco_passage_25_716872635 31.86750057159353 1084233 what does android sdk tools do 10 msmarco_passage_51_243443377 31.84192874401346 1045527 what is surety bond and how it works 1 msmarco_passage_08_778398929 31.52397939349297 1045527 what is surety bond and how it works 2 msmarco_passage_61_818864346 29.612773898218762 1045527 what is surety bond and how it works 3 msmarco_passage_61_818970153 29.586421541253088 1045527 what is surety bond and how it works 4 msmarco_passage_51_549539070 29.005061630571223 1045527 what is surety bond and how it works 5 msmarco_passage_51_367220875 28.904500061787495 1045527 what is surety bond and how it works 6 msmarco_passage_33_469958141 28.904500061787495 1045527 what is surety bond and how it works 7 msmarco_passage_33_469957325 28.904500061787495 1045527 what is surety bond and how it works 8 msmarco_passage_51_367220060 28.904500061787495 1045527 what is surety bond and how it works 9 msmarco_passage_64_620928519 28.890277196233974 1045527 what is surety bond and how it works 10 msmarco_passage_54_39355384 28.85525192346469 1029124 what is upsell 1 msmarco_passage_63_369499164 20.540309691203447 1029124 what is upsell 2 msmarco_passage_63_369497484 20.540309691203447 1029124 what is upsell 3 msmarco_passage_63_369504091 20.540309691203447 1029124 what is upsell 4 msmarco_passage_33_613302984 19.69432548057164 1029124 what is upsell 5 msmarco_passage_32_123532271 19.37145943632134 1029124 what is upsell 6 msmarco_passage_68_232435787 19.366033817770493 1029124 what is upsell 7 msmarco_passage_60_563784887 18.888455971073036 1029124 what is upsell 8 msmarco_passage_69_149731813 18.703955727295305 1029124 what is upsell 9 msmarco_passage_46_741963687 18.66342877815764 1029124 what is upsell 10 msmarco_passage_48_263193322 18.550425162535888 1084478 what does a fingernail consist of 1 msmarco_passage_08_22594858 18.887940813690392 1084478 what does a fingernail consist of 2 msmarco_passage_53_353132860 18.887940813690392 1084478 what does a fingernail consist of 3 msmarco_passage_01_534636350 18.01939727600497 1084478 what does a fingernail consist of 4 msmarco_passage_13_374059068 17.811127163351486 1084478 what does a fingernail consist of 5 msmarco_passage_37_336758954 17.811127163351486 1084478 what does a fingernail consist of 6 msmarco_passage_15_649074625 17.47640947225348 1084478 what does a fingernail consist of 7 msmarco_passage_48_774788454 17.409216701819002 1084478 what does a fingernail consist of 8 msmarco_passage_14_302317350 17.215225078462534 1084478 what does a fingernail consist of 9 msmarco_passage_48_672760194 17.015138339089837 1084478 what does a fingernail consist of 10 msmarco_passage_43_701880180 16.840354799503594 1075156 why was napalm used in the vietnam war 7 msmarco_passage_00_60556471 32.80165334539623 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 1 msmarco_passage_45_424028575 43.864595622645595 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 2 msmarco_passage_46_147990227 41.022868523256875 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 3 msmarco_passage_46_134896828 41.022868523256875 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 4 msmarco_passage_50_184369711 36.871782733040135 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 5 msmarco_passage_02_433578681 36.59775117976239 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 6 msmarco_passage_06_147624460 36.395382818678286 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 7 msmarco_passage_06_160097855 36.395382818678286 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 8 msmarco_passage_45_227168533 36.395382818678286 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 9 msmarco_passage_60_663223656 35.4641774287537 663890 what gives the epa the authority to publish radiation protection guidance for other federal agencies 10 msmarco_passage_54_620111758 35.312329042222416 1057937 confident man definition 1 msmarco_passage_11_181239125 21.316272587247234 1057937 confident man definition 2 msmarco_passage_11_171717148 21.071164268836107 1057937 confident man definition 3 msmarco_passage_53_791799359 20.478991917296483 1057937 confident man definition 4 msmarco_passage_00_773547208 20.224164552399458 1057937 confident man definition 5 msmarco_passage_02_492874962 20.089841886218267 1057937 confident man definition 6 msmarco_passage_45_634746295 19.592899263235022 1057937 confident man definition 7 msmarco_passage_63_838493137 19.335394341408882 1057937 confident man definition 8 msmarco_passage_02_490966622 19.042791514063268 1057937 confident man definition 9 msmarco_passage_02_491563036 18.968845202402576 1057937 confident man definition 10 msmarco_passage_68_637197476 18.90532801383057 910150 what type of account is valuation allowance 1 msmarco_passage_36_266873643 27.22832845871459 910150 what type of account is valuation allowance 2 msmarco_passage_36_266872742 26.239158670050553 910150 what type of account is valuation allowance 3 msmarco_passage_62_25364318 26.224781316329096 910150 what type of account is valuation allowance 4 msmarco_passage_62_25360406 26.090525515322074 910150 what type of account is valuation allowance 5 msmarco_passage_36_266851153 24.955667235561414 910150 what type of account is valuation allowance 6 msmarco_passage_36_266873173 24.63309994823156 910150 what type of account is valuation allowance 7 msmarco_passage_62_25360896 23.931735286016867 910150 what type of account is valuation allowance 8 msmarco_passage_21_855410045 23.82160450310937 910150 what type of account is valuation allowance 9 msmarco_passage_62_25328707 23.743961414236086 910150 what type of account is valuation allowance 10 msmarco_passage_12_430237024 23.42578444056842 1090550 songhai definition 1 msmarco_passage_08_118998557 24.118771202320712 1090550 songhai definition 2 msmarco_passage_30_791770299 21.76891940446011 1090550 songhai definition 3 msmarco_passage_06_386006436 21.339681698257042 1090550 songhai definition 4 msmarco_passage_61_839157854 20.927096388672027 1090550 songhai definition 5 msmarco_passage_30_642821639 20.71861000046766 1090550 songhai definition 6 msmarco_passage_08_119000422 20.65985680439554 1090550 songhai definition 7 msmarco_passage_05_840950002 20.52919162194754 1090550 songhai definition 8 msmarco_passage_44_477217300 20.52919162194754 1090550 songhai definition 9 msmarco_passage_37_201593882 20.51984226068717 1090550 songhai definition 10 msmarco_passage_30_671337839 20.514236730366104 455273 modality cardinal meaning 1 msmarco_passage_21_521041676 26.531351230126937 455273 modality cardinal meaning 2 msmarco_passage_13_727852126 24.195238609250147 455273 modality cardinal meaning 3 msmarco_passage_18_842921918 23.917567997940807 455273 modality cardinal meaning 4 msmarco_passage_18_591255957 23.917567997940807 455273 modality cardinal meaning 5 msmarco_passage_20_459462792 23.917567997940807 455273 modality cardinal meaning 6 msmarco_passage_17_779685614 23.917567997940807 455273 modality cardinal meaning 7 msmarco_passage_18_32881891 23.917567997940807 455273 modality cardinal meaning 8 msmarco_passage_18_32850878 23.917567997940807 455273 modality cardinal meaning 9 msmarco_passage_17_437749239 23.917567997940807 455273 modality cardinal meaning 10 msmarco_passage_18_290157876 23.917567997940807 682190 what is a dry chemical fire extinguisher 1 msmarco_passage_22_344498738 39.32889902307009 682190 what is a dry chemical fire extinguisher 2 msmarco_passage_06_337596411 39.32889902307009 682190 what is a dry chemical fire extinguisher 3 msmarco_passage_22_344508552 39.32889902307009 682190 what is a dry chemical fire extinguisher 4 msmarco_passage_06_337616457 39.32889902307009 682190 what is a dry chemical fire extinguisher 5 msmarco_passage_06_337605674 39.32889902307009 682190 what is a dry chemical fire extinguisher 6 msmarco_passage_46_189726596 39.32889902307009 682190 what is a dry chemical fire extinguisher 7 msmarco_passage_46_189736387 39.32889902307009 682190 what is a dry chemical fire extinguisher 8 msmarco_passage_46_189712933 39.32889902307009 682190 what is a dry chemical fire extinguisher 9 msmarco_passage_46_189728338 38.95511917422808 682190 what is a dry chemical fire extinguisher 10 msmarco_passage_06_337607416 38.95511917422808 1075156 why was napalm used in the vietnam war 1 msmarco_passage_03_353944817 37.83752601774931 1075156 why was napalm used in the vietnam war 2 msmarco_passage_34_542936166 37.83752601774931 1075156 why was napalm used in the vietnam war 3 msmarco_passage_11_424173885 33.95327620909559 1075156 why was napalm used in the vietnam war 4 msmarco_passage_11_424173390 33.95327620909559 1075156 why was napalm used in the vietnam war 5 msmarco_passage_18_846850448 33.17052502507429 1075156 why was napalm used in the vietnam war 6 msmarco_passage_18_846922363 33.17052502507429 1075156 why was napalm used in the vietnam war 8 msmarco_passage_00_60559528 32.80165334539623 1075156 why was napalm used in the vietnam war 9 msmarco_passage_19_786043035 32.38526761089956 1075156 why was napalm used in the vietnam war 10 msmarco_passage_11_424174375 31.916434732202227 766272 what is lispro 1 msmarco_passage_49_749063411 20.45246783119179 766272 what is lispro 2 msmarco_passage_49_749109998 20.45246783119179 766272 what is lispro 3 msmarco_passage_49_749169815 20.45246783119179 766272 what is lispro 4 msmarco_passage_25_665114683 20.45246783119179 766272 what is lispro 5 msmarco_passage_49_749143924 20.45246783119179 766272 what is lispro 6 msmarco_passage_49_749120925 20.45246783119179 766272 what is lispro 7 msmarco_passage_49_749162957 20.45246783119179 766272 what is lispro 8 msmarco_passage_49_749052423 20.45246783119179 766272 what is lispro 9 msmarco_passage_49_749041101 20.45246783119179 766272 what is lispro 10 msmarco_passage_49_749099751 20.45246783119179 984948 why are the nerves in my nose tingling 1 msmarco_passage_09_791844294 28.43521079163436 984948 why are the nerves in my nose tingling 2 msmarco_passage_62_97808075 28.43521079163436 984948 why are the nerves in my nose tingling 3 msmarco_passage_50_577730706 26.953079586630544 984948 why are the nerves in my nose tingling 4 msmarco_passage_11_62616641 26.953079586630544 984948 why are the nerves in my nose tingling 5 msmarco_passage_48_831095889 25.33770660976288 984948 why are the nerves in my nose tingling 6 msmarco_passage_09_791845030 24.934352585955104 984948 why are the nerves in my nose tingling 7 msmarco_passage_62_97808809 24.934352585955104 984948 why are the nerves in my nose tingling 8 msmarco_passage_66_862252165 24.129689461542597 984948 why are the nerves in my nose tingling 9 msmarco_passage_36_156257763 24.129689461542597 984948 why are the nerves in my nose tingling 10 msmarco_passage_03_572397301 24.129689461542597 496717 should patient medical records be private 1 msmarco_passage_40_630736848 26.874530510521527 496717 should patient medical records be private 2 msmarco_passage_31_456468070 25.763734799715824 496717 should patient medical records be private 3 msmarco_passage_16_150315555 24.269401399983295 496717 should patient medical records be private 4 msmarco_passage_39_228712495 23.95206234487364 496717 should patient medical records be private 5 msmarco_passage_45_197129403 23.82479020348827 496717 should patient medical records be private 6 msmarco_passage_36_465069027 23.586478577883263 496717 should patient medical records be private 7 msmarco_passage_40_739412812 23.52908842536786 496717 should patient medical records be private 8 msmarco_passage_68_84603691 23.453000853467138 496717 should patient medical records be private 9 msmarco_passage_37_225289897 23.2288704161621 496717 should patient medical records be private 10 msmarco_passage_60_289444162 23.188316302866266 561448 what are mandrakes 1 msmarco_passage_56_235252031 21.780124614079735 561448 what are mandrakes 2 msmarco_passage_47_324118054 20.806007023961254 561448 what are mandrakes 3 msmarco_passage_63_292491206 20.65587197352055 561448 what are mandrakes 4 msmarco_passage_01_490005434 20.03053451688995 561448 what are mandrakes 5 msmarco_passage_16_764692463 20.03053451688995 561448 what are mandrakes 6 msmarco_passage_23_849867399 20.03053451688995 561448 what are mandrakes 7 msmarco_passage_23_849868269 19.909983244114898 561448 what are mandrakes 8 msmarco_passage_01_490006306 19.909983244114898 561448 what are mandrakes 9 msmarco_passage_36_369987356 19.760200020703643 561448 what are mandrakes 10 msmarco_passage_16_765428496 19.60407061806408 338713 how set up automatic reply in outlook 1 msmarco_passage_33_416955952 36.76489627781269 338713 how set up automatic reply in outlook 2 msmarco_passage_33_365285644 36.76489627781269 338713 how set up automatic reply in outlook 3 msmarco_passage_33_405280229 36.76489627781269 338713 how set up automatic reply in outlook 4 msmarco_passage_49_74686749 32.975781736851324 338713 how set up automatic reply in outlook 5 msmarco_passage_33_368532646 32.751790377789675 338713 how set up automatic reply in outlook 6 msmarco_passage_16_559040713 32.71747402764346 338713 how set up automatic reply in outlook 7 msmarco_passage_33_408039563 32.03436070596892 338713 how set up automatic reply in outlook 8 msmarco_passage_49_74689306 31.985138241973836 338713 how set up automatic reply in outlook 9 msmarco_passage_52_806944300 31.371726586544302 338713 how set up automatic reply in outlook 10 msmarco_passage_33_378604782 31.29744616286873 787784 what is provocative allergy testing 1 msmarco_passage_36_728473659 28.169445929451435 787784 what is provocative allergy testing 2 msmarco_passage_55_741200641 27.676950829551348 787784 what is provocative allergy testing 3 msmarco_passage_55_741227545 27.3650538864037 787784 what is provocative allergy testing 4 msmarco_passage_55_741222405 27.3650538864037 787784 what is provocative allergy testing 5 msmarco_passage_25_832062055 27.122147305647157 787784 what is provocative allergy testing 6 msmarco_passage_05_783288312 26.240686057570507 787784 what is provocative allergy testing 7 msmarco_passage_10_702178387 26.07754060122819 787784 what is provocative allergy testing 8 msmarco_passage_55_741223112 25.50117482235914 787784 what is provocative allergy testing 9 msmarco_passage_55_741228252 25.50117482235914 787784 what is provocative allergy testing 10 msmarco_passage_57_421853223 25.20796002021365 187818 flash definition 1 msmarco_passage_00_206431728 17.804890826061502 187818 flash definition 2 msmarco_passage_10_69646046 17.5131908180108 187818 flash definition 3 msmarco_passage_08_96148554 17.21155834787691 187818 flash definition 4 msmarco_passage_53_802958650 16.74689850598571 187818 flash definition 5 msmarco_passage_53_802955427 16.50862307172595 187818 flash definition 6 msmarco_passage_29_592889174 16.478903182438117 187818 flash definition 7 msmarco_passage_66_598226967 16.478903182438117 187818 flash definition 8 msmarco_passage_14_456664510 16.36422374560754 187818 flash definition 9 msmarco_passage_00_206431167 16.363107706605728 187818 flash definition 10 msmarco_passage_29_605487868 16.135478677777858 1062603 why did italy attack ethiopia in 1935 1 msmarco_passage_51_502715032 38.60229057099018 1062603 why did italy attack ethiopia in 1935 2 msmarco_passage_22_217626779 36.487392765216015 1062603 why did italy attack ethiopia in 1935 3 msmarco_passage_11_236363042 35.830155382895974 1062603 why did italy attack ethiopia in 1935 4 msmarco_passage_11_236365197 35.181464119352 1062603 why did italy attack ethiopia in 1935 5 msmarco_passage_00_647023416 33.73158919858038 1062603 why did italy attack ethiopia in 1935 6 msmarco_passage_68_740516086 32.773334347781955 1062603 why did italy attack ethiopia in 1935 7 msmarco_passage_14_117959001 32.389696647333274 1062603 why did italy attack ethiopia in 1935 8 msmarco_passage_36_523320536 32.38827187631192 1062603 why did italy attack ethiopia in 1935 9 msmarco_passage_68_740516263 32.13616925794206 1062603 why did italy attack ethiopia in 1935 10 msmarco_passage_48_87135522 31.598208399585552 302878 how much caffeine is in lipton tea bags 1 msmarco_passage_22_462696290 46.101886170466855 302878 how much caffeine is in lipton tea bags 2 msmarco_passage_22_458649754 45.25619425509516 302878 how much caffeine is in lipton tea bags 3 msmarco_passage_40_506097721 44.28704623135644 302878 how much caffeine is in lipton tea bags 4 msmarco_passage_01_499699981 44.00077234415084 302878 how much caffeine is in lipton tea bags 5 msmarco_passage_12_201658461 43.38501522776977 302878 how much caffeine is in lipton tea bags 6 msmarco_passage_04_719666830 43.218087025717885 302878 how much caffeine is in lipton tea bags 7 msmarco_passage_40_506626112 43.218087025717885 302878 how much caffeine is in lipton tea bags 8 msmarco_passage_40_506095864 43.151075938321526 302878 how much caffeine is in lipton tea bags 9 msmarco_passage_40_506093878 43.151075938321526 302878 how much caffeine is in lipton tea bags 10 msmarco_passage_40_506097314 41.50791732500375 1027919 what kind idea had wovoka 1 msmarco_passage_03_617193417 22.137441151532855 1027919 what kind idea had wovoka 2 msmarco_passage_06_152433088 21.99855013242938 1027919 what kind idea had wovoka 3 msmarco_passage_09_771074299 21.142969012098355 1027919 what kind idea had wovoka 4 msmarco_passage_33_40621112 20.98584577158578 1027919 what kind idea had wovoka 5 msmarco_passage_33_40612098 20.695289803465997 1027919 what kind idea had wovoka 6 msmarco_passage_03_617193005 20.67850262227683 1027919 what kind idea had wovoka 7 msmarco_passage_14_764788300 20.52818234734269 1027919 what kind idea had wovoka 8 msmarco_passage_04_20276949 20.250077222666384 1027919 what kind idea had wovoka 9 msmarco_passage_06_152424205 20.234004268759584 1027919 what kind idea had wovoka 10 msmarco_passage_10_423706790 20.234004268759584 47588 average wpm typing speed 1 msmarco_passage_62_181173608 38.69803242708866 47588 average wpm typing speed 2 msmarco_passage_65_172334844 38.06552596721505 47588 average wpm typing speed 3 msmarco_passage_33_640232258 37.917044786246095 47588 average wpm typing speed 4 msmarco_passage_06_183010202 36.50950511377481 47588 average wpm typing speed 5 msmarco_passage_33_640232721 36.22427694118272 47588 average wpm typing speed 6 msmarco_passage_29_145033387 35.41113307608036 47588 average wpm typing speed 7 msmarco_passage_58_209958063 35.33729660756808 47588 average wpm typing speed 8 msmarco_passage_58_134060261 35.326905889525406 47588 average wpm typing speed 9 msmarco_passage_58_214484847 35.05239682254367 47588 average wpm typing speed 10 msmarco_passage_66_607146082 35.03743367601223 1094062 balloon meaning 1 msmarco_passage_45_225230934 19.284399913904345 1094062 balloon meaning 2 msmarco_passage_04_304017174 19.21494066017955 1094062 balloon meaning 3 msmarco_passage_04_304017592 18.819280021533324 1094062 balloon meaning 4 msmarco_passage_04_304217172 18.61026452094601 1094062 balloon meaning 5 msmarco_passage_35_376137992 17.811439150623656 1094062 balloon meaning 6 msmarco_passage_35_376130117 17.757522888370687 1094062 balloon meaning 7 msmarco_passage_35_376137088 17.728324777059232 1094062 balloon meaning 8 msmarco_passage_31_566048944 17.723971710902553 1094062 balloon meaning 9 msmarco_passage_11_755999774 17.577680573075224 1094062 balloon meaning 10 msmarco_passage_59_410411877 17.522329289239707 1067659 what is a seller's contribution 1 msmarco_passage_11_524776334 19.97974768792395 1067659 what is a seller's contribution 2 msmarco_passage_58_793388717 19.70807213575764 1067659 what is a seller's contribution 3 msmarco_passage_28_350841698 19.494837955515084 1067659 what is a seller's contribution 4 msmarco_passage_28_350767564 19.01159889978474 1067659 what is a seller's contribution 5 msmarco_passage_02_323314136 18.76774849237258 1067659 what is a seller's contribution 6 msmarco_passage_59_370160781 18.584573361861842 1067659 what is a seller's contribution 7 msmarco_passage_02_323314601 18.310389724013845 1067659 what is a seller's contribution 8 msmarco_passage_49_292837276 18.310389724013845 1067659 what is a seller's contribution 9 msmarco_passage_31_450241688 18.19106537863359 1067659 what is a seller's contribution 10 msmarco_passage_31_450240153 18.19106537863359 1073365 what is a front company 1 msmarco_passage_50_804571317 14.722712088567672 1073365 what is a front company 2 msmarco_passage_04_756494480 14.53559306402951 1073365 what is a front company 3 msmarco_passage_40_665328732 14.53559306402951 1073365 what is a front company 4 msmarco_passage_40_665276089 14.53559306402951 1073365 what is a front company 5 msmarco_passage_40_665241539 13.862019934887153 1073365 what is a front company 6 msmarco_passage_40_665337124 13.862019934887153 1073365 what is a front company 7 msmarco_passage_40_665328113 13.778072115051952 1073365 what is a front company 8 msmarco_passage_40_665275470 13.778072115051952 1073365 what is a front company 9 msmarco_passage_04_756493860 13.778072115051952 1073365 what is a front company 10 msmarco_passage_50_804571947 13.732883463328962 259239 how long does pink eye last bacterial 1 msmarco_passage_53_502276216 32.5053079538099 259239 how long does pink eye last bacterial 2 msmarco_passage_53_502269510 32.5053079538099 259239 how long does pink eye last bacterial 3 msmarco_passage_68_822832878 30.813179800509282 259239 how long does pink eye last bacterial 4 msmarco_passage_48_520034231 30.429275700733555 259239 how long does pink eye last bacterial 5 msmarco_passage_48_530117177 30.429275700733555 259239 how long does pink eye last bacterial 6 msmarco_passage_29_223238501 30.303744279378265 259239 how long does pink eye last bacterial 7 msmarco_passage_52_802516051 29.570616949123185 259239 how long does pink eye last bacterial 8 msmarco_passage_53_502275740 28.923104257227536 259239 how long does pink eye last bacterial 9 msmarco_passage_53_502269034 28.923104257227536 259239 how long does pink eye last bacterial 10 msmarco_passage_52_594523557 28.717963819020696 999567 where is alepotrypa cave 1 msmarco_passage_27_793251800 37.300572071077674 999567 where is alepotrypa cave 2 msmarco_passage_17_792293448 35.685501449852545 999567 where is alepotrypa cave 3 msmarco_passage_17_792288289 30.427692559624006 999567 where is alepotrypa cave 4 msmarco_passage_09_473431990 28.950506477630398 999567 where is alepotrypa cave 5 msmarco_passage_17_792292127 26.217027641324265 999567 where is alepotrypa cave 6 msmarco_passage_05_408060647 25.914378376041327 999567 where is alepotrypa cave 7 msmarco_passage_05_408062024 25.88322052042605 999567 where is alepotrypa cave 8 msmarco_passage_38_358091931 25.402496395408903 999567 where is alepotrypa cave 9 msmarco_passage_05_408064372 25.01392031504264 999567 where is alepotrypa cave 10 msmarco_passage_09_473433727 24.271373210961436 1087309 what are smacks 1 msmarco_passage_25_413844905 18.73558882331683 1087309 what are smacks 2 msmarco_passage_25_413840556 18.561105170229112 1087309 what are smacks 3 msmarco_passage_25_413838463 18.446577103162852 1087309 what are smacks 4 msmarco_passage_25_374696279 18.29293911941963 1087309 what are smacks 5 msmarco_passage_25_413837621 17.50561433353197 1087309 what are smacks 6 msmarco_passage_25_413834907 17.50561433353197 1087309 what are smacks 7 msmarco_passage_38_583212226 17.03814646740785 1087309 what are smacks 8 msmarco_passage_53_836139089 17.009288428537616 1087309 what are smacks 9 msmarco_passage_20_807983120 16.653962176236085 1087309 what are smacks 10 msmarco_passage_12_607294962 16.38319744834522 160339 do you have to collect copays 1 msmarco_passage_67_56070328 20.75145574203097 160339 do you have to collect copays 2 msmarco_passage_67_56069902 20.55095111927999 160339 do you have to collect copays 3 msmarco_passage_08_61608997 20.34918487195106 160339 do you have to collect copays 4 msmarco_passage_48_279609480 19.714455870621528 160339 do you have to collect copays 5 msmarco_passage_02_558029929 19.567262137836067 160339 do you have to collect copays 6 msmarco_passage_01_498069485 19.39804122258562 160339 do you have to collect copays 7 msmarco_passage_24_29779573 19.39804122258562 160339 do you have to collect copays 8 msmarco_passage_65_635888249 19.305033858807224 160339 do you have to collect copays 9 msmarco_passage_48_490715531 19.10531782346469 160339 do you have to collect copays 10 msmarco_passage_30_312103348 19.044546840993988 695238 what is a podcast livestream 1 msmarco_passage_62_6865356 21.807880265657257 695238 what is a podcast livestream 2 msmarco_passage_56_663185709 21.49952979866346 695238 what is a podcast livestream 3 msmarco_passage_31_117227388 21.49952979866346 695238 what is a podcast livestream 4 msmarco_passage_35_337925442 20.834346534730784 695238 what is a podcast livestream 5 msmarco_passage_32_636041411 20.625610716571703 695238 what is a podcast livestream 6 msmarco_passage_30_56844139 20.384699708929904 695238 what is a podcast livestream 7 msmarco_passage_30_56841485 20.384699708929904 695238 what is a podcast livestream 8 msmarco_passage_09_236891271 20.147670720843237 695238 what is a podcast livestream 9 msmarco_passage_59_382870901 20.147670720843237 695238 what is a podcast livestream 10 msmarco_passage_31_31420553 20.147670720843237 471983 partner enablement definition 1 msmarco_passage_41_302651185 22.445954386935178 471983 partner enablement definition 2 msmarco_passage_29_477935601 17.842447341327524 471983 partner enablement definition 3 msmarco_passage_52_249550884 17.42140724037873 471983 partner enablement definition 4 msmarco_passage_46_425603620 17.335465615913492 471983 partner enablement definition 5 msmarco_passage_18_584344955 17.335465615913492 471983 partner enablement definition 6 msmarco_passage_13_114736389 17.061920311687036 471983 partner enablement definition 7 msmarco_passage_55_233836741 16.870034614743535 471983 partner enablement definition 8 msmarco_passage_65_191421179 16.81999667726201 471983 partner enablement definition 9 msmarco_passage_52_249551158 16.663425922476392 471983 partner enablement definition 10 msmarco_passage_66_133895153 16.55602908431304 208265 how big are table tennis tops 1 msmarco_passage_32_651233467 26.012000915532976 208265 how big are table tennis tops 2 msmarco_passage_57_204529078 25.99810263517058 208265 how big are table tennis tops 3 msmarco_passage_33_495838566 25.459631469711923 208265 how big are table tennis tops 4 msmarco_passage_33_535569520 25.3453120627993 208265 how big are table tennis tops 5 msmarco_passage_33_535568545 25.14972002452454 208265 how big are table tennis tops 6 msmarco_passage_39_169616298 24.713635279476943 208265 how big are table tennis tops 7 msmarco_passage_57_204533676 24.542421038960395 208265 how big are table tennis tops 8 msmarco_passage_37_706059185 24.472536913056835 208265 how big are table tennis tops 9 msmarco_passage_32_651238186 24.16940626955804 208265 how big are table tennis tops 10 msmarco_passage_39_169604913 24.16013459253717 1100937 echo alexa what does it do 1 msmarco_passage_37_864000271 26.35488227866122 1100937 echo alexa what does it do 2 msmarco_passage_32_391021483 26.220989280569672 1100937 echo alexa what does it do 3 msmarco_passage_37_694904239 26.19335226324142 1100937 echo alexa what does it do 4 msmarco_passage_37_699357515 25.963517339715708 1100937 echo alexa what does it do 5 msmarco_passage_53_446113763 25.815025389064907 1100937 echo alexa what does it do 6 msmarco_passage_34_718272883 25.709001517239034 1100937 echo alexa what does it do 7 msmarco_passage_62_264756565 25.612695428366962 1100937 echo alexa what does it do 8 msmarco_passage_34_718273802 25.603226500852593 1100937 echo alexa what does it do 9 msmarco_passage_41_767279849 25.347545995906458 1100937 echo alexa what does it do 10 msmarco_passage_24_679585742 25.266472811047784 303045 how much can a hoverboard cost 1 msmarco_passage_47_128804048 29.611223553326163 303045 how much can a hoverboard cost 2 msmarco_passage_49_587599262 29.048505972407806 303045 how much can a hoverboard cost 3 msmarco_passage_43_235006181 26.758034321784628 303045 how much can a hoverboard cost 4 msmarco_passage_29_665450578 26.24876173542977 303045 how much can a hoverboard cost 5 msmarco_passage_43_235007619 25.10507844057332 303045 how much can a hoverboard cost 6 msmarco_passage_49_703502125 24.81494923856326 303045 how much can a hoverboard cost 7 msmarco_passage_22_594484769 24.485734198986908 303045 how much can a hoverboard cost 8 msmarco_passage_25_104060557 23.56214675968424 303045 how much can a hoverboard cost 9 msmarco_passage_39_211874948 23.29685014904846 303045 how much can a hoverboard cost 10 msmarco_passage_43_235004810 23.256395812708078 994941 where is the centromere in replicated chromosomes 1 msmarco_passage_31_606345610 39.3191105118764 994941 where is the centromere in replicated chromosomes 2 msmarco_passage_29_590172866 38.41898437074095 994941 where is the centromere in replicated chromosomes 3 msmarco_passage_31_606342238 38.142153808863206 994941 where is the centromere in replicated chromosomes 4 msmarco_passage_30_715516782 37.4238150716849 994941 where is the centromere in replicated chromosomes 5 msmarco_passage_30_753465029 36.93231252966717 994941 where is the centromere in replicated chromosomes 6 msmarco_passage_30_715517929 36.81121665501797 994941 where is the centromere in replicated chromosomes 7 msmarco_passage_51_700705413 36.732899192784835 994941 where is the centromere in replicated chromosomes 8 msmarco_passage_51_699447910 36.732899192784835 994941 where is the centromere in replicated chromosomes 9 msmarco_passage_05_662830379 36.580653556036964 994941 where is the centromere in replicated chromosomes 10 msmarco_passage_43_529713823 36.580653556036964 1079535 what grade on final do i need 1 msmarco_passage_40_517053169 21.249313122331355 1079535 what grade on final do i need 2 msmarco_passage_23_563569069 21.01854398281597 1079535 what grade on final do i need 3 msmarco_passage_62_772300020 20.817204349512025 1079535 what grade on final do i need 4 msmarco_passage_47_534586562 20.810822204740973 1079535 what grade on final do i need 5 msmarco_passage_52_376983950 20.6201411867634 1079535 what grade on final do i need 6 msmarco_passage_03_265199708 20.555734593589285 1079535 what grade on final do i need 7 msmarco_passage_47_534587581 20.487671575255643 1079535 what grade on final do i need 8 msmarco_passage_58_718713690 20.358664912906782 1079535 what grade on final do i need 9 msmarco_passage_23_563143271 20.34195111158372 1079535 what grade on final do i need 10 msmarco_passage_68_676387036 20.32049653347433 46095 average time for hair to grow 1 msmarco_passage_00_351917831 22.904466817207595 46095 average time for hair to grow 2 msmarco_passage_40_406374986 22.765726861142237 46095 average time for hair to grow 3 msmarco_passage_11_350462917 22.47104573549827 46095 average time for hair to grow 4 msmarco_passage_40_385979612 22.103222599647427 46095 average time for hair to grow 5 msmarco_passage_06_690539878 21.952589707655015 46095 average time for hair to grow 6 msmarco_passage_36_215663844 21.932006234319747 46095 average time for hair to grow 7 msmarco_passage_40_477787932 21.71071888735461 46095 average time for hair to grow 8 msmarco_passage_23_447690279 21.700037752969962 46095 average time for hair to grow 9 msmarco_passage_23_794392599 21.493293300274182 46095 average time for hair to grow 10 msmarco_passage_58_151188876 21.480702012017375 1085422 what county is new london tx 1 msmarco_passage_49_381560578 27.7050989603384 1085422 what county is new london tx 2 msmarco_passage_35_171795858 26.968858739646706 1085422 what county is new london tx 3 msmarco_passage_05_250074468 25.721526507065622 1085422 what county is new london tx 4 msmarco_passage_15_391341358 25.003184592565475 1085422 what county is new london tx 5 msmarco_passage_09_801650772 24.697830673447804 1085422 what county is new london tx 6 msmarco_passage_05_305788621 24.183487272473908 1085422 what county is new london tx 7 msmarco_passage_46_801995708 24.136755594905956 1085422 what county is new london tx 8 msmarco_passage_05_305784562 23.466524792153702 1085422 what county is new london tx 9 msmarco_passage_59_31002509 23.425693289903556 1085422 what county is new london tx 10 msmarco_passage_09_76587827 23.372799290440987 996414 where was linda louise kaufman born 1 msmarco_passage_47_446983840 28.24150542225817 996414 where was linda louise kaufman born 2 msmarco_passage_36_280036796 27.038927010859208 996414 where was linda louise kaufman born 3 msmarco_passage_14_18418750 26.926710158879317 996414 where was linda louise kaufman born 4 msmarco_passage_02_61461321 26.721134082115658 996414 where was linda louise kaufman born 5 msmarco_passage_14_18412867 26.319988262495166 996414 where was linda louise kaufman born 6 msmarco_passage_19_629142858 25.822610620529296 996414 where was linda louise kaufman born 7 msmarco_passage_63_47079864 25.371064704518318 996414 where was linda louise kaufman born 8 msmarco_passage_28_110618245 25.17901783709167 996414 where was linda louise kaufman born 9 msmarco_passage_47_446985535 25.131008123485067 996414 where was linda louise kaufman born 10 msmarco_passage_43_655917251 25.066066902301586 998093 where is the trailer park boys set 1 msmarco_passage_55_224103248 31.86791390456539 998093 where is the trailer park boys set 2 msmarco_passage_21_344655148 28.623458326053438 998093 where is the trailer park boys set 3 msmarco_passage_21_344644020 28.623458326053438 998093 where is the trailer park boys set 4 msmarco_passage_21_64726889 28.623458326053438 998093 where is the trailer park boys set 5 msmarco_passage_61_847499313 28.469152131202534 998093 where is the trailer park boys set 6 msmarco_passage_62_795875532 27.93375416767186 998093 where is the trailer park boys set 7 msmarco_passage_59_131181909 27.641941126305106 998093 where is the trailer park boys set 8 msmarco_passage_58_385564666 27.585358164147856 998093 where is the trailer park boys set 9 msmarco_passage_61_847498604 27.51195006143438 998093 where is the trailer park boys set 10 msmarco_passage_59_131182243 27.370368287547183 1021446 which transaction is a deposit 1 msmarco_passage_44_525944524 19.911204426631826 1021446 which transaction is a deposit 2 msmarco_passage_11_140349594 19.57832024007567 1021446 which transaction is a deposit 3 msmarco_passage_30_565286789 19.33389393198642 1021446 which transaction is a deposit 4 msmarco_passage_46_296977498 19.31845682966444 1021446 which transaction is a deposit 5 msmarco_passage_26_310204763 19.317591983557122 1021446 which transaction is a deposit 6 msmarco_passage_44_525897084 19.228119499449065 1021446 which transaction is a deposit 7 msmarco_passage_39_128941684 19.194862195831004 1021446 which transaction is a deposit 8 msmarco_passage_39_822911349 18.97784362580971 1021446 which transaction is a deposit 9 msmarco_passage_22_543569887 18.940587600309975 1021446 which transaction is a deposit 10 msmarco_passage_51_63989744 18.784209140345194 9926 about how many fans does mexico soccer team 1 msmarco_passage_61_627583998 29.887553996456802 9926 about how many fans does mexico soccer team 2 msmarco_passage_07_567934211 29.32698061744721 9926 about how many fans does mexico soccer team 3 msmarco_passage_04_121368109 28.37111628733753 9926 about how many fans does mexico soccer team 4 msmarco_passage_52_76303216 27.99682883505836 9926 about how many fans does mexico soccer team 5 msmarco_passage_61_627582436 27.82066754353835 9926 about how many fans does mexico soccer team 6 msmarco_passage_05_731095271 27.2070124867741 9926 about how many fans does mexico soccer team 7 msmarco_passage_63_349689720 27.043967662842025 9926 about how many fans does mexico soccer team 8 msmarco_passage_61_63745505 26.803194218001426 9926 about how many fans does mexico soccer team 9 msmarco_passage_09_550523410 26.803194218001426 9926 about how many fans does mexico soccer team 10 msmarco_passage_27_351379838 26.746956978062393 1051475 what is mca and get weekly paychecks fake? 1 msmarco_passage_25_150456224 35.64921653627234 1051475 what is mca and get weekly paychecks fake? 2 msmarco_passage_12_271828222 33.770529311316906 1051475 what is mca and get weekly paychecks fake? 3 msmarco_passage_25_150456520 33.307466376999436 1051475 what is mca and get weekly paychecks fake? 4 msmarco_passage_02_489014746 32.42460143171573 1051475 what is mca and get weekly paychecks fake? 5 msmarco_passage_02_489015145 28.295689848569683 1051475 what is mca and get weekly paychecks fake? 6 msmarco_passage_04_119181415 23.5553639426488 1051475 what is mca and get weekly paychecks fake? 7 msmarco_passage_26_626172716 23.53473051181793 1051475 what is mca and get weekly paychecks fake? 8 msmarco_passage_12_271828649 23.215983761334044 1051475 what is mca and get weekly paychecks fake? 9 msmarco_passage_00_853676319 23.019972135566398 1051475 what is mca and get weekly paychecks fake? 10 msmarco_passage_00_853679582 22.838060814555774 801907 what is table associated for draw originals in sap 1 msmarco_passage_09_399690400 24.90110872587139 801907 what is table associated for draw originals in sap 2 msmarco_passage_31_501191686 23.673664699790464 801907 what is table associated for draw originals in sap 3 msmarco_passage_31_500870142 23.49431464180985 801907 what is table associated for draw originals in sap 4 msmarco_passage_31_500819137 23.329467788497574 801907 what is table associated for draw originals in sap 5 msmarco_passage_31_500962244 23.324019691959162 801907 what is table associated for draw originals in sap 6 msmarco_passage_31_500886236 23.16133686038256 801907 what is table associated for draw originals in sap 7 msmarco_passage_31_501065431 23.11435440274325 801907 what is table associated for draw originals in sap 8 msmarco_passage_31_500945168 23.084742342517742 801907 what is table associated for draw originals in sap 9 msmarco_passage_31_500931857 23.041767641435875 801907 what is table associated for draw originals in sap 10 msmarco_passage_31_501229924 22.93442180871255 837372 what is the official site for free credit reports 1 msmarco_passage_38_45835152 29.39385535260503 837372 what is the official site for free credit reports 2 msmarco_passage_67_921545084 29.12326254333436 837372 what is the official site for free credit reports 3 msmarco_passage_62_620862653 29.045361690797993 837372 what is the official site for free credit reports 4 msmarco_passage_47_635128358 28.83312770313918 837372 what is the official site for free credit reports 5 msmarco_passage_38_347980758 28.58880614343041 837372 what is the official site for free credit reports 6 msmarco_passage_42_654557793 28.297885104991188 837372 what is the official site for free credit reports 7 msmarco_passage_42_654548092 28.297885104991188 837372 what is the official site for free credit reports 8 msmarco_passage_42_654896697 28.297885104991188 837372 what is the official site for free credit reports 9 msmarco_passage_42_654334580 28.297885104991188 837372 what is the official site for free credit reports 10 msmarco_passage_42_654446405 28.297885104991188 99556 copperhead snake bite effects 1 msmarco_passage_25_264776618 38.603449024995655 99556 copperhead snake bite effects 2 msmarco_passage_12_28027261 38.32455030897924 99556 copperhead snake bite effects 3 msmarco_passage_10_529067710 38.270954130440785 99556 copperhead snake bite effects 4 msmarco_passage_25_264774662 37.09433246535454 99556 copperhead snake bite effects 5 msmarco_passage_12_16771948 36.96142996815669 99556 copperhead snake bite effects 6 msmarco_passage_26_312284206 36.83546516322701 99556 copperhead snake bite effects 7 msmarco_passage_66_162595897 36.62163998638559 99556 copperhead snake bite effects 8 msmarco_passage_29_148495131 36.13733248320163 99556 copperhead snake bite effects 9 msmarco_passage_61_28216376 35.97395776983754 99556 copperhead snake bite effects 10 msmarco_passage_12_31350863 35.915020081623084 285537 how many horsepower needed to plane a flat bottom boat 1 msmarco_passage_01_331499805 39.610096196069904 285537 how many horsepower needed to plane a flat bottom boat 2 msmarco_passage_01_331501876 37.55104794340079 285537 how many horsepower needed to plane a flat bottom boat 3 msmarco_passage_14_15039584 34.828335718390065 285537 how many horsepower needed to plane a flat bottom boat 4 msmarco_passage_01_331503254 34.465925066049124 285537 how many horsepower needed to plane a flat bottom boat 5 msmarco_passage_34_824288656 33.97512759514698 285537 how many horsepower needed to plane a flat bottom boat 6 msmarco_passage_46_427192271 32.98896643803535 285537 how many horsepower needed to plane a flat bottom boat 7 msmarco_passage_55_671410227 32.72600003398627 285537 how many horsepower needed to plane a flat bottom boat 8 msmarco_passage_01_336492967 31.83377092698649 285537 how many horsepower needed to plane a flat bottom boat 9 msmarco_passage_46_427077310 31.461955458196506 285537 how many horsepower needed to plane a flat bottom boat 10 msmarco_passage_46_427043867 31.158339156769078 1058442 what is cmc program 1 msmarco_passage_07_186914499 22.56519277830963 1058442 what is cmc program 2 msmarco_passage_50_249891252 22.56519277830963 1058442 what is cmc program 3 msmarco_passage_17_43625220 21.566050600848122 1058442 what is cmc program 4 msmarco_passage_30_320187402 21.061605737878015 1058442 what is cmc program 5 msmarco_passage_12_603963624 20.836215380271724 1058442 what is cmc program 6 msmarco_passage_18_262410530 20.11278450200781 1058442 what is cmc program 7 msmarco_passage_00_601148279 19.933328394890218 1058442 what is cmc program 8 msmarco_passage_28_177128467 19.629405375161213 1058442 what is cmc program 9 msmarco_passage_01_426486212 19.5436836138828 1058442 what is cmc program 10 msmarco_passage_00_820400724 19.5436836138828 1101535 diff types of fuses 1 msmarco_passage_45_154600659 20.418705851769833 1101535 diff types of fuses 2 msmarco_passage_48_749115573 20.093299042567804 1101535 diff types of fuses 3 msmarco_passage_53_20452713 19.92433570436841 1101535 diff types of fuses 4 msmarco_passage_45_86536962 19.78763998196473 1101535 diff types of fuses 5 msmarco_passage_06_94820784 19.718419523405185 1101535 diff types of fuses 6 msmarco_passage_42_39125954 19.69579570462922 1101535 diff types of fuses 7 msmarco_passage_32_754505970 19.624725284619664 1101535 diff types of fuses 8 msmarco_passage_15_33000775 19.4777778469565 1101535 diff types of fuses 9 msmarco_passage_45_107856448 19.406990462482135 1101535 diff types of fuses 10 msmarco_passage_45_107854739 19.38620485678652 1067764 what is a sales cycle process? 1 msmarco_passage_13_654777857 21.847612267162226 1067764 what is a sales cycle process? 2 msmarco_passage_03_697520948 21.359210519348686 1067764 what is a sales cycle process? 3 msmarco_passage_52_779496775 21.136813498432133 1067764 what is a sales cycle process? 4 msmarco_passage_57_617770967 21.136813498432133 1067764 what is a sales cycle process? 5 msmarco_passage_46_634423153 21.007365255782467 1067764 what is a sales cycle process? 6 msmarco_passage_26_166679772 20.962721578457987 1067764 what is a sales cycle process? 7 msmarco_passage_26_494735728 20.79187428635835 1067764 what is a sales cycle process? 8 msmarco_passage_53_204805063 20.79187428635835 1067764 what is a sales cycle process? 9 msmarco_passage_59_629993092 20.76506379420391 1067764 what is a sales cycle process? 10 msmarco_passage_08_470074767 20.735613815230117 49802 belizean cuisine 1 msmarco_passage_17_848762665 27.208914487964392 49802 belizean cuisine 2 msmarco_passage_04_475620085 24.564040563833593 49802 belizean cuisine 3 msmarco_passage_59_695961757 23.75443276221856 49802 belizean cuisine 4 msmarco_passage_39_111262643 22.998141920187415 49802 belizean cuisine 5 msmarco_passage_39_111235667 22.28991007334067 49802 belizean cuisine 6 msmarco_passage_19_712672997 22.025759161408818 49802 belizean cuisine 7 msmarco_passage_18_121282839 21.72762494249121 49802 belizean cuisine 8 msmarco_passage_66_332263067 21.509748688353334 49802 belizean cuisine 9 msmarco_passage_56_432624143 20.745560081433002 49802 belizean cuisine 10 msmarco_passage_33_845631664 20.55570881111817 458235 most knockouts in boxing 1 msmarco_passage_63_712273894 23.759865587651895 458235 most knockouts in boxing 2 msmarco_passage_45_31914575 23.154659007014956 458235 most knockouts in boxing 3 msmarco_passage_49_670915860 22.859485509388577 458235 most knockouts in boxing 4 msmarco_passage_45_32023723 22.732627650868338 458235 most knockouts in boxing 5 msmarco_passage_49_781658833 22.71242082467878 458235 most knockouts in boxing 6 msmarco_passage_63_690405489 22.668447790231426 458235 most knockouts in boxing 7 msmarco_passage_61_351434925 22.189506529056004 458235 most knockouts in boxing 8 msmarco_passage_45_31112629 22.162247564210404 458235 most knockouts in boxing 9 msmarco_passage_39_683300087 22.0039391690558 458235 most knockouts in boxing 10 msmarco_passage_61_351433226 21.802070171936265 1000459 where do black widow spiders live in the us? 1 msmarco_passage_62_511049585 36.69874828919266 1000459 where do black widow spiders live in the us? 2 msmarco_passage_56_74166269 35.99369797092612 1000459 where do black widow spiders live in the us? 3 msmarco_passage_53_588634191 35.99369797092612 1000459 where do black widow spiders live in the us? 4 msmarco_passage_11_371592655 35.245761560966386 1000459 where do black widow spiders live in the us? 5 msmarco_passage_44_176731444 35.06135362110064 1000459 where do black widow spiders live in the us? 6 msmarco_passage_16_799532830 35.06135362110064 1000459 where do black widow spiders live in the us? 7 msmarco_passage_04_226868933 34.94915178324902 1000459 where do black widow spiders live in the us? 8 msmarco_passage_29_371109503 34.89110254007177 1000459 where do black widow spiders live in the us? 9 msmarco_passage_29_686705389 34.66357333886216 1000459 where do black widow spiders live in the us? 10 msmarco_passage_02_577974156 34.66357333886216 1089804 the largest wide area network (wan) in existence is the _____. 1 msmarco_passage_61_545645780 42.103256883767536 1089804 the largest wide area network (wan) in existence is the _____. 2 msmarco_passage_33_317842362 39.99835985378361 1089804 the largest wide area network (wan) in existence is the _____. 3 msmarco_passage_62_325785456 39.31065597316386 1089804 the largest wide area network (wan) in existence is the _____. 4 msmarco_passage_41_575135222 37.823362594759686 1089804 the largest wide area network (wan) in existence is the _____. 5 msmarco_passage_17_162160112 36.94275471622877 1089804 the largest wide area network (wan) in existence is the _____. 6 msmarco_passage_42_215106462 36.58685098697474 1089804 the largest wide area network (wan) in existence is the _____. 7 msmarco_passage_28_545833646 36.33638119483156 1089804 the largest wide area network (wan) in existence is the _____. 8 msmarco_passage_42_215105008 36.26049317530002 1089804 the largest wide area network (wan) in existence is the _____. 9 msmarco_passage_30_786633050 35.66291213199211 1089804 the largest wide area network (wan) in existence is the _____. 10 msmarco_passage_50_21171368 35.10787513442796 174592 dogs that are good with kids 1 msmarco_passage_12_638413335 22.16803207866075 174592 dogs that are good with kids 2 msmarco_passage_10_540769780 21.037113311707706 174592 dogs that are good with kids 3 msmarco_passage_14_484434262 20.532798954345648 174592 dogs that are good with kids 4 msmarco_passage_10_540768804 20.39335085958657 174592 dogs that are good with kids 5 msmarco_passage_58_464507068 20.3294373103206 174592 dogs that are good with kids 6 msmarco_passage_10_540768519 20.20172618371412 174592 dogs that are good with kids 7 msmarco_passage_14_854872846 19.8765130969216 174592 dogs that are good with kids 8 msmarco_passage_29_713455645 19.727701193825496 174592 dogs that are good with kids 9 msmarco_passage_14_854871768 19.663421227021665 174592 dogs that are good with kids 10 msmarco_passage_08_801457932 19.566436246796826 528760 types of metal plating 1 msmarco_passage_36_380350411 23.374284296963182 528760 types of metal plating 2 msmarco_passage_42_405292663 22.28063432837576 528760 types of metal plating 3 msmarco_passage_04_354477733 21.552656647844824 528760 types of metal plating 4 msmarco_passage_14_390870406 21.355194160122657 528760 types of metal plating 5 msmarco_passage_60_90053664 21.26074325384249 528760 types of metal plating 6 msmarco_passage_58_370840978 21.225706440165148 528760 types of metal plating 7 msmarco_passage_43_446983324 21.187913207037038 528760 types of metal plating 8 msmarco_passage_05_612810785 21.187913207037038 528760 types of metal plating 9 msmarco_passage_10_557627785 21.145927282277 528760 types of metal plating 10 msmarco_passage_32_57633837 20.899919522120463 65583 can chewing gum cause ulcers 1 msmarco_passage_07_390492036 34.68562205185284 65583 can chewing gum cause ulcers 2 msmarco_passage_16_869806383 29.56950002316148 65583 can chewing gum cause ulcers 3 msmarco_passage_00_834369290 29.56950002316148 65583 can chewing gum cause ulcers 4 msmarco_passage_06_730381460 28.845441961419414 65583 can chewing gum cause ulcers 5 msmarco_passage_48_338607831 28.793750647848576 65583 can chewing gum cause ulcers 6 msmarco_passage_50_554415150 28.46358872462996 65583 can chewing gum cause ulcers 7 msmarco_passage_48_636104214 28.106471885503744 65583 can chewing gum cause ulcers 8 msmarco_passage_24_188458625 28.106471885503744 65583 can chewing gum cause ulcers 9 msmarco_passage_44_208973700 27.88184413766801 65583 can chewing gum cause ulcers 10 msmarco_passage_55_413561351 27.737714029203488 970242 where do I find drivers on windows 10 1 msmarco_passage_38_340926979 25.72236108148287 970242 where do I find drivers on windows 10 2 msmarco_passage_43_227732278 25.03032448088821 970242 where do I find drivers on windows 10 3 msmarco_passage_43_232024346 25.03032448088821 970242 where do I find drivers on windows 10 4 msmarco_passage_43_232023949 24.918335338666587 970242 where do I find drivers on windows 10 5 msmarco_passage_44_220202751 24.86426440025345 970242 where do I find drivers on windows 10 6 msmarco_passage_37_114617414 24.821474630685152 970242 where do I find drivers on windows 10 7 msmarco_passage_33_787171366 24.572216540373432 970242 where do I find drivers on windows 10 8 msmarco_passage_38_146550088 24.435612589005324 970242 where do I find drivers on windows 10 9 msmarco_passage_00_178178202 24.20780844583803 970242 where do I find drivers on windows 10 10 msmarco_passage_12_129550008 24.20780844583803 1096475 how many years of college do you need to be a park ranger 1 msmarco_passage_08_717054254 27.599226028899576 1096475 how many years of college do you need to be a park ranger 2 msmarco_passage_56_510612103 27.599226028899576 1096475 how many years of college do you need to be a park ranger 3 msmarco_passage_06_83743762 27.55030184174863 1096475 how many years of college do you need to be a park ranger 4 msmarco_passage_00_571365407 27.55030184174863 1096475 how many years of college do you need to be a park ranger 5 msmarco_passage_62_799289800 27.55030184174863 1096475 how many years of college do you need to be a park ranger 6 msmarco_passage_39_72019551 27.544441078083075 1096475 how many years of college do you need to be a park ranger 7 msmarco_passage_08_717030127 27.46497786767864 1096475 how many years of college do you need to be a park ranger 8 msmarco_passage_56_510347290 27.46497786767864 1096475 how many years of college do you need to be a park ranger 9 msmarco_passage_56_510689468 26.545954732780906 1096475 how many years of college do you need to be a park ranger 10 msmarco_passage_56_506004585 26.089186995130373 1051571 what is marilyn manson music called 1 msmarco_passage_04_191507789 38.633421763110384 1051571 what is marilyn manson music called 2 msmarco_passage_63_22572760 38.046122412519665 1051571 what is marilyn manson music called 3 msmarco_passage_50_378824607 37.777076866232754 1051571 what is marilyn manson music called 4 msmarco_passage_25_797973913 35.94141992430089 1051571 what is marilyn manson music called 5 msmarco_passage_35_356209634 35.696593285402166 1051571 what is marilyn manson music called 6 msmarco_passage_07_201884111 34.70860203250325 1051571 what is marilyn manson music called 7 msmarco_passage_50_307447198 34.70860203250325 1051571 what is marilyn manson music called 8 msmarco_passage_07_201883640 34.359865623892844 1051571 what is marilyn manson music called 9 msmarco_passage_50_307446725 34.359865623892844 1051571 what is marilyn manson music called 10 msmarco_passage_53_88598978 34.29356452006264 61531 campagnolo bikes 1 msmarco_passage_42_854609499 28.821600239400432 61531 campagnolo bikes 2 msmarco_passage_29_586567673 26.982223381266863 61531 campagnolo bikes 3 msmarco_passage_18_270269495 26.712315118703437 61531 campagnolo bikes 4 msmarco_passage_50_122510673 26.18838051303773 61531 campagnolo bikes 5 msmarco_passage_50_122510219 25.684603416118055 61531 campagnolo bikes 6 msmarco_passage_18_270280973 25.139965562928005 61531 campagnolo bikes 7 msmarco_passage_39_447761715 24.964259771268182 61531 campagnolo bikes 8 msmarco_passage_07_675967606 24.410703002711884 61531 campagnolo bikes 9 msmarco_passage_18_270285307 24.301669241626843 61531 campagnolo bikes 10 msmarco_passage_18_270274208 24.301669241626843 810324 what is the cause of myoma? 1 msmarco_passage_53_753930511 26.572215345146475 810324 what is the cause of myoma? 2 msmarco_passage_25_814595045 25.37272695837926 810324 what is the cause of myoma? 3 msmarco_passage_53_753932355 25.232278577380164 810324 what is the cause of myoma? 4 msmarco_passage_25_473726402 25.210238463229324 810324 what is the cause of myoma? 5 msmarco_passage_06_343534390 24.295773062795455 810324 what is the cause of myoma? 6 msmarco_passage_25_814595422 24.291800268807116 810324 what is the cause of myoma? 7 msmarco_passage_53_753928215 23.609272234733194 1100010 hemelse modder 1 msmarco_passage_50_10348394 40.29518160453387 810324 what is the cause of myoma? 8 msmarco_passage_25_473731212 22.851301985107515 810324 what is the cause of myoma? 9 msmarco_passage_31_637217780 22.513635963497876 810324 what is the cause of myoma? 10 msmarco_passage_25_473728138 22.288427822529325 448035 meaning of interval 1 msmarco_passage_33_392924370 18.227394932193906 448035 meaning of interval 2 msmarco_passage_51_694269474 18.083202751133392 448035 meaning of interval 3 msmarco_passage_33_392936479 18.07648889516924 448035 meaning of interval 4 msmarco_passage_03_211891931 18.07648889516924 448035 meaning of interval 5 msmarco_passage_51_694280079 17.906568127267057 448035 meaning of interval 6 msmarco_passage_27_370167380 17.820885096470636 448035 meaning of interval 7 msmarco_passage_38_393349128 17.8034758791414 448035 meaning of interval 8 msmarco_passage_61_517705453 17.72863270811598 448035 meaning of interval 9 msmarco_passage_13_846009152 17.603789219414463 448035 meaning of interval 10 msmarco_passage_27_370164466 17.55410358951526 248086 how long do you have to go college to become nurse 1 msmarco_passage_04_98247750 26.144794002966506 248086 how long do you have to go college to become nurse 2 msmarco_passage_40_742220284 25.594317651066504 248086 how long do you have to go college to become nurse 3 msmarco_passage_36_467271511 25.136477476271672 248086 how long do you have to go college to become nurse 4 msmarco_passage_58_724073042 24.817008196991576 248086 how long do you have to go college to become nurse 5 msmarco_passage_09_138407426 24.817008196991576 248086 how long do you have to go college to become nurse 6 msmarco_passage_59_47238886 24.486538125141244 248086 how long do you have to go college to become nurse 7 msmarco_passage_04_98248179 24.453727350754182 248086 how long do you have to go college to become nurse 8 msmarco_passage_00_550548641 23.645950038241303 248086 how long do you have to go college to become nurse 9 msmarco_passage_12_251524318 23.143777149041544 248086 how long do you have to go college to become nurse 10 msmarco_passage_00_101046319 23.056177188132057 1012431 what was the company's net revenues for the year? 1 msmarco_passage_59_571639206 25.638653255184188 1012431 what was the company's net revenues for the year? 2 msmarco_passage_58_784465348 25.367155724729024 1012431 what was the company's net revenues for the year? 3 msmarco_passage_09_153167497 25.367155724729024 1012431 what was the company's net revenues for the year? 4 msmarco_passage_50_491016318 25.15996494411212 1012431 what was the company's net revenues for the year? 5 msmarco_passage_66_382548926 25.049476405277616 1012431 what was the company's net revenues for the year? 6 msmarco_passage_30_570056443 25.01475248128376 1012431 what was the company's net revenues for the year? 7 msmarco_passage_02_696832275 25.01475248128376 1012431 what was the company's net revenues for the year? 8 msmarco_passage_47_637537205 24.961473144284465 1012431 what was the company's net revenues for the year? 9 msmarco_passage_47_637611672 24.87917625376666 1012431 what was the company's net revenues for the year? 10 msmarco_passage_43_641081864 24.788344404375664 1083727 what does gyrene mean 1 msmarco_passage_63_161360043 22.086841613488662 1083727 what does gyrene mean 2 msmarco_passage_18_54901386 22.086841613488662 1083727 what does gyrene mean 3 msmarco_passage_10_501294486 21.653630916484655 1083727 what does gyrene mean 4 msmarco_passage_27_258667476 21.034767888152512 1083727 what does gyrene mean 5 msmarco_passage_07_826231961 21.034767888152512 1083727 what does gyrene mean 6 msmarco_passage_15_105466958 19.937231655682844 1083727 what does gyrene mean 7 msmarco_passage_65_600556629 17.98812004853915 1083727 what does gyrene mean 8 msmarco_passage_66_589976077 17.1498164447555 1083727 what does gyrene mean 9 msmarco_passage_57_76322234 17.1498164447555 1083727 what does gyrene mean 10 msmarco_passage_66_589987292 17.1498164447555 1083597 what does it mean centcom 1 msmarco_passage_00_714948136 25.928344091030624 1083597 what does it mean centcom 2 msmarco_passage_41_266354533 20.815013380449116 1083597 what does it mean centcom 3 msmarco_passage_41_266358566 20.49370221742631 1083597 what does it mean centcom 4 msmarco_passage_65_645112681 20.445511001265622 1083597 what does it mean centcom 5 msmarco_passage_41_266328956 20.02991425860676 1083597 what does it mean centcom 6 msmarco_passage_00_714947682 19.983273677765805 1083597 what does it mean centcom 7 msmarco_passage_41_266329338 19.879948178146833 1083597 what does it mean centcom 8 msmarco_passage_12_271282951 19.784430964872662 1083597 what does it mean centcom 9 msmarco_passage_41_266315476 19.586653497360686 1083597 what does it mean centcom 10 msmarco_passage_04_820790231 19.586653497360686 715588 what is an interlude 1 msmarco_passage_02_63715081 19.374762955236534 715588 what is an interlude 2 msmarco_passage_17_492057328 19.11855493905774 715588 what is an interlude 3 msmarco_passage_19_379259690 18.802145896875217 715588 what is an interlude 4 msmarco_passage_43_188870324 18.60267794624512 715588 what is an interlude 5 msmarco_passage_26_636709681 18.419176730457263 715588 what is an interlude 6 msmarco_passage_26_636700977 18.419176730457263 715588 what is an interlude 7 msmarco_passage_46_844849208 18.239260351788783 715588 what is an interlude 8 msmarco_passage_15_822603276 18.13425271769323 715588 what is an interlude 9 msmarco_passage_43_706078300 18.089544217756366 715588 what is an interlude 10 msmarco_passage_65_87977560 18.06282477760074 46081 average time for an appraisal 1 msmarco_passage_31_57209940 21.920369739071308 46081 average time for an appraisal 2 msmarco_passage_58_839601830 21.222921315858468 46081 average time for an appraisal 3 msmarco_passage_09_159248063 21.222921315858468 46081 average time for an appraisal 4 msmarco_passage_03_747631350 20.54081110025436 46081 average time for an appraisal 5 msmarco_passage_53_390589389 20.41654708823699 46081 average time for an appraisal 6 msmarco_passage_39_34702489 20.133655492396112 46081 average time for an appraisal 7 msmarco_passage_46_415180656 20.060440342263178 46081 average time for an appraisal 8 msmarco_passage_06_381616357 20.060440342263178 46081 average time for an appraisal 9 msmarco_passage_00_195906103 19.772030550911744 46081 average time for an appraisal 10 msmarco_passage_30_687901298 19.761367141562793 1100010 hemelse modder 2 msmarco_passage_50_10348781 30.975103959735357 1100010 hemelse modder 3 msmarco_passage_57_143720005 25.770771085354422 1100010 hemelse modder 4 msmarco_passage_08_450932604 17.275777539427107 1100010 hemelse modder 5 msmarco_passage_19_267819052 17.275777539427107 1100010 hemelse modder 6 msmarco_passage_55_383381379 17.145456241742163 1100010 hemelse modder 7 msmarco_passage_55_389007984 17.017086402656783 1100010 hemelse modder 8 msmarco_passage_28_522901913 17.017086402656783 1100010 hemelse modder 9 msmarco_passage_53_818170812 16.890624515580377 1100010 hemelse modder 10 msmarco_passage_15_333128711 16.643256942745065 1098646 how long can it take to heal a peptic ulcer 1 msmarco_passage_48_836260218 41.062433123430104 1098646 how long can it take to heal a peptic ulcer 2 msmarco_passage_44_258358157 39.90366948505652 1098646 how long can it take to heal a peptic ulcer 3 msmarco_passage_44_258364485 35.92158367669552 1098646 how long can it take to heal a peptic ulcer 4 msmarco_passage_08_129422932 35.302665887445244 1098646 how long can it take to heal a peptic ulcer 5 msmarco_passage_48_837876453 35.164035212087285 1098646 how long can it take to heal a peptic ulcer 6 msmarco_passage_07_450597005 33.747798186621466 1098646 how long can it take to heal a peptic ulcer 7 msmarco_passage_65_94415096 33.618766512275606 1098646 how long can it take to heal a peptic ulcer 8 msmarco_passage_22_746425360 33.26142856968835 1098646 how long can it take to heal a peptic ulcer 9 msmarco_passage_44_344566752 33.08550837820629 1098646 how long can it take to heal a peptic ulcer 10 msmarco_passage_10_407573568 33.02897420210814 1086385 what can be reason of excess urination 1 msmarco_passage_08_424115782 23.84293382062746 1086385 what can be reason of excess urination 2 msmarco_passage_55_285404584 23.84293382062746 1086385 what can be reason of excess urination 3 msmarco_passage_55_273405129 23.84293382062746 1086385 what can be reason of excess urination 4 msmarco_passage_02_59449297 23.538357499730893 1086385 what can be reason of excess urination 5 msmarco_passage_44_463916829 22.617863964246702 1086385 what can be reason of excess urination 6 msmarco_passage_11_709325712 22.23348202849963 1086385 what can be reason of excess urination 7 msmarco_passage_29_457304052 22.223071800262353 1086385 what can be reason of excess urination 8 msmarco_passage_44_553153932 21.87092485838952 1086385 what can be reason of excess urination 9 msmarco_passage_69_22998859 21.848865314273716 1086385 what can be reason of excess urination 10 msmarco_passage_66_164476831 21.323876000256607 687375 what is a hydromyelia 1 msmarco_passage_11_52130936 22.911243561400152 687375 what is a hydromyelia 2 msmarco_passage_56_557787178 22.90152831837151 687375 what is a hydromyelia 3 msmarco_passage_56_557789191 22.641516805356023 687375 what is a hydromyelia 4 msmarco_passage_56_557787660 22.641516805356023 687375 what is a hydromyelia 5 msmarco_passage_25_117973289 22.396626870933357 687375 what is a hydromyelia 6 msmarco_passage_56_557788684 22.387343072332087 687375 what is a hydromyelia 7 msmarco_passage_56_557788164 22.262384277247676 687375 what is a hydromyelia 8 msmarco_passage_18_320948663 22.230187115547135 687375 what is a hydromyelia 9 msmarco_passage_21_642487793 22.230187115547135 687375 what is a hydromyelia 10 msmarco_passage_21_642492803 22.230187115547135 624503 what do potatoes grow from 1 msmarco_passage_05_298885207 20.676653915829696 624503 what do potatoes grow from 2 msmarco_passage_24_736178218 19.710909878738462 624503 what do potatoes grow from 3 msmarco_passage_01_617918664 19.60978632252112 624503 what do potatoes grow from 4 msmarco_passage_24_739289507 19.60978632252112 624503 what do potatoes grow from 5 msmarco_passage_26_646321654 19.589300617453766 624503 what do potatoes grow from 6 msmarco_passage_30_380153019 19.488002776023713 624503 what do potatoes grow from 7 msmarco_passage_06_641724963 19.46604260506997 624503 what do potatoes grow from 8 msmarco_passage_30_380430225 19.41353579245731 624503 what do potatoes grow from 9 msmarco_passage_26_646320641 19.37271866893701 624503 what do potatoes grow from 10 msmarco_passage_47_248166346 19.325233743859695 849596 what is the synonym for prudent 1 msmarco_passage_09_778353401 24.35146259457158 849596 what is the synonym for prudent 2 msmarco_passage_15_95858017 23.416071041292795 849596 what is the synonym for prudent 3 msmarco_passage_66_296126643 22.926618745026758 849596 what is the synonym for prudent 4 msmarco_passage_07_768944081 22.820332744030278 849596 what is the synonym for prudent 5 msmarco_passage_63_638116916 22.54556319065569 849596 what is the synonym for prudent 6 msmarco_passage_53_857182900 22.24330888424094 849596 what is the synonym for prudent 7 msmarco_passage_09_778353821 22.143877033619905 849596 what is the synonym for prudent 8 msmarco_passage_65_208854619 22.045193276966977 849596 what is the synonym for prudent 9 msmarco_passage_13_801258889 22.045193276966977 849596 what is the synonym for prudent 10 msmarco_passage_62_29103745 21.86961361009186 798945 what is spotting 1 msmarco_passage_02_161674795 10.262156762731692 798945 what is spotting 2 msmarco_passage_25_395292623 10.186067567300833 798945 what is spotting 3 msmarco_passage_43_465185645 10.182607756480685 798945 what is spotting 4 msmarco_passage_56_824637205 10.157523620361026 798945 what is spotting 5 msmarco_passage_25_425024045 10.120543660002015 798945 what is spotting 6 msmarco_passage_25_393197289 10.120543660002015 798945 what is spotting 7 msmarco_passage_26_305163535 10.120543660002015 798945 what is spotting 8 msmarco_passage_68_80868798 10.08979446529081 798945 what is spotting 9 msmarco_passage_07_848199974 10.087369424795911 798945 what is spotting 10 msmarco_passage_05_626727234 10.063546071883808 57411 business identification number bin 1 msmarco_passage_56_154032595 31.1997524774841 57411 business identification number bin 2 msmarco_passage_18_241235971 30.40231811146375 57411 business identification number bin 3 msmarco_passage_23_809307313 29.88549994949161 57411 business identification number bin 4 msmarco_passage_32_540907683 29.3575564721858 57411 business identification number bin 5 msmarco_passage_23_237764122 27.980207801939486 57411 business identification number bin 6 msmarco_passage_13_363863586 27.43909872326013 107812 cost of us passport 10 msmarco_passage_27_33194239 21.38045311127897 57411 business identification number bin 7 msmarco_passage_56_650682663 27.218236853103576 57411 business identification number bin 8 msmarco_passage_56_292521005 27.211245061326508 57411 business identification number bin 9 msmarco_passage_12_363759360 27.14915854226692 57411 business identification number bin 10 msmarco_passage_00_60799193 27.083159298118932 1012547 what was stalin scorched earth policy 1 msmarco_passage_26_794820559 32.3633154006849 1012547 what was stalin scorched earth policy 2 msmarco_passage_49_234206470 31.81419121083387 1012547 what was stalin scorched earth policy 3 msmarco_passage_15_517702734 29.370294577678056 1012547 what was stalin scorched earth policy 4 msmarco_passage_11_718828386 28.448868856451362 1012547 what was stalin scorched earth policy 5 msmarco_passage_26_794821608 28.050276200010394 1012547 what was stalin scorched earth policy 6 msmarco_passage_51_109217654 27.979194266188465 1012547 what was stalin scorched earth policy 7 msmarco_passage_61_241477146 27.285685254529028 1012547 what was stalin scorched earth policy 8 msmarco_passage_37_823648054 27.15254481536787 1012547 what was stalin scorched earth policy 9 msmarco_passage_11_718830618 26.91881045813379 1012547 what was stalin scorched earth policy 10 msmarco_passage_03_97028441 26.91333090078992 1097195 how many electrons and protons and neutrons in gold? 1 msmarco_passage_04_109998558 47.78308555851602 1097195 how many electrons and protons and neutrons in gold? 2 msmarco_passage_12_220913137 47.59731721795841 1097195 how many electrons and protons and neutrons in gold? 3 msmarco_passage_14_126567216 47.55948726447698 1097195 how many electrons and protons and neutrons in gold? 4 msmarco_passage_04_110013289 46.98658354851611 1097195 how many electrons and protons and neutrons in gold? 5 msmarco_passage_04_108669331 46.258419314071396 1097195 how many electrons and protons and neutrons in gold? 6 msmarco_passage_04_109995793 46.258419314071396 1097195 how many electrons and protons and neutrons in gold? 7 msmarco_passage_04_207494343 46.197517312310396 1097195 how many electrons and protons and neutrons in gold? 8 msmarco_passage_04_108669650 46.197517312310396 1097195 how many electrons and protons and neutrons in gold? 9 msmarco_passage_04_109994262 42.648458982813274 1097195 how many electrons and protons and neutrons in gold? 10 msmarco_passage_04_108668353 42.648458982813274 1029617 what is tobramycin ophthalmic used for in cats 1 msmarco_passage_35_636796350 38.907477093968815 1029617 what is tobramycin ophthalmic used for in cats 2 msmarco_passage_35_636796647 38.27772344551538 1029617 what is tobramycin ophthalmic used for in cats 3 msmarco_passage_44_363084162 37.785549305818236 1029617 what is tobramycin ophthalmic used for in cats 4 msmarco_passage_56_813195938 37.06814570254191 1029617 what is tobramycin ophthalmic used for in cats 5 msmarco_passage_56_813199744 37.06814570254191 1029617 what is tobramycin ophthalmic used for in cats 6 msmarco_passage_36_634722899 36.708961051138814 1029617 what is tobramycin ophthalmic used for in cats 7 msmarco_passage_66_154725744 36.42479200662336 1029617 what is tobramycin ophthalmic used for in cats 8 msmarco_passage_45_608493861 35.86725501671189 1029617 what is tobramycin ophthalmic used for in cats 9 msmarco_passage_09_292132872 35.0877124211991 1029617 what is tobramycin ophthalmic used for in cats 10 msmarco_passage_45_176813172 35.0877124211991 75698 can windows media player play amr files 1 msmarco_passage_04_539346980 46.15651605593093 75698 can windows media player play amr files 2 msmarco_passage_04_539346400 43.76782213818467 75698 can windows media player play amr files 3 msmarco_passage_62_395603812 43.70344211209972 75698 can windows media player play amr files 4 msmarco_passage_26_770226670 42.754250824435324 75698 can windows media player play amr files 5 msmarco_passage_36_699708698 41.27095657734292 75698 can windows media player play amr files 6 msmarco_passage_21_838737519 40.775511728724396 75698 can windows media player play amr files 7 msmarco_passage_52_424284081 40.74111122931921 75698 can windows media player play amr files 8 msmarco_passage_37_281153830 39.79726595264582 75698 can windows media player play amr files 9 msmarco_passage_58_82625196 38.9804834720211 75698 can windows media player play amr files 10 msmarco_passage_36_699706808 38.80911992394723 474234 phantom pain definition 1 msmarco_passage_02_24275547 23.68414249489776 474234 phantom pain definition 2 msmarco_passage_37_713774050 23.547383396242147 474234 phantom pain definition 3 msmarco_passage_54_759834383 22.962997093107848 474234 phantom pain definition 4 msmarco_passage_25_314108119 22.889499928975813 474234 phantom pain definition 5 msmarco_passage_24_197710015 21.90397459475815 474234 phantom pain definition 6 msmarco_passage_24_191418671 21.90397459475815 474234 phantom pain definition 7 msmarco_passage_55_414510746 21.90397459475815 474234 phantom pain definition 8 msmarco_passage_04_746596728 21.80551773104548 474234 phantom pain definition 9 msmarco_passage_04_746492806 21.80551773104548 474234 phantom pain definition 10 msmarco_passage_53_344601508 21.731188952414904 1576 60x40 slab cost 1 msmarco_passage_06_786367940 21.98009630938327 1576 60x40 slab cost 2 msmarco_passage_49_299619029 21.1976143787251 1576 60x40 slab cost 3 msmarco_passage_24_589395225 20.350773280166607 1576 60x40 slab cost 4 msmarco_passage_49_299034307 20.287802096350738 1576 60x40 slab cost 5 msmarco_passage_07_19218800 20.287802096350738 1576 60x40 slab cost 6 msmarco_passage_45_639303589 20.237825933330477 1576 60x40 slab cost 7 msmarco_passage_04_171421551 20.136273165921274 1576 60x40 slab cost 8 msmarco_passage_49_387080872 19.89719774340731 1576 60x40 slab cost 9 msmarco_passage_24_589432853 19.89719774340731 1576 60x40 slab cost 10 msmarco_passage_35_364453271 19.754581688903507 107812 cost of us passport 1 msmarco_passage_08_722180872 24.703487516327023 107812 cost of us passport 2 msmarco_passage_08_722188790 24.247277064777677 107812 cost of us passport 3 msmarco_passage_15_542556800 24.18171976703131 107812 cost of us passport 4 msmarco_passage_15_540196443 22.599787829348166 107812 cost of us passport 5 msmarco_passage_31_421815423 22.200205387824443 107812 cost of us passport 6 msmarco_passage_05_363141417 21.802307154098177 107812 cost of us passport 7 msmarco_passage_56_539020845 21.644034085499293 107812 cost of us passport 8 msmarco_passage_29_485505347 21.575088468149016 107812 cost of us passport 9 msmarco_passage_08_722181344 21.400657186048043 788035 what is ptb 1 msmarco_passage_04_13210671 23.258928567565807 788035 what is ptb 2 msmarco_passage_04_13209802 22.989362961232878 788035 what is ptb 3 msmarco_passage_46_234354766 22.008037928433883 788035 what is ptb 4 msmarco_passage_20_473645835 21.826414103300955 788035 what is ptb 5 msmarco_passage_21_684992966 21.826414103300955 788035 what is ptb 6 msmarco_passage_20_478784671 21.826414103300955 788035 what is ptb 7 msmarco_passage_20_478797866 21.826414103300955 788035 what is ptb 8 msmarco_passage_55_736249481 21.528319919432132 788035 what is ptb 9 msmarco_passage_34_633870304 21.320067972092808 788035 what is ptb 10 msmarco_passage_46_234356432 21.22310371563201 404051 is beer or wine more fattening 1 msmarco_passage_02_400034630 37.471151538974766 404051 is beer or wine more fattening 2 msmarco_passage_34_855149227 32.84138543151284 404051 is beer or wine more fattening 3 msmarco_passage_34_855151515 30.60074968782067 404051 is beer or wine more fattening 4 msmarco_passage_39_696775500 28.852752107554814 404051 is beer or wine more fattening 5 msmarco_passage_12_181262991 28.613837291623184 404051 is beer or wine more fattening 6 msmarco_passage_35_328588031 27.55229155557492 404051 is beer or wine more fattening 7 msmarco_passage_02_400038122 27.417478513960283 404051 is beer or wine more fattening 8 msmarco_passage_58_145816972 26.787876174337036 404051 is beer or wine more fattening 9 msmarco_passage_12_197442871 26.40304003605192 404051 is beer or wine more fattening 10 msmarco_passage_01_407254460 26.08750077161079 608197 what county is lake harris florida 1 msmarco_passage_26_401155399 31.959806116325417 608197 what county is lake harris florida 2 msmarco_passage_52_11285653 31.933747750383162 608197 what county is lake harris florida 3 msmarco_passage_07_545580912 31.933747750383162 608197 what county is lake harris florida 4 msmarco_passage_60_393287932 29.769934891245803 608197 what county is lake harris florida 5 msmarco_passage_46_429556113 29.38893962916606 608197 what county is lake harris florida 6 msmarco_passage_52_9041771 28.37849925965133 608197 what county is lake harris florida 7 msmarco_passage_07_542989721 27.99275467591142 608197 what county is lake harris florida 8 msmarco_passage_07_543901656 27.807237200258033 608197 what county is lake harris florida 9 msmarco_passage_07_543900743 27.673228610870158 608197 what county is lake harris florida 10 msmarco_passage_69_77485317 27.643667960046855 591898 what causes sores on my head 1 msmarco_passage_50_575617866 22.327284933176294 591898 what causes sores on my head 2 msmarco_passage_24_24330235 21.602078906192673 591898 what causes sores on my head 3 msmarco_passage_53_597505637 20.80124869380444 591898 what causes sores on my head 4 msmarco_passage_08_88781199 20.80124869380444 591898 what causes sores on my head 5 msmarco_passage_08_88768991 20.80124869380444 591898 what causes sores on my head 6 msmarco_passage_08_88792249 20.80124869380444 591898 what causes sores on my head 7 msmarco_passage_50_575615102 20.64741843584332 591898 what causes sores on my head 8 msmarco_passage_41_174421067 20.620329135613975 591898 what causes sores on my head 9 msmarco_passage_53_699654109 20.592057255116128 591898 what causes sores on my head 10 msmarco_passage_64_655980718 20.259026082796225 1033092 what is the region for merl luxembourg 1 msmarco_passage_47_460607464 31.224353997242392 1033092 what is the region for merl luxembourg 2 msmarco_passage_68_215940785 25.483905667669053 1033092 what is the region for merl luxembourg 3 msmarco_passage_07_810165252 24.84019597948987 1033092 what is the region for merl luxembourg 4 msmarco_passage_08_345355935 23.925932242520496 1033092 what is the region for merl luxembourg 5 msmarco_passage_55_1757394 23.925932242520496 1033092 what is the region for merl luxembourg 6 msmarco_passage_19_141637215 22.702687306663123 1033092 what is the region for merl luxembourg 7 msmarco_passage_68_179824503 21.148666572026766 1033092 what is the region for merl luxembourg 8 msmarco_passage_10_565711299 21.068644676922702 1033092 what is the region for merl luxembourg 9 msmarco_passage_04_21521832 20.853588562316627 1033092 what is the region for merl luxembourg 10 msmarco_passage_30_173504737 20.746415176845616 30956 average annual income for rn in illinois 1 msmarco_passage_04_666484453 31.27960909937952 30956 average annual income for rn in illinois 2 msmarco_passage_40_224162921 31.27960909937952 30956 average annual income for rn in illinois 3 msmarco_passage_55_708926424 29.972446987633848 30956 average annual income for rn in illinois 4 msmarco_passage_53_281618931 29.181241384272077 30956 average annual income for rn in illinois 5 msmarco_passage_41_725127146 28.739899594704386 30956 average annual income for rn in illinois 6 msmarco_passage_10_335695105 28.023316489527907 30956 average annual income for rn in illinois 7 msmarco_passage_64_473110118 28.023316489527907 30956 average annual income for rn in illinois 8 msmarco_passage_10_335678798 27.880231256418263 30956 average annual income for rn in illinois 9 msmarco_passage_64_473100110 27.880231256418263 30956 average annual income for rn in illinois 10 msmarco_passage_61_497211769 27.687002962964613 1039195 who is the actress that played gretchen schwartz on breaking bad 1 msmarco_passage_00_437191514 47.422018691697644 1039195 who is the actress that played gretchen schwartz on breaking bad 2 msmarco_passage_19_148809993 39.97129668504975 1039195 who is the actress that played gretchen schwartz on breaking bad 3 msmarco_passage_58_693536812 38.90387267931111 1039195 who is the actress that played gretchen schwartz on breaking bad 4 msmarco_passage_58_693527704 38.90387267931111 1039195 who is the actress that played gretchen schwartz on breaking bad 5 msmarco_passage_62_571358285 36.38983500633573 1039195 who is the actress that played gretchen schwartz on breaking bad 6 msmarco_passage_18_814484867 33.66896180314005 1039195 who is the actress that played gretchen schwartz on breaking bad 7 msmarco_passage_31_668546750 31.285001628569404 1039195 who is the actress that played gretchen schwartz on breaking bad 8 msmarco_passage_64_353881036 30.82064843906756 1039195 who is the actress that played gretchen schwartz on breaking bad 9 msmarco_passage_18_208191216 30.82064843906756 1039195 who is the actress that played gretchen schwartz on breaking bad 10 msmarco_passage_03_355830279 30.36987813248325 901355 what temp to roast brussel sprouts 1 msmarco_passage_51_468263817 40.380610914486226 901355 what temp to roast brussel sprouts 2 msmarco_passage_00_121355007 40.33468036504517 901355 what temp to roast brussel sprouts 3 msmarco_passage_14_412142209 39.79267402195019 901355 what temp to roast brussel sprouts 4 msmarco_passage_09_489759463 39.79267402195019 901355 what temp to roast brussel sprouts 5 msmarco_passage_63_427598567 39.60603481627224 901355 what temp to roast brussel sprouts 6 msmarco_passage_37_443795901 39.59557873274046 901355 what temp to roast brussel sprouts 7 msmarco_passage_37_460275186 39.4040668914082 901355 what temp to roast brussel sprouts 8 msmarco_passage_15_492717845 39.00833191765568 901355 what temp to roast brussel sprouts 9 msmarco_passage_47_33349524 38.89489947082079 901355 what temp to roast brussel sprouts 10 msmarco_passage_37_457370886 38.840487806858754 1099178 how does sperm develop 1 msmarco_passage_29_187541118 20.038955136335332 1099178 how does sperm develop 2 msmarco_passage_15_572690529 19.573913566249516 1099178 how does sperm develop 3 msmarco_passage_12_197498904 19.020469833471473 1099178 how does sperm develop 4 msmarco_passage_29_188010860 19.020469833471473 1099178 how does sperm develop 5 msmarco_passage_04_93863179 19.002574532998416 1099178 how does sperm develop 6 msmarco_passage_13_372058247 18.887742792841753 1099178 how does sperm develop 7 msmarco_passage_04_566014158 18.843822417493712 1099178 how does sperm develop 8 msmarco_passage_11_254307231 18.429110489436276 1099178 how does sperm develop 9 msmarco_passage_11_184888325 18.429110489436276 1099178 how does sperm develop 10 msmarco_passage_15_618107201 18.42312008096872 909506 what tv channel does the lottery 1 msmarco_passage_60_440887698 26.23051474938039 909506 what tv channel does the lottery 2 msmarco_passage_07_493971054 24.726747482182496 909506 what tv channel does the lottery 3 msmarco_passage_60_623706498 24.564866348264218 909506 what tv channel does the lottery 4 msmarco_passage_24_340355578 24.452317944828188 909506 what tv channel does the lottery 5 msmarco_passage_24_340354169 23.008322569683536 909506 what tv channel does the lottery 6 msmarco_passage_68_237525587 22.93118391496753 909506 what tv channel does the lottery 7 msmarco_passage_21_498237262 22.580482025354925 909506 what tv channel does the lottery 8 msmarco_passage_11_139506408 22.460747600258635 909506 what tv channel does the lottery 9 msmarco_passage_07_147603895 22.424283436697724 909506 what tv channel does the lottery 10 msmarco_passage_24_347354907 22.29343232580645 555750 what are boot devices? 1 msmarco_passage_16_615870841 18.953377539109372 555750 what are boot devices? 2 msmarco_passage_63_321513311 18.848545332105175 555750 what are boot devices? 3 msmarco_passage_39_600210320 18.725949964644148 555750 what are boot devices? 4 msmarco_passage_03_263869204 18.59167660467643 555750 what are boot devices? 5 msmarco_passage_54_173876777 18.2346247039036 555750 what are boot devices? 6 msmarco_passage_05_456861833 18.20878022623906 555750 what are boot devices? 7 msmarco_passage_38_312677014 18.193136933609956 555750 what are boot devices? 8 msmarco_passage_32_178249959 18.155782912632297 555750 what are boot devices? 9 msmarco_passage_50_695209673 18.0962470283606 555750 what are boot devices? 10 msmarco_passage_50_756037565 18.0962470283606 866428 what kind of battery does a boat take 1 msmarco_passage_30_78868262 24.973246864040338 866428 what kind of battery does a boat take 2 msmarco_passage_54_753435630 24.09061897564564 866428 what kind of battery does a boat take 3 msmarco_passage_28_718216846 22.28902338784451 866428 what kind of battery does a boat take 4 msmarco_passage_29_333705826 22.11854679402347 866428 what kind of battery does a boat take 5 msmarco_passage_14_319435765 21.92467093293547 866428 what kind of battery does a boat take 6 msmarco_passage_37_557592915 21.79589414626856 866428 what kind of battery does a boat take 7 msmarco_passage_10_825077552 21.449304286357222 866428 what kind of battery does a boat take 8 msmarco_passage_67_434529772 21.449304286357222 866428 what kind of battery does a boat take 9 msmarco_passage_12_843380582 20.777479313496713 866428 what kind of battery does a boat take 10 msmarco_passage_48_492733214 20.739587859919574 1066116 why do women get period cramps 1 msmarco_passage_12_210378323 29.20301603514426 1066116 why do women get period cramps 2 msmarco_passage_53_622606559 28.879790423745337 1066116 why do women get period cramps 3 msmarco_passage_07_459510046 28.27052771816562 1066116 why do women get period cramps 4 msmarco_passage_34_403027932 27.718632965248233 1066116 why do women get period cramps 5 msmarco_passage_53_592285434 27.522060072769833 1066116 why do women get period cramps 6 msmarco_passage_49_475740157 27.51566252018209 1066116 why do women get period cramps 7 msmarco_passage_56_410432073 26.978973373694128 1066116 why do women get period cramps 8 msmarco_passage_08_45700047 26.978066520869312 1066116 why do women get period cramps 9 msmarco_passage_15_310858580 26.67513662944775 1066116 why do women get period cramps 10 msmarco_passage_00_606705426 26.67513662944775 1083517 what does labor burden generally include 1 msmarco_passage_13_556589627 25.430918319395886 1083517 what does labor burden generally include 2 msmarco_passage_07_130176774 24.21961487190692 1083517 what does labor burden generally include 3 msmarco_passage_53_194084421 24.00354444611576 1083517 what does labor burden generally include 4 msmarco_passage_37_46952008 23.60955691114483 1083517 what does labor burden generally include 5 msmarco_passage_07_130179769 23.50489884614401 1083517 what does labor burden generally include 6 msmarco_passage_37_3983631 22.94464775608646 1083517 what does labor burden generally include 7 msmarco_passage_03_762842086 22.94464775608646 1083517 what does labor burden generally include 8 msmarco_passage_04_650901713 22.629886924600505 1083517 what does labor burden generally include 9 msmarco_passage_13_470078616 22.422905968551653 1083517 what does labor burden generally include 10 msmarco_passage_46_335813915 22.36322124308753 703383 what is a thoracentesis 1 msmarco_passage_07_53470346 22.351189774765487 703383 what is a thoracentesis 2 msmarco_passage_51_675207037 21.352484913245664 703383 what is a thoracentesis 3 msmarco_passage_11_678622999 20.412847000103127 703383 what is a thoracentesis 4 msmarco_passage_68_82940685 19.70200389012523 703383 what is a thoracentesis 5 msmarco_passage_24_209699359 19.684482818443676 703383 what is a thoracentesis 6 msmarco_passage_24_196515002 19.684482818443676 703383 what is a thoracentesis 7 msmarco_passage_40_586558557 19.675733963578825 703383 what is a thoracentesis 8 msmarco_passage_27_434743668 19.558026421818997 703383 what is a thoracentesis 9 msmarco_passage_39_314033604 19.558026421818997 703383 what is a thoracentesis 10 msmarco_passage_39_314039432 19.558026421818997 662282 what form of currency does belize 1 msmarco_passage_68_171736390 26.71305630239648 662282 what form of currency does belize 2 msmarco_passage_02_67806359 26.196969591427738 662282 what form of currency does belize 3 msmarco_passage_54_644436792 25.093531299964027 662282 what form of currency does belize 4 msmarco_passage_38_73892003 24.52378495956991 662282 what form of currency does belize 5 msmarco_passage_11_800858176 24.398025951002538 662282 what form of currency does belize 6 msmarco_passage_54_644431202 24.097030902021125 662282 what form of currency does belize 7 msmarco_passage_02_67804346 24.052828272691166 662282 what form of currency does belize 8 msmarco_passage_13_52237349 24.049957610217668 662282 what form of currency does belize 9 msmarco_passage_39_110850261 24.049957610217668 662282 what form of currency does belize 10 msmarco_passage_39_111123311 24.020385873094007 237561 how is the element magnesium used 1 msmarco_passage_51_233113666 23.1996712065978 237561 how is the element magnesium used 2 msmarco_passage_21_670775215 22.53674095162934 237561 how is the element magnesium used 3 msmarco_passage_29_595330802 22.172851430409146 237561 how is the element magnesium used 4 msmarco_passage_52_652531534 21.74275344879426 237561 how is the element magnesium used 5 msmarco_passage_07_719578063 21.74275344879426 237561 how is the element magnesium used 6 msmarco_passage_04_48634321 21.595092431996825 237561 how is the element magnesium used 7 msmarco_passage_37_807828609 21.595092431996825 237561 how is the element magnesium used 8 msmarco_passage_00_289282934 21.551942656404034 237561 how is the element magnesium used 9 msmarco_passage_04_192923631 21.28277870775261 237561 how is the element magnesium used 10 msmarco_passage_24_246801377 21.11570173725525 119168 define coherent 1 msmarco_passage_05_667232358 22.23587174725447 119168 define coherent 2 msmarco_passage_02_29888635 21.630196961387394 119168 define coherent 3 msmarco_passage_43_530460989 20.527580680951065 119168 define coherent 4 msmarco_passage_53_298501709 20.44679935118276 119168 define coherent 5 msmarco_passage_18_414075909 20.369570414641153 119168 define coherent 6 msmarco_passage_64_594796547 20.161876329614802 119168 define coherent 7 msmarco_passage_18_414068572 19.220133800334455 119168 define coherent 8 msmarco_passage_32_537426578 19.17750832165039 119168 define coherent 9 msmarco_passage_30_40629552 19.17750832165039 119168 define coherent 10 msmarco_passage_18_256051640 19.00057202397966 663950 what gland secrete epinephrine 1 msmarco_passage_55_856435744 31.894371495615843 663950 what gland secrete epinephrine 2 msmarco_passage_21_742461138 30.262972685906185 663950 what gland secrete epinephrine 3 msmarco_passage_04_210626947 30.18133602663213 663950 what gland secrete epinephrine 4 msmarco_passage_39_472011281 30.075873854144433 663950 what gland secrete epinephrine 5 msmarco_passage_61_677618807 30.075873854144433 663950 what gland secrete epinephrine 6 msmarco_passage_26_169187845 29.95951852570174 663950 what gland secrete epinephrine 7 msmarco_passage_30_589992535 29.66279599044495 663950 what gland secrete epinephrine 8 msmarco_passage_12_539833057 29.571596029145194 663950 what gland secrete epinephrine 9 msmarco_passage_21_745489356 29.518755691859603 663950 what gland secrete epinephrine 10 msmarco_passage_38_390492626 29.48982345133082 253678 how long does it take for collections to come off credit report 1 msmarco_passage_42_623351467 30.35624535414894 253678 how long does it take for collections to come off credit report 2 msmarco_passage_12_220739368 27.009381322167577 253678 how long does it take for collections to come off credit report 3 msmarco_passage_13_216050950 25.875382664535017 253678 how long does it take for collections to come off credit report 4 msmarco_passage_12_220741991 25.68555051816563 253678 how long does it take for collections to come off credit report 5 msmarco_passage_35_828963818 24.723345420121866 253678 how long does it take for collections to come off credit report 6 msmarco_passage_57_502831817 24.57334340075997 253678 how long does it take for collections to come off credit report 7 msmarco_passage_41_749996370 24.480548635859712 253678 how long does it take for collections to come off credit report 8 msmarco_passage_42_626305946 24.421079601660754 253678 how long does it take for collections to come off credit report 9 msmarco_passage_60_369019044 24.285050016264698 253678 how long does it take for collections to come off credit report 10 msmarco_passage_59_708283095 24.187882912187057 1090165 synonyms of ignorant 1 msmarco_passage_53_854972038 21.14734160970874 1090165 synonyms of ignorant 2 msmarco_passage_53_854973709 20.75818882152305 1090165 synonyms of ignorant 3 msmarco_passage_02_36721149 20.574873418363083 1090165 synonyms of ignorant 4 msmarco_passage_62_33732901 20.5679261497771 1090165 synonyms of ignorant 5 msmarco_passage_10_692921143 20.38339949651502 1090165 synonyms of ignorant 6 msmarco_passage_53_854972609 20.38339949651502 1090165 synonyms of ignorant 7 msmarco_passage_53_854972433 20.283357948695556 1090165 synonyms of ignorant 8 msmarco_passage_62_40437972 20.022177531815323 1090165 synonyms of ignorant 9 msmarco_passage_62_561630539 20.021005972044556 1090165 synonyms of ignorant 10 msmarco_passage_62_33734373 19.93682279052394 619675 what did rosalind franklin discover 1 msmarco_passage_64_239436176 34.36221176853928 619675 what did rosalind franklin discover 2 msmarco_passage_37_794075836 31.991775660473703 619675 what did rosalind franklin discover 3 msmarco_passage_53_727022638 31.639394225302954 619675 what did rosalind franklin discover 4 msmarco_passage_57_797354725 31.639394225302954 619675 what did rosalind franklin discover 5 msmarco_passage_37_794066757 29.90319681946388 619675 what did rosalind franklin discover 6 msmarco_passage_53_727023024 29.68203852674111 619675 what did rosalind franklin discover 7 msmarco_passage_20_647213632 29.379706587163568 619675 what did rosalind franklin discover 8 msmarco_passage_20_647170225 29.379706587163568 619675 what did rosalind franklin discover 9 msmarco_passage_21_105363939 28.75503329273782 619675 what did rosalind franklin discover 10 msmarco_passage_60_228549788 28.526601651148972 804103 what is the actual us independence day 1 msmarco_passage_68_172262585 20.637920203608154 804103 what is the actual us independence day 2 msmarco_passage_66_285771592 20.505891832698097 804103 what is the actual us independence day 3 msmarco_passage_07_263825215 20.407788831835234 804103 what is the actual us independence day 4 msmarco_passage_04_205977067 19.3806631879701 804103 what is the actual us independence day 5 msmarco_passage_45_336523919 19.19647112699552 804103 what is the actual us independence day 6 msmarco_passage_64_844718600 19.12467920577297 804103 what is the actual us independence day 7 msmarco_passage_43_482767431 19.122404393815536 804103 what is the actual us independence day 8 msmarco_passage_50_34425574 19.02274752074402 804103 what is the actual us independence day 9 msmarco_passage_58_56316814 18.980877637465223 804103 what is the actual us independence day 10 msmarco_passage_29_236594026 18.877590826990172 878959 what moon phases makes neap tides 1 msmarco_passage_04_90417110 51.052982952559965 878959 what moon phases makes neap tides 2 msmarco_passage_04_214885807 50.27444881966254 878959 what moon phases makes neap tides 3 msmarco_passage_04_214884889 50.00438226084216 878959 what moon phases makes neap tides 4 msmarco_passage_04_205535076 49.96112140258313 878959 what moon phases makes neap tides 5 msmarco_passage_04_175521307 49.867198046556354 878959 what moon phases makes neap tides 6 msmarco_passage_04_205535493 49.867198046556354 878959 what moon phases makes neap tides 7 msmarco_passage_04_214883593 48.942521965206886 878959 what moon phases makes neap tides 8 msmarco_passage_04_214884055 48.29599739794793 878959 what moon phases makes neap tides 9 msmarco_passage_04_154041432 47.86909909167531 878959 what moon phases makes neap tides 10 msmarco_passage_04_214882240 47.674003725866406 1095631 how much money will americans spend for easter 1 msmarco_passage_10_60759819 26.74251815825871 1095631 how much money will americans spend for easter 2 msmarco_passage_10_60758181 26.595680245246708 1095631 how much money will americans spend for easter 3 msmarco_passage_07_94378210 26.071366644607387 1095631 how much money will americans spend for easter 4 msmarco_passage_04_112816202 25.554080936396936 1095631 how much money will americans spend for easter 5 msmarco_passage_06_433946408 25.118759816564754 1095631 how much money will americans spend for easter 6 msmarco_passage_04_116194943 24.851235777542303 1095631 how much money will americans spend for easter 7 msmarco_passage_47_65059492 24.765680408499495 1095631 how much money will americans spend for easter 8 msmarco_passage_65_522386395 24.26567777981775 1095631 how much money will americans spend for easter 9 msmarco_passage_57_651133367 23.84763448438498 1095631 how much money will americans spend for easter 10 msmarco_passage_11_620410671 23.80941788862053 979787 where is chedun town songjiang district 1 msmarco_passage_64_695850341 25.835740213254784 979787 where is chedun town songjiang district 2 msmarco_passage_38_276853228 25.809768657749714 979787 where is chedun town songjiang district 3 msmarco_passage_64_695849844 25.306907973937054 979787 where is chedun town songjiang district 4 msmarco_passage_40_446163642 24.643334541820526 979787 where is chedun town songjiang district 5 msmarco_passage_19_696814026 24.522988552257935 979787 where is chedun town songjiang district 6 msmarco_passage_60_470665447 23.014264804980094 979787 where is chedun town songjiang district 7 msmarco_passage_64_815701123 22.995249102004642 979787 where is chedun town songjiang district 8 msmarco_passage_18_357479113 22.71844696062268 979787 where is chedun town songjiang district 9 msmarco_passage_38_824685065 22.522248981364726 979787 where is chedun town songjiang district 10 msmarco_passage_38_824685653 22.522248981364726 988787 who is the judge michael day district court of south dakota 1 msmarco_passage_21_423175514 39.84943277673479 988787 who is the judge michael day district court of south dakota 2 msmarco_passage_21_439140054 38.94411623463443 988787 who is the judge michael day district court of south dakota 3 msmarco_passage_21_423175067 36.787816036095364 988787 who is the judge michael day district court of south dakota 4 msmarco_passage_47_499778951 35.358855589398914 988787 who is the judge michael day district court of south dakota 5 msmarco_passage_51_140187476 34.939018044891014 988787 who is the judge michael day district court of south dakota 6 msmarco_passage_51_140196226 34.9158262978882 988787 who is the judge michael day district court of south dakota 7 msmarco_passage_12_774362218 34.86566729599392 988787 who is the judge michael day district court of south dakota 8 msmarco_passage_53_795604302 34.58609513424951 988787 who is the judge michael day district court of south dakota 9 msmarco_passage_01_147781076 34.58569039690724 988787 who is the judge michael day district court of south dakota 10 msmarco_passage_21_516334695 34.58569039690724 393881 in what county in ga is canton' 1 msmarco_passage_42_455745461 31.88286612160355 393881 in what county in ga is canton' 2 msmarco_passage_42_471526741 31.88286612160355 393881 in what county in ga is canton' 3 msmarco_passage_42_502815323 31.543571647494126 393881 in what county in ga is canton' 4 msmarco_passage_68_742279934 30.151309214159834 393881 in what county in ga is canton' 5 msmarco_passage_64_631325911 29.797581294325997 393881 in what county in ga is canton' 6 msmarco_passage_42_461936659 29.71541784856419 393881 in what county in ga is canton' 7 msmarco_passage_00_530085991 29.572543027260412 393881 in what county in ga is canton' 8 msmarco_passage_42_455752684 29.49989175214195 393881 in what county in ga is canton' 9 msmarco_passage_42_471537695 29.49989175214195 393881 in what county in ga is canton' 10 msmarco_passage_42_471545873 29.49989175214195 782549 what is pennsylvania's minimum wage 1 msmarco_passage_45_207307747 31.77248944904839 782549 what is pennsylvania's minimum wage 2 msmarco_passage_08_187858261 31.182347578334866 782549 what is pennsylvania's minimum wage 3 msmarco_passage_22_677152386 30.856993194236335 782549 what is pennsylvania's minimum wage 4 msmarco_passage_06_137804216 30.721748386808933 782549 what is pennsylvania's minimum wage 5 msmarco_passage_45_207290596 30.721748386808933 782549 what is pennsylvania's minimum wage 6 msmarco_passage_08_187857885 30.01954335775975 782549 what is pennsylvania's minimum wage 7 msmarco_passage_46_374373547 29.53800373784135 782549 what is pennsylvania's minimum wage 8 msmarco_passage_26_453136447 29.453900674093518 782549 what is pennsylvania's minimum wage 9 msmarco_passage_08_187715072 29.4412725516381 782549 what is pennsylvania's minimum wage 10 msmarco_passage_54_157159006 29.4412725516381 384845 how to use a selfie stick 1 msmarco_passage_02_111841477 28.159544961254987 384845 how to use a selfie stick 2 msmarco_passage_32_396901018 27.72525454894186 384845 how to use a selfie stick 3 msmarco_passage_03_6588528 26.919394090503715 384845 how to use a selfie stick 4 msmarco_passage_50_404433142 26.768411987099583 384845 how to use a selfie stick 5 msmarco_passage_47_377675601 26.205078780255977 384845 how to use a selfie stick 6 msmarco_passage_07_772549239 26.159057359247733 384845 how to use a selfie stick 7 msmarco_passage_22_398378598 26.033048401461887 384845 how to use a selfie stick 8 msmarco_passage_50_188960361 25.86350309055204 384845 how to use a selfie stick 9 msmarco_passage_68_629730114 25.531620527611487 384845 how to use a selfie stick 10 msmarco_passage_46_88090295 25.481560432761675 681264 what is a digital perm 1 msmarco_passage_69_41038683 26.829915979931396 681264 what is a digital perm 2 msmarco_passage_18_605907769 26.31841427683937 681264 what is a digital perm 3 msmarco_passage_29_848364267 25.95311674362685 681264 what is a digital perm 4 msmarco_passage_12_863730339 25.829405516470047 681264 what is a digital perm 5 msmarco_passage_12_863734609 25.639006736349074 681264 what is a digital perm 6 msmarco_passage_12_863733238 25.386568345078125 681264 what is a digital perm 7 msmarco_passage_39_48898214 25.26417649152422 681264 what is a digital perm 8 msmarco_passage_30_295863735 25.19840874964554 681264 what is a digital perm 9 msmarco_passage_04_71351168 25.017297518271477 681264 what is a digital perm 10 msmarco_passage_04_71352980 24.813779625751074 808200 what is the best description of an adaptation 1 msmarco_passage_28_416497841 16.9967339475076 808200 what is the best description of an adaptation 2 msmarco_passage_28_414871386 16.9967339475076 808200 what is the best description of an adaptation 3 msmarco_passage_11_270889351 16.770433889668872 808200 what is the best description of an adaptation 4 msmarco_passage_50_404266465 16.58777245534814 808200 what is the best description of an adaptation 5 msmarco_passage_59_832251394 16.579945823425177 808200 what is the best description of an adaptation 6 msmarco_passage_56_418154356 16.562550805731235 808200 what is the best description of an adaptation 7 msmarco_passage_33_282414568 16.436296606461205 808200 what is the best description of an adaptation 8 msmarco_passage_16_593989509 16.436296606461205 808200 what is the best description of an adaptation 9 msmarco_passage_43_584192338 16.165913199854366 808200 what is the best description of an adaptation 10 msmarco_passage_33_272846415 16.16239160406173 1099805 honey in south carolina 1 msmarco_passage_13_42996398 23.57778719300157 1099805 honey in south carolina 2 msmarco_passage_16_360259699 23.490721082758128 1099805 honey in south carolina 3 msmarco_passage_32_563667734 23.405845763561956 1099805 honey in south carolina 4 msmarco_passage_13_212415978 22.551183887080185 1099805 honey in south carolina 5 msmarco_passage_16_360259280 22.28044350342994 1099805 honey in south carolina 6 msmarco_passage_14_578856840 22.059376138018763 1099805 honey in south carolina 7 msmarco_passage_16_360210678 21.854681064937616 1099805 honey in south carolina 8 msmarco_passage_63_439129790 21.799437562367032 1099805 honey in south carolina 9 msmarco_passage_16_360255366 21.664981108274116 1099805 honey in south carolina 10 msmarco_passage_55_641376067 21.560248498428297 1084354 what does abc stand for in liquor store 1 msmarco_passage_52_676279970 33.879183747086834 1084354 what does abc stand for in liquor store 2 msmarco_passage_26_21004403 30.94218537692703 1084354 what does abc stand for in liquor store 3 msmarco_passage_19_635273374 28.58176730913307 1084354 what does abc stand for in liquor store 4 msmarco_passage_20_268143351 28.58176730913307 1084354 what does abc stand for in liquor store 5 msmarco_passage_68_475877739 27.930780444820478 1084354 what does abc stand for in liquor store 6 msmarco_passage_68_522280958 27.8300447711997 1084354 what does abc stand for in liquor store 7 msmarco_passage_22_500842030 27.78805418347291 1084354 what does abc stand for in liquor store 8 msmarco_passage_68_687821382 27.220320577485822 1084354 what does abc stand for in liquor store 9 msmarco_passage_04_179214192 27.214495541470086 1084354 what does abc stand for in liquor store 10 msmarco_passage_17_790857744 27.022832924419724 282530 how many doctors are in the us 1 msmarco_passage_27_746964997 17.15144547087517 282530 how many doctors are in the us 2 msmarco_passage_15_692438419 16.67422290674855 282530 how many doctors are in the us 3 msmarco_passage_48_840088460 15.91448958481425 282530 how many doctors are in the us 4 msmarco_passage_17_300306729 15.914096311458724 282530 how many doctors are in the us 5 msmarco_passage_48_832142415 15.825423990892645 282530 how many doctors are in the us 6 msmarco_passage_34_25889131 15.70051743891668 282530 how many doctors are in the us 7 msmarco_passage_48_834066697 15.581628374802946 282530 how many doctors are in the us 8 msmarco_passage_55_254390076 15.561120648965712 282530 how many doctors are in the us 9 msmarco_passage_55_254312396 15.561120648965712 282530 how many doctors are in the us 10 msmarco_passage_01_682309661 15.482569103420714 281704 how many days to defrost 1 msmarco_passage_37_27895086 21.024442022369367 281704 how many days to defrost 2 msmarco_passage_65_564615112 20.568700592592137 281704 how many days to defrost 3 msmarco_passage_29_35349143 19.708065979115986 281704 how many days to defrost 4 msmarco_passage_14_571984688 19.675783988577372 281704 how many days to defrost 5 msmarco_passage_43_219371785 19.48764979973412 281704 how many days to defrost 6 msmarco_passage_05_522028844 19.48764979973412 281704 how many days to defrost 7 msmarco_passage_11_521296784 19.32217395554734 281704 how many days to defrost 8 msmarco_passage_46_410173656 19.30933956953952 281704 how many days to defrost 9 msmarco_passage_59_139593025 19.284950592814432 281704 how many days to defrost 10 msmarco_passage_59_139601418 19.284950592814432 683193 what is a file compression mean in computer software? 1 msmarco_passage_55_130513244 29.545707744449913 683193 what is a file compression mean in computer software? 2 msmarco_passage_25_437953104 28.512849540478182 683193 what is a file compression mean in computer software? 3 msmarco_passage_51_243870946 28.27364979538848 683193 what is a file compression mean in computer software? 4 msmarco_passage_59_128846387 27.79928261907962 683193 what is a file compression mean in computer software? 5 msmarco_passage_29_168571545 27.727811577655345 683193 what is a file compression mean in computer software? 6 msmarco_passage_42_221768998 27.395497825096097 683193 what is a file compression mean in computer software? 7 msmarco_passage_56_536958663 26.95710349940021 683193 what is a file compression mean in computer software? 8 msmarco_passage_48_370788089 26.648206671744674 683193 what is a file compression mean in computer software? 9 msmarco_passage_43_311547904 26.58895260650992 683193 what is a file compression mean in computer software? 10 msmarco_passage_33_743546987 26.467306841077683 185276 fastest car in the world motor tr 1 msmarco_passage_31_319843669 28.05090061692746 185276 fastest car in the world motor tr 2 msmarco_passage_13_234101889 27.078618087344857 185276 fastest car in the world motor tr 3 msmarco_passage_28_225192486 26.927015570167537 185276 fastest car in the world motor tr 4 msmarco_passage_66_606026031 26.778527686263686 185276 fastest car in the world motor tr 5 msmarco_passage_00_688161265 26.73680247665995 185276 fastest car in the world motor tr 6 msmarco_passage_00_688160965 26.3828347645779 185276 fastest car in the world motor tr 7 msmarco_passage_66_606042903 26.234719771414316 185276 fastest car in the world motor tr 8 msmarco_passage_00_688159991 26.081666889657207 185276 fastest car in the world motor tr 9 msmarco_passage_54_395480454 26.042089664709028 185276 fastest car in the world motor tr 10 msmarco_passage_43_341121993 25.832924899478687 108037 cost of wooden shutters 1 msmarco_passage_57_133213246 25.219011815330894 108037 cost of wooden shutters 2 msmarco_passage_25_116092911 25.04440709208247 108037 cost of wooden shutters 3 msmarco_passage_11_801948180 24.304547949597826 108037 cost of wooden shutters 4 msmarco_passage_49_293444730 24.271443494846228 108037 cost of wooden shutters 5 msmarco_passage_07_17728573 24.271443494846228 108037 cost of wooden shutters 6 msmarco_passage_35_856425078 23.782248011857323 108037 cost of wooden shutters 7 msmarco_passage_01_615116094 23.630939310807847 108037 cost of wooden shutters 8 msmarco_passage_69_103132820 23.53733592051777 108037 cost of wooden shutters 9 msmarco_passage_03_637038443 23.33119821940135 108037 cost of wooden shutters 10 msmarco_passage_58_393033578 23.305871322424032 469535 optimum performance definition 1 msmarco_passage_00_752483365 22.571299124304446 469535 optimum performance definition 2 msmarco_passage_16_286052860 22.571299124304446 469535 optimum performance definition 3 msmarco_passage_06_199009349 19.919613844761773 469535 optimum performance definition 4 msmarco_passage_02_492269526 19.919613844761773 469535 optimum performance definition 5 msmarco_passage_60_820572232 19.609578632117344 469535 optimum performance definition 6 msmarco_passage_26_586961289 19.390217133085066 469535 optimum performance definition 7 msmarco_passage_26_599656177 19.390217133085066 469535 optimum performance definition 8 msmarco_passage_65_604667001 19.36194473023026 469535 optimum performance definition 9 msmarco_passage_01_677615539 19.28928860599241 469535 optimum performance definition 10 msmarco_passage_02_495948192 19.084730944194163 188134 flower shops in pueblo colorado 1 msmarco_passage_06_396731313 32.6480533245767 188134 flower shops in pueblo colorado 2 msmarco_passage_64_629137957 27.1155655923586 188134 flower shops in pueblo colorado 3 msmarco_passage_42_504278148 26.808101542500523 188134 flower shops in pueblo colorado 4 msmarco_passage_64_629138429 26.74545997813467 188134 flower shops in pueblo colorado 5 msmarco_passage_42_504303415 26.694266836412496 188134 flower shops in pueblo colorado 6 msmarco_passage_42_504300682 26.694266836412496 188134 flower shops in pueblo colorado 7 msmarco_passage_03_455299053 26.488109666569684 188134 flower shops in pueblo colorado 8 msmarco_passage_49_544219656 26.025136513275346 188134 flower shops in pueblo colorado 9 msmarco_passage_64_716665750 25.922616831360884 188134 flower shops in pueblo colorado 10 msmarco_passage_04_735017536 25.919549157620718 100661 cost estimation technique 1 msmarco_passage_04_656390036 24.63183804120724 100661 cost estimation technique 2 msmarco_passage_57_604119752 23.906356502321266 100661 cost estimation technique 3 msmarco_passage_40_35829534 23.588475695637573 100661 cost estimation technique 4 msmarco_passage_60_689599857 23.467021739123965 100661 cost estimation technique 5 msmarco_passage_43_102754831 22.59110048800562 100661 cost estimation technique 6 msmarco_passage_40_35831894 22.576778706877743 100661 cost estimation technique 7 msmarco_passage_30_87475043 22.557756395093236 100661 cost estimation technique 8 msmarco_passage_02_619492298 22.557756395093236 100661 cost estimation technique 9 msmarco_passage_13_724409699 22.290764045322234 100661 cost estimation technique 10 msmarco_passage_13_724407961 22.213166181207306 996317 where is the ky derby held? 1 msmarco_passage_32_647591905 29.036014779154655 996317 where is the ky derby held? 2 msmarco_passage_32_647404775 29.036014779154655 996317 where is the ky derby held? 3 msmarco_passage_67_2648881 28.814672964337934 996317 where is the ky derby held? 4 msmarco_passage_37_757682225 28.20324907108551 996317 where is the ky derby held? 5 msmarco_passage_34_873475077 27.921595846844053 996317 where is the ky derby held? 6 msmarco_passage_03_410593389 27.921595846844053 996317 where is the ky derby held? 7 msmarco_passage_04_101758453 27.864631022802506 996317 where is the ky derby held? 8 msmarco_passage_10_390473462 25.932276936278168 996317 where is the ky derby held? 9 msmarco_passage_47_723799458 25.824393802900918 996317 where is the ky derby held? 10 msmarco_passage_29_654589982 25.636748281494267 954711 when should sutures be removed 1 msmarco_passage_06_128612007 23.06866798039124 954711 when should sutures be removed 2 msmarco_passage_06_128604112 23.06866798039124 954711 when should sutures be removed 3 msmarco_passage_06_128619902 23.06866798039124 954711 when should sutures be removed 4 msmarco_passage_45_187187142 23.06866798039124 954711 when should sutures be removed 5 msmarco_passage_45_187195019 23.06866798039124 954711 when should sutures be removed 6 msmarco_passage_45_187179265 23.06866798039124 954711 when should sutures be removed 7 msmarco_passage_08_769138395 22.86638784030188 954711 when should sutures be removed 8 msmarco_passage_08_769137960 22.04206711690354 954711 when should sutures be removed 9 msmarco_passage_53_453174632 21.900870385899793 954711 when should sutures be removed 10 msmarco_passage_36_790956653 21.900870385899793 605169 what county is dixon mo in 1 msmarco_passage_41_367839410 32.326538261988944 605169 what county is dixon mo in 2 msmarco_passage_50_615527795 31.940685824892114 605169 what county is dixon mo in 3 msmarco_passage_53_110916829 30.519851371055648 605169 what county is dixon mo in 4 msmarco_passage_50_615526918 28.595067870312683 605169 what county is dixon mo in 5 msmarco_passage_07_547484762 27.05606645541156 605169 what county is dixon mo in 6 msmarco_passage_07_547483200 26.332569987209105 605169 what county is dixon mo in 7 msmarco_passage_42_519851145 26.055174568709386 605169 what county is dixon mo in 8 msmarco_passage_38_722558212 25.20723085797306 605169 what county is dixon mo in 9 msmarco_passage_38_722075250 24.963359783073734 605169 what county is dixon mo in 10 msmarco_passage_56_168789587 24.408642324382388 666694 what happens to the boiling point when a solute is added to a solution 1 msmarco_passage_04_204008558 42.93951958880611 666694 what happens to the boiling point when a solute is added to a solution 2 msmarco_passage_04_94822446 41.23644273147734 666694 what happens to the boiling point when a solute is added to a solution 3 msmarco_passage_14_814123781 38.800115331833254 666694 what happens to the boiling point when a solute is added to a solution 4 msmarco_passage_14_814023810 38.800115331833254 666694 what happens to the boiling point when a solute is added to a solution 5 msmarco_passage_58_592017787 38.08833855214362 666694 what happens to the boiling point when a solute is added to a solution 6 msmarco_passage_14_821076748 37.99406142529094 666694 what happens to the boiling point when a solute is added to a solution 7 msmarco_passage_14_804838241 37.99406142529094 666694 what happens to the boiling point when a solute is added to a solution 8 msmarco_passage_14_374118537 37.38754610297187 666694 what happens to the boiling point when a solute is added to a solution 9 msmarco_passage_33_111954444 36.633353093491976 666694 what happens to the boiling point when a solute is added to a solution 10 msmarco_passage_14_374114756 36.533660560425304 90941 chateau meaning 1 msmarco_passage_00_60418592 20.914919040093192 90941 chateau meaning 2 msmarco_passage_00_60419290 19.633997188203097 90941 chateau meaning 3 msmarco_passage_06_287219425 19.25472823399912 90941 chateau meaning 4 msmarco_passage_66_774308979 19.093146624285833 90941 chateau meaning 5 msmarco_passage_06_287220309 18.967457415594037 90941 chateau meaning 6 msmarco_passage_66_774308110 18.946806643986587 90941 chateau meaning 7 msmarco_passage_67_867644804 18.92666497595726 90941 chateau meaning 8 msmarco_passage_63_788515445 18.254069415537618 90941 chateau meaning 9 msmarco_passage_67_867647280 18.006288198890424 90941 chateau meaning 10 msmarco_passage_35_43497200 17.92205342212032 196232 government does do 1 msmarco_passage_04_224380879 9.316862587859559 196232 government does do 2 msmarco_passage_04_88553377 9.21421962564773 196232 government does do 3 msmarco_passage_04_224380012 9.21421962564773 196232 government does do 4 msmarco_passage_16_92886151 9.175171583574583 196232 government does do 5 msmarco_passage_04_133085047 9.168889607393119 196232 government does do 6 msmarco_passage_04_173915071 9.168889607393119 196232 government does do 7 msmarco_passage_04_224380452 9.12682448515367 196232 government does do 8 msmarco_passage_63_262066351 9.100966228336988 196232 government does do 9 msmarco_passage_63_204447101 9.100966228336988 196232 government does do 10 msmarco_passage_63_132000831 9.100966228336988 1090877 sawdust bricks used for what 1 msmarco_passage_49_157514814 27.595593592155918 1090877 sawdust bricks used for what 2 msmarco_passage_33_482629859 25.93991673001466 1090877 sawdust bricks used for what 3 msmarco_passage_59_504923788 23.653583744521214 1090877 sawdust bricks used for what 4 msmarco_passage_58_78395177 23.415375187261276 1090877 sawdust bricks used for what 5 msmarco_passage_00_521362035 23.185740928479593 1090877 sawdust bricks used for what 6 msmarco_passage_47_553454680 23.185740928479593 1090877 sawdust bricks used for what 7 msmarco_passage_23_416532224 23.185740928479593 1090877 sawdust bricks used for what 8 msmarco_passage_22_460816999 23.10645877728194 1090877 sawdust bricks used for what 9 msmarco_passage_04_489936248 22.46954111117674 1090877 sawdust bricks used for what 10 msmarco_passage_34_430350347 22.06822813215788 464663 nitrocellulose cas number 1 msmarco_passage_07_734379189 27.758248145261405 464663 nitrocellulose cas number 2 msmarco_passage_07_734374829 25.144941618454887 464663 nitrocellulose cas number 3 msmarco_passage_40_860949887 21.09020407136695 464663 nitrocellulose cas number 4 msmarco_passage_38_587405022 21.010052838764615 464663 nitrocellulose cas number 5 msmarco_passage_50_560616708 20.872349803654465 464663 nitrocellulose cas number 6 msmarco_passage_40_860943547 20.860210625690115 464663 nitrocellulose cas number 7 msmarco_passage_40_860955913 20.860210625690115 464663 nitrocellulose cas number 8 msmarco_passage_33_277845402 20.860210625690115 464663 nitrocellulose cas number 9 msmarco_passage_04_779227311 20.860210625690115 464663 nitrocellulose cas number 10 msmarco_passage_46_138448973 20.626620772277054 924895 what woah means 1 msmarco_passage_23_252879034 23.767701287703115 924895 what woah means 2 msmarco_passage_23_277603990 22.698331712187805 924895 what woah means 3 msmarco_passage_23_273521326 22.562700662356924 924895 what woah means 4 msmarco_passage_23_278948669 22.522170049664403 924895 what woah means 5 msmarco_passage_54_26347308 22.12818530119767 924895 what woah means 6 msmarco_passage_23_277103791 22.04585184207996 924895 what woah means 7 msmarco_passage_23_252881560 21.914084907142943 924895 what woah means 8 msmarco_passage_23_255304543 21.914083063727922 924895 what woah means 9 msmarco_passage_23_251360369 21.89413114558035 924895 what woah means 10 msmarco_passage_10_242349414 21.844020584476926 97964 contributory aged parent visa cost 1 msmarco_passage_04_342922465 43.94911467507004 97964 contributory aged parent visa cost 2 msmarco_passage_04_342917752 43.94911467507004 97964 contributory aged parent visa cost 3 msmarco_passage_04_342409889 42.978076521017876 97964 contributory aged parent visa cost 4 msmarco_passage_04_342920890 40.69820168151095 97964 contributory aged parent visa cost 5 msmarco_passage_04_342920443 40.51844733312434 97964 contributory aged parent visa cost 6 msmarco_passage_03_434424626 39.86772954003205 97964 contributory aged parent visa cost 7 msmarco_passage_04_342922289 38.69108027431427 97964 contributory aged parent visa cost 8 msmarco_passage_04_342923404 38.5279050009557 97964 contributory aged parent visa cost 9 msmarco_passage_04_342919197 38.16551259966059 97964 contributory aged parent visa cost 10 msmarco_passage_04_342409633 38.089814881332714 1089925 the bancorp routing number 1 msmarco_passage_38_699282427 34.547423764439756 1089925 the bancorp routing number 2 msmarco_passage_65_550759624 34.26783008565925 1089925 the bancorp routing number 3 msmarco_passage_65_550761190 33.222141315495165 1089925 the bancorp routing number 4 msmarco_passage_65_550758115 32.52339623980617 1089925 the bancorp routing number 5 msmarco_passage_38_699236654 32.48095355586644 1089925 the bancorp routing number 6 msmarco_passage_03_481299623 32.21471464424179 1089925 the bancorp routing number 7 msmarco_passage_35_471822591 32.21471464424179 1089925 the bancorp routing number 8 msmarco_passage_03_481298333 32.21471464424179 1089925 the bancorp routing number 9 msmarco_passage_12_799325540 32.08915795300779 1089925 the bancorp routing number 10 msmarco_passage_12_799494895 31.375812924258923 1087425 what are pitchforks 1 msmarco_passage_17_597016827 19.054857786519744 1087425 what are pitchforks 2 msmarco_passage_64_573130784 18.719435823829034 1087425 what are pitchforks 3 msmarco_passage_22_37790206 18.719435823829034 1087425 what are pitchforks 4 msmarco_passage_21_730092821 18.637417380196595 1087425 what are pitchforks 5 msmarco_passage_34_846525456 18.439696999222235 1087425 what are pitchforks 6 msmarco_passage_59_348747198 18.168195775610258 1087425 what are pitchforks 7 msmarco_passage_00_528801589 18.040289537486046 1087425 what are pitchforks 8 msmarco_passage_02_422804576 18.040289537486046 1087425 what are pitchforks 9 msmarco_passage_47_376142034 17.716518418499017 1087425 what are pitchforks 10 msmarco_passage_27_442896406 17.716518418499017 1074949 what is a dapple dog 1 msmarco_passage_38_29528794 26.2651706088802 1074949 what is a dapple dog 2 msmarco_passage_38_29528300 24.501961786185174 1074949 what is a dapple dog 3 msmarco_passage_05_389751718 24.047095385470044 1074949 what is a dapple dog 4 msmarco_passage_43_70378217 23.537785992761094 1074949 what is a dapple dog 5 msmarco_passage_05_453473933 23.537785992761094 1074949 what is a dapple dog 6 msmarco_passage_05_389759563 23.485515861907093 1074949 what is a dapple dog 7 msmarco_passage_59_54446572 23.338532196693162 1074949 what is a dapple dog 8 msmarco_passage_05_389750803 22.933638688819777 1074949 what is a dapple dog 9 msmarco_passage_34_256800359 22.933638688819777 1074949 what is a dapple dog 10 msmarco_passage_38_29529171 22.846241745205457 9454 a simple definition for the word bias. 1 msmarco_passage_51_809610163 22.418797458144642 9454 a simple definition for the word bias. 2 msmarco_passage_58_351206062 21.968813349547325 9454 a simple definition for the word bias. 3 msmarco_passage_45_328122876 21.55527472249786 9454 a simple definition for the word bias. 4 msmarco_passage_23_299336919 21.164345963082837 9454 a simple definition for the word bias. 5 msmarco_passage_61_511173314 20.805886317346 9454 a simple definition for the word bias. 6 msmarco_passage_09_685219014 20.805886317346 9454 a simple definition for the word bias. 7 msmarco_passage_43_559218852 20.753802134143832 9454 a simple definition for the word bias. 8 msmarco_passage_05_669916004 20.753802134143832 9454 a simple definition for the word bias. 9 msmarco_passage_00_171769078 20.686090186233937 9454 a simple definition for the word bias. 10 msmarco_passage_16_408566297 20.562704821736713 1098809 how is the gem painite made 1 msmarco_passage_23_232155467 32.041189774548606 1098809 how is the gem painite made 2 msmarco_passage_57_145547634 31.004110036089315 1098809 how is the gem painite made 3 msmarco_passage_57_145548386 30.72447250803087 1098809 how is the gem painite made 4 msmarco_passage_23_232156337 29.34363579859516 1098809 how is the gem painite made 5 msmarco_passage_47_428058673 29.305632510135492 1098809 how is the gem painite made 6 msmarco_passage_57_145549673 29.063534552829317 1098809 how is the gem painite made 7 msmarco_passage_23_232154554 28.520859362803403 1098809 how is the gem painite made 8 msmarco_passage_57_145546633 27.97672353883653 1098809 how is the gem painite made 9 msmarco_passage_47_424358093 27.78093305908515 1098809 how is the gem painite made 10 msmarco_passage_32_635711048 26.966758728749056 449750 meaning of the name elodie 1 msmarco_passage_54_789654516 31.808545472681473 449750 meaning of the name elodie 2 msmarco_passage_10_198080730 31.491312131696727 449750 meaning of the name elodie 3 msmarco_passage_54_789654068 31.012111304495757 449750 meaning of the name elodie 4 msmarco_passage_04_383332066 29.488031753893562 449750 meaning of the name elodie 5 msmarco_passage_10_198079304 29.10468559213256 449750 meaning of the name elodie 6 msmarco_passage_10_198073832 28.58071719045555 449750 meaning of the name elodie 7 msmarco_passage_54_375494176 28.363582625053557 449750 meaning of the name elodie 8 msmarco_passage_10_263165643 27.511207955817223 449750 meaning of the name elodie 9 msmarco_passage_10_198074902 27.434988811990404 449750 meaning of the name elodie 10 msmarco_passage_10_198074306 27.39591608180586 646179 what does pro forma mean 1 msmarco_passage_42_126405019 32.048760493996795 646179 what does pro forma mean 2 msmarco_passage_05_615047289 29.580609421253495 646179 what does pro forma mean 3 msmarco_passage_13_451081021 29.245814643433864 646179 what does pro forma mean 4 msmarco_passage_59_276537884 28.72056685913408 646179 what does pro forma mean 5 msmarco_passage_05_615319652 28.674786067760415 646179 what does pro forma mean 6 msmarco_passage_52_132532718 28.497060942715116 646179 what does pro forma mean 7 msmarco_passage_59_667387576 28.457036889852798 646179 what does pro forma mean 8 msmarco_passage_52_132537682 28.340033481706044 646179 what does pro forma mean 9 msmarco_passage_25_363801254 28.190595494560064 646179 what does pro forma mean 10 msmarco_passage_14_348618428 28.074588566730462 358150 how to find a bathroom remodel contractor 1 msmarco_passage_05_753784861 35.25151958985455 358150 how to find a bathroom remodel contractor 2 msmarco_passage_38_862111968 34.636051324272614 358150 how to find a bathroom remodel contractor 3 msmarco_passage_04_424555698 34.51753866439962 358150 how to find a bathroom remodel contractor 4 msmarco_passage_04_424518630 34.08416780909665 358150 how to find a bathroom remodel contractor 5 msmarco_passage_64_269026357 34.078802617944476 358150 how to find a bathroom remodel contractor 6 msmarco_passage_06_594737520 34.07608172737167 358150 how to find a bathroom remodel contractor 7 msmarco_passage_05_753858928 33.4754905578911 358150 how to find a bathroom remodel contractor 8 msmarco_passage_50_430721758 32.84815409437237 358150 how to find a bathroom remodel contractor 9 msmarco_passage_31_260620158 32.83890678198205 358150 how to find a bathroom remodel contractor 10 msmarco_passage_30_58371050 32.304484037231724 758901 what is inductance 1 msmarco_passage_05_595131396 14.778523749326748 758901 what is inductance 2 msmarco_passage_05_595134944 14.778523749326748 758901 what is inductance 3 msmarco_passage_43_425056321 14.778523749326748 758901 what is inductance 4 msmarco_passage_02_207098175 14.735052580146693 758901 what is inductance 5 msmarco_passage_27_492381886 14.735052580146693 758901 what is inductance 6 msmarco_passage_02_32056304 14.735052580146693 758901 what is inductance 7 msmarco_passage_23_361193 14.415345036516133 758901 what is inductance 8 msmarco_passage_22_870728850 14.415345036516133 758901 what is inductance 9 msmarco_passage_22_871534596 14.415345036516133 758901 what is inductance 10 msmarco_passage_22_870851458 14.415345036516133 1090146 tachycardia effects on cardiac output 1 msmarco_passage_17_326760560 36.95480426715332 1090146 tachycardia effects on cardiac output 2 msmarco_passage_12_269684936 32.68446625899447 1090146 tachycardia effects on cardiac output 3 msmarco_passage_60_55096166 30.62692028745232 1090146 tachycardia effects on cardiac output 4 msmarco_passage_30_591122199 30.521167689052163 1090146 tachycardia effects on cardiac output 5 msmarco_passage_30_628279851 29.84050344282813 1090146 tachycardia effects on cardiac output 6 msmarco_passage_60_222728285 29.477037429143923 1090146 tachycardia effects on cardiac output 7 msmarco_passage_55_15475763 28.642016411781952 1090146 tachycardia effects on cardiac output 8 msmarco_passage_44_434353576 28.553475820140534 1090146 tachycardia effects on cardiac output 9 msmarco_passage_12_269684694 28.374192199063177 1090146 tachycardia effects on cardiac output 10 msmarco_passage_25_814855598 28.276159372510794 160255 do you give a toast at rehearsal dinner 1 msmarco_passage_01_342767960 41.41667375796797 160255 do you give a toast at rehearsal dinner 2 msmarco_passage_60_616032708 39.67919682378322 160255 do you give a toast at rehearsal dinner 3 msmarco_passage_03_566057373 38.74592111967186 160255 do you give a toast at rehearsal dinner 4 msmarco_passage_58_836714100 38.73179759396922 160255 do you give a toast at rehearsal dinner 5 msmarco_passage_45_581175180 37.03335503879471 160255 do you give a toast at rehearsal dinner 6 msmarco_passage_66_782260379 36.65444395538698 160255 do you give a toast at rehearsal dinner 7 msmarco_passage_66_770457793 36.1296379746121 160255 do you give a toast at rehearsal dinner 8 msmarco_passage_66_782255656 35.20715489924235 160255 do you give a toast at rehearsal dinner 9 msmarco_passage_03_566055031 34.70764843481999 160255 do you give a toast at rehearsal dinner 10 msmarco_passage_03_566055977 34.636953839793286 1058601 what is chromosome coiling 1 msmarco_passage_14_130619663 23.623207250128452 1058601 what is chromosome coiling 2 msmarco_passage_30_724805615 23.170836649241757 1058601 what is chromosome coiling 3 msmarco_passage_29_148561304 22.974155789323948 1058601 what is chromosome coiling 4 msmarco_passage_04_184948719 22.79904428459171 1058601 what is chromosome coiling 5 msmarco_passage_30_791483853 22.613523804343146 1058601 what is chromosome coiling 6 msmarco_passage_41_378866767 22.483383984005464 1058601 what is chromosome coiling 7 msmarco_passage_51_812405295 22.43768056721725 1058601 what is chromosome coiling 8 msmarco_passage_05_663894575 22.43768056721725 1058601 what is chromosome coiling 9 msmarco_passage_43_533913665 22.43768056721725 1058601 what is chromosome coiling 10 msmarco_passage_23_691827188 22.43768056721725 371695 how to order certificate of eligibility 1 msmarco_passage_65_552096351 22.99906880013109 371695 how to order certificate of eligibility 2 msmarco_passage_65_551963105 22.503448313171486 371695 how to order certificate of eligibility 3 msmarco_passage_61_482238141 22.013346997736974 371695 how to order certificate of eligibility 4 msmarco_passage_09_680420835 22.013346997736974 371695 how to order certificate of eligibility 5 msmarco_passage_65_572783954 22.008515767680297 371695 how to order certificate of eligibility 6 msmarco_passage_65_691518969 21.88538064807829 371695 how to order certificate of eligibility 7 msmarco_passage_54_121999881 21.687403416127207 371695 how to order certificate of eligibility 8 msmarco_passage_59_758975639 21.111251814552446 371695 how to order certificate of eligibility 9 msmarco_passage_64_338864155 21.057964099271025 371695 how to order certificate of eligibility 10 msmarco_passage_67_544972315 20.861916293135693 506181 symmetry math definition 1 msmarco_passage_44_621881484 28.99909450209269 506181 symmetry math definition 2 msmarco_passage_44_621881910 27.93595235578558 506181 symmetry math definition 3 msmarco_passage_14_388113013 27.3056603221735 506181 symmetry math definition 4 msmarco_passage_07_158205410 27.049213326841507 506181 symmetry math definition 5 msmarco_passage_14_387808446 27.048464027094703 506181 symmetry math definition 6 msmarco_passage_07_158206391 26.65821285612587 506181 symmetry math definition 7 msmarco_passage_03_179593802 26.46326009768387 506181 symmetry math definition 8 msmarco_passage_44_621966157 26.28363640023152 506181 symmetry math definition 9 msmarco_passage_44_621976637 25.53140956232306 506181 symmetry math definition 10 msmarco_passage_44_621977351 25.247634274259504 900696 what team was shelton benjamin a part of wwe 1 msmarco_passage_63_669812928 38.9741590642407 900696 what team was shelton benjamin a part of wwe 2 msmarco_passage_09_63368132 38.6182585638652 900696 what team was shelton benjamin a part of wwe 3 msmarco_passage_68_264602491 36.935836436263656 900696 what team was shelton benjamin a part of wwe 4 msmarco_passage_68_264668205 36.935836436263656 900696 what team was shelton benjamin a part of wwe 5 msmarco_passage_63_670370544 36.479291088288285 900696 what team was shelton benjamin a part of wwe 6 msmarco_passage_13_518931795 36.00138746572684 900696 what team was shelton benjamin a part of wwe 7 msmarco_passage_11_143862784 34.632816140717665 900696 what team was shelton benjamin a part of wwe 8 msmarco_passage_68_264945096 34.61373304236272 900696 what team was shelton benjamin a part of wwe 9 msmarco_passage_19_489362570 34.59704468144898 900696 what team was shelton benjamin a part of wwe 10 msmarco_passage_58_676367253 33.95033771133815 702790 what is a tanf 1 msmarco_passage_62_99109344 19.85087157010833 702790 what is a tanf 2 msmarco_passage_62_99439715 19.670481252195366 702790 what is a tanf 3 msmarco_passage_62_99441041 19.611077504386145 702790 what is a tanf 4 msmarco_passage_62_99440572 19.377007608783348 702790 what is a tanf 5 msmarco_passage_41_46979500 18.08766400881107 702790 what is a tanf 6 msmarco_passage_49_574827048 17.937775025478004 702790 what is a tanf 7 msmarco_passage_04_796995278 17.881784837406627 702790 what is a tanf 8 msmarco_passage_61_423726535 17.79034982742735 702790 what is a tanf 9 msmarco_passage_09_665086960 17.79034982742735 702790 what is a tanf 10 msmarco_passage_62_99235829 17.70626038911348 1053716 what is ibm chat 1 msmarco_passage_23_327251910 24.362271285749863 1053716 what is ibm chat 2 msmarco_passage_16_198040081 21.07751348592509 1053716 what is ibm chat 3 msmarco_passage_23_327250780 20.986150422029 1053716 what is ibm chat 4 msmarco_passage_23_327250452 19.940016806218253 1053716 what is ibm chat 5 msmarco_passage_31_485346405 19.786577689557287 1053716 what is ibm chat 6 msmarco_passage_50_26390229 19.75596561458496 1053716 what is ibm chat 7 msmarco_passage_36_599102354 19.505771508004607 1053716 what is ibm chat 8 msmarco_passage_47_476295682 19.076578046687366 1053716 what is ibm chat 9 msmarco_passage_23_327251146 18.526440205060233 1053716 what is ibm chat 10 msmarco_passage_23_868876830 18.133043604090563 1094364 interchange definition 1 msmarco_passage_68_244609107 17.810937050807443 1094364 interchange definition 2 msmarco_passage_37_106671830 17.451004259310526 1094364 interchange definition 3 msmarco_passage_02_689310705 17.1429465374608 1094364 interchange definition 4 msmarco_passage_53_793906928 17.1429465374608 1094364 interchange definition 5 msmarco_passage_05_685570048 17.105330731736483 1094364 interchange definition 6 msmarco_passage_33_78832652 17.023809598517108 1094364 interchange definition 7 msmarco_passage_53_811546058 16.953743374711692 1094364 interchange definition 8 msmarco_passage_37_13622438 16.932035438745142 1094364 interchange definition 9 msmarco_passage_53_793907152 16.80925276633324 1094364 interchange definition 10 msmarco_passage_51_641278353 16.63284829468357 1096823 how many people died in the parkland school shooting 1 msmarco_passage_55_637736830 37.02203657448422 1096823 how many people died in the parkland school shooting 2 msmarco_passage_55_637737229 36.253953046676614 1096823 how many people died in the parkland school shooting 3 msmarco_passage_37_769433936 34.8762889784639 1096823 how many people died in the parkland school shooting 4 msmarco_passage_11_400717554 32.78090821376462 1096823 how many people died in the parkland school shooting 5 msmarco_passage_15_681733705 32.405667324717776 1096823 how many people died in the parkland school shooting 6 msmarco_passage_64_163988034 32.162155628262 1096823 how many people died in the parkland school shooting 7 msmarco_passage_40_309313299 31.620200638482054 1096823 how many people died in the parkland school shooting 8 msmarco_passage_44_834765714 31.606818791966674 1096823 how many people died in the parkland school shooting 9 msmarco_passage_44_834644266 31.606818791966674 1096823 how many people died in the parkland school shooting 10 msmarco_passage_00_30611271 30.67145433064144 79891 can you substitute chocolate chips for semi-sweet 1 msmarco_passage_05_555996782 66.61533285776964 79891 can you substitute chocolate chips for semi-sweet 2 msmarco_passage_46_394221685 63.3781490698231 79891 can you substitute chocolate chips for semi-sweet 3 msmarco_passage_45_53678201 58.08022686555849 79891 can you substitute chocolate chips for semi-sweet 4 msmarco_passage_62_564825525 57.64860562352943 79891 can you substitute chocolate chips for semi-sweet 5 msmarco_passage_54_735842662 57.57157984695039 79891 can you substitute chocolate chips for semi-sweet 6 msmarco_passage_40_539824659 56.46705544208075 79891 can you substitute chocolate chips for semi-sweet 7 msmarco_passage_00_119030582 55.60985376678514 79891 can you substitute chocolate chips for semi-sweet 8 msmarco_passage_37_463911033 55.60985376678514 79891 can you substitute chocolate chips for semi-sweet 9 msmarco_passage_05_555997168 55.469188403571465 79891 can you substitute chocolate chips for semi-sweet 10 msmarco_passage_36_183788747 54.97621907393163 278606 how many carbs in cool whip 1 msmarco_passage_09_623519417 30.874356191070056 278606 how many carbs in cool whip 2 msmarco_passage_27_90979220 30.62938817020602 278606 how many carbs in cool whip 3 msmarco_passage_00_485352768 29.84048270804513 278606 how many carbs in cool whip 4 msmarco_passage_25_724018127 29.512629123998252 278606 how many carbs in cool whip 5 msmarco_passage_00_485378222 29.01237204696904 278606 how many carbs in cool whip 6 msmarco_passage_00_485376920 28.168292960783894 278606 how many carbs in cool whip 7 msmarco_passage_09_524158131 27.681647387712847 278606 how many carbs in cool whip 8 msmarco_passage_27_89576045 27.481429030334617 278606 how many carbs in cool whip 9 msmarco_passage_52_758689209 27.481429030334617 278606 how many carbs in cool whip 10 msmarco_passage_44_667674294 27.40060615640749 1085967 what character did michael b. jordan in creed 1 msmarco_passage_46_329251004 44.519425080083494 1085967 what character did michael b. jordan in creed 2 msmarco_passage_54_601571254 41.86234998165448 1085967 what character did michael b. jordan in creed 3 msmarco_passage_54_601575359 41.415745382898166 1085967 what character did michael b. jordan in creed 4 msmarco_passage_17_760255220 38.694964266040444 1085967 what character did michael b. jordan in creed 5 msmarco_passage_27_711645710 38.56035316298333 1085967 what character did michael b. jordan in creed 6 msmarco_passage_08_549459667 38.27250798415668 1085967 what character did michael b. jordan in creed 7 msmarco_passage_28_170995339 37.809068183214116 1085967 what character did michael b. jordan in creed 8 msmarco_passage_25_498654854 37.73589209699133 1085967 what character did michael b. jordan in creed 9 msmarco_passage_39_477549761 37.73589209699133 1085967 what character did michael b. jordan in creed 10 msmarco_passage_28_170995979 37.28133531743467 724121 what is bioflavonoids 1 msmarco_passage_11_73789641 19.6531541035536 724121 what is bioflavonoids 2 msmarco_passage_34_745057267 19.177313665641147 724121 what is bioflavonoids 3 msmarco_passage_45_175121009 19.16609120443793 724121 what is bioflavonoids 4 msmarco_passage_61_844773521 19.077980028056746 724121 what is bioflavonoids 5 msmarco_passage_59_530276275 19.032035556194085 724121 what is bioflavonoids 6 msmarco_passage_09_274392210 19.032035556194085 724121 what is bioflavonoids 7 msmarco_passage_45_604032386 19.029510970460763 724121 what is bioflavonoids 8 msmarco_passage_64_313152983 19.029510970460763 724121 what is bioflavonoids 9 msmarco_passage_67_81731973 18.943668801640232 724121 what is bioflavonoids 10 msmarco_passage_45_180975850 18.89984216495411 763878 what is kyc? 1 msmarco_passage_46_270234673 20.123783824102592 763878 what is kyc? 2 msmarco_passage_46_269655635 20.123783824102592 763878 what is kyc? 3 msmarco_passage_53_324656206 19.777340917999847 763878 what is kyc? 4 msmarco_passage_28_75530535 19.730179276180795 763878 what is kyc? 5 msmarco_passage_46_270353631 19.69258602311825 763878 what is kyc? 6 msmarco_passage_46_270374261 19.69258602311825 763878 what is kyc? 7 msmarco_passage_46_270363993 19.442624573480497 763878 what is kyc? 8 msmarco_passage_46_270374718 19.364323355947356 763878 what is kyc? 9 msmarco_passage_46_270354088 19.364323355947356 763878 what is kyc? 10 msmarco_passage_38_677442548 19.17044032758524 1009023 what's the difference between yeast and bread machine yeast 1 msmarco_passage_34_572748463 48.45561631529958 1009023 what's the difference between yeast and bread machine yeast 2 msmarco_passage_65_762279707 48.05742321401754 1009023 what's the difference between yeast and bread machine yeast 3 msmarco_passage_34_572746418 47.22322194737679 1009023 what's the difference between yeast and bread machine yeast 4 msmarco_passage_34_507366934 46.311339150745496 1009023 what's the difference between yeast and bread machine yeast 5 msmarco_passage_34_572746051 46.29999729542047 1009023 what's the difference between yeast and bread machine yeast 6 msmarco_passage_14_174384383 45.981218825781 1009023 what's the difference between yeast and bread machine yeast 7 msmarco_passage_00_436974297 45.981218825781 1009023 what's the difference between yeast and bread machine yeast 8 msmarco_passage_14_174523367 44.66287583076147 1009023 what's the difference between yeast and bread machine yeast 9 msmarco_passage_39_856867335 44.61236651389741 1009023 what's the difference between yeast and bread machine yeast 10 msmarco_passage_00_778939367 44.39957766703069 1025188 who can write a death certificate 1 msmarco_passage_26_595373749 21.702222089947423 1025188 who can write a death certificate 2 msmarco_passage_41_169630971 21.624024010311196 1025188 who can write a death certificate 3 msmarco_passage_15_93480998 21.56598735127155 1025188 who can write a death certificate 4 msmarco_passage_15_93479934 20.997635116961774 1025188 who can write a death certificate 5 msmarco_passage_46_36062978 20.83162573820903 1025188 who can write a death certificate 6 msmarco_passage_65_496379353 20.354191543215528 1025188 who can write a death certificate 7 msmarco_passage_06_453438947 20.247576126089356 1025188 who can write a death certificate 8 msmarco_passage_55_513611493 19.15190007024667 1025188 who can write a death certificate 9 msmarco_passage_62_443562294 19.15190007024667 1025188 who can write a death certificate 10 msmarco_passage_17_240405009 19.12108070646998 696918 what is a qph file 1 msmarco_passage_09_571160918 30.533093110786304 696918 what is a qph file 2 msmarco_passage_09_209694470 30.18421377218926 696918 what is a qph file 3 msmarco_passage_59_281947480 30.18421377218926 696918 what is a qph file 4 msmarco_passage_09_209694892 30.001027306917305 696918 what is a qph file 5 msmarco_passage_59_281947901 30.001027306917305 696918 what is a qph file 6 msmarco_passage_09_209696029 29.357068888300844 696918 what is a qph file 7 msmarco_passage_59_281949036 29.357068888300844 696918 what is a qph file 8 msmarco_passage_46_228356695 29.319796570378816 696918 what is a qph file 9 msmarco_passage_09_209696470 29.247856003789195 696918 what is a qph file 10 msmarco_passage_59_281949476 29.247856003789195 1035719 airplane definition of drag 1 msmarco_passage_53_798004652 25.437574022655465 1035719 airplane definition of drag 2 msmarco_passage_04_224700011 24.311470621343986 1035719 airplane definition of drag 3 msmarco_passage_62_192287612 23.338208418018752 1035719 airplane definition of drag 4 msmarco_passage_08_835158909 23.209335685834045 1035719 airplane definition of drag 5 msmarco_passage_55_677029631 22.73201188022651 1035719 airplane definition of drag 6 msmarco_passage_44_778651093 22.640174768852155 1035719 airplane definition of drag 7 msmarco_passage_54_814903525 22.58540624923358 1035719 airplane definition of drag 8 msmarco_passage_54_814902227 22.58540624923358 1035719 airplane definition of drag 9 msmarco_passage_25_121887181 22.40315088394823 1035719 airplane definition of drag 10 msmarco_passage_07_431702157 22.33384075785203 80590 can you use denture tablets to clean a brace 1 msmarco_passage_61_546890403 34.11670997715102 80590 can you use denture tablets to clean a brace 2 msmarco_passage_61_546888801 32.080507865346185 80590 can you use denture tablets to clean a brace 3 msmarco_passage_57_572682410 31.84455980442201 80590 can you use denture tablets to clean a brace 4 msmarco_passage_12_643016719 31.7653793196168 80590 can you use denture tablets to clean a brace 5 msmarco_passage_03_380638174 31.717536685567968 80590 can you use denture tablets to clean a brace 6 msmarco_passage_34_675194564 31.717536685567968 80590 can you use denture tablets to clean a brace 7 msmarco_passage_56_276585437 31.580799682228513 80590 can you use denture tablets to clean a brace 8 msmarco_passage_12_547872881 31.447149163794087 80590 can you use denture tablets to clean a brace 9 msmarco_passage_00_240128140 31.447149163794087 80590 can you use denture tablets to clean a brace 10 msmarco_passage_58_751061261 31.348522332031543 292094 how many people die from snake bites a year? 1 msmarco_passage_28_32052217 35.933271811245426 292094 how many people die from snake bites a year? 2 msmarco_passage_04_344687020 35.75675710423282 292094 how many people die from snake bites a year? 3 msmarco_passage_59_826105770 34.25532281366844 292094 how many people die from snake bites a year? 4 msmarco_passage_59_826105174 34.02929311552765 292094 how many people die from snake bites a year? 5 msmarco_passage_21_2465866 33.81423125286316 292094 how many people die from snake bites a year? 6 msmarco_passage_03_503869831 33.37684732164054 292094 how many people die from snake bites a year? 7 msmarco_passage_11_769356115 33.141894278647925 292094 how many people die from snake bites a year? 8 msmarco_passage_00_844937246 33.12775612505992 292094 how many people die from snake bites a year? 9 msmarco_passage_11_207121935 32.79899690216332 292094 how many people die from snake bites a year? 10 msmarco_passage_17_43247363 32.49315879451484 1029016 who invented the first camera 1 msmarco_passage_04_217353893 26.291776884828977 1029016 who invented the first camera 2 msmarco_passage_04_183705395 24.46083470044632 1029016 who invented the first camera 3 msmarco_passage_04_217353437 23.488655729461374 1029016 who invented the first camera 4 msmarco_passage_04_207117267 23.33345298795295 1029016 who invented the first camera 5 msmarco_passage_04_183704477 23.31986335676564 1029016 who invented the first camera 6 msmarco_passage_58_544683387 22.92407067986663 1029016 who invented the first camera 7 msmarco_passage_58_490394792 22.92407067986663 1029016 who invented the first camera 8 msmarco_passage_58_513767241 22.92407067986663 1029016 who invented the first camera 9 msmarco_passage_68_181746486 22.692630440522855 1029016 who invented the first camera 10 msmarco_passage_11_72825426 22.567757664029145 461491 name meaning paige 1 msmarco_passage_51_561528735 28.019397759747687 461491 name meaning paige 2 msmarco_passage_08_43364658 27.829832218776808 461491 name meaning paige 3 msmarco_passage_67_18683859 27.61075127444516 461491 name meaning paige 4 msmarco_passage_51_561529338 26.625681844733045 461491 name meaning paige 5 msmarco_passage_08_681260103 26.357470566767585 461491 name meaning paige 6 msmarco_passage_54_793053125 26.256245633947174 461491 name meaning paige 7 msmarco_passage_04_384319836 25.389611192184816 461491 name meaning paige 8 msmarco_passage_38_642161222 25.389611192184816 461491 name meaning paige 9 msmarco_passage_54_793053616 25.008397060239027 461491 name meaning paige 10 msmarco_passage_09_457618125 24.850137105472285 1087122 what are the color of the vietnam service ribbon 1 msmarco_passage_57_244617520 32.582526302235834 1087122 what are the color of the vietnam service ribbon 2 msmarco_passage_53_426099447 27.026471718666663 1087122 what are the color of the vietnam service ribbon 3 msmarco_passage_35_774136276 26.788267894310067 1087122 what are the color of the vietnam service ribbon 4 msmarco_passage_66_159982600 26.741049851939927 1087122 what are the color of the vietnam service ribbon 5 msmarco_passage_57_244619681 26.413517891381815 1087122 what are the color of the vietnam service ribbon 6 msmarco_passage_14_583019933 26.244533459878774 1087122 what are the color of the vietnam service ribbon 7 msmarco_passage_12_236748587 25.958328061314646 1087122 what are the color of the vietnam service ribbon 8 msmarco_passage_44_684275100 25.92773898250864 1087122 what are the color of the vietnam service ribbon 9 msmarco_passage_12_236748344 25.57737272244338 1087122 what are the color of the vietnam service ribbon 10 msmarco_passage_57_114037788 25.504551801673323 792900 what is rylon 1 msmarco_passage_01_771481307 23.42935986451113 792900 what is rylon 2 msmarco_passage_35_680103705 22.9606232652586 792900 what is rylon 3 msmarco_passage_23_59013149 22.07725163971805 792900 what is rylon 4 msmarco_passage_14_840088461 21.45807883581007 792900 what is rylon 5 msmarco_passage_01_771482471 20.00347666584023 792900 what is rylon 6 msmarco_passage_59_570195957 19.00948692295977 792900 what is rylon 7 msmarco_passage_68_217847095 17.29107092643955 792900 what is rylon 8 msmarco_passage_01_771482728 17.034425137191374 792900 what is rylon 9 msmarco_passage_03_120393864 17.034425137191374 792900 what is rylon 10 msmarco_passage_15_398618638 16.543330498727173 1101977 why must oil be used with oil immersion lens 1 msmarco_passage_09_260204397 45.35494357766797 1101977 why must oil be used with oil immersion lens 2 msmarco_passage_59_479067913 45.35494357766797 1101977 why must oil be used with oil immersion lens 3 msmarco_passage_54_72138416 43.16620428670809 1101977 why must oil be used with oil immersion lens 4 msmarco_passage_31_383820163 43.03981381991248 1101977 why must oil be used with oil immersion lens 5 msmarco_passage_54_72136916 42.286760034501725 1101977 why must oil be used with oil immersion lens 6 msmarco_passage_22_473417050 41.66608828456587 1101977 why must oil be used with oil immersion lens 7 msmarco_passage_31_383821170 41.64856603978498 1101977 why must oil be used with oil immersion lens 8 msmarco_passage_04_227531177 41.2984458684537 1101977 why must oil be used with oil immersion lens 9 msmarco_passage_20_272547852 41.13520592878511 1101977 why must oil be used with oil immersion lens 10 msmarco_passage_01_457897735 40.571232259394485 969264 where did slavery exist 1 msmarco_passage_25_192307329 20.315991197372902 969264 where did slavery exist 2 msmarco_passage_16_351386496 20.315991197372902 969264 where did slavery exist 3 msmarco_passage_08_692400565 19.97390967565427 969264 where did slavery exist 4 msmarco_passage_29_508078832 19.901489957559985 969264 where did slavery exist 5 msmarco_passage_04_653278875 19.63436442017133 969264 where did slavery exist 6 msmarco_passage_40_24158385 19.63436442017133 969264 where did slavery exist 7 msmarco_passage_09_618881080 19.600346875227373 969264 where did slavery exist 8 msmarco_passage_58_104001069 19.06344434695137 969264 where did slavery exist 9 msmarco_passage_45_243354706 19.04173478763804 969264 where did slavery exist 10 msmarco_passage_55_844628218 19.011790090971093 904007 what those pending mean 1 msmarco_passage_31_861264218 18.74132280547917 904007 what those pending mean 2 msmarco_passage_25_15385770 18.6432644137942 904007 what those pending mean 3 msmarco_passage_48_182460095 18.6432644137942 904007 what those pending mean 4 msmarco_passage_34_319763083 18.509999222675155 904007 what those pending mean 5 msmarco_passage_48_182463420 18.408790600234806 904007 what those pending mean 6 msmarco_passage_31_365139121 18.386319710129932 904007 what those pending mean 7 msmarco_passage_12_242116775 18.386319710129932 904007 what those pending mean 8 msmarco_passage_25_15386655 18.292477934037002 904007 what those pending mean 9 msmarco_passage_59_159121403 18.233742328827613 904007 what those pending mean 10 msmarco_passage_04_144883810 18.179657836959414 195693 golf cost 1 msmarco_passage_34_236210807 17.932517560863303 195693 golf cost 2 msmarco_passage_52_674559282 17.85459689507011 195693 golf cost 3 msmarco_passage_52_674561538 17.85459689507011 195693 golf cost 4 msmarco_passage_49_608407127 17.757779203330735 195693 golf cost 5 msmarco_passage_23_496404370 17.64665141402129 195693 golf cost 6 msmarco_passage_52_674566720 17.605483734405784 195693 golf cost 7 msmarco_passage_49_607750118 17.399074803250823 195693 golf cost 8 msmarco_passage_07_74878246 17.399074803250823 195693 golf cost 9 msmarco_passage_49_612598465 17.399074803250823 195693 golf cost 10 msmarco_passage_23_502109682 17.23489132120219 810680 what is the charter of french liberties 1 msmarco_passage_00_754093650 26.35777458307752 810680 what is the charter of french liberties 2 msmarco_passage_00_754087385 25.2821099824694 810680 what is the charter of french liberties 3 msmarco_passage_02_476877675 24.838660184355085 810680 what is the charter of french liberties 4 msmarco_passage_00_754090765 23.382610825895643 810680 what is the charter of french liberties 5 msmarco_passage_40_157823390 22.950376140133557 810680 what is the charter of french liberties 6 msmarco_passage_43_183538038 22.41316905343198 810680 what is the charter of french liberties 7 msmarco_passage_05_508955668 22.41316905343198 810680 what is the charter of french liberties 8 msmarco_passage_05_486713173 22.41316905343198 810680 what is the charter of french liberties 9 msmarco_passage_43_192395750 22.41316905343198 810680 what is the charter of french liberties 10 msmarco_passage_54_347872813 22.286568873988923 1043702 who make kenmore ranges 1 msmarco_passage_16_369801365 24.97739336701026 1043702 who make kenmore ranges 2 msmarco_passage_00_779965999 24.97739336701026 1043702 who make kenmore ranges 3 msmarco_passage_04_487363352 24.142638775288653 1043702 who make kenmore ranges 4 msmarco_passage_60_314670274 23.649453173953546 1043702 who make kenmore ranges 5 msmarco_passage_51_493103356 23.564829034662875 1043702 who make kenmore ranges 6 msmarco_passage_60_314498354 23.56056065734696 1043702 who make kenmore ranges 7 msmarco_passage_52_867239225 23.277573078010533 1043702 who make kenmore ranges 8 msmarco_passage_60_314534412 23.214924964761842 1043702 who make kenmore ranges 9 msmarco_passage_60_313967339 23.156703820155347 1043702 who make kenmore ranges 10 msmarco_passage_07_460832991 23.140644609067177 1085762 what contains capsaicin 1 msmarco_passage_24_266437804 22.117943382233605 1085762 what contains capsaicin 2 msmarco_passage_49_100098594 21.564145607637904 1085762 what contains capsaicin 3 msmarco_passage_56_764645913 21.31299862006224 1085762 what contains capsaicin 4 msmarco_passage_11_466309221 21.31040635984396 1085762 what contains capsaicin 5 msmarco_passage_65_796746334 21.301766794459184 1085762 what contains capsaicin 6 msmarco_passage_66_30959554 21.301766794459184 1085762 what contains capsaicin 7 msmarco_passage_00_128083314 21.03787388849919 1085762 what contains capsaicin 8 msmarco_passage_65_843996671 21.03787388849919 1085762 what contains capsaicin 9 msmarco_passage_56_765208187 21.00418360127215 1085762 what contains capsaicin 10 msmarco_passage_16_828128490 20.922843252172143 1025270 what mammals eat birds 1 msmarco_passage_22_141810073 27.566780226615577 1025270 what mammals eat birds 2 msmarco_passage_04_144012571 26.512536649711297 1025270 what mammals eat birds 3 msmarco_passage_06_593060124 26.40691111582692 1025270 what mammals eat birds 4 msmarco_passage_57_854912356 26.347950984848325 1025270 what mammals eat birds 5 msmarco_passage_03_775223948 26.25600836237131 1025270 what mammals eat birds 6 msmarco_passage_11_190474759 24.970223442096277 1025270 what mammals eat birds 7 msmarco_passage_45_441540643 24.95420799999071 1025270 what mammals eat birds 8 msmarco_passage_36_320950892 24.875674072137834 1025270 what mammals eat birds 9 msmarco_passage_31_186298212 24.70948242921791 1025270 what mammals eat birds 10 msmarco_passage_59_144002744 24.70948242921791 1034587 cures that work for mites on dogs homemade 1 msmarco_passage_34_763432782 28.09965308204204 1034587 cures that work for mites on dogs homemade 2 msmarco_passage_02_539922390 26.737492968412226 1034587 cures that work for mites on dogs homemade 3 msmarco_passage_34_257625955 26.47326261751401 1034587 cures that work for mites on dogs homemade 4 msmarco_passage_47_508381458 26.137814315797506 1034587 cures that work for mites on dogs homemade 5 msmarco_passage_35_792022540 26.066011519159797 1034587 cures that work for mites on dogs homemade 6 msmarco_passage_49_771911728 26.062424863197485 1034587 cures that work for mites on dogs homemade 7 msmarco_passage_24_185576573 25.934077163497825 1034587 cures that work for mites on dogs homemade 8 msmarco_passage_63_732240452 25.88326421111 682205 what is a ds1 cable 1 msmarco_passage_21_92832811 24.081142383905593 1034587 cures that work for mites on dogs homemade 9 msmarco_passage_17_169987313 25.529009314710457 1034587 cures that work for mites on dogs homemade 10 msmarco_passage_34_763431900 24.49627176852099 477639 population of georgetown co 1 msmarco_passage_38_195600554 24.337200424157636 477639 population of georgetown co 2 msmarco_passage_38_195598176 24.078349648313655 477639 population of georgetown co 3 msmarco_passage_04_266913929 23.70482935239363 477639 population of georgetown co 4 msmarco_passage_08_254415272 23.686196964520093 477639 population of georgetown co 5 msmarco_passage_30_193335865 21.46040191043273 477639 population of georgetown co 6 msmarco_passage_04_31179246 21.2095773755006 477639 population of georgetown co 7 msmarco_passage_49_381155277 21.18898304413758 477639 population of georgetown co 8 msmarco_passage_21_536848596 21.09670713938783 477639 population of georgetown co 9 msmarco_passage_08_254413478 21.08482467765092 477639 population of georgetown co 10 msmarco_passage_36_807341593 20.883995566628688 166043 does estrogen make me start my period 1 msmarco_passage_48_774535297 22.356744692082344 166043 does estrogen make me start my period 2 msmarco_passage_65_794389790 22.319878368695896 166043 does estrogen make me start my period 3 msmarco_passage_66_10442931 22.319878368695896 166043 does estrogen make me start my period 4 msmarco_passage_66_658433762 22.171635033976234 166043 does estrogen make me start my period 5 msmarco_passage_66_662345165 22.171635033976234 166043 does estrogen make me start my period 6 msmarco_passage_10_648542370 22.171635033976234 166043 does estrogen make me start my period 7 msmarco_passage_66_661436390 22.171635033976234 166043 does estrogen make me start my period 8 msmarco_passage_65_768311466 21.97716620246175 166043 does estrogen make me start my period 9 msmarco_passage_03_463122084 21.742219171889893 166043 does estrogen make me start my period 10 msmarco_passage_00_467236850 21.742219171889893 272605 how long to wait to swim after tattoo 1 msmarco_passage_55_287098690 29.221237730717228 272605 how long to wait to swim after tattoo 2 msmarco_passage_08_424607928 29.221237730717228 272605 how long to wait to swim after tattoo 3 msmarco_passage_55_274975967 29.221237730717228 272605 how long to wait to swim after tattoo 4 msmarco_passage_40_472722410 28.960052052530834 272605 how long to wait to swim after tattoo 5 msmarco_passage_03_246346979 28.960052052530834 272605 how long to wait to swim after tattoo 6 msmarco_passage_12_226687876 28.48930753936367 272605 how long to wait to swim after tattoo 7 msmarco_passage_52_57188409 27.280741727992755 272605 how long to wait to swim after tattoo 8 msmarco_passage_12_226686213 26.592959717892974 272605 how long to wait to swim after tattoo 9 msmarco_passage_48_677829092 26.520962733870554 272605 how long to wait to swim after tattoo 10 msmarco_passage_40_472724160 26.071279108126635 338040 how popular is the name kaitlyn 1 msmarco_passage_08_675108661 30.63702421207243 338040 how popular is the name kaitlyn 2 msmarco_passage_08_675109704 30.629027427870426 338040 how popular is the name kaitlyn 3 msmarco_passage_54_791110431 30.13312617130859 338040 how popular is the name kaitlyn 4 msmarco_passage_54_791107630 29.494922644581315 338040 how popular is the name kaitlyn 5 msmarco_passage_38_614624726 27.815689703707495 338040 how popular is the name kaitlyn 6 msmarco_passage_38_637461089 27.53061200013512 338040 how popular is the name kaitlyn 7 msmarco_passage_63_505070613 27.302908286932585 338040 how popular is the name kaitlyn 8 msmarco_passage_10_199138538 27.302908286932585 338040 how popular is the name kaitlyn 9 msmarco_passage_54_791110041 27.06261483404503 338040 how popular is the name kaitlyn 10 msmarco_passage_54_791106814 26.847798554436665 856568 what is tone and tenor? 1 msmarco_passage_30_687461471 25.897454940737514 856568 what is tone and tenor? 2 msmarco_passage_00_271572454 23.87157763191849 856568 what is tone and tenor? 3 msmarco_passage_58_325884051 23.851903888664292 856568 what is tone and tenor? 4 msmarco_passage_22_720628477 23.69981072472303 856568 what is tone and tenor? 5 msmarco_passage_61_536921387 22.722177700818754 856568 what is tone and tenor? 6 msmarco_passage_40_215905781 22.654946058031506 856568 what is tone and tenor? 7 msmarco_passage_43_164968581 22.28724985501691 856568 what is tone and tenor? 8 msmarco_passage_22_719725045 22.26513039668326 856568 what is tone and tenor? 9 msmarco_passage_11_181677550 22.22632512650223 856568 what is tone and tenor? 10 msmarco_passage_33_563612525 22.187908591243197 447340 meaning of black flowers 1 msmarco_passage_23_181651804 21.603880313096752 447340 meaning of black flowers 2 msmarco_passage_52_810068019 21.142697793784713 447340 meaning of black flowers 3 msmarco_passage_64_509325599 21.04306735135762 447340 meaning of black flowers 4 msmarco_passage_06_396380821 20.6596337774601 447340 meaning of black flowers 5 msmarco_passage_26_476293679 20.09725619004574 447340 meaning of black flowers 6 msmarco_passage_01_306848031 20.001686010763432 447340 meaning of black flowers 7 msmarco_passage_06_396391890 19.42514125973704 447340 meaning of black flowers 8 msmarco_passage_28_320007359 19.299361737467738 447340 meaning of black flowers 9 msmarco_passage_28_319895521 19.299361737467738 447340 meaning of black flowers 10 msmarco_passage_23_181649483 19.23437005127688 1084330 what does actor in dumb mean 1 msmarco_passage_58_690324825 23.967084538984835 1084330 what does actor in dumb mean 2 msmarco_passage_58_690324647 23.47308291847972 1084330 what does actor in dumb mean 3 msmarco_passage_02_491454523 20.966390472314146 1084330 what does actor in dumb mean 4 msmarco_passage_18_663425152 20.35024082365767 1084330 what does actor in dumb mean 5 msmarco_passage_27_287444361 19.774117376076155 1084330 what does actor in dumb mean 6 msmarco_passage_50_286698426 19.774117376076155 1084330 what does actor in dumb mean 7 msmarco_passage_50_337303352 19.774117376076155 1084330 what does actor in dumb mean 8 msmarco_passage_68_86266347 19.71374104093994 1084330 what does actor in dumb mean 9 msmarco_passage_58_561388226 19.564958612597703 1084330 what does actor in dumb mean 10 msmarco_passage_06_179919348 19.557697798492548 715508 what is an integrated studies degree 1 msmarco_passage_59_856054491 21.764086795993983 715508 what is an integrated studies degree 2 msmarco_passage_19_870427197 21.71218923474529 715508 what is an integrated studies degree 3 msmarco_passage_19_870366163 21.71218923474529 715508 what is an integrated studies degree 4 msmarco_passage_20_879602 21.71218923474529 715508 what is an integrated studies degree 5 msmarco_passage_17_515390361 21.71218923474529 715508 what is an integrated studies degree 6 msmarco_passage_19_756475739 21.29959239283556 715508 what is an integrated studies degree 7 msmarco_passage_18_68403093 20.95542088103369 715508 what is an integrated studies degree 8 msmarco_passage_51_656363981 20.433531680973047 715508 what is an integrated studies degree 9 msmarco_passage_08_272762301 20.37641223859852 715508 what is an integrated studies degree 10 msmarco_passage_37_371329126 20.061685996982032 563347 what are plastids 1 msmarco_passage_29_597519642 20.155761384009995 563347 what are plastids 2 msmarco_passage_30_26505321 19.893926345887873 563347 what are plastids 3 msmarco_passage_29_597520476 19.77668896855651 563347 what are plastids 4 msmarco_passage_00_465819116 19.745992825623663 563347 what are plastids 5 msmarco_passage_14_370537022 19.745992825623663 563347 what are plastids 6 msmarco_passage_14_155010413 19.745992825623663 563347 what are plastids 7 msmarco_passage_53_775938156 19.547293650069385 563347 what are plastids 8 msmarco_passage_05_698122277 19.449436151526967 563347 what are plastids 9 msmarco_passage_27_736922313 19.406241409014548 563347 what are plastids 10 msmarco_passage_27_725748663 19.166891329457332 778139 what is nrn 1 msmarco_passage_10_74977280 23.979659742478162 778139 what is nrn 2 msmarco_passage_41_498236336 22.100325666413703 778139 what is nrn 3 msmarco_passage_41_498222717 21.771850059879622 778139 what is nrn 4 msmarco_passage_11_509078342 20.695276458049104 778139 what is nrn 5 msmarco_passage_28_584881897 20.351789600396394 778139 what is nrn 6 msmarco_passage_31_201542199 20.351789600396394 778139 what is nrn 7 msmarco_passage_28_584877779 20.351789600396394 778139 what is nrn 8 msmarco_passage_41_498238360 20.14190837138574 778139 what is nrn 9 msmarco_passage_50_474842118 19.734870084432764 778139 what is nrn 10 msmarco_passage_41_498237555 19.734870084432764 1060040 what is betts 1 msmarco_passage_27_303315613 18.961852123074916 1060040 what is betts 2 msmarco_passage_49_554187952 18.203693126834587 1060040 what is betts 3 msmarco_passage_15_799123082 17.87400273340002 1060040 what is betts 4 msmarco_passage_66_564300058 17.87400273340002 1060040 what is betts 5 msmarco_passage_18_660010365 17.838969828652587 1060040 what is betts 6 msmarco_passage_24_375693429 17.741089597019048 1060040 what is betts 7 msmarco_passage_49_554189885 17.32764366652871 1060040 what is betts 8 msmarco_passage_13_430508116 17.28397380716261 1060040 what is betts 9 msmarco_passage_64_74291560 17.216082683520657 1060040 what is betts 10 msmarco_passage_26_610869609 17.216082683520657 1071545 what is a labia? 1 msmarco_passage_44_588932002 19.6085159760117 1071545 what is a labia? 2 msmarco_passage_05_151915208 18.776026067878988 1071545 what is a labia? 3 msmarco_passage_41_669666185 18.776026067878988 1071545 what is a labia? 4 msmarco_passage_41_504124433 18.648998550518925 1071545 what is a labia? 5 msmarco_passage_48_602919780 18.632764736703542 1071545 what is a labia? 6 msmarco_passage_19_564184775 18.61300256721177 1071545 what is a labia? 7 msmarco_passage_21_667476727 18.61300256721177 1071545 what is a labia? 8 msmarco_passage_19_564086304 18.61300256721177 1071545 what is a labia? 9 msmarco_passage_19_564075596 18.61300256721177 1071545 what is a labia? 10 msmarco_passage_11_249299802 18.5989123777768 779674 what is opc ua 1 msmarco_passage_55_416067106 36.506290382612434 779674 what is opc ua 2 msmarco_passage_55_416068003 35.83279593719461 779674 what is opc ua 3 msmarco_passage_53_308402780 35.19857673292718 779674 what is opc ua 4 msmarco_passage_16_504575053 35.18325937981959 779674 what is opc ua 5 msmarco_passage_34_8929419 34.947588914245394 779674 what is opc ua 6 msmarco_passage_09_258353623 34.63696082447649 779674 what is opc ua 7 msmarco_passage_61_390710384 34.50132193474124 779674 what is opc ua 8 msmarco_passage_53_307200102 33.830522289659775 779674 what is opc ua 9 msmarco_passage_25_617695783 33.7606260249574 779674 what is opc ua 10 msmarco_passage_09_258351489 33.668783960212934 577131 what are year quarters 1 msmarco_passage_24_51553551 16.639532853993426 577131 what are year quarters 2 msmarco_passage_24_51550650 16.499631540969006 577131 what are year quarters 3 msmarco_passage_24_51548781 16.494385227970447 577131 what are year quarters 4 msmarco_passage_24_51555245 16.295670760824585 577131 what are year quarters 5 msmarco_passage_07_369000254 16.011588557713658 577131 what are year quarters 6 msmarco_passage_51_105537846 16.011588557713658 577131 what are year quarters 7 msmarco_passage_31_807695565 15.797731086021896 577131 what are year quarters 8 msmarco_passage_42_410410371 15.657651982209144 577131 what are year quarters 9 msmarco_passage_21_731215723 15.542722225475105 577131 what are year quarters 10 msmarco_passage_36_577722940 15.468771617436037 247194 how long do you bake muffins 1 msmarco_passage_29_35344026 28.70317705630668 247194 how long do you bake muffins 2 msmarco_passage_42_579196710 27.323846456348733 247194 how long do you bake muffins 3 msmarco_passage_54_646892802 27.323846456348733 247194 how long do you bake muffins 4 msmarco_passage_04_98385022 27.163996036055174 247194 how long do you bake muffins 5 msmarco_passage_04_98387531 26.01224976254724 247194 how long do you bake muffins 6 msmarco_passage_33_497601365 25.382797211477058 247194 how long do you bake muffins 7 msmarco_passage_58_150734038 25.382797211477058 247194 how long do you bake muffins 8 msmarco_passage_53_228317490 25.229865814269978 247194 how long do you bake muffins 9 msmarco_passage_04_98383661 24.780094478369737 247194 how long do you bake muffins 10 msmarco_passage_38_680818125 24.586213198532896 1056758 what is depreciation of products 1 msmarco_passage_11_477287483 19.625372435057656 1056758 what is depreciation of products 2 msmarco_passage_66_385658009 19.45715099319829 1056758 what is depreciation of products 3 msmarco_passage_00_800367921 19.250049307622607 1056758 what is depreciation of products 4 msmarco_passage_22_552251243 19.1510557149413 1056758 what is depreciation of products 5 msmarco_passage_14_344960425 19.135037215768243 1056758 what is depreciation of products 6 msmarco_passage_46_630204971 19.008731755033 1056758 what is depreciation of products 7 msmarco_passage_22_552252532 18.88269534306889 1056758 what is depreciation of products 8 msmarco_passage_00_825069164 18.88269534306889 1056758 what is depreciation of products 9 msmarco_passage_16_660267379 18.88269534306889 1056758 what is depreciation of products 10 msmarco_passage_22_552250787 18.763371791420216 670829 what industry is hershey in? 1 msmarco_passage_14_496884075 21.6002000336501 670829 what industry is hershey in? 2 msmarco_passage_19_550110397 21.390455185903306 670829 what industry is hershey in? 3 msmarco_passage_21_200997683 21.390455185903306 670829 what industry is hershey in? 4 msmarco_passage_19_223759653 21.390455185903306 670829 what industry is hershey in? 5 msmarco_passage_21_697324713 21.390455185903306 670829 what industry is hershey in? 6 msmarco_passage_36_804561151 21.270683477307053 670829 what industry is hershey in? 7 msmarco_passage_08_153085772 21.235683870951753 670829 what industry is hershey in? 8 msmarco_passage_54_39793536 21.235683870951753 670829 what industry is hershey in? 9 msmarco_passage_08_153081447 21.235683870951753 670829 what industry is hershey in? 10 msmarco_passage_45_404833283 21.141228024616534 210442 how can i get more energy while pregnant 1 msmarco_passage_27_408426285 19.245995429244317 210442 how can i get more energy while pregnant 2 msmarco_passage_27_184428427 18.543293928190355 210442 how can i get more energy while pregnant 3 msmarco_passage_04_374560954 18.365657285231535 210442 how can i get more energy while pregnant 4 msmarco_passage_38_632290060 18.365657285231535 210442 how can i get more energy while pregnant 5 msmarco_passage_44_287509383 18.08503179782233 210442 how can i get more energy while pregnant 6 msmarco_passage_63_61267761 17.706131141242178 210442 how can i get more energy while pregnant 7 msmarco_passage_16_736432467 17.5985839264511 210442 how can i get more energy while pregnant 8 msmarco_passage_29_462221088 17.44994952051608 210442 how can i get more energy while pregnant 9 msmarco_passage_41_623832377 17.134456830507194 210442 how can i get more energy while pregnant 10 msmarco_passage_49_746889574 17.031473093778956 1099823 homes in seminole florida 1 msmarco_passage_07_549963896 28.218140430151823 1099823 homes in seminole florida 2 msmarco_passage_46_429528814 28.020721117493814 1099823 homes in seminole florida 3 msmarco_passage_46_429529333 27.195859230141476 1099823 homes in seminole florida 4 msmarco_passage_20_741425234 26.827224507436235 1099823 homes in seminole florida 5 msmarco_passage_09_421495379 26.483503451879287 1099823 homes in seminole florida 6 msmarco_passage_57_409384429 26.11226300822122 1099823 homes in seminole florida 7 msmarco_passage_59_84711420 26.099035663432737 1099823 homes in seminole florida 8 msmarco_passage_35_33973660 26.02165610971704 1099823 homes in seminole florida 9 msmarco_passage_03_701545378 25.896062301437436 1099823 homes in seminole florida 10 msmarco_passage_54_446046028 25.880064979083837 864507 what is your mandible 1 msmarco_passage_28_227130400 17.835797555875736 864507 what is your mandible 2 msmarco_passage_30_796404920 17.835797555875736 864507 what is your mandible 3 msmarco_passage_02_221081775 17.31290020937144 864507 what is your mandible 4 msmarco_passage_37_792418201 17.235085590930538 864507 what is your mandible 5 msmarco_passage_37_792116246 17.235085590930538 864507 what is your mandible 6 msmarco_passage_36_782131598 17.223544522478342 864507 what is your mandible 7 msmarco_passage_04_132336346 17.206261871485626 864507 what is your mandible 8 msmarco_passage_44_589005561 16.66909597470129 864507 what is your mandible 9 msmarco_passage_25_289605110 16.622980505852833 864507 what is your mandible 10 msmarco_passage_51_652167428 16.622980505852833 1100167 guayana venezuela 1 msmarco_passage_19_156809020 36.9233385339036 1100167 guayana venezuela 2 msmarco_passage_11_783937022 35.54609403584799 1100167 guayana venezuela 3 msmarco_passage_11_113262365 34.79224545563046 1100167 guayana venezuela 4 msmarco_passage_68_149194463 34.79224545563046 1100167 guayana venezuela 5 msmarco_passage_68_158258545 34.79224545563046 1100167 guayana venezuela 6 msmarco_passage_68_167422622 32.595454981511566 1100167 guayana venezuela 7 msmarco_passage_21_858673205 32.08981926230514 1100167 guayana venezuela 8 msmarco_passage_11_783932915 30.99317112454124 1100167 guayana venezuela 9 msmarco_passage_37_123027200 30.690813271724906 1100167 guayana venezuela 10 msmarco_passage_01_162795827 30.394973384188738 693101 what is a note for mortgage 1 msmarco_passage_44_8640721 18.849576681306257 693101 what is a note for mortgage 2 msmarco_passage_07_606106910 18.45656502523145 693101 what is a note for mortgage 3 msmarco_passage_52_284603601 18.45656502523145 693101 what is a note for mortgage 4 msmarco_passage_38_46857413 18.03621880396549 693101 what is a note for mortgage 5 msmarco_passage_24_713075413 17.975301147433978 693101 what is a note for mortgage 6 msmarco_passage_01_612765643 17.975301147433978 693101 what is a note for mortgage 7 msmarco_passage_11_841875870 17.892941129169852 693101 what is a note for mortgage 8 msmarco_passage_10_497668195 17.892941129169852 693101 what is a note for mortgage 9 msmarco_passage_65_592219654 17.892941129169852 693101 what is a note for mortgage 10 msmarco_passage_54_367126366 17.74874734500783 1086708 what are the two major subdivisions of the nervous system? 1 msmarco_passage_39_507334056 36.84215772156506 1086708 what are the two major subdivisions of the nervous system? 2 msmarco_passage_02_378646580 36.39163702756206 1086708 what are the two major subdivisions of the nervous system? 3 msmarco_passage_15_620844212 34.97023320801079 1086708 what are the two major subdivisions of the nervous system? 4 msmarco_passage_15_569393595 33.96587335628043 1086708 what are the two major subdivisions of the nervous system? 5 msmarco_passage_60_278924658 33.43201480358287 1086708 what are the two major subdivisions of the nervous system? 6 msmarco_passage_02_362551358 33.41149762849307 1086708 what are the two major subdivisions of the nervous system? 7 msmarco_passage_41_378316440 33.07144941850092 1086708 what are the two major subdivisions of the nervous system? 8 msmarco_passage_56_80576315 33.03187814374629 1086708 what are the two major subdivisions of the nervous system? 9 msmarco_passage_61_681402377 32.9779101739027 1086708 what are the two major subdivisions of the nervous system? 10 msmarco_passage_50_862091965 32.693264924149545 1040312 what is the difference of islands and continents 1 msmarco_passage_65_134228250 25.905684355639277 1040312 what is the difference of islands and continents 2 msmarco_passage_04_173328465 25.71083210925568 1040312 what is the difference of islands and continents 3 msmarco_passage_26_221975746 25.321643869467565 1040312 what is the difference of islands and continents 4 msmarco_passage_58_595032601 24.192605603442747 1040312 what is the difference of islands and continents 5 msmarco_passage_43_563710513 23.609281733688903 1040312 what is the difference of islands and continents 6 msmarco_passage_05_675302565 23.609281733688903 1040312 what is the difference of islands and continents 7 msmarco_passage_58_595027773 23.36531544335655 1040312 what is the difference of islands and continents 8 msmarco_passage_43_542984328 23.352873481041936 1040312 what is the difference of islands and continents 9 msmarco_passage_05_666094788 23.352873481041936 1040312 what is the difference of islands and continents 10 msmarco_passage_05_675303805 23.145897009397924 1011328 what were the effects of pearl harbor on the u.s.'s economy 1 msmarco_passage_32_20010547 32.13977936233781 1011328 what were the effects of pearl harbor on the u.s.'s economy 2 msmarco_passage_28_652243185 30.180717710771173 1011328 what were the effects of pearl harbor on the u.s.'s economy 3 msmarco_passage_31_31651472 29.44131050491676 1011328 what were the effects of pearl harbor on the u.s.'s economy 4 msmarco_passage_25_206601961 28.808365274368647 1011328 what were the effects of pearl harbor on the u.s.'s economy 5 msmarco_passage_61_674687179 28.54928078548767 1011328 what were the effects of pearl harbor on the u.s.'s economy 6 msmarco_passage_29_584477240 28.34798256319198 1011328 what were the effects of pearl harbor on the u.s.'s economy 7 msmarco_passage_01_586263069 27.81945091032146 1011328 what were the effects of pearl harbor on the u.s.'s economy 8 msmarco_passage_64_212487919 27.81945091032146 1011328 what were the effects of pearl harbor on the u.s.'s economy 9 msmarco_passage_25_624167461 27.688317010147447 1011328 what were the effects of pearl harbor on the u.s.'s economy 10 msmarco_passage_45_856785466 27.628159158781497 820161 what is the elevation of white pass in washington 1 msmarco_passage_06_593458555 28.42877915230958 820161 what is the elevation of white pass in washington 2 msmarco_passage_35_539917461 24.87224743257235 820161 what is the elevation of white pass in washington 3 msmarco_passage_47_697379660 24.295410576708043 820161 what is the elevation of white pass in washington 4 msmarco_passage_20_813907061 23.39810038788012 820161 what is the elevation of white pass in washington 5 msmarco_passage_35_531841508 22.427346879988725 820161 what is the elevation of white pass in washington 6 msmarco_passage_06_593461307 21.24416127707494 820161 what is the elevation of white pass in washington 7 msmarco_passage_21_517011315 21.06475709289129 820161 what is the elevation of white pass in washington 8 msmarco_passage_35_531843403 20.580766811867917 820161 what is the elevation of white pass in washington 9 msmarco_passage_38_651978107 20.265478943868334 820161 what is the elevation of white pass in washington 10 msmarco_passage_38_651989551 20.265478943868334 167156 does highster mobile really work 1 msmarco_passage_57_60621354 30.851005948891785 167156 does highster mobile really work 2 msmarco_passage_57_60620880 30.743209600558394 167156 does highster mobile really work 3 msmarco_passage_42_21440090 30.007552506910315 167156 does highster mobile really work 4 msmarco_passage_34_786643728 29.956723263770705 167156 does highster mobile really work 5 msmarco_passage_42_21757368 29.94068554870834 167156 does highster mobile really work 6 msmarco_passage_32_679211878 29.62703245099802 167156 does highster mobile really work 7 msmarco_passage_31_460387754 29.289017044374663 167156 does highster mobile really work 8 msmarco_passage_57_60618667 29.195532228461417 167156 does highster mobile really work 9 msmarco_passage_49_163104947 28.401940394670667 167156 does highster mobile really work 10 msmarco_passage_57_60614638 28.358762033335815 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 1 msmarco_passage_10_401503058 63.07457911699997 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 2 msmarco_passage_10_401504025 63.07457911699997 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 3 msmarco_passage_65_56627978 63.07457911699997 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 4 msmarco_passage_65_56627017 63.07457911699997 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 5 msmarco_passage_10_401504654 58.87336743086753 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 6 msmarco_passage_65_56628603 58.87336743086753 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 7 msmarco_passage_53_228457121 50.49575579933485 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 8 msmarco_passage_12_166562871 49.20440571787317 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 9 msmarco_passage_10_401502605 48.93492995428708 1017892 which president is credited with the inspiration for maxwell's house slogan good to the last drop 10 msmarco_passage_65_56626566 48.93492995428708 709342 what is alcohol used for 1 msmarco_passage_11_804832187 13.525040873445818 709342 what is alcohol used for 2 msmarco_passage_35_619497472 13.496553985099867 709342 what is alcohol used for 3 msmarco_passage_07_150286080 13.41131401406107 709342 what is alcohol used for 4 msmarco_passage_50_48115931 13.41131401406107 709342 what is alcohol used for 5 msmarco_passage_25_311818907 13.380177256705014 709342 what is alcohol used for 6 msmarco_passage_32_176038092 13.3668653399531 709342 what is alcohol used for 7 msmarco_passage_30_629265633 13.3668653399531 709342 what is alcohol used for 8 msmarco_passage_32_363048477 13.352884367163265 709342 what is alcohol used for 9 msmarco_passage_04_163912234 13.33830422984272 709342 what is alcohol used for 10 msmarco_passage_01_500789044 13.33830422984272 1057251 who were george washington's parents 1 msmarco_passage_01_476857710 24.307451983587647 1057251 who were george washington's parents 2 msmarco_passage_35_865325868 24.307451983587647 1057251 who were george washington's parents 3 msmarco_passage_06_856877893 23.295471186548586 1057251 who were george washington's parents 4 msmarco_passage_17_217110300 22.9587627299803 1057251 who were george washington's parents 5 msmarco_passage_45_351270134 22.770420463900074 1057251 who were george washington's parents 6 msmarco_passage_45_738484557 22.591800586562677 1057251 who were george washington's parents 7 msmarco_passage_04_81099491 22.45424162124965 1057251 who were george washington's parents 8 msmarco_passage_04_201021565 22.41753950687385 1057251 who were george washington's parents 9 msmarco_passage_22_216763199 22.32953902329929 1057251 who were george washington's parents 10 msmarco_passage_26_244822288 22.22466736407042 1057139 what is database software? quizlet 1 msmarco_passage_58_21570068 20.542597308994342 1057139 what is database software? quizlet 2 msmarco_passage_57_834961443 19.874768852488103 1057139 what is database software? quizlet 3 msmarco_passage_47_122654900 19.67427497547086 1057139 what is database software? quizlet 4 msmarco_passage_30_700565638 19.568609661686338 1057139 what is database software? quizlet 5 msmarco_passage_56_411673556 19.31176609451512 1057139 what is database software? quizlet 6 msmarco_passage_16_664494422 18.85791264822178 1057139 what is database software? quizlet 7 msmarco_passage_30_670479008 18.784422271642654 1057139 what is database software? quizlet 8 msmarco_passage_47_118797155 18.47325498194034 1057139 what is database software? quizlet 9 msmarco_passage_62_317329034 18.344114362900235 1057139 what is database software? quizlet 10 msmarco_passage_62_317327699 18.336436020795304 578607 what beach is close to disneyland california 1 msmarco_passage_35_59303874 27.013994740855125 578607 what beach is close to disneyland california 2 msmarco_passage_69_130460703 26.47802472019702 578607 what beach is close to disneyland california 3 msmarco_passage_61_235818846 25.947358407553857 578607 what beach is close to disneyland california 4 msmarco_passage_10_378552686 25.864079778077553 578607 what beach is close to disneyland california 5 msmarco_passage_64_632018135 25.864079778077553 578607 what beach is close to disneyland california 6 msmarco_passage_09_317397653 25.827126832292773 578607 what beach is close to disneyland california 7 msmarco_passage_07_819360567 25.715530940825445 578607 what beach is close to disneyland california 8 msmarco_passage_69_130459028 25.651057827711902 578607 what beach is close to disneyland california 9 msmarco_passage_09_506790091 25.423237711275544 578607 what beach is close to disneyland california 10 msmarco_passage_44_773612306 25.28291738860387 82293 capital expenditure definition example 1 msmarco_passage_57_200703914 29.690108152044814 82293 capital expenditure definition example 2 msmarco_passage_01_284021136 29.56086004194217 82293 capital expenditure definition example 3 msmarco_passage_66_374294612 28.942210134293536 82293 capital expenditure definition example 4 msmarco_passage_26_268062411 28.861227267261725 82293 capital expenditure definition example 5 msmarco_passage_43_295120233 28.256922100753336 82293 capital expenditure definition example 6 msmarco_passage_03_148290451 27.87979669519532 82293 capital expenditure definition example 7 msmarco_passage_03_148691287 27.87979669519532 82293 capital expenditure definition example 8 msmarco_passage_66_369730233 27.859570091977357 82293 capital expenditure definition example 9 msmarco_passage_00_868491659 27.79533818347735 82293 capital expenditure definition example 10 msmarco_passage_03_753195386 27.483928233822898 764691 what is lavf 1 msmarco_passage_04_833939974 20.95438368221607 764691 what is lavf 2 msmarco_passage_25_226707561 19.67854584764544 764691 what is lavf 3 msmarco_passage_22_714069147 17.043042360936454 764691 what is lavf 4 msmarco_passage_12_244309098 16.790077979694782 764691 what is lavf 5 msmarco_passage_01_98103450 15.849105343131082 764691 what is lavf 6 msmarco_passage_18_788348725 15.849105343131082 764691 what is lavf 7 msmarco_passage_19_589040673 15.849105343131082 764691 what is lavf 8 msmarco_passage_04_833940452 14.811497459979188 764691 what is lavf 9 msmarco_passage_25_226708065 14.251680764126483 764691 what is lavf 10 msmarco_passage_28_858521485 14.251680764126483 156688 do hormones diffuse through the membrane 1 msmarco_passage_30_760556025 29.670297713272348 156688 do hormones diffuse through the membrane 2 msmarco_passage_30_768384313 28.723722326829208 156688 do hormones diffuse through the membrane 3 msmarco_passage_13_381315734 28.45669945317735 156688 do hormones diffuse through the membrane 4 msmarco_passage_13_381320689 28.19197912250722 156688 do hormones diffuse through the membrane 5 msmarco_passage_14_368108653 27.455647935379595 156688 do hormones diffuse through the membrane 6 msmarco_passage_30_690297755 27.031061784474808 156688 do hormones diffuse through the membrane 7 msmarco_passage_13_373287103 27.009048673067532 156688 do hormones diffuse through the membrane 8 msmarco_passage_29_187719761 27.009048673067532 156688 do hormones diffuse through the membrane 9 msmarco_passage_27_427013078 27.009048673067532 156688 do hormones diffuse through the membrane 10 msmarco_passage_04_224283856 26.961576035976694 885081 what part of the cell does glycolysis occur 1 msmarco_passage_04_209462339 36.83750278710906 885081 what part of the cell does glycolysis occur 2 msmarco_passage_04_209462850 34.72390546137574 885081 what part of the cell does glycolysis occur 3 msmarco_passage_04_224872887 34.4760708214412 885081 what part of the cell does glycolysis occur 4 msmarco_passage_04_209463349 31.093744058511916 885081 what part of the cell does glycolysis occur 5 msmarco_passage_04_224872070 30.82960392374728 885081 what part of the cell does glycolysis occur 6 msmarco_passage_04_209463848 30.741919176295482 682205 what is a ds1 cable 2 msmarco_passage_05_835764297 22.837251826194006 885081 what part of the cell does glycolysis occur 7 msmarco_passage_29_164782392 30.05433605296811 885081 what part of the cell does glycolysis occur 8 msmarco_passage_30_614539645 30.040375186227696 885081 what part of the cell does glycolysis occur 9 msmarco_passage_04_210109358 29.71007933596222 885081 what part of the cell does glycolysis occur 10 msmarco_passage_04_209311996 29.488232321576607 174273 does yellow mucus mean infection 1 msmarco_passage_48_806342171 32.73088885913542 174273 does yellow mucus mean infection 2 msmarco_passage_49_7794781 32.088564969448484 174273 does yellow mucus mean infection 3 msmarco_passage_35_85318219 32.00438325419084 174273 does yellow mucus mean infection 4 msmarco_passage_35_777340272 31.856918435190746 174273 does yellow mucus mean infection 5 msmarco_passage_06_559294120 31.856918435190746 174273 does yellow mucus mean infection 6 msmarco_passage_35_777273501 31.856918435190746 174273 does yellow mucus mean infection 7 msmarco_passage_35_777200188 31.856918435190746 174273 does yellow mucus mean infection 8 msmarco_passage_47_521763481 31.856918435190746 174273 does yellow mucus mean infection 9 msmarco_passage_53_67389884 31.482936119261158 174273 does yellow mucus mean infection 10 msmarco_passage_03_233284298 29.774890443386838 282397 how many digits are chinese zip codes 1 msmarco_passage_25_492913997 28.050757053069646 282397 how many digits are chinese zip codes 2 msmarco_passage_13_547663003 27.200808373026717 282397 how many digits are chinese zip codes 3 msmarco_passage_45_791237690 26.88110841852056 282397 how many digits are chinese zip codes 4 msmarco_passage_68_747205237 26.627784204851622 282397 how many digits are chinese zip codes 5 msmarco_passage_07_806603553 26.51597829077126 282397 how many digits are chinese zip codes 6 msmarco_passage_07_805422285 26.51597829077126 282397 how many digits are chinese zip codes 7 msmarco_passage_07_806795225 26.51597829077126 282397 how many digits are chinese zip codes 8 msmarco_passage_07_806457741 26.51597829077126 282397 how many digits are chinese zip codes 9 msmarco_passage_07_806035518 26.51597829077126 282397 how many digits are chinese zip codes 10 msmarco_passage_07_806811882 26.51597829077126 889104 what program is syntphelper 1 msmarco_passage_60_585693786 21.419713681318058 889104 what program is syntphelper 2 msmarco_passage_60_585705993 21.419713681318058 889104 what program is syntphelper 3 msmarco_passage_06_347884810 20.379244484824884 889104 what program is syntphelper 4 msmarco_passage_46_244392517 20.379244484824884 889104 what program is syntphelper 5 msmarco_passage_60_585700463 18.701266137863662 889104 what program is syntphelper 6 msmarco_passage_09_471587816 18.701266137863662 889104 what program is syntphelper 7 msmarco_passage_62_460478072 7.671418743120185 889104 what program is syntphelper 8 msmarco_passage_62_460936750 7.671418743120185 889104 what program is syntphelper 9 msmarco_passage_36_292624764 7.6627083817999 889104 what program is syntphelper 10 msmarco_passage_62_461498428 7.635978104192576 853646 what is the weather in panama city panama? 1 msmarco_passage_36_55667902 42.23453616950049 853646 what is the weather in panama city panama? 2 msmarco_passage_36_49906040 41.69517160195404 853646 what is the weather in panama city panama? 3 msmarco_passage_29_411823386 41.3767665408305 853646 what is the weather in panama city panama? 4 msmarco_passage_08_705182641 40.617270887690324 853646 what is the weather in panama city panama? 5 msmarco_passage_68_260670754 40.504715416061934 853646 what is the weather in panama city panama? 6 msmarco_passage_64_711878054 39.83361014634956 853646 what is the weather in panama city panama? 7 msmarco_passage_64_711853665 39.83361014634956 853646 what is the weather in panama city panama? 8 msmarco_passage_64_808294100 39.745927862317004 853646 what is the weather in panama city panama? 9 msmarco_passage_64_711865907 39.54418639107332 853646 what is the weather in panama city panama? 10 msmarco_passage_64_719154861 39.54418639107332 440362 linking verb synonym 1 msmarco_passage_11_100714303 24.258291859207045 440362 linking verb synonym 2 msmarco_passage_15_86711633 23.686594561554436 440362 linking verb synonym 3 msmarco_passage_58_415186565 23.228800879034033 440362 linking verb synonym 4 msmarco_passage_42_126224293 23.09483433727418 440362 linking verb synonym 5 msmarco_passage_42_116385004 23.071039218763943 440362 linking verb synonym 6 msmarco_passage_53_855731059 23.038891496983446 440362 linking verb synonym 7 msmarco_passage_42_122650320 22.879620714083146 440362 linking verb synonym 8 msmarco_passage_66_294622666 22.825729389998607 440362 linking verb synonym 9 msmarco_passage_42_123484318 22.81829590732324 440362 linking verb synonym 10 msmarco_passage_42_128906427 22.680085319896854 1100190 green card definition 1 msmarco_passage_30_687468062 20.478446383527718 1100190 green card definition 2 msmarco_passage_05_674520401 20.395380076185468 1100190 green card definition 3 msmarco_passage_23_626896794 20.25305578644103 1100190 green card definition 4 msmarco_passage_67_62225958 20.184665030723444 1100190 green card definition 5 msmarco_passage_55_549665837 20.07226843793893 1100190 green card definition 6 msmarco_passage_30_589777842 19.811902042883748 1100190 green card definition 7 msmarco_passage_05_674514705 19.810910103626135 1100190 green card definition 8 msmarco_passage_48_227440953 19.629733573322458 1100190 green card definition 9 msmarco_passage_30_679419997 19.574152987949326 1100190 green card definition 10 msmarco_passage_30_782535948 19.516214731449793 1088628 weather in cedartown ga 1 msmarco_passage_36_55531265 38.25247022174034 1088628 weather in cedartown ga 2 msmarco_passage_04_7469255 35.18537939613218 1088628 weather in cedartown ga 3 msmarco_passage_37_508552919 35.18537939613218 1088628 weather in cedartown ga 4 msmarco_passage_42_461286070 32.54651438843091 1088628 weather in cedartown ga 5 msmarco_passage_68_345881127 31.536228116491607 1088628 weather in cedartown ga 6 msmarco_passage_42_506100063 31.227928705286402 1088628 weather in cedartown ga 7 msmarco_passage_53_106502322 30.865016946619424 1088628 weather in cedartown ga 8 msmarco_passage_61_595759146 29.96606147933779 1088628 weather in cedartown ga 9 msmarco_passage_54_40345070 29.604843698053667 1088628 weather in cedartown ga 10 msmarco_passage_46_795336458 29.022106846230653 682205 what is a ds1 cable 3 msmarco_passage_44_466076817 22.142785055873752 682205 what is a ds1 cable 4 msmarco_passage_05_835763822 21.70341346330897 682205 what is a ds1 cable 5 msmarco_passage_05_466287086 21.70341346330897 682205 what is a ds1 cable 6 msmarco_passage_43_126581708 21.70341346330897 682205 what is a ds1 cable 7 msmarco_passage_09_106142902 21.350778915386652 682205 what is a ds1 cable 8 msmarco_passage_44_466081640 21.281568777326587 682205 what is a ds1 cable 9 msmarco_passage_62_58659341 20.9918969499737 682205 what is a ds1 cable 10 msmarco_passage_22_367052182 20.9918969499737 459481 most used automation program 1 msmarco_passage_13_755734707 19.149765669653412 459481 most used automation program 2 msmarco_passage_04_302233334 18.528221257980103 459481 most used automation program 3 msmarco_passage_31_865844588 18.323814212652575 459481 most used automation program 4 msmarco_passage_08_642414927 18.288436863244307 459481 most used automation program 5 msmarco_passage_59_693744768 18.20265914145154 459481 most used automation program 6 msmarco_passage_03_795199433 18.18024790135702 459481 most used automation program 7 msmarco_passage_03_795199922 18.18024790135702 459481 most used automation program 8 msmarco_passage_45_104749272 18.13496627630131 459481 most used automation program 9 msmarco_passage_12_658776726 18.06659216048132 459481 most used automation program 10 msmarco_passage_24_516552055 17.8427501973946 1089787 the meaning of haploid cell 1 msmarco_passage_31_591262082 27.67567487345077 1089787 the meaning of haploid cell 2 msmarco_passage_04_199767487 27.33722985419987 1089787 the meaning of haploid cell 3 msmarco_passage_12_534799167 27.216146171145773 1089787 the meaning of haploid cell 4 msmarco_passage_04_178850242 26.92049191569042 1089787 the meaning of haploid cell 5 msmarco_passage_04_144441755 26.857677078913316 1089787 the meaning of haploid cell 6 msmarco_passage_39_517875697 26.480738073774386 1089787 the meaning of haploid cell 7 msmarco_passage_00_341854554 26.460638613991275 1089787 the meaning of haploid cell 8 msmarco_passage_00_341200143 26.460638613991275 1089787 the meaning of haploid cell 9 msmarco_passage_64_161031423 26.460638613991275 1089787 the meaning of haploid cell 10 msmarco_passage_04_105075745 26.3332224457071 986411 who was president of us during ww 1 msmarco_passage_04_218684253 23.79687365122273 986411 who was president of us during ww 2 msmarco_passage_05_692466941 23.015900116008712 986411 who was president of us during ww 3 msmarco_passage_06_294434538 22.150331286940713 986411 who was president of us during ww 4 msmarco_passage_44_679249450 22.146065917190004 986411 who was president of us during ww 5 msmarco_passage_44_679248919 21.969749017584668 986411 who was president of us during ww 6 msmarco_passage_44_679252628 21.405861881375216 986411 who was president of us during ww 7 msmarco_passage_44_679248452 21.35657149653062 986411 who was president of us during ww 8 msmarco_passage_66_821576867 21.31695318977662 986411 who was president of us during ww 9 msmarco_passage_66_821206098 21.31695318977662 986411 who was president of us during ww 10 msmarco_passage_00_574945574 21.18858467319991 194531 geranium rose essential oil benefits 1 msmarco_passage_38_228925055 46.124130075848655 194531 geranium rose essential oil benefits 2 msmarco_passage_31_822064681 42.785802632357345 194531 geranium rose essential oil benefits 3 msmarco_passage_56_306445845 42.24785551662448 194531 geranium rose essential oil benefits 4 msmarco_passage_66_106387312 41.49335309096914 194531 geranium rose essential oil benefits 5 msmarco_passage_06_744330686 40.775140286224065 194531 geranium rose essential oil benefits 6 msmarco_passage_06_272166825 40.75789075078014 194531 geranium rose essential oil benefits 7 msmarco_passage_61_544649107 40.613409359205114 194531 geranium rose essential oil benefits 8 msmarco_passage_44_19329323 40.548022424534146 194531 geranium rose essential oil benefits 9 msmarco_passage_31_822064297 40.38004041484876 194531 geranium rose essential oil benefits 10 msmarco_passage_26_863247443 40.317414419532206 778095 what is npt standard 1 msmarco_passage_45_304418052 24.48132331526334 778095 what is npt standard 2 msmarco_passage_39_341894582 23.281333544611606 778095 what is npt standard 3 msmarco_passage_41_6013021 23.07503121963547 778095 what is npt standard 4 msmarco_passage_41_5992550 23.07503121963547 778095 what is npt standard 5 msmarco_passage_50_485308628 22.854030432622128 778095 what is npt standard 6 msmarco_passage_47_555339284 22.84558315529255 778095 what is npt standard 7 msmarco_passage_08_838344487 22.521415105435786 778095 what is npt standard 8 msmarco_passage_50_485307892 22.52077303229359 778095 what is npt standard 9 msmarco_passage_59_74919716 22.20458685561551 778095 what is npt standard 10 msmarco_passage_32_256969635 22.20458685561551 1006922 which chromosome controls sex characteristics 1 msmarco_passage_30_611189344 33.76851581928745 1006922 which chromosome controls sex characteristics 2 msmarco_passage_04_564164383 31.444389424275254 1006922 which chromosome controls sex characteristics 3 msmarco_passage_39_506413550 31.444389424275254 1006922 which chromosome controls sex characteristics 4 msmarco_passage_40_121485219 31.291878880041153 1006922 which chromosome controls sex characteristics 5 msmarco_passage_02_74840649 31.09143136347297 1006922 which chromosome controls sex characteristics 6 msmarco_passage_13_413944053 30.174471618595568 1006922 which chromosome controls sex characteristics 7 msmarco_passage_05_661864523 30.095562560681913 1006922 which chromosome controls sex characteristics 8 msmarco_passage_43_525869583 30.095562560681913 1006922 which chromosome controls sex characteristics 9 msmarco_passage_30_718091883 30.051260489578677 1006922 which chromosome controls sex characteristics 10 msmarco_passage_03_501574319 29.707467440664587 635125 what does corrective action in a job mean 1 msmarco_passage_08_449474198 21.580753817607693 635125 what does corrective action in a job mean 2 msmarco_passage_39_557437813 20.507404294420823 635125 what does corrective action in a job mean 3 msmarco_passage_17_191130443 20.104591137430667 635125 what does corrective action in a job mean 4 msmarco_passage_57_812571682 19.7888286325271 635125 what does corrective action in a job mean 5 msmarco_passage_08_288057560 19.37798033152478 635125 what does corrective action in a job mean 6 msmarco_passage_08_296688693 19.35215665806391 635125 what does corrective action in a job mean 7 msmarco_passage_23_526165562 19.195307803840613 635125 what does corrective action in a job mean 8 msmarco_passage_27_102102428 19.168701219262726 635125 what does corrective action in a job mean 9 msmarco_passage_67_17658481 19.128925671926723 635125 what does corrective action in a job mean 10 msmarco_passage_10_715084087 19.128925671926723 1092417 most well known alkaline earth metals 1 msmarco_passage_29_674378066 34.703131106740415 1092417 most well known alkaline earth metals 2 msmarco_passage_36_373233307 34.616238276639976 1092417 most well known alkaline earth metals 3 msmarco_passage_65_726267888 32.801321011924365 1092417 most well known alkaline earth metals 4 msmarco_passage_41_381887291 32.57684267615002 1092417 most well known alkaline earth metals 5 msmarco_passage_45_169220149 32.50211801575287 1092417 most well known alkaline earth metals 6 msmarco_passage_02_489424664 31.8390822353348 1092417 most well known alkaline earth metals 7 msmarco_passage_58_411244002 31.748182607083255 1092417 most well known alkaline earth metals 8 msmarco_passage_58_411244924 31.602029057514493 1092417 most well known alkaline earth metals 9 msmarco_passage_04_221221033 31.49795426524109 1092417 most well known alkaline earth metals 10 msmarco_passage_45_169222339 31.280678032649405 1089293 types of open channel flow 1 msmarco_passage_45_154129883 26.273476594018558 1089293 types of open channel flow 2 msmarco_passage_36_834198314 25.56435395769781 1089293 types of open channel flow 3 msmarco_passage_34_87296491 25.506464494135603 1089293 types of open channel flow 4 msmarco_passage_34_87294356 25.178350774015648 1089293 types of open channel flow 5 msmarco_passage_20_289859203 25.119572635090748 1089293 types of open channel flow 6 msmarco_passage_06_7158643 24.959594197331143 1089293 types of open channel flow 7 msmarco_passage_36_834204291 24.64522257635934 1089293 types of open channel flow 8 msmarco_passage_40_33472845 24.500418493842623 1089293 types of open channel flow 9 msmarco_passage_40_33480357 24.500418493842623 1089293 types of open channel flow 10 msmarco_passage_40_33432043 24.470294584658163 39360 average new rn salary in mass. 1 msmarco_passage_55_711086621 30.504483307879276 39360 average new rn salary in mass. 2 msmarco_passage_10_335678798 29.072736682110012 39360 average new rn salary in mass. 3 msmarco_passage_64_473100110 29.072736682110012 39360 average new rn salary in mass. 4 msmarco_passage_64_473110118 28.708988723636335 39360 average new rn salary in mass. 5 msmarco_passage_10_335695105 28.708988723636335 39360 average new rn salary in mass. 6 msmarco_passage_10_335617286 28.580857777552737 39360 average new rn salary in mass. 7 msmarco_passage_64_473044025 28.580857777552737 39360 average new rn salary in mass. 8 msmarco_passage_10_335581476 28.382395965383058 39360 average new rn salary in mass. 9 msmarco_passage_64_473012171 28.382395965383058 39360 average new rn salary in mass. 10 msmarco_passage_55_711105788 28.22805952357257 282411 how many digits does a vin have 1 msmarco_passage_25_774343091 23.947471018299936 282411 how many digits does a vin have 2 msmarco_passage_60_311702365 23.814140180165694 282411 how many digits does a vin have 3 msmarco_passage_41_628291806 23.79458456833158 282411 how many digits does a vin have 4 msmarco_passage_60_310401954 23.52111439144231 282411 how many digits does a vin have 5 msmarco_passage_14_631704654 23.424507183225465 282411 how many digits does a vin have 6 msmarco_passage_08_530483712 23.307092142319874 282411 how many digits does a vin have 7 msmarco_passage_16_333643052 23.08242722798454 282411 how many digits does a vin have 8 msmarco_passage_08_615023539 23.066045589573427 282411 how many digits does a vin have 9 msmarco_passage_48_497517511 23.034382927609904 282411 how many digits does a vin have 10 msmarco_passage_25_780796353 22.79651600558421 93649 clutch baseball definition 1 msmarco_passage_18_405142724 21.23738464689517 93649 clutch baseball definition 2 msmarco_passage_44_235985434 21.01356513172507 93649 clutch baseball definition 3 msmarco_passage_02_490763378 20.451771280995487 93649 clutch baseball definition 4 msmarco_passage_02_490766020 20.409608107393403 93649 clutch baseball definition 5 msmarco_passage_34_340748574 20.399012082287012 93649 clutch baseball definition 6 msmarco_passage_28_87408098 20.11276943945715 93649 clutch baseball definition 7 msmarco_passage_26_536110016 19.993381818461437 93649 clutch baseball definition 8 msmarco_passage_43_485577622 19.95914765782576 93649 clutch baseball definition 9 msmarco_passage_30_242769963 19.822951461356844 93649 clutch baseball definition 10 msmarco_passage_53_790247277 19.590089830861192 853471 what is the weather in death valley 1 msmarco_passage_06_867843295 26.13547830441334 853471 what is the weather in death valley 2 msmarco_passage_67_45204631 25.70772064772968 853471 what is the weather in death valley 3 msmarco_passage_67_45204057 25.529782621322195 853471 what is the weather in death valley 4 msmarco_passage_14_502724443 25.1049882781771 853471 what is the weather in death valley 5 msmarco_passage_67_45198863 23.97926642722107 853471 what is the weather in death valley 6 msmarco_passage_67_45199598 23.97926642722107 853471 what is the weather in death valley 7 msmarco_passage_67_45198483 23.97926642722107 853471 what is the weather in death valley 8 msmarco_passage_67_45199229 23.97926642722107 853471 what is the weather in death valley 9 msmarco_passage_05_536160207 23.73450042788792 853471 what is the weather in death valley 10 msmarco_passage_54_865963323 23.706666385972937 1084403 what does a rightward shift in a supply curve indicate 1 msmarco_passage_44_503972973 45.613763586208634 1084403 what does a rightward shift in a supply curve indicate 2 msmarco_passage_30_790272506 42.879669958590846 1084403 what does a rightward shift in a supply curve indicate 3 msmarco_passage_04_127354029 42.69007526623424 1084403 what does a rightward shift in a supply curve indicate 4 msmarco_passage_64_467328554 42.53584056982541 1084403 what does a rightward shift in a supply curve indicate 5 msmarco_passage_35_300431009 42.48153349841207 1084403 what does a rightward shift in a supply curve indicate 6 msmarco_passage_11_712790059 42.35084682198081 1084403 what does a rightward shift in a supply curve indicate 7 msmarco_passage_04_35996391 42.09094113825499 1084403 what does a rightward shift in a supply curve indicate 8 msmarco_passage_30_585107657 41.456255925598796 1084403 what does a rightward shift in a supply curve indicate 9 msmarco_passage_40_381206880 41.22861445527251 1084403 what does a rightward shift in a supply curve indicate 10 msmarco_passage_64_467328155 41.10503271585269 734979 what is coronary cta 1 msmarco_passage_65_382063059 30.154575513823346 734979 what is coronary cta 2 msmarco_passage_08_260014109 28.526202745086035 734979 what is coronary cta 3 msmarco_passage_08_260014985 28.351444849398526 734979 what is coronary cta 4 msmarco_passage_06_462049586 27.64138623841693 734979 what is coronary cta 5 msmarco_passage_51_669646966 27.568838877498518 734979 what is coronary cta 6 msmarco_passage_41_194446926 27.48560045757887 734979 what is coronary cta 7 msmarco_passage_51_669649111 26.581154250515507 734979 what is coronary cta 8 msmarco_passage_66_617382698 26.163215143729275 734979 what is coronary cta 9 msmarco_passage_10_662630471 26.163215143729275 734979 what is coronary cta 10 msmarco_passage_66_690129551 26.163215143729275 1058425 who wrote just one look at you 1 msmarco_passage_31_645548617 15.863203372994676 1058425 who wrote just one look at you 2 msmarco_passage_21_76994110 15.625263504381994 1058425 who wrote just one look at you 3 msmarco_passage_38_538359407 15.568841803490319 1058425 who wrote just one look at you 4 msmarco_passage_49_725772328 15.567860822624061 1058425 who wrote just one look at you 5 msmarco_passage_39_385602951 15.333448851685558 1058425 who wrote just one look at you 6 msmarco_passage_20_110761690 15.164982475656139 1058425 who wrote just one look at you 7 msmarco_passage_16_42522070 15.164982475656139 1058425 who wrote just one look at you 8 msmarco_passage_22_453574591 15.11161644884093 1058425 who wrote just one look at you 9 msmarco_passage_24_632219070 15.11161644884093 1058425 who wrote just one look at you 10 msmarco_passage_43_39337417 15.046734552523658 691507 what is a montadale sheep 1 msmarco_passage_59_457609763 36.384591438063524 691507 what is a montadale sheep 2 msmarco_passage_59_457610088 34.013124885661966 691507 what is a montadale sheep 3 msmarco_passage_05_507641755 32.66564520044323 691507 what is a montadale sheep 4 msmarco_passage_59_457610937 31.420861808125114 691507 what is a montadale sheep 5 msmarco_passage_59_458124581 30.38041853674024 691507 what is a montadale sheep 6 msmarco_passage_05_507641006 27.944881754090396 691507 what is a montadale sheep 7 msmarco_passage_59_457613752 27.577673030901643 691507 what is a montadale sheep 8 msmarco_passage_45_834434361 27.3149687359649 691507 what is a montadale sheep 9 msmarco_passage_26_212478270 25.88153542712131 691507 what is a montadale sheep 10 msmarco_passage_07_501226976 25.88153542712131 1057334 what is custom software 1 msmarco_passage_68_723246815 15.983774769511077 1057334 what is custom software 2 msmarco_passage_28_192486855 15.807076846202666 1057334 what is custom software 3 msmarco_passage_21_712450435 15.769954862197821 1057334 what is custom software 4 msmarco_passage_06_198606454 15.764403375328515 1057334 what is custom software 5 msmarco_passage_15_71283345 15.764403375328515 1057334 what is custom software 6 msmarco_passage_45_474663058 15.764403375328515 1057334 what is custom software 7 msmarco_passage_48_300547123 15.764403375328515 1057334 what is custom software 8 msmarco_passage_68_723254724 15.730218446361643 1057334 what is custom software 9 msmarco_passage_68_723226729 15.730218446361643 1057334 what is custom software 10 msmarco_passage_59_240974985 15.713717277199365 770167 what is meant urgency 1 msmarco_passage_68_677331074 19.677215697319927 770167 what is meant urgency 2 msmarco_passage_53_845272522 19.420024457115908 770167 what is meant urgency 3 msmarco_passage_58_304119924 17.140685665859305 770167 what is meant urgency 4 msmarco_passage_13_528904824 16.67986147770215 770167 what is meant urgency 5 msmarco_passage_08_865610464 16.42471351534562 770167 what is meant urgency 6 msmarco_passage_41_650936544 16.42471351534562 770167 what is meant urgency 7 msmarco_passage_53_267947370 16.42471351534562 770167 what is meant urgency 8 msmarco_passage_61_216633615 16.42471351534562 770167 what is meant urgency 9 msmarco_passage_02_525832931 16.17725382537241 770167 what is meant urgency 10 msmarco_passage_68_669839782 16.17725382537241 1088903 volatile meaning? 1 msmarco_passage_51_17329379 18.18827492175461 1088903 volatile meaning? 2 msmarco_passage_51_94958208 18.123515250089625 1088903 volatile meaning? 3 msmarco_passage_51_9982837 17.99630317091383 1088903 volatile meaning? 4 msmarco_passage_51_10000783 17.99630317091383 1088903 volatile meaning? 5 msmarco_passage_37_514800373 17.85220851903975 1088903 volatile meaning? 6 msmarco_passage_37_514800748 17.85220851903975 1088903 volatile meaning? 7 msmarco_passage_37_514805480 17.85220851903975 1088903 volatile meaning? 8 msmarco_passage_57_407596601 17.737280497376904 1088903 volatile meaning? 9 msmarco_passage_50_185031772 17.720578780295806 1088903 volatile meaning? 10 msmarco_passage_27_41906660 17.701828493120196 1064206 color of urine 1 msmarco_passage_03_231725874 18.471550705980917 1064206 color of urine 2 msmarco_passage_33_525425170 18.471550705980917 1064206 color of urine 3 msmarco_passage_33_525428597 18.410617015273264 1064206 color of urine 4 msmarco_passage_03_231729293 18.410617015273264 1064206 color of urine 5 msmarco_passage_01_498705472 18.361687308821946 1064206 color of urine 6 msmarco_passage_03_231730143 18.361687308821946 1064206 color of urine 7 msmarco_passage_33_525429449 18.361687308821946 1064206 color of urine 8 msmarco_passage_27_401775916 18.31869789392387 1064206 color of urine 9 msmarco_passage_41_448762154 18.24426234230489 1064206 color of urine 10 msmarco_passage_03_231720203 18.243617566073766 1084910 what disease do roof rats cause 1 msmarco_passage_12_755221322 29.645051334898714 1084910 what disease do roof rats cause 2 msmarco_passage_12_299725443 29.4741270931258 1084910 what disease do roof rats cause 3 msmarco_passage_56_317639941 29.123875970430664 1084910 what disease do roof rats cause 4 msmarco_passage_56_800323882 28.87071139376774 1084910 what disease do roof rats cause 5 msmarco_passage_12_299730372 28.771060894177847 1084910 what disease do roof rats cause 6 msmarco_passage_44_126933802 27.931976825854164 1084910 what disease do roof rats cause 7 msmarco_passage_12_755224871 26.99740499926578 1084910 what disease do roof rats cause 8 msmarco_passage_12_299724980 26.4720652057758 1084910 what disease do roof rats cause 9 msmarco_passage_12_755221827 26.190731187965056 1084910 what disease do roof rats cause 10 msmarco_passage_56_804767841 26.190515453139383 735387 what is county for seattle? 1 msmarco_passage_09_407462722 20.082175463991806 735387 what is county for seattle? 2 msmarco_passage_66_267727501 20.021818671994453 735387 what is county for seattle? 3 msmarco_passage_66_266641887 20.021818671994453 735387 what is county for seattle? 4 msmarco_passage_65_92824955 18.8611350129834 735387 what is county for seattle? 5 msmarco_passage_42_482493820 18.796582963309707 735387 what is county for seattle? 6 msmarco_passage_37_381575720 18.67143555908395 735387 what is county for seattle? 7 msmarco_passage_66_827254074 18.617035688853473 735387 what is county for seattle? 8 msmarco_passage_05_760955461 18.535726810960167 735387 what is county for seattle? 9 msmarco_passage_21_862842170 18.512289239835226 735387 what is county for seattle? 10 msmarco_passage_42_469048559 18.402094416167003 791140 what is respite 1 msmarco_passage_62_37669193 17.45151768147503 791140 what is respite 2 msmarco_passage_54_592397788 17.25684934515111 791140 what is respite 3 msmarco_passage_43_240856474 17.189786295647828 791140 what is respite 4 msmarco_passage_05_512534967 17.003948297691817 791140 what is respite 5 msmarco_passage_37_212293715 16.97514818132582 791140 what is respite 6 msmarco_passage_12_433627357 16.67937241169111 791140 what is respite 7 msmarco_passage_20_597704099 16.513204012260818 791140 what is respite 8 msmarco_passage_17_541536554 16.513204012260818 791140 what is respite 9 msmarco_passage_46_816067072 16.461158065463437 791140 what is respite 10 msmarco_passage_60_386641374 16.43135524916496 186390 fenugreek supplement benefits 1 msmarco_passage_45_543703142 29.924661603073837 186390 fenugreek supplement benefits 2 msmarco_passage_06_778934981 29.12060210940553 186390 fenugreek supplement benefits 3 msmarco_passage_06_777731698 29.12060210940553 186390 fenugreek supplement benefits 4 msmarco_passage_49_192897 29.12060210940553 186390 fenugreek supplement benefits 5 msmarco_passage_39_536460204 28.180872058562557 186390 fenugreek supplement benefits 6 msmarco_passage_45_543696219 26.6119508661523 186390 fenugreek supplement benefits 7 msmarco_passage_13_730688649 26.242058350120956 186390 fenugreek supplement benefits 8 msmarco_passage_05_655406714 25.854967104039368 186390 fenugreek supplement benefits 9 msmarco_passage_40_471672112 25.84512439753502 186390 fenugreek supplement benefits 10 msmarco_passage_43_502718004 25.475544214289638 444790 makeover cost 1 msmarco_passage_58_822389404 20.754707707111766 444790 makeover cost 2 msmarco_passage_68_632985959 20.57886412810547 444790 makeover cost 3 msmarco_passage_58_822398664 20.207653984579572 444790 makeover cost 4 msmarco_passage_57_159740806 20.130972809308712 444790 makeover cost 5 msmarco_passage_63_566776186 20.015180908878573 444790 makeover cost 6 msmarco_passage_58_822390656 19.79989012454463 444790 makeover cost 7 msmarco_passage_10_528929784 19.42146044554478 444790 makeover cost 8 msmarco_passage_36_655217166 19.043376680816397 444790 makeover cost 9 msmarco_passage_27_703867457 18.924880959267487 444790 makeover cost 10 msmarco_passage_06_814287916 18.82973490166189 877810 what metal are most watches made from 1 msmarco_passage_06_269212735 19.92118407804866 877810 what metal are most watches made from 2 msmarco_passage_34_487481937 19.92118407804866 877810 what metal are most watches made from 3 msmarco_passage_06_269210680 19.8355705863892 877810 what metal are most watches made from 4 msmarco_passage_21_518748949 19.515039166274825 877810 what metal are most watches made from 5 msmarco_passage_20_153640676 19.515039166274825 877810 what metal are most watches made from 6 msmarco_passage_06_269212107 19.450579909733186 877810 what metal are most watches made from 7 msmarco_passage_06_269212421 19.450579909733186 877810 what metal are most watches made from 8 msmarco_passage_54_619231721 19.440222149487628 877810 what metal are most watches made from 9 msmarco_passage_67_756078059 19.245757945775708 877810 what metal are most watches made from 10 msmarco_passage_11_11549569 19.245757945775708 701335 what is a staple food of guatemala? 1 msmarco_passage_62_757683383 29.91130390352671 701335 what is a staple food of guatemala? 2 msmarco_passage_62_757691382 27.214307317176733 701335 what is a staple food of guatemala? 3 msmarco_passage_06_412727592 27.00738259061688 701335 what is a staple food of guatemala? 4 msmarco_passage_56_50642140 25.446998133230117 701335 what is a staple food of guatemala? 5 msmarco_passage_62_757688315 24.74342244968262 701335 what is a staple food of guatemala? 6 msmarco_passage_06_412734093 23.250443606424025 701335 what is a staple food of guatemala? 7 msmarco_passage_47_233270207 22.88522650493418 701335 what is a staple food of guatemala? 8 msmarco_passage_45_198102949 22.70799348124914 701335 what is a staple food of guatemala? 9 msmarco_passage_64_709941432 22.434989186122703 701335 what is a staple food of guatemala? 10 msmarco_passage_64_709932997 22.434989186122703 411953 is hctz safe for kidneys 1 msmarco_passage_44_326798557 27.642371775436875 411953 is hctz safe for kidneys 2 msmarco_passage_00_770533268 26.46345643252271 411953 is hctz safe for kidneys 3 msmarco_passage_59_285696676 25.247051073922634 411953 is hctz safe for kidneys 4 msmarco_passage_02_589458527 24.9152650717108 411953 is hctz safe for kidneys 5 msmarco_passage_51_132257723 23.965823908489924 411953 is hctz safe for kidneys 6 msmarco_passage_44_326804753 23.801557419376298 411953 is hctz safe for kidneys 7 msmarco_passage_35_82762399 23.73966322544714 411953 is hctz safe for kidneys 8 msmarco_passage_51_131668719 23.51773110450193 411953 is hctz safe for kidneys 9 msmarco_passage_51_131967672 23.340393512367893 411953 is hctz safe for kidneys 10 msmarco_passage_07_378670812 23.340393512367893 998802 where is iron located in a plant body 1 msmarco_passage_54_690734771 21.179057962424153 998802 where is iron located in a plant body 2 msmarco_passage_61_701581055 21.11249716782996 998802 where is iron located in a plant body 3 msmarco_passage_49_5162399 20.760827655122668 998802 where is iron located in a plant body 4 msmarco_passage_65_741092693 20.68553543774814 998802 where is iron located in a plant body 5 msmarco_passage_68_199751703 20.42622718938843 998802 where is iron located in a plant body 6 msmarco_passage_48_230790892 20.28832494290525 998802 where is iron located in a plant body 7 msmarco_passage_55_555275582 20.116663306106865 998802 where is iron located in a plant body 8 msmarco_passage_08_499258265 20.116663306106865 998802 where is iron located in a plant body 9 msmarco_passage_43_113507141 19.80024085252472 998802 where is iron located in a plant body 10 msmarco_passage_00_230449770 19.723307836045095 436100 largest cave in the world 1 msmarco_passage_54_865291199 26.351721444059777 436100 largest cave in the world 2 msmarco_passage_35_72225164 25.161806718724485 436100 largest cave in the world 3 msmarco_passage_36_504879056 25.15344249501849 436100 largest cave in the world 4 msmarco_passage_54_848254991 25.01953538607904 436100 largest cave in the world 5 msmarco_passage_05_184715376 24.915640529167078 436100 largest cave in the world 6 msmarco_passage_36_504877848 24.567574113258893 436100 largest cave in the world 7 msmarco_passage_61_849238552 24.376033268648303 436100 largest cave in the world 8 msmarco_passage_36_504878651 24.18762086140023 436100 largest cave in the world 9 msmarco_passage_31_252933105 24.141987592237662 436100 largest cave in the world 10 msmarco_passage_50_544244044 24.04788049643578 1090841 scriptural definition of idol 1 msmarco_passage_10_251869748 29.543797256533274 1090841 scriptural definition of idol 2 msmarco_passage_10_251872415 27.291865771904234 1090841 scriptural definition of idol 3 msmarco_passage_28_335688485 23.985773502102624 1090841 scriptural definition of idol 4 msmarco_passage_10_251879834 23.982533081898872 1090841 scriptural definition of idol 5 msmarco_passage_10_251871506 23.325156519626777 1090841 scriptural definition of idol 6 msmarco_passage_00_656291453 20.680983694200677 1090841 scriptural definition of idol 7 msmarco_passage_42_39685572 20.43865623430813 1090841 scriptural definition of idol 8 msmarco_passage_21_615984278 20.069277620517767 1090841 scriptural definition of idol 9 msmarco_passage_21_295408641 20.069277620517767 1090841 scriptural definition of idol 10 msmarco_passage_59_224237115 19.603157374227187 1093717 kamikaze definition 1 msmarco_passage_07_587124416 25.043252852223915 1093717 kamikaze definition 2 msmarco_passage_53_812901131 24.38716259426087 1093717 kamikaze definition 3 msmarco_passage_53_812900357 22.687651806142345 1093717 kamikaze definition 4 msmarco_passage_04_21888024 21.29304443842188 1093717 kamikaze definition 5 msmarco_passage_34_681388643 20.896608757940168 1093717 kamikaze definition 6 msmarco_passage_05_597923716 20.51479214653915 1093717 kamikaze definition 7 msmarco_passage_43_428785585 20.51479214653915 1093717 kamikaze definition 8 msmarco_passage_07_587124183 20.274706802284992 1093717 kamikaze definition 9 msmarco_passage_11_100133659 19.9807715749907 1093717 kamikaze definition 10 msmarco_passage_04_21889353 19.859360217648046 392936 implantation bleeding how far along 1 msmarco_passage_57_447434265 25.14874090083821 392936 implantation bleeding how far along 2 msmarco_passage_40_18885100 25.02226110198087 392936 implantation bleeding how far along 3 msmarco_passage_14_206835337 25.02226110198087 392936 implantation bleeding how far along 4 msmarco_passage_14_206469822 25.02226110198087 392936 implantation bleeding how far along 5 msmarco_passage_08_308662865 24.857626702585986 392936 implantation bleeding how far along 6 msmarco_passage_08_308663737 24.857626702585986 392936 implantation bleeding how far along 7 msmarco_passage_02_206807819 24.75910367385999 392936 implantation bleeding how far along 8 msmarco_passage_57_447436734 24.387279944584527 392936 implantation bleeding how far along 9 msmarco_passage_45_321723559 24.299854644961517 392936 implantation bleeding how far along 10 msmarco_passage_49_261936745 24.264962432054823 116431 deed restrictions 1 msmarco_passage_00_173219817 22.688685566966782 116431 deed restrictions 2 msmarco_passage_46_605636880 22.660521954195648 116431 deed restrictions 3 msmarco_passage_38_425172359 22.40802094648269 116431 deed restrictions 4 msmarco_passage_30_661400114 22.399512241256033 116431 deed restrictions 5 msmarco_passage_30_114011522 22.283885061323893 116431 deed restrictions 6 msmarco_passage_06_78545534 22.278604030774545 116431 deed restrictions 7 msmarco_passage_30_104828905 22.278604030774545 116431 deed restrictions 8 msmarco_passage_11_60755308 22.006019559034684 116431 deed restrictions 9 msmarco_passage_04_666440320 21.839072463059253 116431 deed restrictions 10 msmarco_passage_39_656090545 21.753093511771212 1083085 what does safety mean 1 msmarco_passage_12_266283286 14.470829278559394 1083085 what does safety mean 2 msmarco_passage_61_671217231 14.386898909094358 1083085 what does safety mean 3 msmarco_passage_42_715671545 14.153932096556176 1083085 what does safety mean 4 msmarco_passage_59_585999964 13.956014948446242 1083085 what does safety mean 5 msmarco_passage_59_585562088 13.956014948446242 1083085 what does safety mean 6 msmarco_passage_13_530738512 13.746586365565335 1083085 what does safety mean 7 msmarco_passage_03_391656514 13.626774864999735 1083085 what does safety mean 8 msmarco_passage_68_621930324 13.625448373664323 1083085 what does safety mean 9 msmarco_passage_11_164027250 13.625448373664323 1083085 what does safety mean 10 msmarco_passage_59_585933621 13.465852550973983 750821 what is german smear 1 msmarco_passage_12_712410073 23.63273525202565 750821 what is german smear 2 msmarco_passage_51_373594501 22.995087093034712 750821 what is german smear 3 msmarco_passage_32_257748374 22.65331270951658 750821 what is german smear 4 msmarco_passage_12_712410553 22.472516936017684 750821 what is german smear 5 msmarco_passage_22_492294229 22.321549074817135 750821 what is german smear 6 msmarco_passage_14_200761207 22.278474734514887 750821 what is german smear 7 msmarco_passage_59_132863832 22.15928484054576 750821 what is german smear 8 msmarco_passage_10_219666521 21.99936270735498 750821 what is german smear 9 msmarco_passage_59_132864349 21.686344796717723 750821 what is german smear 10 msmarco_passage_51_373588754 21.53315257659252 483521 pseudocoelom definition 1 msmarco_passage_43_513940538 25.1475653449478 483521 pseudocoelom definition 2 msmarco_passage_43_564010970 25.1475653449478 483521 pseudocoelom definition 3 msmarco_passage_53_828371863 24.44455493843914 483521 pseudocoelom definition 4 msmarco_passage_53_800581341 24.44455493843914 483521 pseudocoelom definition 5 msmarco_passage_29_590368643 24.26073816099514 483521 pseudocoelom definition 6 msmarco_passage_06_385084105 23.174407122668136 483521 pseudocoelom definition 7 msmarco_passage_44_585741325 22.592056039993064 483521 pseudocoelom definition 8 msmarco_passage_05_866729381 22.592056039993064 483521 pseudocoelom definition 9 msmarco_passage_43_564014807 22.423360327134674 483521 pseudocoelom definition 10 msmarco_passage_13_410642876 21.90393595797886 994688 which feature do gas giants have that terrestrial planets do not? 1 msmarco_passage_04_121565806 45.25740391157354 994688 which feature do gas giants have that terrestrial planets do not? 2 msmarco_passage_04_134524494 43.23708673717737 994688 which feature do gas giants have that terrestrial planets do not? 3 msmarco_passage_30_751671879 42.57545311978094 994688 which feature do gas giants have that terrestrial planets do not? 4 msmarco_passage_04_121564976 41.819273768850344 994688 which feature do gas giants have that terrestrial planets do not? 5 msmarco_passage_17_247797909 41.56085272972056 994688 which feature do gas giants have that terrestrial planets do not? 6 msmarco_passage_04_143711270 40.704473631435874 994688 which feature do gas giants have that terrestrial planets do not? 7 msmarco_passage_27_85451480 40.60079378494733 994688 which feature do gas giants have that terrestrial planets do not? 8 msmarco_passage_04_121565347 40.581062800529494 994688 which feature do gas giants have that terrestrial planets do not? 9 msmarco_passage_30_597360877 39.87936511747927 994688 which feature do gas giants have that terrestrial planets do not? 10 msmarco_passage_32_512770381 39.092972490780596 135516 definition of infraction 1 msmarco_passage_42_122464496 22.345340114208703 135516 definition of infraction 2 msmarco_passage_53_810908835 21.642196087505983 135516 definition of infraction 3 msmarco_passage_16_90511014 21.077615863406628 135516 definition of infraction 4 msmarco_passage_43_188563374 20.910523888930495 135516 definition of infraction 5 msmarco_passage_05_504503789 20.910523888930495 135516 definition of infraction 6 msmarco_passage_16_93495607 19.809144557891834 135516 definition of infraction 7 msmarco_passage_11_175715460 19.515242501695596 135516 definition of infraction 8 msmarco_passage_68_640179755 19.515242501695596 135516 definition of infraction 9 msmarco_passage_68_115874255 19.28332758149802 135516 definition of infraction 10 msmarco_passage_43_188563784 19.283013652535477 1096947 how many microgram in a milligram 1 msmarco_passage_04_213810377 35.9148876607021 1096947 how many microgram in a milligram 2 msmarco_passage_04_213810836 35.34252241137407 1096947 how many microgram in a milligram 3 msmarco_passage_04_91472204 34.39243546730482 1096947 how many microgram in a milligram 4 msmarco_passage_04_91471696 33.397202888559384 1096947 how many microgram in a milligram 5 msmarco_passage_38_299311675 33.22254251546717 1096947 how many microgram in a milligram 6 msmarco_passage_38_299311199 33.12926971711174 1096947 how many microgram in a milligram 7 msmarco_passage_04_91474349 32.95659400689638 1096947 how many microgram in a milligram 8 msmarco_passage_40_514921185 32.6087784973383 1096947 how many microgram in a milligram 9 msmarco_passage_04_91469687 32.491125846097496 1096947 how many microgram in a milligram 10 msmarco_passage_42_346566055 32.46360068252579 1091360 price of forever stamp now 1 msmarco_passage_14_246869729 29.248236377743176 1091360 price of forever stamp now 2 msmarco_passage_02_844614143 28.28558942273103 1091360 price of forever stamp now 3 msmarco_passage_11_420096148 28.273376224395633 1091360 price of forever stamp now 4 msmarco_passage_00_36039892 28.273376224395633 1091360 price of forever stamp now 5 msmarco_passage_04_85985036 28.273376224395633 1091360 price of forever stamp now 6 msmarco_passage_60_702830190 28.064513172558307 1091360 price of forever stamp now 7 msmarco_passage_22_508106925 27.84320019018343 1091360 price of forever stamp now 8 msmarco_passage_26_171005452 27.84320019018343 1091360 price of forever stamp now 9 msmarco_passage_26_423422336 27.698113634156122 1091360 price of forever stamp now 10 msmarco_passage_58_143159103 27.67798369767972 532142 unemployment rate honolulu hawaii 1 msmarco_passage_10_445990362 33.20265025711979 532142 unemployment rate honolulu hawaii 2 msmarco_passage_56_767731163 31.584761484694106 532142 unemployment rate honolulu hawaii 3 msmarco_passage_10_727169325 29.841714520273378 532142 unemployment rate honolulu hawaii 4 msmarco_passage_61_460570417 29.499192361145113 532142 unemployment rate honolulu hawaii 5 msmarco_passage_68_300827203 29.1643786369096 532142 unemployment rate honolulu hawaii 6 msmarco_passage_38_512655872 28.931117349298066 532142 unemployment rate honolulu hawaii 7 msmarco_passage_59_632867482 28.858605114897323 532142 unemployment rate honolulu hawaii 8 msmarco_passage_41_587745900 28.77359496766076 532142 unemployment rate honolulu hawaii 9 msmarco_passage_41_586047554 28.602017663021535 532142 unemployment rate honolulu hawaii 10 msmarco_passage_04_505630307 28.40615569583349 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 1 msmarco_passage_04_81573716 65.22288895207114 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 2 msmarco_passage_58_773382397 63.728510752928486 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 3 msmarco_passage_55_268643301 63.63131618772519 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 4 msmarco_passage_55_279995168 63.63131618772519 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 5 msmarco_passage_04_81574529 62.97614027789048 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 6 msmarco_passage_24_184260564 62.629098792668934 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 7 msmarco_passage_06_110888775 62.19635673803782 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 8 msmarco_passage_04_561504519 61.76047948334406 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 9 msmarco_passage_27_733461087 61.668131090603744 1040959 what is the difference between a narrow spectrum antibiotic and a broad spectrum antibiotic 10 msmarco_passage_24_184264295 61.37631060429055 1092605 monster types 1 msmarco_passage_69_96471052 17.45753486407209 1092605 monster types 2 msmarco_passage_16_814239843 17.3989615237889 1092605 monster types 3 msmarco_passage_28_523726066 17.388417941090445 1092605 monster types 4 msmarco_passage_28_108108322 17.265789313469156 1092605 monster types 5 msmarco_passage_69_94485002 17.264144769888766 1092605 monster types 6 msmarco_passage_48_11697130 17.06989643095636 1092605 monster types 7 msmarco_passage_54_457557576 16.950666542053618 1092605 monster types 8 msmarco_passage_11_256990553 16.737801544657643 1092605 monster types 9 msmarco_passage_69_95203959 16.737801544657643 1092605 monster types 10 msmarco_passage_67_572658419 16.725807010292087 342115 how therapist refer to dr for evaluation 1 msmarco_passage_49_11988267 22.898062405771277 342115 how therapist refer to dr for evaluation 2 msmarco_passage_07_525123585 22.625429302327056 342115 how therapist refer to dr for evaluation 3 msmarco_passage_48_833251438 22.625429302327056 342115 how therapist refer to dr for evaluation 4 msmarco_passage_22_264836431 22.60758673469706 342115 how therapist refer to dr for evaluation 5 msmarco_passage_37_117675321 21.652496524102187 342115 how therapist refer to dr for evaluation 6 msmarco_passage_65_836435935 21.613676311413883 342115 how therapist refer to dr for evaluation 7 msmarco_passage_66_92927615 21.613676311413883 342115 how therapist refer to dr for evaluation 8 msmarco_passage_45_616804293 21.56890491461867 342115 how therapist refer to dr for evaluation 9 msmarco_passage_28_861910255 21.484435054179947 342115 how therapist refer to dr for evaluation 10 msmarco_passage_39_287722091 21.163939476825576 893275 what settings do i use to charge a battery that in my car? 1 msmarco_passage_28_155816990 28.47997165968584 893275 what settings do i use to charge a battery that in my car? 2 msmarco_passage_27_420700220 27.956535263005996 893275 what settings do i use to charge a battery that in my car? 3 msmarco_passage_67_222856065 27.784762393918196 893275 what settings do i use to charge a battery that in my car? 4 msmarco_passage_12_844066738 27.381195620835314 893275 what settings do i use to charge a battery that in my car? 5 msmarco_passage_30_79793800 27.29722409511376 893275 what settings do i use to charge a battery that in my car? 6 msmarco_passage_32_119849310 27.27792935364417 893275 what settings do i use to charge a battery that in my car? 7 msmarco_passage_38_667625082 27.108379389258495 893275 what settings do i use to charge a battery that in my car? 8 msmarco_passage_36_371915008 26.973056499403874 893275 what settings do i use to charge a battery that in my car? 9 msmarco_passage_68_657726180 26.80293332648719 893275 what settings do i use to charge a battery that in my car? 10 msmarco_passage_62_523634349 26.742310848712894 1026799 what kind of match is necessary for an organ transplant 1 msmarco_passage_39_472463697 29.1716021500749 1026799 what kind of match is necessary for an organ transplant 2 msmarco_passage_56_303386095 26.54224440472808 1026799 what kind of match is necessary for an organ transplant 3 msmarco_passage_60_155524367 25.467248377801546 1026799 what kind of match is necessary for an organ transplant 4 msmarco_passage_20_157640601 25.03874842787343 1026799 what kind of match is necessary for an organ transplant 5 msmarco_passage_11_73278125 24.907937759167936 1026799 what kind of match is necessary for an organ transplant 6 msmarco_passage_42_391141633 24.651944725689933 1026799 what kind of match is necessary for an organ transplant 7 msmarco_passage_54_710537391 24.503079762909493 1026799 what kind of match is necessary for an organ transplant 8 msmarco_passage_02_140262247 24.401358988189063 1026799 what kind of match is necessary for an organ transplant 9 msmarco_passage_53_282078694 24.401358988189063 1026799 what kind of match is necessary for an organ transplant 10 msmarco_passage_35_395273068 24.08940762205936 1033912 who is jack valentine 1 msmarco_passage_64_296204215 23.84026580722825 1033912 who is jack valentine 2 msmarco_passage_04_438898169 22.76665456851894 1033912 who is jack valentine 3 msmarco_passage_49_719821730 21.51496969706193 1033912 who is jack valentine 4 msmarco_passage_49_718760401 21.053920068878757 1033912 who is jack valentine 5 msmarco_passage_33_175449900 20.238191167903164 1033912 who is jack valentine 6 msmarco_passage_50_330893436 20.191876531647964 1033912 who is jack valentine 7 msmarco_passage_07_283133225 19.597812174744718 1033912 who is jack valentine 8 msmarco_passage_01_484952156 19.35664400646175 1033912 who is jack valentine 9 msmarco_passage_48_252301875 19.03548711785851 1033912 who is jack valentine 10 msmarco_passage_06_563415953 18.78526841460286 timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/load.sql000066400000000000000000000074561525762047700263500ustar00rootroot00000000000000-- MS MARCO v2 Passage Ranking - Load and Index -- Loads the full 138M passage collection and creates BM25 index -- -- Usage: -- DATA_DIR=/path/to/data psql -f load.sql -- -- The DATA_DIR environment variable should point to the directory containing: -- - collection.tsv (passage_id, passage_text) -- - passv2_dev_queries.tsv (query_id, query_text) -- - passv2_dev_qrels.tsv (query_id, 0, passage_id, relevance) \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO v2 Passage Ranking - Data Loading ===' \echo 'Loading ~138M passages from MS MARCO v2 collection' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_v2_passages CASCADE; DROP TABLE IF EXISTS msmarco_v2_queries CASCADE; DROP TABLE IF EXISTS msmarco_v2_qrels CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_v2_passages ( passage_id TEXT PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_v2_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_v2_qrels ( query_id INTEGER NOT NULL, passage_id TEXT NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages (this is the big one - 138M rows) -- Convert TSV to CSV to avoid issues with \. sequences in text format \echo 'Loading passages (this will take a long time for 138M rows)...' \copy msmarco_v2_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \"\\\"\" \$1 \"\\\"\", \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries \echo 'Loading queries...' \copy msmarco_v2_queries(query_id, query_text) FROM PROGRAM 'cat "$DATA_DIR/passv2_dev_queries.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments (qrels format: query_id, 0, passage_id, relevance) \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id TEXT, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/passv2_dev_qrels.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_v2_qrels (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_v2_passages UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_v2_queries UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_v2_qrels ORDER BY metric; -- Show sample data \echo '' \echo 'Sample passages:' SELECT passage_id, LEFT(passage_text, 100) || '...' as passage_preview FROM msmarco_v2_passages LIMIT 3; \echo '' \echo 'Sample queries:' SELECT query_id, query_text FROM msmarco_v2_queries LIMIT 5; -- Create BM25 index \echo '' \echo '=== Building BM25 Index ===' \echo 'Creating BM25 index on ~138M passages (this will take a long time)...' CREATE INDEX msmarco_v2_bm25_idx ON msmarco_v2_passages USING bm25(passage_text) WITH (text_config='english'); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_v2_bm25_idx')) as index_size, pg_relation_size('msmarco_v2_bm25_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_v2_passages')) as table_size, pg_total_relation_size('msmarco_v2_passages') as table_bytes; -- Segment statistics \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('msmarco_v2_bm25_idx'); \echo '' \echo '=== MS MARCO v2 Load Complete ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/load_data_only.sql000066400000000000000000000056611525762047700303760ustar00rootroot00000000000000-- MS MARCO v2 Passage Ranking - Data Loading Only (no index) -- Loads the full 138M passage collection without creating the index -- -- Usage: -- DATA_DIR=/path/to/data psql -f load_data_only.sql -- -- After loading, run parallel_scaling.sql to benchmark different worker counts \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO v2 Passage Ranking - Data Loading ===' \echo 'Loading ~138M passages from MS MARCO v2 collection (no index)' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_v2_passages CASCADE; DROP TABLE IF EXISTS msmarco_v2_queries CASCADE; DROP TABLE IF EXISTS msmarco_v2_qrels CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_v2_passages ( passage_id TEXT PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_v2_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_v2_qrels ( query_id INTEGER NOT NULL, passage_id TEXT NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages (this is the big one - 138M rows) \echo 'Loading passages (this will take a long time for 138M rows)...' \copy msmarco_v2_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \"\\\"\" \$1 \"\\\"\", \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries \echo 'Loading queries...' \copy msmarco_v2_queries(query_id, query_text) FROM PROGRAM 'cat "$DATA_DIR/passv2_dev_queries.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id TEXT, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/passv2_dev_qrels.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_v2_qrels (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Analyze for accurate statistics \echo 'Analyzing tables...' ANALYZE msmarco_v2_passages; ANALYZE msmarco_v2_queries; ANALYZE msmarco_v2_qrels; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_v2_passages UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_v2_queries UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_v2_qrels ORDER BY metric; -- Report table size \echo '' \echo '=== Table Size ===' SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_v2_passages')) as table_size, pg_total_relation_size('msmarco_v2_passages') as table_bytes; \echo '' \echo '=== Data Loading Complete ===' \echo 'Run parallel_scaling.sql to benchmark different worker counts' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/paradedb/000077500000000000000000000000001525762047700264365ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/paradedb/load.sql000066400000000000000000000107411525762047700301010ustar00rootroot00000000000000-- MS MARCO v2 Passage Ranking - Load and Index (ParadeDB) -- Loads the full 138M passage collection and creates ParadeDB BM25 index -- -- Usage: -- DATA_DIR=/path/to/data psql -U postgres postgres -f load.sql -- -- The DATA_DIR environment variable should point to the directory containing: -- - collection.tsv (passage_id, passage_text) -- - passv2_dev_queries.tsv (query_id, query_text) -- - passv2_dev_qrels.tsv (query_id, 0, passage_id, relevance) \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO v2 Passage Ranking - Data Loading (ParadeDB) ===' \echo 'Loading ~138M passages from MS MARCO v2 collection' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS msmarco_v2_passages_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_v2_queries_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_v2_qrels_paradedb CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_v2_passages_paradedb ( passage_id TEXT PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_v2_queries_paradedb ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_v2_qrels_paradedb ( query_id INTEGER NOT NULL, passage_id TEXT NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages (this is the big one - 138M rows) -- Convert TSV to CSV to avoid issues with \. sequences in text format \echo 'Loading passages (this will take a long time for 138M rows)...' \copy msmarco_v2_passages_paradedb(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \"\\\"\" \$1 \"\\\"\", \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries \echo 'Loading queries...' \copy msmarco_v2_queries_paradedb(query_id, query_text) FROM PROGRAM 'cat "$DATA_DIR/passv2_dev_queries.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments (qrels format: query_id, 0, passage_id, relevance) \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id TEXT, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/passv2_dev_qrels.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_v2_qrels_paradedb (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_v2_passages_paradedb UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_v2_queries_paradedb UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_v2_qrels_paradedb ORDER BY metric; -- Show sample data \echo '' \echo 'Sample passages:' SELECT passage_id, LEFT(passage_text, 100) || '...' as passage_preview FROM msmarco_v2_passages_paradedb LIMIT 3; \echo '' \echo 'Sample queries:' SELECT query_id, query_text FROM msmarco_v2_queries_paradedb LIMIT 5; -- Create ParadeDB BM25 index with English tokenizer (stopwords + stemming) \echo '' \echo '=== Building ParadeDB BM25 Index ===' \echo 'Creating ParadeDB BM25 index on ~138M passages (this will take a long time)...' CREATE INDEX msmarco_v2_paradedb_idx ON msmarco_v2_passages_paradedb USING bm25 (passage_id, passage_text) WITH ( key_field = 'passage_id', text_fields = '{ "passage_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Compact index segments (ParadeDB merges segments during VACUUM) \echo 'INDEX_VACUUM:' VACUUM msmarco_v2_passages_paradedb; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_v2_paradedb_idx')) as index_size, pg_relation_size('msmarco_v2_paradedb_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_v2_passages_paradedb')) as table_size, pg_total_relation_size('msmarco_v2_passages_paradedb') as table_bytes; \echo '' \echo '=== MS MARCO v2 Load Complete (ParadeDB) ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/paradedb/queries.sql000066400000000000000000000313371525762047700306430ustar00rootroot00000000000000-- MS MARCO v2 Passage Ranking - Query Benchmarks (ParadeDB) -- Runs various query workloads against the indexed MS MARCO v2 collection using ParadeDB -- Outputs structured timing data for historical tracking \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO v2 Query Benchmarks (ParadeDB) ===' \echo '' -- Load benchmark queries (pre-sampled by token count) \echo 'Loading benchmark queries...' DROP TABLE IF EXISTS benchmark_queries_paradedb; CREATE TABLE benchmark_queries_paradedb ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy benchmark_queries_paradedb FROM 'benchmarks/datasets/msmarco-v2/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') -- Verify load SELECT 'Loaded ' || COUNT(*) || ' benchmark queries' as status FROM benchmark_queries_paradedb; SELECT token_bucket, COUNT(*) as count FROM benchmark_queries_paradedb GROUP BY token_bucket ORDER BY token_bucket; -- Warm up: run queries from each bucket to ensure index is cached \echo '' \echo 'Warming up index...' DO $$ DECLARE q record; BEGIN FOR q IN SELECT query_text FROM benchmark_queries_paradedb ORDER BY random() LIMIT 50 LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10' USING q.query_text; END LOOP; END; $$; -- ============================================================ -- Benchmark 1: Query Latency by Token Count -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency by Token Count ===' \echo 'Running 100 queries per token bucket, reporting p50/p95/p99' \echo '' -- Temp table to capture per-bucket results for weighted average CREATE TEMP TABLE bucket_results ( bucket int, p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint ); -- Function to benchmark a bucket and return percentiles -- Uses ParadeDB match() to handle arbitrary query strings CREATE OR REPLACE FUNCTION benchmark_bucket_paradedb(bucket int) RETURNS TABLE(p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint) AS $$ DECLARE q record; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; n int; result_count bigint; results_sum bigint := 0; BEGIN times := ARRAY[]::numeric[]; FOR q IN SELECT query_text FROM benchmark_queries_paradedb WHERE token_bucket = bucket ORDER BY query_id LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT COUNT(*) FROM (SELECT passage_id FROM msmarco_v2_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10) t' INTO result_count USING q.query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); results_sum := results_sum + result_count; END LOOP; n := array_length(times, 1); SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; p50_ms := sorted_times[(n + 1) / 2]; p95_ms := sorted_times[(n * 95 + 99) / 100]; p99_ms := sorted_times[(n * 99 + 99) / 100]; avg_ms := (SELECT AVG(t) FROM unnest(times) t); num_queries := n; total_results := results_sum; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run benchmarks for each token bucket (results saved for weighted avg) \echo 'Token bucket 1 (1 search token):' INSERT INTO bucket_results SELECT 1, * FROM benchmark_bucket_paradedb(1); SELECT 'LATENCY_BUCKET_1: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 1; \echo '' \echo 'Token bucket 2 (2 search tokens):' INSERT INTO bucket_results SELECT 2, * FROM benchmark_bucket_paradedb(2); SELECT 'LATENCY_BUCKET_2: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 2; \echo '' \echo 'Token bucket 3 (3 search tokens):' INSERT INTO bucket_results SELECT 3, * FROM benchmark_bucket_paradedb(3); SELECT 'LATENCY_BUCKET_3: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 3; \echo '' \echo 'Token bucket 4 (4 search tokens):' INSERT INTO bucket_results SELECT 4, * FROM benchmark_bucket_paradedb(4); SELECT 'LATENCY_BUCKET_4: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 4; \echo '' \echo 'Token bucket 5 (5 search tokens):' INSERT INTO bucket_results SELECT 5, * FROM benchmark_bucket_paradedb(5); SELECT 'LATENCY_BUCKET_5: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 5; \echo '' \echo 'Token bucket 6 (6 search tokens):' INSERT INTO bucket_results SELECT 6, * FROM benchmark_bucket_paradedb(6); SELECT 'LATENCY_BUCKET_6: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 6; \echo '' \echo 'Token bucket 7 (7 search tokens):' INSERT INTO bucket_results SELECT 7, * FROM benchmark_bucket_paradedb(7); SELECT 'LATENCY_BUCKET_7: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 7; \echo '' \echo 'Token bucket 8 (8+ search tokens):' INSERT INTO bucket_results SELECT 8, * FROM benchmark_bucket_paradedb(8); SELECT 'LATENCY_BUCKET_8: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 8; DROP FUNCTION benchmark_bucket_paradedb; -- ============================================================ -- Weighted-Average Query Latency (MS-MARCO v1 distribution) -- ============================================================ -- Weights from observed lexeme distribution in 1,010,905 MS-MARCO -- v1 queries after to_tsvector: 1=3.5%, 2=16.3%, 3=30.2%, -- 4=26.1%, 5=14.2%, 6=5.9%, 7=2.2%, 8+=1.5% \echo '' \echo '=== Weighted-Average Query Latency & Throughput ===' SELECT 'WEIGHTED_LATENCY_RESULT: weighted_p50=' || round(SUM(b.p50_ms * w.weight) / SUM(w.weight), 2) || 'ms weighted_avg=' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || 'ms' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; SELECT 'WEIGHTED_THROUGHPUT_RESULT: ' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || ' ms/query' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; DROP TABLE bucket_results; -- ============================================================ -- Benchmark 2: Query Throughput (3 iterations) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (3 iterations) ===' -- Helper function for throughput benchmark -- Uses ParadeDB match() to handle special characters in query strings CREATE OR REPLACE FUNCTION benchmark_throughput_paradedb(iterations int DEFAULT 3) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric, queries_run int) AS $$ DECLARE q record; i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; query_count int; BEGIN SELECT COUNT(*) INTO query_count FROM benchmark_queries_paradedb; -- Warmup: run all queries once FOR q IN SELECT query_text FROM benchmark_queries_paradedb ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10' USING q.query_text; END LOOP; -- Timed iterations times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); FOR q IN SELECT query_text FROM benchmark_queries_paradedb ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10' USING q.query_text; END LOOP; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; queries_run := query_count; RETURN NEXT; END; $$ LANGUAGE plpgsql; SELECT 'Execution Time: ' || round(median_ms / queries_run, 3) || ' ms (min=' || round(min_ms / queries_run, 3) || ', max=' || round(max_ms / queries_run, 3) || ')' as result, 'THROUGHPUT_RESULT: ' || queries_run || ' queries in ' || round(median_ms, 2) || ' ms (avg ' || round(median_ms / queries_run, 2) || ' ms/query)' as summary FROM benchmark_throughput_paradedb(); DROP FUNCTION benchmark_throughput_paradedb; -- ============================================================ -- Benchmark 3: Concurrent Throughput (pgbench, weighted) -- ============================================================ -- Uses a weighted query pool matching MS-MARCO v1 lexeme distribution -- so that query mix reflects real-world traffic (72.6% have 2-4 lexemes). -- Run separately via pgbench: -- pgbench -n -c 16 -j 16 -T 60 \ -- -f benchmarks/datasets/msmarco-v2/pgbench-throughput-paradedb.sql -- -- Setup: create weighted_query_pool table first: \echo '' \echo '=== Benchmark 3: Concurrent Throughput Setup ===' \echo 'Creating weighted query pool (MS-MARCO v1 distribution)...' DROP TABLE IF EXISTS weighted_query_pool; CREATE TABLE weighted_query_pool AS WITH numbered AS ( SELECT query_text, token_bucket, ROW_NUMBER() OVER (PARTITION BY token_bucket ORDER BY query_id) as rn, COUNT(*) OVER (PARTITION BY token_bucket) as bucket_size FROM benchmark_queries_paradedb ) SELECT n.query_text, n.token_bucket, gs.i as pool_idx FROM (VALUES (1, 35), (2, 163), (3, 302), (4, 261), (5, 142), (6, 59), (7, 22), (8, 16)) AS w(bucket, target), LATERAL generate_series(1, w.target) AS gs(i), LATERAL ( SELECT query_text, token_bucket FROM numbered WHERE token_bucket = w.bucket AND rn = ((gs.i - 1) % bucket_size) + 1 ) n; ALTER TABLE weighted_query_pool ADD COLUMN id SERIAL; CREATE INDEX ON weighted_query_pool(id); SELECT 'WEIGHTED_POOL: ' || COUNT(*) || ' queries (' || string_agg(bucket || '=' || cnt, ', ' ORDER BY bucket) || ')' as pool_summary FROM (SELECT token_bucket::text as bucket, COUNT(*)::text as cnt FROM weighted_query_pool GROUP BY token_bucket) t; \echo '' \echo 'Weighted query pool created. Run concurrent benchmark with:' \echo ' pgbench -n -c 16 -j 16 -T 60 -f benchmarks/datasets/msmarco-v2/pgbench-throughput-paradedb.sql' -- ============================================================ -- Index Statistics -- ============================================================ \echo '' \echo '=== Index Statistics ===' SELECT 'msmarco_v2_paradedb_idx' as index_name, pg_size_pretty(pg_relation_size('msmarco_v2_paradedb_idx')) as index_size, pg_size_pretty(pg_relation_size('msmarco_v2_passages_paradedb')) as table_size, (SELECT COUNT(*) FROM msmarco_v2_passages_paradedb) as num_documents; -- Cleanup (keep weighted_query_pool for pgbench) DROP TABLE benchmark_queries_paradedb; \echo '' \echo '=== MS MARCO v2 Query Benchmarks Complete (ParadeDB) ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/pgbench-throughput-paradedb.sql000066400000000000000000000003571525762047700327770ustar00rootroot00000000000000\set qid random(1, 1000) SELECT passage_id FROM msmarco_v2_passages WHERE passage_text @@@ paradedb.match('passage_text', (SELECT query_text FROM weighted_query_pool WHERE id = :qid)) ORDER BY paradedb.score(passage_id) DESC LIMIT 10; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/pgbench-throughput.sql000066400000000000000000000003231525762047700312300ustar00rootroot00000000000000\set qid random(1, 1000) SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query( (SELECT query_text FROM weighted_query_pool WHERE id = :qid), 'msmarco_v2_bm25_idx') LIMIT 10; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/power_paradedb.sql000066400000000000000000000003671525762047700304010ustar00rootroot00000000000000\set qid random(1, 691) SELECT passage_id FROM msmarco_v2_passages WHERE passage_text @@@ paradedb.match( 'passage_text', (SELECT query_text FROM benchmark_queries_power WHERE id = :qid)) ORDER BY paradedb.score(passage_id) DESC LIMIT 10; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/power_tapir.sql000066400000000000000000000003261525762047700277510ustar00rootroot00000000000000\set qid random(1, 691) SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query( (SELECT query_text FROM benchmark_queries_power WHERE id = :qid), 'msmarco_v2_bm25_idx') LIMIT 10; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/precompute_ground_truth.sql000066400000000000000000000342161525762047700324120ustar00rootroot00000000000000-- MS-MARCO v2 Ground Truth Precomputation -- Computes BM25 scores using the exact formula for validation purposes -- Output: ground_truth.tsv with (query_id, query_text, rank, doc_id, score) -- -- IMPORTANT: Different Postgres versions may produce different ground truth -- due to Snowball stemmer changes (e.g., PG17 vs PG18). After generating, -- rename the output to ground_truth_pgNN.tsv for the target version. -- -- Prerequisites: -- - msmarco_v2_passages table loaded (index NOT required) -- - msmarco_v2_queries table loaded -- -- Usage: -- psql -p PORT -f precompute_ground_truth.sql -- -- This is a ONE-TIME computation. Results are saved and reused for -- validation. At 138M rows, full validation of all 3,903 dev queries -- is infeasible (~1.5 min/query = ~4 days). Instead we validate 400 -- queries: 10 curated high-frequency term queries (to detect the -- block_count overflow bug fixed by PR #266), 10 curated -- low-frequency term queries (baseline correctness), and 380 -- randomly sampled queries from the full dev set. -- -- Note: Uses single-pass doc_freq computation (~13 min total) instead -- of one COUNT(*) per term (~13 min each) as in v1. \set ON_ERROR_STOP on \timing on -- Ensure validation functions exist (for fieldnorm_quantize) \i test/sql/validation.sql -- Mark fieldnorm_quantize as PARALLEL SAFE so that queries using it -- can leverage parallel workers. It's a pure computation (no side -- effects, no transaction state). ALTER FUNCTION fieldnorm_quantize(int) PARALLEL SAFE; -- ============================================================ -- Step 0: Select validation queries (400 total) -- ============================================================ -- 10 hand-picked high-frequency term queries (terms with -- doc_freq > 8.4M, the uint16 block_count overflow threshold). -- 10 hand-picked low-frequency term queries (rare/specific). -- 380 randomly sampled from the remaining dev queries. -- Random seed is fixed (0.42) for reproducibility. \echo 'Creating validation queries...' DROP TABLE IF EXISTS validation_queries; CREATE TABLE validation_queries ( query_id INTEGER, query_text TEXT, category TEXT -- 'high_freq', 'low_freq', or 'random' ); -- High-frequency term queries (terms likely > 8.4M postings) INSERT INTO validation_queries SELECT q.query_id, q.query_text, 'high_freq' FROM msmarco_v2_queries q WHERE q.query_id IN ( 107077, -- 'cost of slate' (cost) 195693, -- 'golf cost' (cost) 46081, -- 'average time for an appraisal' (time) 167156, -- 'does highster mobile really work' (work) 166043, -- 'does estrogen make me start my period' (make) 80590, -- 'can you use denture tablets...' (use) 46095, -- 'average time for hair to grow' (time) 109276, -- 'cost to dig a pond' (cost) 108037, -- 'cost of wooden shutters' (cost) 102695 -- 'cost of bamboo hr' (cost) ); -- Low-frequency term queries (rare/specific terms) INSERT INTO validation_queries SELECT q.query_id, q.query_text, 'low_freq' FROM msmarco_v2_queries q WHERE q.query_id IN ( 49802, -- 'belizean cuisine' 61531, -- 'campagnolo bikes' 558263, -- 'what are endotoxins?' 561448, -- 'what are mandrakes' 563347, -- 'what are plastids' 687375, -- 'what is a hydromyelia' 699510, -- 'what is a serigraph' 703383, -- 'what is a thoracentesis' 724121, -- 'what is bioflavonoids' 724571 -- 'what is blackleg' ); -- Random sample of 380 queries (fixed seed for reproducibility) INSERT INTO validation_queries SELECT q.query_id, q.query_text, 'random' FROM msmarco_v2_queries q WHERE q.query_id NOT IN ( SELECT vq.query_id FROM validation_queries vq ) ORDER BY hashint4(q.query_id + 42) LIMIT 380; DO $$ DECLARE v_high int; v_low int; v_random int; BEGIN SELECT COUNT(*) INTO v_high FROM validation_queries WHERE category = 'high_freq'; SELECT COUNT(*) INTO v_low FROM validation_queries WHERE category = 'low_freq'; SELECT COUNT(*) INTO v_random FROM validation_queries WHERE category = 'random'; RAISE NOTICE 'Selected % high-freq + % low-freq + % random' ' = % queries', v_high, v_low, v_random, v_high + v_low + v_random; END; $$; \echo '' \echo 'Validation queries by category:' SELECT category, COUNT(*) as count FROM validation_queries GROUP BY category ORDER BY category; -- ============================================================ -- Step 1: Compute corpus statistics -- ============================================================ -- Scans the full 138M-row table to count tokens per document. -- Documents with empty tsvectors (no indexable terms) are excluded, -- matching the index behavior. -- -- Uses LATERAL unnest join (not correlated subquery) so Postgres -- can parallelize with Gather Merge. Tune parallel costs to -- ensure workers are used on this CPU-heavy scan. \echo '' \echo 'Step 1: Computing corpus statistics (parallel scan)...' SET max_parallel_workers_per_gather = 15; SET parallel_tuple_cost = 0; SET parallel_setup_cost = 0; SET min_parallel_table_scan_size = 0; DROP TABLE IF EXISTS corpus_stats; CREATE TABLE corpus_stats AS SELECT COUNT(*)::bigint as total_docs, SUM(doc_len)::bigint as total_len, (SUM(doc_len)::float8 / COUNT(*)::float8) as avg_doc_len FROM ( SELECT p.passage_id, SUM(array_length( string_to_array(t.positions::text, ','), 1 )) as doc_len FROM msmarco_v2_passages p, LATERAL unnest(to_tsvector('english', p.passage_text)) AS t(lexeme, positions, weights) GROUP BY p.passage_id ) dl; RESET max_parallel_workers_per_gather; RESET parallel_tuple_cost; RESET parallel_setup_cost; RESET min_parallel_table_scan_size; DO $$ DECLARE r record; BEGIN SELECT * INTO r FROM corpus_stats; RAISE NOTICE 'Corpus stats: total_docs=%, total_len=%, ' 'avg_doc_len=%', r.total_docs, r.total_len, round(r.avg_doc_len::numeric, 4); END; $$; -- ============================================================ -- Step 2: Precompute document frequencies (single-pass) -- ============================================================ -- At 138M rows, one COUNT(*) per term takes ~13 min each. -- Instead, compute all term doc_freqs in one table scan using -- LATERAL unnest + GROUP BY. This takes ~13 min total. \echo '' \echo 'Step 2: Precomputing document frequencies (single-pass, ~13 min)...' -- First collect all unique query terms DROP TABLE IF EXISTS unique_query_terms; CREATE TABLE unique_query_terms AS SELECT DISTINCT qt.lexeme::text as term FROM validation_queries vq, LATERAL unnest(to_tsvector('english', vq.query_text)) as qt(lexeme, positions, weights); SELECT 'Computing df for ' || COUNT(*) || ' unique query terms' as status FROM unique_query_terms; -- Single-pass: scan entire corpus once, unnest tsvectors, filter -- to only our query terms, count distinct passage_ids per term SET max_parallel_workers_per_gather = 15; SET parallel_tuple_cost = 0; SET parallel_setup_cost = 0; SET min_parallel_table_scan_size = 0; DROP TABLE IF EXISTS term_doc_freq; CREATE TABLE term_doc_freq AS SELECT t.lexeme::text as term, COUNT(DISTINCT p.passage_id) as df FROM msmarco_v2_passages p, LATERAL unnest(to_tsvector('english', p.passage_text)) AS t(lexeme, positions, weights) WHERE t.lexeme::text IN (SELECT term FROM unique_query_terms) GROUP BY t.lexeme; RESET max_parallel_workers_per_gather; RESET parallel_tuple_cost; RESET parallel_setup_cost; RESET min_parallel_table_scan_size; SELECT 'Computed df for ' || COUNT(*) || ' terms' as status FROM term_doc_freq; \echo '' \echo 'Document frequencies (sorted by df):' SELECT term, df FROM term_doc_freq ORDER BY df DESC; DROP TABLE unique_query_terms; -- ============================================================ -- Step 3: Materialize document vectors -- ============================================================ -- Pre-compute tsvectors and quantized doc lengths for all docs -- matching any query term. This avoids recomputing to_tsvector -- per query and eliminates correlated subqueries in scoring. -- -- Strategy: single parallel scan of 138M rows, unnest tsvectors, -- filter to docs containing any query term, extract per-term tf -- and quantized doc length. The result is a flat table suitable -- for join-based BM25 scoring. \echo '' \echo 'Step 3: Materializing document term frequencies...' SET max_parallel_workers_per_gather = 15; SET parallel_tuple_cost = 0; SET parallel_setup_cost = 0; SET min_parallel_table_scan_size = 0; -- 3a: Extract per-document term frequencies for query terms. -- One row per (passage_id, term). Single parallel scan that -- unnests all tsvectors and filters to query terms only. \echo ' 3a: Extracting term frequencies (~20 min)...' DROP TABLE IF EXISTS doc_term_data; CREATE TABLE doc_term_data AS SELECT p.passage_id, t.lexeme::text as term, array_length( string_to_array(t.positions::text, ','), 1 ) as tf FROM msmarco_v2_passages p, LATERAL unnest(to_tsvector('english', p.passage_text)) AS t(lexeme, positions, weights) WHERE t.lexeme::text IN (SELECT term FROM term_doc_freq); SELECT 'Extracted ' || COUNT(*) || ' (doc, term) pairs' as status FROM doc_term_data; CREATE INDEX ON doc_term_data (passage_id, term); ANALYZE doc_term_data; -- 3b: Compute quantized doc lengths for ALL documents. -- Full parallel scan — no joins that would block parallelism. -- Requires fieldnorm_quantize to be marked PARALLEL SAFE. \echo ' 3b: Computing quantized doc lengths (~18 min)...' DROP TABLE IF EXISTS doc_lengths; CREATE TABLE doc_lengths AS SELECT p.passage_id, fieldnorm_quantize( SUM(array_length( string_to_array(t.positions::text, ','), 1 ))::int ) as doc_len_q FROM msmarco_v2_passages p, LATERAL unnest(to_tsvector('english', p.passage_text)) AS t(lexeme, positions, weights) GROUP BY p.passage_id; SELECT 'Computed doc lengths for ' || COUNT(*) || ' passages' as status FROM doc_lengths; CREATE INDEX ON doc_lengths (passage_id); ANALYZE doc_lengths; RESET max_parallel_workers_per_gather; RESET parallel_tuple_cost; RESET parallel_setup_cost; RESET min_parallel_table_scan_size; -- ============================================================ -- Step 4: Compute ground truth using flat joins -- ============================================================ -- No correlated subqueries. For each query, join doc_term_data -- and doc_lengths to compute BM25 scores with SUM/GROUP BY. \echo '' \echo 'Step 4: Computing ground truth (400 queries, flat joins)...' DROP TABLE IF EXISTS ground_truth; CREATE TABLE ground_truth ( query_id int, query_text text, rank int, doc_id text, score float8 ); DO $$ DECLARE q record; v_count int := 0; v_total int; v_start timestamp; v_elapsed interval; v_total_docs bigint; v_avg_doc_len float8; v_k1 float8 := 1.2; v_b float8 := 0.75; BEGIN SELECT COUNT(*) INTO v_total FROM validation_queries; SELECT cs.total_docs, cs.avg_doc_len INTO v_total_docs, v_avg_doc_len FROM corpus_stats cs; v_start := clock_timestamp(); FOR q IN SELECT query_id, query_text, category FROM validation_queries ORDER BY category, query_id LOOP v_count := v_count + 1; RAISE NOTICE '[%/%] % - Query %: %...', v_count, v_total, q.category, q.query_id, left(q.query_text, 40); INSERT INTO ground_truth WITH query_terms AS ( SELECT qt.lexeme::text as term, array_length( string_to_array(qt.positions::text, ','), 1 ) as query_freq FROM unnest( to_tsvector('english', q.query_text) ) as qt(lexeme, positions, weights) ), doc_scores AS ( SELECT dt.passage_id, SUM( -- IDF ln(1.0 + (v_total_docs - tdf.df + 0.5) / (tdf.df + 0.5)) -- TF saturation * (dt.tf * (v_k1 + 1.0)) / (dt.tf + v_k1 * (1.0 - v_b + v_b * dl.doc_len_q / v_avg_doc_len)) -- Query term frequency * qt.query_freq ) as bm25_score FROM query_terms qt JOIN term_doc_freq tdf ON tdf.term = qt.term JOIN doc_term_data dt ON dt.term = qt.term JOIN doc_lengths dl ON dl.passage_id = dt.passage_id GROUP BY dt.passage_id ), ranked AS ( SELECT passage_id, bm25_score, ROW_NUMBER() OVER ( ORDER BY bm25_score DESC ) as rn FROM doc_scores WHERE bm25_score > 0 ) SELECT q.query_id, q.query_text, rn::int, passage_id::text, bm25_score::float8 FROM ranked WHERE rn <= 10; v_elapsed := clock_timestamp() - v_start; RAISE NOTICE ' Elapsed: %, avg %/query', v_elapsed, v_elapsed / v_count; END LOOP; v_elapsed := clock_timestamp() - v_start; RAISE NOTICE 'Completed % queries in %', v_count, v_elapsed; END; $$; -- Export to TSV \echo '' \echo 'Exporting ground truth to TSV...' \copy ground_truth TO 'benchmarks/datasets/msmarco-v2/ground_truth.tsv' WITH (FORMAT text, DELIMITER E'\t', HEADER true) SELECT 'Exported ' || COUNT(*) || ' rows for ' || COUNT(DISTINCT query_id) || ' queries' as status FROM ground_truth; -- Show summary per category \echo '' \echo 'Results per category:' SELECT vq.category, COUNT(DISTINCT gt.query_id) as queries, COUNT(*) as result_rows, round(AVG(gt.score)::numeric, 4) as avg_score FROM ground_truth gt JOIN validation_queries vq ON vq.query_id = gt.query_id GROUP BY vq.category ORDER BY vq.category; -- Cleanup DROP TABLE IF EXISTS validation_queries; DROP TABLE IF EXISTS doc_term_data; DROP TABLE IF EXISTS doc_lengths; \echo '' \echo '=== Ground Truth Precomputation Complete ===' \echo 'Output: benchmarks/datasets/msmarco-v2/ground_truth.tsv' \echo 'Note: term_doc_freq and corpus_stats tables retained' \echo ' for debugging' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/profile_build.sh000077500000000000000000000055341525762047700300610ustar00rootroot00000000000000#!/usr/bin/env bash # # profile_build.sh - Profile parallel index build with perf + flamegraph # # Usage: # ./profile_build.sh [database] [table] [column] # # Defaults: msmarco_v2 / passages / passage # # Prerequisites: # - perf installed (linux-tools-common / linux-tools-$(uname -r)) # - Postgres running, data loaded # - Run as a user with perf_event access (or root) # # Output: # - flamegraph_build.svg Flame graph of the build # - perf.data Raw perf recording # - pidstat_build.log CPU/IO stats during build set -euo pipefail DB="${1:-msmarco_v2}" TABLE="${2:-passages}" COLUMN="${3:-passage}" INDEX_NAME="idx_${TABLE}_bm25" FLAMEGRAPH_DIR="./FlameGraph" # Clone FlameGraph if not present if [ ! -d "$FLAMEGRAPH_DIR" ]; then echo "Cloning FlameGraph repository..." git clone --depth 1 \ https://github.com/brendangregg/FlameGraph.git \ "$FLAMEGRAPH_DIR" fi # Check for perf if ! command -v perf &>/dev/null; then echo "ERROR: perf not found. Install with:" echo " sudo apt install linux-tools-common linux-tools-\$(uname -r)" exit 1 fi # Drop existing index if present echo "Dropping existing index (if any)..." psql -d "$DB" -c "DROP INDEX IF EXISTS $INDEX_NAME;" 2>/dev/null || true # System-wide perf recording captures the leader backend AND all # parallel worker processes (workers are forked by the postmaster, # not by the backend, so -p --inherit misses them). echo "Starting system-wide perf record..." sudo perf record -ag --call-graph dwarf,32768 \ -o perf.data & PERF_PID=$! sleep 1 # Start pidstat for all postgres processes pidstat -C postgres -dru 5 > pidstat_build.log 2>&1 & PIDSTAT_PID=$! # Run CREATE INDEX echo "Running CREATE INDEX..." START_TIME=$(date +%s) psql -d "$DB" -c " SET maintenance_work_mem = '8GB'; SET max_parallel_maintenance_workers = 4; CREATE INDEX $INDEX_NAME ON $TABLE USING bm25 ($COLUMN) WITH (text_config = 'english'); " END_TIME=$(date +%s) ELAPSED=$((END_TIME - START_TIME)) echo "CREATE INDEX completed in ${ELAPSED}s" # Stop perf and pidstat sudo kill -INT "$PERF_PID" 2>/dev/null || true wait "$PERF_PID" 2>/dev/null || true kill "$PIDSTAT_PID" 2>/dev/null || true wait "$PIDSTAT_PID" 2>/dev/null || true # Generate flame graph (filtered to postgres processes only) echo "Generating flame graph..." sudo perf script -i perf.data --comms=postgres \ | "$FLAMEGRAPH_DIR/stackcollapse-perf.pl" \ | "$FLAMEGRAPH_DIR/flamegraph.pl" \ --title "Parallel Index Build (${ELAPSED}s)" \ --width 1600 \ > flamegraph_build.svg echo "" echo "=== Output files ===" echo " flamegraph_build.svg - Flame graph (postgres only)" echo " flamegraph_build_all.svg - Flame graph (all processes)" echo " perf.data - Raw perf data" echo " pidstat_build.log - CPU/IO stats" echo "" echo "View: open flamegraph_build.svg" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/profile_queries.sql000066400000000000000000000077211525762047700306210ustar00rootroot00000000000000-- Profile slow queries (5-8+ token buckets) to find optimization targets. -- Outputs per-query wall-clock timing + BMW stats from the server log. -- -- Usage: -- SET pg_textsearch.log_bmw_stats = true; -- SET log_min_messages = LOG; -- \i benchmarks/datasets/msmarco-v2/profile_queries.sql -- -- Then grep the PG log for "BMW stats" lines to correlate with output. \set ON_ERROR_STOP on \timing on -- Enable BMW diagnostics SET pg_textsearch.log_bmw_stats = true; SET client_min_messages = LOG; -- Load benchmark queries DROP TABLE IF EXISTS benchmark_queries; CREATE TABLE benchmark_queries ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy benchmark_queries FROM 'benchmarks/datasets/msmarco-v2/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') -- Warm up: touch each bucket once DO $$ DECLARE q record; BEGIN FOR q IN SELECT DISTINCT ON (token_bucket) query_text FROM benchmark_queries WHERE token_bucket >= 5 ORDER BY token_bucket, query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_v2_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; END; $$; \echo '' \echo '=== Profiling Buckets 5-8 ===' -- Per-query timing with BMW stats (logged server-side) CREATE TEMP TABLE profile_results ( query_id int, token_bucket int, query_text text, elapsed_ms numeric, num_results int ); DO $$ DECLARE q record; t0 timestamp; t1 timestamp; cnt int; BEGIN FOR q IN SELECT query_id, query_text, token_bucket FROM benchmark_queries WHERE token_bucket >= 5 ORDER BY token_bucket, query_id LOOP t0 := clock_timestamp(); EXECUTE 'SELECT COUNT(*) FROM ( SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_v2_bm25_idx'') LIMIT 10 ) t' INTO cnt USING q.query_text; t1 := clock_timestamp(); INSERT INTO profile_results VALUES ( q.query_id, q.token_bucket, q.query_text, EXTRACT(EPOCH FROM (t1 - t0)) * 1000, cnt ); END LOOP; END; $$; -- Summary by bucket \echo '' \echo '=== Per-Bucket Summary ===' SELECT token_bucket, COUNT(*) AS n, round(percentile_cont(0.50) WITHIN GROUP (ORDER BY elapsed_ms)::numeric, 2) AS p50_ms, round(percentile_cont(0.95) WITHIN GROUP (ORDER BY elapsed_ms)::numeric, 2) AS p95_ms, round(AVG(elapsed_ms), 2) AS avg_ms, round(MIN(elapsed_ms), 2) AS min_ms, round(MAX(elapsed_ms), 2) AS max_ms FROM profile_results GROUP BY token_bucket ORDER BY token_bucket; -- Slowest 20 queries \echo '' \echo '=== Slowest 20 Queries ===' SELECT query_id, token_bucket AS bucket, round(elapsed_ms, 2) AS ms, LEFT(query_text, 70) AS query FROM profile_results ORDER BY elapsed_ms DESC LIMIT 20; -- Distribution: how many queries in each latency band? \echo '' \echo '=== Latency Distribution (buckets 5-8) ===' SELECT CASE WHEN elapsed_ms < 100 THEN ' <100ms' WHEN elapsed_ms < 200 THEN ' 100-200ms' WHEN elapsed_ms < 300 THEN ' 200-300ms' WHEN elapsed_ms < 400 THEN ' 300-400ms' WHEN elapsed_ms < 500 THEN ' 400-500ms' WHEN elapsed_ms < 750 THEN ' 500-750ms' ELSE ' 750ms+' END AS band, COUNT(*) AS n, round(100.0 * COUNT(*) / SUM(COUNT(*)) OVER (), 1) AS pct FROM profile_results GROUP BY 1 ORDER BY 1; -- Token count vs elapsed correlation \echo '' \echo '=== Per-Query Detail (buckets 5-8) ===' SELECT query_id, token_bucket AS bucket, round(elapsed_ms, 2) AS ms, query_text FROM profile_results ORDER BY token_bucket, elapsed_ms DESC; DROP TABLE profile_results; DROP TABLE benchmark_queries; \echo '' \echo '=== Profiling Complete ===' \echo 'Check PG log for BMW stats: grep "BMW stats" ' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/queries.sql000066400000000000000000000300601525762047700270710ustar00rootroot00000000000000-- MS MARCO v2 Passage Ranking - Query Benchmarks -- Runs various query workloads against the indexed MS MARCO v2 collection -- Outputs structured timing data for historical tracking \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO v2 Query Benchmarks ===' \echo '' -- Load benchmark queries (pre-sampled by token count) \echo 'Loading benchmark queries...' DROP TABLE IF EXISTS benchmark_queries; CREATE TABLE benchmark_queries ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy benchmark_queries FROM 'benchmarks/datasets/msmarco-v2/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') -- Verify load SELECT 'Loaded ' || COUNT(*) || ' benchmark queries' as status FROM benchmark_queries; SELECT token_bucket, COUNT(*) as count FROM benchmark_queries GROUP BY token_bucket ORDER BY token_bucket; -- Warm up: run queries from each bucket to ensure index is cached \echo '' \echo 'Warming up index...' DO $$ DECLARE q record; BEGIN FOR q IN SELECT query_text FROM benchmark_queries ORDER BY random() LIMIT 50 LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_v2_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; END; $$; -- ============================================================ -- Benchmark 1: Query Latency by Token Count -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency by Token Count ===' \echo 'Running 100 queries per token bucket, reporting p50/p95/p99' \echo '' -- Temp table to capture per-bucket results for weighted average CREATE TEMP TABLE bucket_results ( bucket int, p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint ); -- Function to benchmark a bucket and return percentiles CREATE OR REPLACE FUNCTION benchmark_bucket(bucket int) RETURNS TABLE(p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint) AS $$ DECLARE q record; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; n int; result_count bigint; results_sum bigint := 0; BEGIN times := ARRAY[]::numeric[]; FOR q IN SELECT query_text FROM benchmark_queries WHERE token_bucket = bucket ORDER BY query_id LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT COUNT(*) FROM (SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_v2_bm25_idx'') LIMIT 10) t' INTO result_count USING q.query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); results_sum := results_sum + result_count; END LOOP; n := array_length(times, 1); SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; p50_ms := sorted_times[(n + 1) / 2]; p95_ms := sorted_times[(n * 95 + 99) / 100]; p99_ms := sorted_times[(n * 99 + 99) / 100]; avg_ms := (SELECT AVG(t) FROM unnest(times) t); num_queries := n; total_results := results_sum; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run benchmarks for each token bucket (results saved for weighted avg) \echo 'Token bucket 1 (1 search token):' INSERT INTO bucket_results SELECT 1, * FROM benchmark_bucket(1); SELECT 'LATENCY_BUCKET_1: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 1; \echo '' \echo 'Token bucket 2 (2 search tokens):' INSERT INTO bucket_results SELECT 2, * FROM benchmark_bucket(2); SELECT 'LATENCY_BUCKET_2: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 2; \echo '' \echo 'Token bucket 3 (3 search tokens):' INSERT INTO bucket_results SELECT 3, * FROM benchmark_bucket(3); SELECT 'LATENCY_BUCKET_3: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 3; \echo '' \echo 'Token bucket 4 (4 search tokens):' INSERT INTO bucket_results SELECT 4, * FROM benchmark_bucket(4); SELECT 'LATENCY_BUCKET_4: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 4; \echo '' \echo 'Token bucket 5 (5 search tokens):' INSERT INTO bucket_results SELECT 5, * FROM benchmark_bucket(5); SELECT 'LATENCY_BUCKET_5: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 5; \echo '' \echo 'Token bucket 6 (6 search tokens):' INSERT INTO bucket_results SELECT 6, * FROM benchmark_bucket(6); SELECT 'LATENCY_BUCKET_6: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 6; \echo '' \echo 'Token bucket 7 (7 search tokens):' INSERT INTO bucket_results SELECT 7, * FROM benchmark_bucket(7); SELECT 'LATENCY_BUCKET_7: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 7; \echo '' \echo 'Token bucket 8 (8+ search tokens):' INSERT INTO bucket_results SELECT 8, * FROM benchmark_bucket(8); SELECT 'LATENCY_BUCKET_8: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 8; DROP FUNCTION benchmark_bucket; -- ============================================================ -- Weighted-Average Query Latency (MS-MARCO v1 distribution) -- ============================================================ -- Weights from observed lexeme distribution in 1,010,905 MS-MARCO -- v1 queries after to_tsvector: 1=3.5%, 2=16.3%, 3=30.2%, -- 4=26.1%, 5=14.2%, 6=5.9%, 7=2.2%, 8+=1.5% \echo '' \echo '=== Weighted-Average Query Latency & Throughput ===' SELECT 'WEIGHTED_LATENCY_RESULT: weighted_p50=' || round(SUM(b.p50_ms * w.weight) / SUM(w.weight), 2) || 'ms weighted_avg=' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || 'ms' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; SELECT 'WEIGHTED_THROUGHPUT_RESULT: ' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || ' ms/query' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; DROP TABLE bucket_results; -- ============================================================ -- Benchmark 2: Query Throughput (3 iterations) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (3 iterations) ===' -- Helper function for throughput benchmark CREATE OR REPLACE FUNCTION benchmark_throughput(iterations int DEFAULT 3) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric, queries_run int) AS $$ DECLARE q record; i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; query_count int; BEGIN SELECT COUNT(*) INTO query_count FROM benchmark_queries; -- Warmup: run all queries once FOR q IN SELECT query_text FROM benchmark_queries ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_v2_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; -- Timed iterations times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); FOR q IN SELECT query_text FROM benchmark_queries ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_v2_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; queries_run := query_count; RETURN NEXT; END; $$ LANGUAGE plpgsql; SELECT 'Execution Time: ' || round(median_ms / queries_run, 3) || ' ms (min=' || round(min_ms / queries_run, 3) || ', max=' || round(max_ms / queries_run, 3) || ')' as result, 'THROUGHPUT_RESULT: ' || queries_run || ' queries in ' || round(median_ms, 2) || ' ms (avg ' || round(median_ms / queries_run, 2) || ' ms/query)' as summary FROM benchmark_throughput(); DROP FUNCTION benchmark_throughput; -- ============================================================ -- Benchmark 3: Concurrent Throughput (pgbench, weighted) -- ============================================================ -- Uses a weighted query pool matching MS-MARCO v1 lexeme distribution -- so that query mix reflects real-world traffic (72.6% have 2-4 lexemes). -- Run separately via pgbench: -- pgbench -n -c 16 -j 16 -T 60 \ -- -f benchmarks/datasets/msmarco-v2/pgbench-throughput.sql -- -- Setup: create weighted_query_pool table first: \echo '' \echo '=== Benchmark 3: Concurrent Throughput Setup ===' \echo 'Creating weighted query pool (MS-MARCO v1 distribution)...' DROP TABLE IF EXISTS weighted_query_pool; CREATE TABLE weighted_query_pool AS WITH numbered AS ( SELECT query_text, token_bucket, ROW_NUMBER() OVER (PARTITION BY token_bucket ORDER BY query_id) as rn, COUNT(*) OVER (PARTITION BY token_bucket) as bucket_size FROM benchmark_queries ) SELECT n.query_text, n.token_bucket, gs.i as pool_idx FROM (VALUES (1, 35), (2, 163), (3, 302), (4, 261), (5, 142), (6, 59), (7, 22), (8, 16)) AS w(bucket, target), LATERAL generate_series(1, w.target) AS gs(i), LATERAL ( SELECT query_text, token_bucket FROM numbered WHERE token_bucket = w.bucket AND rn = ((gs.i - 1) % bucket_size) + 1 ) n; ALTER TABLE weighted_query_pool ADD COLUMN id SERIAL; CREATE INDEX ON weighted_query_pool(id); SELECT 'WEIGHTED_POOL: ' || COUNT(*) || ' queries (' || string_agg(bucket || '=' || cnt, ', ' ORDER BY bucket) || ')' as pool_summary FROM (SELECT token_bucket::text as bucket, COUNT(*)::text as cnt FROM weighted_query_pool GROUP BY token_bucket) t; \echo '' \echo 'Weighted query pool created. Run concurrent benchmark with:' \echo ' pgbench -n -c 16 -j 16 -T 60 -f benchmarks/datasets/msmarco-v2/pgbench-throughput.sql' -- ============================================================ -- Index Statistics -- ============================================================ \echo '' \echo '=== Index Statistics ===' SELECT 'msmarco_v2_bm25_idx' as index_name, pg_size_pretty(pg_relation_size('msmarco_v2_bm25_idx')) as index_size, pg_size_pretty(pg_relation_size('msmarco_v2_passages')) as table_size, (SELECT COUNT(*) FROM msmarco_v2_passages) as num_documents; -- Cleanup (keep weighted_query_pool for pgbench) DROP TABLE benchmark_queries; \echo '' \echo '=== MS MARCO v2 Query Benchmarks Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/results/000077500000000000000000000000001525762047700263755ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/results/20260304_tapir_latest/000077500000000000000000000000001525762047700320505ustar00rootroot00000000000000summary.md000066400000000000000000000055261525762047700340200ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/results/20260304_tapir_latest# MS-MARCO v2 Benchmark Results (138M passages) Date: 2026-03-10 Commit: main @ fb3b3b1 Machine: c6i.4xlarge (Intel Xeon Platinum 8375C, 8C/16T, 123 GB RAM, NVMe) Postgres: 17.7, shared_buffers=31 GB ## Index Build | Metric | pg_textsearch | System X (ParadeDB v0.21.6) | |--------|---------------|----------------------------| | Build time | 1,057,461 ms (17:37) | 535,169 ms (8:55) | | Workers launched | 15 of 15 requested | 14 | | Index size | 17 GB (18,153,144,320 bytes) | 23 GB (24,876,695,552 bytes) | | Documents | 138,364,158 | 138,364,158 | | Unique terms | 17,373,764 | - | | Avg doc length | 29.34 lexemes | - | | Segments | 1 (L0, 2,214,872 pages) | - | ## Single-Client Query Latency (top-10, LIMIT 10) 691 queries: 100 per bucket 1-6, 53 for bucket 7, 38 for bucket 8+. ### pg_textsearch | Tokens | p50 (ms) | p95 (ms) | p99 (ms) | avg (ms) | n | |--------|----------|----------|----------|----------|---| | 1 | 5.11 | 6.43 | 7.05 | 5.34 | 100 | | 2 | 9.14 | 32.63 | 56.22 | 12.94 | 100 | | 3 | 20.04 | 51.51 | 57.25 | 24.11 | 100 | | 4 | 41.92 | 124.17 | 153.56 | 49.33 | 100 | | 5 | 67.76 | 167.05 | 239.87 | 77.08 | 100 | | 6 | 102.82 | 262.07 | 321.54 | 115.90 | 100 | | 7 | 159.37 | 311.58 | 439.19 | 163.22 | 53 | | 8+ | 177.95 | 404.95 | 421.80 | 185.09 | 38 | Weighted p50: 40.61 ms | Weighted avg: 46.69 ms ### System X (ParadeDB v0.21.6) | Tokens | p50 (ms) | p95 (ms) | p99 (ms) | avg (ms) | n | |--------|----------|----------|----------|----------|---| | 1 | 59.83 | 68.34 | 90.48 | 61.45 | 100 | | 2 | 59.65 | 103.17 | 141.36 | 66.37 | 100 | | 3 | 77.62 | 114.79 | 136.13 | 82.86 | 100 | | 4 | 98.89 | 147.32 | 177.70 | 106.97 | 100 | | 5 | 125.38 | 190.07 | 250.79 | 135.40 | 100 | | 6 | 148.78 | 201.76 | 244.19 | 156.34 | 100 | | 7 | 169.65 | 291.09 | 334.75 | 184.45 | 53 | | 8+ | 190.47 | 310.68 | 428.87 | 206.74 | 38 | Weighted p50: 94.36 ms | Weighted avg: 101.66 ms ## Single-Client Throughput (3 iterations, median) | Metric | pg_textsearch | System X | |--------|---------------|----------| | Total (691 queries) | 43,477 ms | 73,613 ms | | Avg ms/query | 62.92 | 106.53 | | Min ms/query | 62.84 | 105.50 | | Max ms/query | 62.92 | 106.89 | ## Concurrent Throughput (pgbench, 16 clients, 60 s) | Metric | pg_textsearch | System X | |--------|---------------|----------| | TPS | 91.43 | 19.43 | | Avg latency | 174.995 ms | 823.351 ms | | Transactions | 5,526 | 1,180 | | Failed | 0 | 0 | ## Comparison Summary | Metric | pg_textsearch | System X | Ratio | |--------|---------------|----------|-------| | Weighted p50 latency | 40.61 ms | 94.36 ms | 2.3x faster | | Weighted avg latency | 46.69 ms | 101.66 ms | 2.2x faster | | Single-client throughput | 62.92 ms/q | 106.53 ms/q | 1.7x faster | | Concurrent TPS (16 clients) | 91.4 | 19.4 | 4.7x higher | | Index size | 17 GB | 23 GB | 26% smaller | | Build time | 17:37 | 8:55 | 1.9x slower | timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/run_full_benchmark.sh000077500000000000000000000372671525762047700311120ustar00rootroot00000000000000#!/bin/bash # MS MARCO v2 Full Benchmark — pg_textsearch vs ParadeDB (ParadeDB) # # Reproducible benchmark for blog-post results on 138M passages. # # Prerequisites: # - download.sh has been run (collection.tsv exists) # - generate_benchmark_queries.sh has been run (benchmark_queries.tsv) # - pg_textsearch is built and installed (make && sudo make install) # - ParadeDB .deb is installed # - msmarco_v2_passages table is already loaded (load.sql or # load_data_only.sql) # # Usage: ./run_full_benchmark.sh # Steps: # env - capture machine specs, PG config, extensions # build-tapir - build pg_textsearch BM25 index # build-paradedb - copy data + build ParadeDB BM25 index # query-tapir - single-client latency benchmarks # query-paradedb - single-client latency benchmarks # power-tapir - concurrent throughput (pgbench) # power-paradedb - concurrent throughput (pgbench) # summary - side-by-side comparison table # all - run everything in sequence set -euo pipefail # ── Configuration ──────────────────────────────────────────────── PARALLEL_WORKERS=14 MAINT_WORK_MEM='2GB' MIN_PARALLEL_SCAN='64kB' POWER_CLIENTS=16 POWER_DURATION=60 POWER_WARMUP=5 # ───────────────────────────────────────────────────────────────── SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" DATA_DIR="$SCRIPT_DIR/data" TIMESTAMP=$(date +%Y%m%d_%H%M%S) RESULTS_DIR="$SCRIPT_DIR/results/$TIMESTAMP" STEP="${1:-all}" PGPORT="${PGPORT:-5432}" PGDATABASE="${PGDATABASE:-postgres}" export PGPORT PGDATABASE DATA_DIR mkdir -p "$RESULTS_DIR" log() { echo "$(date +%H:%M:%S) $*"; } # ── Prerequisites ──────────────────────────────────────────────── check_prereqs() { if [ ! -f "$DATA_DIR/collection.tsv" ]; then echo "ERROR: collection.tsv not found. Run ./download.sh first." exit 1 fi if [ ! -f "$SCRIPT_DIR/benchmark_queries.tsv" ]; then echo "ERROR: benchmark_queries.tsv not found." exit 1 fi log "Prerequisites OK" log " Collection: $(wc -l < "$DATA_DIR/collection.tsv") passages" log " Benchmark queries: $(wc -l < "$SCRIPT_DIR/benchmark_queries.tsv")" } # ── step: env ──────────────────────────────────────────────────── step_env() { local out="$RESULTS_DIR/env.log" log "Capturing environment → $out" { echo "=== Machine ===" echo "Date: $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo "Hostname: $(hostname)" echo "CPU: $(nproc) cores" lscpu | grep -E 'Model name|Socket|Core|Thread|MHz' || true echo "RAM: $(free -h | awk '/^Mem:/{print $2}')" echo "Disk:" df -h /nvme 2>/dev/null || df -h / echo "" echo "=== Postgres ===" psql -c "SELECT version();" psql -c "SHOW data_directory;" psql -c "SHOW shared_buffers;" psql -c "SHOW work_mem;" psql -c "SHOW maintenance_work_mem;" psql -c "SHOW max_parallel_maintenance_workers;" psql -c "SHOW max_parallel_workers_per_gather;" psql -c "SHOW max_parallel_workers;" psql -c "SHOW shared_preload_libraries;" echo "" echo "=== Extensions ===" psql -c "SELECT name, default_version, installed_version FROM pg_available_extensions WHERE name IN ('pg_textsearch','pg_search') ORDER BY name;" echo "" echo "=== Dataset ===" psql -t -c "SELECT 'Passages: ' || COUNT(*) FROM msmarco_v2_passages;" psql -t -c "SELECT 'Table size: ' || pg_size_pretty(pg_total_relation_size('msmarco_v2_passages'));" echo "" echo "=== Benchmark Config ===" echo "PARALLEL_WORKERS=$PARALLEL_WORKERS" echo "MAINT_WORK_MEM=$MAINT_WORK_MEM" echo "MIN_PARALLEL_SCAN=$MIN_PARALLEL_SCAN" echo "POWER_CLIENTS=$POWER_CLIENTS" echo "POWER_DURATION=${POWER_DURATION}s" } 2>&1 | tee "$out" } # ── step: build-tapir ──────────────────────────────────────────── step_build_tapir() { local out="$RESULTS_DIR/build_tapir.log" log "Building pg_textsearch index → $out" # Drop pg_search to avoid bm25 AM name collision psql -c "DROP EXTENSION IF EXISTS pg_search CASCADE;" 2>/dev/null || true # Drop page cache for cold-start fairness if [ -w /proc/sys/vm/drop_caches ]; then sync echo 3 > /proc/sys/vm/drop_caches log "Page cache dropped" else log "WARNING: cannot drop page cache (not root)" fi psql -v ON_ERROR_STOP=1 <&1 | tee "$out" \timing on CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET max_parallel_maintenance_workers = $PARALLEL_WORKERS; SET maintenance_work_mem = '$MAINT_WORK_MEM'; SET min_parallel_table_scan_size = '$MIN_PARALLEL_SCAN'; DROP INDEX IF EXISTS msmarco_v2_bm25_idx; \echo '' \echo '=== Building pg_textsearch BM25 Index ===' \echo 'parallel_workers=$PARALLEL_WORKERS maint_work_mem=$MAINT_WORK_MEM min_parallel_scan=$MIN_PARALLEL_SCAN' CREATE INDEX msmarco_v2_bm25_idx ON msmarco_v2_passages USING bm25(passage_text) WITH (text_config='english'); \echo '' \echo '=== Sizes ===' SELECT 'INDEX_SIZE_TAPIR: ' || pg_size_pretty(pg_relation_size('msmarco_v2_bm25_idx')) || ' (' || pg_relation_size('msmarco_v2_bm25_idx') || ' bytes)' as result; SELECT 'TABLE_SIZE: ' || pg_size_pretty(pg_total_relation_size('msmarco_v2_passages')) as result; \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('msmarco_v2_bm25_idx'); \echo '=== build-tapir complete ===' EOSQL log "build-tapir done" } # ── step: build-paradedb ───────────────────────────────────────── step_build_paradedb() { local out="$RESULTS_DIR/build_paradedb.log" log "Building ParadeDB index → $out" # Free disk: drop tapir index before copying data log "Dropping tapir index to free disk space..." psql -c "DROP INDEX IF EXISTS msmarco_v2_bm25_idx;" 2>/dev/null || true # Drop pg_textsearch to avoid bm25 AM name collision with pg_search log "Dropping pg_textsearch extension (bm25 AM conflict)..." psql -c "DROP EXTENSION IF EXISTS pg_textsearch CASCADE;" 2>/dev/null || true # Drop page cache for cold-start fairness if [ -w /proc/sys/vm/drop_caches ]; then sync echo 3 > /proc/sys/vm/drop_caches log "Page cache dropped" fi psql -v ON_ERROR_STOP=1 <&1 | tee "$out" \timing on -- Ensure pg_search extension is available (pg_textsearch already dropped) CREATE EXTENSION IF NOT EXISTS pg_search; -- Apply same parallel GUCs for fairness SET max_parallel_maintenance_workers = $PARALLEL_WORKERS; SET maintenance_work_mem = '$MAINT_WORK_MEM'; SET min_parallel_table_scan_size = '$MIN_PARALLEL_SCAN'; DROP INDEX IF EXISTS msmarco_v2_paradedb_idx; \echo '' \echo '=== Building ParadeDB BM25 Index ===' \echo 'parallel_workers=$PARALLEL_WORKERS maint_work_mem=$MAINT_WORK_MEM min_parallel_scan=$MIN_PARALLEL_SCAN' CREATE INDEX msmarco_v2_paradedb_idx ON msmarco_v2_passages USING bm25 (passage_id, passage_text) WITH ( key_field = 'passage_id', text_fields = '{ "passage_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); \echo 'Running VACUUM to compact segments...' VACUUM msmarco_v2_passages; \echo '' \echo '=== Sizes ===' SELECT 'INDEX_SIZE_SYSTEMX: ' || pg_size_pretty(pg_relation_size('msmarco_v2_paradedb_idx')) || ' (' || pg_relation_size('msmarco_v2_paradedb_idx') || ' bytes)' as result; SELECT 'TABLE_SIZE_SYSTEMX: ' || pg_size_pretty(pg_total_relation_size('msmarco_v2_passages')) as result; \echo '=== build-paradedb complete ===' EOSQL log "build-paradedb done" } # ── step: query-tapir ──────────────────────────────────────────── step_query_tapir() { local out="$RESULTS_DIR/query_tapir.log" log "Running pg_textsearch query benchmarks → $out" cd "$REPO_DIR" psql -f benchmarks/datasets/msmarco-v2/queries.sql 2>&1 | tee "$out" log "query-tapir done" echo "" echo "=== Key Results ===" grep -E "LATENCY_BUCKET|WEIGHTED_|THROUGHPUT_RESULT" "$out" || true } # ── step: query-paradedb ───────────────────────────────────────── step_query_paradedb() { local out="$RESULTS_DIR/query_paradedb.log" log "Running ParadeDB query benchmarks → $out" cd "$REPO_DIR" # Rewrite paradedb queries to use msmarco_v2_passages (same table) sed 's/msmarco_v2_passages_paradedb/msmarco_v2_passages/g' \ benchmarks/datasets/msmarco-v2/paradedb/queries.sql \ | psql 2>&1 | tee "$out" log "query-paradedb done" echo "" echo "=== Key Results ===" grep -E "LATENCY_BUCKET|WEIGHTED_|THROUGHPUT_RESULT" "$out" || true } # ── step: power-tapir ──────────────────────────────────────────── step_power_tapir() { local out="$RESULTS_DIR/power_tapir.log" log "Running pg_textsearch power test → $out" log " clients=$POWER_CLIENTS duration=${POWER_DURATION}s" # Set up the power-test query table psql -v ON_ERROR_STOP=1 \ -f "$SCRIPT_DIR/setup_power_test.sql" 2>&1 \ | tee "$RESULTS_DIR/setup_power_test.log" pgbench -n \ -c "$POWER_CLIENTS" \ -j "$POWER_CLIENTS" \ -T "$POWER_DURATION" \ -M prepared \ -f "$SCRIPT_DIR/power_tapir.sql" \ 2>&1 | tee "$out" local tps tps=$(grep -oP 'tps = \K[0-9.]+' "$out" | tail -1) echo "POWER_TPS_TAPIR: $tps" | tee -a "$out" log "power-tapir done" } # ── step: power-paradedb ───────────────────────────────────────── step_power_paradedb() { local out="$RESULTS_DIR/power_paradedb.log" log "Running ParadeDB power test → $out" log " clients=$POWER_CLIENTS duration=${POWER_DURATION}s" # Set up the power-test query table (idempotent) psql -v ON_ERROR_STOP=1 \ -f "$SCRIPT_DIR/setup_power_test.sql" 2>&1 \ | tee "$RESULTS_DIR/setup_power_test.log" pgbench -n \ -c "$POWER_CLIENTS" \ -j "$POWER_CLIENTS" \ -T "$POWER_DURATION" \ -M prepared \ -f "$SCRIPT_DIR/power_paradedb.sql" \ 2>&1 | tee "$out" local tps tps=$(grep -oP 'tps = \K[0-9.]+' "$out" | tail -1) echo "POWER_TPS_SYSTEMX: $tps" | tee -a "$out" log "power-paradedb done" } # ── step: summary ──────────────────────────────────────────────── step_summary() { local out="$RESULTS_DIR/summary.log" log "Generating summary → $out" # Find the latest result files (use explicit dir or latest) local rdir="$RESULTS_DIR" # If query logs don't exist in this run's dir, find the latest if [ ! -f "$rdir/query_tapir.log" ]; then rdir=$(ls -dt "$SCRIPT_DIR/results"/*/ 2>/dev/null | head -1) if [ -z "$rdir" ]; then echo "ERROR: no result logs found" exit 1 fi log "Using results from: $rdir" fi { echo "==============================================" echo " MS-MARCO v2 Benchmark Summary" echo " $(date -u +%Y-%m-%dT%H:%M:%SZ)" echo " Results dir: $rdir" echo "==============================================" echo "" # --- Build Times --- echo "=== Index Build Time ===" local bt_tapir bt_paradedb bt_tapir=$(grep -oP 'CREATE INDEX.*Time: \K[0-9.]+' \ "$rdir/build_tapir.log" 2>/dev/null || echo "N/A") bt_paradedb=$(grep -oP 'CREATE INDEX.*Time: \K[0-9.]+' \ "$rdir/build_paradedb.log" 2>/dev/null || echo "N/A") printf " %-20s %s\n" "pg_textsearch:" "${bt_tapir} ms" printf " %-20s %s\n" "ParadeDB:" "${bt_paradedb} ms" echo "" # --- Index Sizes --- echo "=== Index Size ===" grep "INDEX_SIZE_TAPIR:" "$rdir/build_tapir.log" 2>/dev/null \ | sed 's/^/ /' || true grep "INDEX_SIZE_SYSTEMX:" "$rdir/build_paradedb.log" 2>/dev/null \ | sed 's/^/ /' || true echo "" # --- Single-Client Latency --- echo "=== Single-Client Query Latency ===" echo " --- pg_textsearch ---" grep -E "LATENCY_BUCKET" "$rdir/query_tapir.log" 2>/dev/null \ | sed 's/^/ /' || echo " (not available)" echo "" grep -E "WEIGHTED_LATENCY" "$rdir/query_tapir.log" 2>/dev/null \ | sed 's/^/ /' || true echo "" echo " --- ParadeDB ---" grep -E "LATENCY_BUCKET" "$rdir/query_paradedb.log" 2>/dev/null \ | sed 's/^/ /' || echo " (not available)" echo "" grep -E "WEIGHTED_LATENCY" "$rdir/query_paradedb.log" 2>/dev/null \ | sed 's/^/ /' || true echo "" # --- Throughput (sequential) --- echo "=== Sequential Throughput ===" grep "THROUGHPUT_RESULT:" "$rdir/query_tapir.log" 2>/dev/null \ | sed 's/^/ pg_textsearch /' || true grep "THROUGHPUT_RESULT:" "$rdir/query_paradedb.log" 2>/dev/null \ | sed 's/^/ ParadeDB /' || true echo "" # --- Power Test (concurrent) --- echo "=== Concurrent Throughput (pgbench, ${POWER_CLIENTS} clients) ===" grep "POWER_TPS_TAPIR:" "$rdir/power_tapir.log" 2>/dev/null \ | sed 's/^/ /' || echo " (not available)" grep "POWER_TPS_SYSTEMX:" "$rdir/power_paradedb.log" 2>/dev/null \ | sed 's/^/ /' || echo " (not available)" echo "" echo "==============================================" } 2>&1 | tee "$out" } # ── Main ───────────────────────────────────────────────────────── echo "=== MS-MARCO v2 Full Benchmark ===" echo "Timestamp: $TIMESTAMP" echo "Step: $STEP" echo "Results: $RESULTS_DIR" echo "" check_prereqs case "$STEP" in env) step_env ;; build-tapir) step_env; step_build_tapir ;; build-paradedb) step_env; step_build_paradedb ;; query-tapir) step_query_tapir ;; query-paradedb) step_query_paradedb ;; power-tapir) step_power_tapir ;; power-paradedb) step_power_paradedb ;; summary) step_summary ;; all) step_env step_build_tapir step_query_tapir step_power_tapir step_build_paradedb step_query_paradedb step_power_paradedb step_summary ;; *) echo "Unknown step: $STEP" echo "Steps: env build-tapir build-paradedb query-tapir" \ "query-paradedb power-tapir power-paradedb summary all" exit 1 ;; esac echo "" log "Done. Results in $RESULTS_DIR" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/run_v2_benchmark.sh000077500000000000000000000203211525762047700304560ustar00rootroot00000000000000#!/bin/bash # MS MARCO v2 Full Benchmark Runner # Runs pg_textsearch and ParadeDB benchmarks sequentially on 138M passages # # Prerequisites: # - download.sh has been run (collection.tsv exists) # - generate_benchmark_queries.sh has been run (benchmark_queries.tsv exists) # - pg_textsearch is built and installed (make && sudo make install) # - ParadeDB .deb is installed # # Usage: ./run_v2_benchmark.sh [step] # Steps: # all - Run everything (default) # load-tapir - Load data + build pg_textsearch index # query-tapir - Run pg_textsearch query benchmarks # load-paradedb - Load data + build ParadeDB index # query-paradedb - Run ParadeDB query benchmarks set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_DIR="$(cd "$SCRIPT_DIR/../../.." && pwd)" DATA_DIR="$SCRIPT_DIR/data" LOG_DIR="/tmp/msmarco-v2-benchmark" STEP="${1:-all}" # Postgres connection PGPORT="${PGPORT:-5432}" PGDATABASE="${PGDATABASE:-postgres}" export DATA_DIR PGPORT PGDATABASE mkdir -p "$LOG_DIR" TIMESTAMP=$(date +%Y%m%d_%H%M%S) echo "=== MS MARCO v2 Benchmark ===" echo "Timestamp: $TIMESTAMP" echo "Step: $STEP" echo "Data dir: $DATA_DIR" echo "PG port: $PGPORT" echo "Log dir: $LOG_DIR" echo "" # Verify prerequisites check_prereqs() { if [ ! -f "$DATA_DIR/collection.tsv" ]; then echo "ERROR: collection.tsv not found. Run ./download.sh first." exit 1 fi if [ ! -f "$SCRIPT_DIR/benchmark_queries.tsv" ]; then echo "ERROR: benchmark_queries.tsv not found. Run ./generate_benchmark_queries.sh first." exit 1 fi echo "Prerequisites OK" echo " Collection: $(wc -l < "$DATA_DIR/collection.tsv") passages" echo " Benchmark queries: $(wc -l < "$SCRIPT_DIR/benchmark_queries.tsv") queries" echo "" } # Verify pg_textsearch version check_tapir_version() { local ver ver=$(psql -t -c "SELECT default_version FROM pg_available_extensions WHERE name = 'pg_textsearch';" 2>/dev/null | tr -d ' ') echo "pg_textsearch version: $ver" if [ "$ver" != "1.5.1" ]; then echo "WARNING: Expected 1.5.1, got '$ver'" fi } # Show system info show_system_info() { echo "=== System Info ===" echo "CPU: $(nproc) cores" echo "RAM: $(free -h | awk '/^Mem:/{print $2}')" echo "Disk (NVMe): $(df -h /nvme | tail -1 | awk '{print $4 " avail of " $2}')" echo "PG data dir: $(psql -t -c 'SHOW data_directory;' | tr -d ' ')" echo "PG version: $(psql -t -c 'SELECT version();' | head -1)" echo "" } load_tapir() { echo "=== Loading pg_textsearch (Tapir) ===" local log="$LOG_DIR/tapir_load_${TIMESTAMP}.log" echo "Log: $log" check_tapir_version # Use max parallelism for index build local ncpu ncpu=$(nproc) local workers=$((ncpu - 2)) if [ "$workers" -lt 2 ]; then workers=2 fi echo "Using $workers parallel maintenance workers" psql -v ON_ERROR_STOP=1 <&1 | tee "$log" \timing on -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Max out parallelism for this session SET max_parallel_maintenance_workers = $workers; SET maintenance_work_mem = '8GB'; -- Verify data is loaded \echo '=== Data Verification ===' SELECT 'Passages:' as what, COUNT(*)::text as n FROM msmarco_v2_passages UNION ALL SELECT 'Queries:', COUNT(*)::text FROM msmarco_v2_queries UNION ALL SELECT 'Qrels:', COUNT(*)::text FROM msmarco_v2_qrels ORDER BY what; -- Drop existing BM25 index if present, then rebuild DROP INDEX IF EXISTS msmarco_v2_bm25_idx; \echo '' \echo '=== Building BM25 Index (parallel workers: $workers) ===' CREATE INDEX msmarco_v2_bm25_idx ON msmarco_v2_passages USING bm25(passage_text) WITH (text_config='english'); -- Report sizes \echo '' \echo '=== Sizes ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_v2_bm25_idx')) as size, pg_relation_size('msmarco_v2_bm25_idx') as bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_v2_passages')) as size, pg_total_relation_size('msmarco_v2_passages') as bytes; \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('msmarco_v2_bm25_idx'); \echo '=== pg_textsearch Load Complete ===' EOSQL echo "" echo "Load log: $log" } load_tapir_index_only() { echo "=== Building pg_textsearch Index Only ===" local log="$LOG_DIR/tapir_index_${TIMESTAMP}.log" echo "Log: $log" check_tapir_version local ncpu ncpu=$(nproc) local workers=$((ncpu - 2)) if [ "$workers" -lt 2 ]; then workers=2 fi echo "Using $workers parallel maintenance workers" psql -v ON_ERROR_STOP=1 <&1 | tee "$log" \timing on SET max_parallel_maintenance_workers = $workers; SET maintenance_work_mem = '8GB'; -- Drop old index if exists DROP INDEX IF EXISTS msmarco_v2_bm25_idx; \echo '=== Building BM25 Index (parallel workers: $workers) ===' CREATE INDEX msmarco_v2_bm25_idx ON msmarco_v2_passages USING bm25(passage_text) WITH (text_config='english'); \echo '' \echo '=== Sizes ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_v2_bm25_idx')) as size, pg_relation_size('msmarco_v2_bm25_idx') as bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_v2_passages')) as size, pg_total_relation_size('msmarco_v2_passages') as bytes; \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('msmarco_v2_bm25_idx'); \echo '=== Index Build Complete ===' EOSQL echo "" echo "Index build log: $log" } query_tapir() { echo "=== Running pg_textsearch Query Benchmarks ===" local log="$LOG_DIR/tapir_queries_${TIMESTAMP}.log" echo "Log: $log" cd "$REPO_DIR" psql -f benchmarks/datasets/msmarco-v2/queries.sql 2>&1 | tee "$log" echo "" echo "Query log: $log" echo "" echo "=== Key Results ===" grep -E "LATENCY_BUCKET|THROUGHPUT_RESULT|INDEX_SIZE" "$log" || true } load_paradedb() { echo "=== Loading ParadeDB (ParadeDB) ===" local log="$LOG_DIR/paradedb_load_${TIMESTAMP}.log" echo "Log: $log" # Drop pg_textsearch first to avoid conflicts psql -c "DROP EXTENSION IF EXISTS pg_textsearch CASCADE;" 2>/dev/null || true cd "$REPO_DIR" psql -f benchmarks/datasets/msmarco-v2/paradedb/load.sql 2>&1 | tee "$log" echo "" echo "Load log: $log" } query_paradedb() { echo "=== Running ParadeDB Query Benchmarks ===" local log="$LOG_DIR/paradedb_queries_${TIMESTAMP}.log" echo "Log: $log" cd "$REPO_DIR" psql -f benchmarks/datasets/msmarco-v2/paradedb/queries.sql 2>&1 | tee "$log" echo "" echo "Query log: $log" echo "" echo "=== Key Results ===" grep -E "LATENCY_BUCKET|THROUGHPUT_RESULT|INDEX_SIZE" "$log" || true } cleanup_tapir() { echo "=== Cleaning up pg_textsearch data ===" psql -c "DROP TABLE IF EXISTS msmarco_v2_passages CASCADE;" psql -c "DROP TABLE IF EXISTS msmarco_v2_queries CASCADE;" psql -c "DROP TABLE IF EXISTS msmarco_v2_qrels CASCADE;" psql -c "DROP EXTENSION IF EXISTS pg_textsearch CASCADE;" echo "Cleanup complete" } cleanup_paradedb() { echo "=== Cleaning up ParadeDB data ===" psql -c "DROP TABLE IF EXISTS msmarco_v2_passages_paradedb CASCADE;" psql -c "DROP TABLE IF EXISTS msmarco_v2_queries_paradedb CASCADE;" psql -c "DROP TABLE IF EXISTS msmarco_v2_qrels_paradedb CASCADE;" psql -c "DROP EXTENSION IF EXISTS pg_search CASCADE;" echo "Cleanup complete" } # Main check_prereqs show_system_info case "$STEP" in all) load_tapir query_tapir cleanup_tapir load_paradedb query_paradedb cleanup_paradedb ;; load-tapir) load_tapir ;; index-tapir) load_tapir_index_only ;; query-tapir) query_tapir ;; load-paradedb) load_paradedb ;; query-paradedb) query_paradedb ;; cleanup-tapir) cleanup_tapir ;; cleanup-paradedb) cleanup_paradedb ;; *) echo "Unknown step: $STEP" echo "Valid steps: all, load-tapir, index-tapir, query-tapir, load-paradedb, query-paradedb, cleanup-tapir, cleanup-paradedb" exit 1 ;; esac echo "" echo "=== Done ===" echo "Logs in: $LOG_DIR" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/setup_power_test.sql000066400000000000000000000017651525762047700310410ustar00rootroot00000000000000-- Set up benchmark_queries_power table for pgbench power tests. -- Remaps sparse MS-MARCO query_ids to dense integers 1..N so that -- pgbench can use \set qid random(1, N). \set ON_ERROR_STOP on -- Idempotent: recreate every time DROP TABLE IF EXISTS benchmark_queries_power; -- Staging table with original columns CREATE TEMP TABLE _bq_raw ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy _bq_raw FROM 'benchmarks/datasets/msmarco-v2/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') -- Dense sequential IDs 1..N CREATE TABLE benchmark_queries_power ( id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); INSERT INTO benchmark_queries_power (id, query_text) SELECT row_number() OVER (ORDER BY query_id)::int, query_text FROM _bq_raw; DROP TABLE _bq_raw; -- Verify SELECT 'Power test queries loaded: ' || COUNT(*) AS status FROM benchmark_queries_power; SELECT 'ID range: ' || MIN(id) || '..' || MAX(id) AS range FROM benchmark_queries_power; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco-v2/validate_queries.sql000066400000000000000000000202211525762047700307400ustar00rootroot00000000000000-- MS-MARCO v2 Query Validation -- Compares Tapir query results against precomputed ground truth -- -- Requires: ground_truth.tsv in the same directory (created by CI or -- precompute_ground_truth.sql, from ground_truth_pgNN.tsv files) -- Usage: psql -p PORT -f validate_queries.sql -- -- Validates: -- 1. Same documents appear in top-10 (order may vary for ties) -- 2. BM25 scores match within absolute tolerance (0.001) -- This accounts for float4 vs float8 precision in the index -- -- Key differences from v1: -- - passage_id is TEXT (not INTEGER) -- - Uses msmarco_v2_passages / msmarco_v2_bm25_idx -- - ground_truth.doc_id is TEXT \set ON_ERROR_STOP on \timing on \echo '=== MS-MARCO v2 BM25 Validation ===' \echo '' -- Load ground truth data \echo 'Loading ground truth...' DROP TABLE IF EXISTS ground_truth; CREATE TABLE ground_truth ( query_id int, query_text text, rank int, doc_id text, score float8 ); \copy ground_truth FROM 'benchmarks/datasets/msmarco-v2/ground_truth.tsv' WITH (FORMAT text, DELIMITER E'\t', HEADER true) SELECT 'Loaded ' || COUNT(DISTINCT query_id) || ' queries, ' || COUNT(*) || ' result rows' as status FROM ground_truth; -- Validation function that handles ties properly. -- Uses absolute tolerance (default 0.001) to account for float4 vs -- float8 precision differences between index scoring and ground -- truth. Rank-boundary ties (missing/extra docs with scores within -- tolerance of the rank-10 cutoff) are counted separately from real -- mismatches. CREATE OR REPLACE FUNCTION validate_single_query_v2( p_query_id int, p_query_text text, p_tolerance float8 DEFAULT 0.001 ) RETURNS TABLE( query_id int, docs_match boolean, scores_match boolean, missing_docs int, extra_docs int, max_score_diff float8, details text ) AS $$ DECLARE v_gt_docs text[]; v_tapir_docs text[]; v_missing text[]; v_extra text[]; v_max_score_diff float8 := 0; v_all_scores_match boolean := true; v_details text := ''; v_gt_min_score float8; v_tapir_min_score float8; v_boundary_tolerance float8; v_real_missing int := 0; r record; BEGIN -- Get Tapir's top-10 results CREATE TEMP TABLE IF NOT EXISTS tapir_results ( doc_id text, score float8 ); TRUNCATE tapir_results; INSERT INTO tapir_results SELECT passage_id, -(passage_text <@> to_bm25query( p_query_text, 'msmarco_v2_bm25_idx' ))::float8 as score FROM msmarco_v2_passages ORDER BY passage_text <@> to_bm25query( p_query_text, 'msmarco_v2_bm25_idx' ) LIMIT 10; -- Get document sets SELECT array_agg(gt.doc_id ORDER BY gt.doc_id) INTO v_gt_docs FROM ground_truth gt WHERE gt.query_id = p_query_id; SELECT array_agg(doc_id ORDER BY doc_id) INTO v_tapir_docs FROM tapir_results; -- Find missing and extra documents SELECT array_agg(d) INTO v_missing FROM unnest(v_gt_docs) d WHERE d NOT IN (SELECT doc_id FROM tapir_results); SELECT array_agg(d) INTO v_extra FROM unnest(v_tapir_docs) d WHERE d NOT IN ( SELECT gt2.doc_id FROM ground_truth gt2 WHERE gt2.query_id = p_query_id ); -- Compare scores for matching documents (absolute tolerance) FOR r IN SELECT gt.doc_id, gt.score as gt_score, t.score as tapir_score, ABS(gt.score - t.score) as abs_diff FROM ground_truth gt JOIN tapir_results t ON gt.doc_id = t.doc_id WHERE gt.query_id = p_query_id LOOP IF r.abs_diff > p_tolerance THEN v_all_scores_match := false; v_details := v_details || 'doc ' || r.doc_id || ': gt=' || ROUND(r.gt_score::numeric, 4)::text || ', tapir=' || ROUND(r.tapir_score::numeric, 4)::text || '; '; END IF; v_max_score_diff := GREATEST( v_max_score_diff, r.abs_diff ); END LOOP; -- Check if missing/extra docs are rank-boundary ties. -- If the missing doc's gt score is within tolerance of the -- rank-10 boundary score, it's a benign tie-break difference. IF v_missing IS NOT NULL AND array_length(v_missing, 1) > 0 THEN SELECT MIN(gt3.score) INTO v_gt_min_score FROM ground_truth gt3 WHERE gt3.query_id = p_query_id; SELECT MIN(score) INTO v_tapir_min_score FROM tapir_results; v_boundary_tolerance := GREATEST( p_tolerance, 0.001 ); FOR r IN SELECT d as doc_id, gt.score as gt_score FROM unnest(v_missing) d JOIN ground_truth gt ON gt.doc_id = d AND gt.query_id = p_query_id LOOP -- Missing doc's score close to boundary = tie-break IF ABS(r.gt_score - v_tapir_min_score) > v_boundary_tolerance THEN v_real_missing := v_real_missing + 1; END IF; END LOOP; END IF; -- Build result query_id := p_query_id; -- docs_match is false only for non-tie missing docs docs_match := (v_real_missing = 0); scores_match := v_all_scores_match; missing_docs := COALESCE(array_length(v_missing, 1), 0); extra_docs := COALESCE(array_length(v_extra, 1), 0); max_score_diff := v_max_score_diff; IF v_missing IS NOT NULL AND array_length(v_missing, 1) > 0 THEN v_details := v_details || 'Missing: ' || array_to_string(v_missing, ',') || '; '; END IF; IF v_extra IS NOT NULL AND array_length(v_extra, 1) > 0 THEN v_details := v_details || 'Extra: ' || array_to_string(v_extra, ',') || '; '; END IF; details := CASE WHEN v_details = '' THEN 'OK' ELSE trim(trailing '; ' from v_details) END; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run validation for all ground truth queries \echo '' \echo '=== Running Validation ===' DROP TABLE IF EXISTS validation_results; CREATE TABLE validation_results AS SELECT v.* FROM ( SELECT DISTINCT gt.query_id, gt.query_text FROM ground_truth gt ) q, LATERAL validate_single_query_v2( q.query_id, q.query_text ) v; -- Summary \echo '' \echo '=== Validation Summary ===' SELECT COUNT(*) as total_queries, SUM(CASE WHEN docs_match THEN 1 ELSE 0 END) as docs_match_count, SUM(CASE WHEN scores_match THEN 1 ELSE 0 END) as scores_match_count, round(100.0 * SUM( CASE WHEN docs_match THEN 1 ELSE 0 END ) / NULLIF(COUNT(*), 0), 1) as docs_match_pct, round(100.0 * SUM( CASE WHEN scores_match THEN 1 ELSE 0 END ) / NULLIF(COUNT(*), 0), 1) as scores_match_pct, round(MAX(max_score_diff)::numeric, 6) as worst_abs_diff FROM validation_results; -- Check for failures and report DO $$ DECLARE v_total int; v_failures int; v_match_pct numeric; BEGIN SELECT COUNT(*) INTO v_total FROM validation_results; -- A query fails if scores differ beyond tolerance -- OR if docs differ for non-tie-break reasons SELECT COUNT(*) INTO v_failures FROM validation_results WHERE NOT scores_match OR NOT docs_match; v_match_pct := 100.0 * (v_total - v_failures) / NULLIF(v_total, 0); IF v_match_pct < 100.0 THEN RAISE NOTICE 'VALIDATION FAILED: % of % queries failed ' '(scores differ beyond 0.001 tolerance or ' 'non-tie doc mismatches)', v_failures, v_total; ELSE RAISE NOTICE 'VALIDATION PASSED: All % queries match ' 'within tolerance', v_total; END IF; END; $$; -- Failed queries detail \echo '' \echo '=== Validation Details (failures only) ===' SELECT query_id, docs_match, scores_match, missing_docs, extra_docs, round(max_score_diff::numeric, 6) as max_abs_diff, left(details, 200) as details FROM validation_results WHERE NOT docs_match OR NOT scores_match ORDER BY missing_docs DESC, max_score_diff DESC LIMIT 20; -- Cleanup DROP TABLE IF EXISTS ground_truth; DROP TABLE IF EXISTS validation_results; DROP TABLE IF EXISTS tapir_results; DROP FUNCTION IF EXISTS validate_single_query_v2; \echo '' \echo '=== Validation Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/000077500000000000000000000000001525762047700243475ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/benchmark_queries.tsv000066400000000000000000001176751525762047700306150ustar00rootroot00000000000000254 +what is cchaps 1 255 +what is centure 1 1724 < symbol 1 4288 How to have sex 1 7354 What is my ethnicity 1 9340 a mutation is 1 9351 a novel is a _____. 1 16453 amendment you can be who you are 1 27302 ash who is ash. 1 92357 churrascaria what is 1 114779 cymol what is 1 122126 define has 1 142802 diabetes what is 1 151608 dimitrov is from 1 157194 do impairment 1 157317 do it yourself carpentry 1 161233 do you smell 1 181036 epel what is 1 196232 government does do 1 198573 harissa what is it 1 201253 hepatologist what is it 1 201374 here lyrics 1 207567 how are marsh 1 215815 how did washington's 1 223417 how do you poison yourself 1 235523 how hot is it? 1 236447 how is coa 1 340101 how states are there 1 346531 how to but bulke 1 350305 how to cko 1 354311 how to delete a 1 355811 how to do cinevouchers 1 356168 how to do the spilts 1 356486 how to draw a s 1 383882 how to trickline 1 388711 how you describe yourself? 1 388892 howmanystagesofcancer 1 389736 hydralazine what is it 1 397933 ionizer what does it do 1 404659 is both an adjective 1 414723 is it tastes or taste? 1 417519 is matter 1 422479 is rez.a.word 1 425501 is tequila 1 430795 is vertical up and down 1 438294 leadership is __________. 1 439694 libmmd.dll what is 1 440465 lipophilic what is 1 489682 rocky where are they now 1 499368 slime what is it 1 509951 synonyms for to 1 536844 victims/witnesses 1 542038 waves waves waves waves 1 548859 what is translucent 1 548895 what is opra 1 548916 what is the meninges 1 549197 what a mammogram 1 549219 what a pathologist does 1 549235 what a pronoun 1 553232 what amalla? 1 554327 what are * called 1 554500 what are accommodation 1 554645 what are afterpains 1 554676 what are airpods 1 554827 what are amiibo for 1 554890 what are anchovies 1 555115 what are arteries? 1 555129 what are artichokes? 1 555359 what are ballas 1 555433 what are bats 1 555558 what are bifidobacterium 1 555563 what are bigos 1 555580 what are bindis 1 555590 what are biokinetics 1 555731 what are boils on 1 555972 what are caches 1 556159 what are casts 1 556328 what are centromeres 1 556585 what are cilium 1 556600 what are citation 1 556731 what are cockroaches 1 556779 what are collagens 1 556952 what are comps? 1 556988 what are conductors 1 557355 what are crumpets 1 557764 what are diplobacilli 1 557925 what are downbursts 1 558263 what are endotoxins? 1 558307 what are enzymes 1 558518 what are exons 1 558567 what are extremities 1 558596 what are factor 1 558759 what are fibroadenomas 1 559009 what are foragers 1 559230 what are gandules 1 559257 what are gbs 1 559354 what are ghb- 1 559671 what are grits? 1 559731 what are gyres? 1 559735 what are habitat 1 83 you are so dutch means what 2 102 % saturation do calculation 2 214 +what can mirna be used for 2 220 +what do pathogic organisms do 2 244 +what is a lmu wall 2 317 +where is highlong park 2 370 . what is the institute for leaders? 2 409 .vbs what it mean 2 1288 3/5 of 60 2 1504 5% of 329300 2 1550 5k how long 2 1661 81 out of what is 27% 2 2216 Benefits of Adenoidectomy 2 2322 CAA definition 2 2773 Do I have any reminder for this week? 2 2942 Ewe/Doe definition 2 3399 How To Draw People 2 3537 How do you pronounce manna 2 3557 How do you spell Erleen 2 4131 How strong is dilaudid 2 4216 How to Smoke Roxicodone 2 4294 How to list of the sensors? 2 5040 Meaning of Saman 2 5260 Nba scores 2 5385 Onomake means what 2 5772 SOAPS definition 2 5923 Somnambulatory definition 2 6527 Types of Octopus 2 6734 Was Luke an Apostle 2 6858 What are some Weather Instruments 2 6863 What are subjunctive phones 2 7222 What is OMG orbit 2 7242 What is a Staffing Agency 2 7368 What is septic disease 2 7451 What is the capital of Qatar? 2 7595 What is the miricale that St.Anthony did 2 7902 What was ww1 cost 2 8477 Why do they have to be electric sheep 2 8642 ____ segregation is legalized segregation 2 8756 ______ are the spaces between nodes. 2 9281 a joule is equal to what 2 9335 a monopolist is a price 2 9671 aast mba 2 9769 abbreviation tsubasa 2 9806 abednego meaning 2 9849 abject defined 2 10018 abraided definition 2 10054 abscess symptoms 2 10222 accentuates definition 2 10312 accordian definition 2 10368 accosting definition 2 11280 adaptogen definition 2 11284 adau definition 2 12108 advisee definition 2 12262 aesthetic function 2 12340 affiliated definition 2 12370 affluent definition 2 12437 africa's population 2 12514 after donating blood what should you do 2 13476 age of aries 2 13671 age of exploration 2 13784 age of metternich 2 13932 age of variability 2 14247 age to be in casino 2 14890 airbender avatar 2 14912 airfoil definition 2 14963 airport outfit 2 15010 ake definition 2 15094 alaine meaning 2 15183 albeit means 2 15400 aleo definition 2 15499 alhe disease 2 15517 alice what does it mean 2 15687 allentown is in what county 2 15812 allopathic lesions 2 15815 allopurinol tablets 2 16017 alternator price 2 16019 alternator system 2 16096 alzheimer's meaning 2 16121 am i too young for perimenipause 2 16351 ambien what is it used for 2 16375 ambreco construction in 2 16399 amcobi payment 2 16407 amdocs crm 2 17006 amn symptoms 2 17010 amnesty don meaning 2 17430 amp definition 2 17442 amphidromic meaning 2 17512 amylase is secreted 2 17746 analects of confucius 2 17751 analiza justo 2 17772 analyst define 2 17793 anamosa is in what township 2 17872 ancestry of obama 2 17991 andre segura 2 18136 anemia that is terminal 2 18174 ange in canada 2 18190 angela inzerillo 2 18196 angella lapukeni 2 18612 annuity definition 2 2 Androgen receptor define 3 58 the knee is what type of joint 3 71 what nationality is the name Factor 3 74 where are protists most abundant in humans 3 184 +is biology a social science 3 321 +where is your siatic nerve located 3 361 . irritability medical definition 3 401 .net culture code 3 537 1/2 cups in tbsp 3 548 10 am PST in AST 3 637 120000 jpy to usd 3 660 136 Bus Timetable 3 697 15 oz can in ml 3 729 17.7 pints to gallons 3 1112 21/2 percent of 4200 3 1297 300 canadian to usd 3 1350 37gram change to Oz. 3 1354 391dm = how many mm 3 1421 401k loan requirements 3 1531 5216 kg to lbs 3 1576 60x40 slab cost 3 1699 9000twd to us dollars 3 1998 Andongson name nationality 3 2042 Are Unicorns Real Animals 3 2057 Arlena name meaning 3 2166 BLESSED BE your name lyrics 3 2262 Bone ends definition 3 2388 Can You Snort Diazepam 5mg 3 2638 DOES EUCLID HAVE COMPUTER ORDINANCES 3 2739 Difference Between Synthroid and Levothyroxine 3 2758 Discuss on how to use other selectors 3 2798 Does Suddenlink Carry ESPN3 3 2828 Donuts Dunkin Calories 3 2936 Eurorack UB1202 Manual 3 3151 Ginevra name meaning 3 3171 Gopinath name pronunciation 3 3267 Hailey name meaning 3 3400 How Well Does Advertising Work 3 3461 How do I add fonts to my pc 3 3606 How far is the sea from wells 3 3663 How long do you vomit from the flu 3 4145 How to Cancel Rdio Account 3 4156 How to Delete Bing from Computer 3 4194 How to Measure Scope Height 3 4209 How to Remove Bing Searches 3 4218 How to Start a Thank You Letter 3 4275 How to estimate cost for gravel 3 4290 How to improve circulation to bones 3 4297 How to make a cardboard Prism 3 4301 How to make bootable windows 3 4308 How to move forward after a break up 3 4352 How to use a nice box? 3 4446 ITIL wiki definition 3 4449 Ice slicer weight 3 4578 Incubation period for gonorrhea 3 4615 Intangible drilling costs 3 4866 Lebanese potato recipe 3 4920 List of Travel Agencies 3 4935 Log Cabin Prices 3 4947 Ludacris Net Worth 3 4969 MEANING OF FAIRGROUNG FREAK 3 4978 MRA Neck Symptoms 3 4979 MRIS what does TLA mean 3 4988 Mace name meaning 3 5124 Most Googled Items by State 3 5225 NON wat does it stand for 3 5420 P.I.D.meaning in medical terms 3 5461 Passive immunity can be obtained from 3 5541 Point Reyes weather 3 5618 Qld average rainfall 3 5656 Randall Delgado Girlfriend 3 5671 Recipe for Homemade Soap 3 5767 SIDE EEFECTS OF ROBAXIN 3 5793 Saint Martin the soldier why is he a saint 3 5858 Shirley maclaine 2015 where is she 3 5861 Should you drink water with the meal 3 5875 Significant places for huhindus 3 5904 Skyworth TV Price 3 6217 The hormone that does the opposite of calcitonin is 3 6357 The term that means childbirth is 3 6490 Translate Bedzie To English 3 6493 Translate zur To English 3 6526 Types of Kinetic Energy 3 6643 Vitamin for Tiredness and Fatigue 3 6660 WHAT COUNTY IS WINCHESTER, OH IN? 3 6787 What Does Fram PH2 Fit 3 6791 What Does Noel Mean in the Bible 3 6806 What Happened to Doo Dads 3 6945 What causes up Welling and down wedding 3 7107 What does the name Alicia mean 3 7123 What does the name Lara' mean 3 7125 What does the name Marley mean 3 7129 What does the name Pelier mean 3 7131 What does the name Roman mean 3 7223 What is Provisional appraiser License 3 7518 What is the direction of blood through the vessels 3 7582 What is the meaning of the name Mitchell 3 7692 What is the size of a cot bed? 3 7783 What other certifications do you need to be a pediatrician 3 7989 When is the Next Blue Moon 3 34 guidelines for mothballing of process plants 4 197 +micheal moore does he wear a wig 4 215 +what causes bacteria inflamation of face 4 232 +what happen when rome was invaded by germania 4 357 . difference between isometric & dynamic exercises 4 362 . what are the president's main duties? explain each 4 364 . what does the c prefacing a date mean 4 427 02 Chevy 1500 Specs 4 579 100g chicken carb content 4 609 115 days until baby is due 4 628 12.25 mbps upload speed 4 682 14th amendment definition during police 4 782 1968 median income in the us 4 809 1995 silver eagle value 4 827 1st trust deed definition 4 846 2 liters is how many ounces 4 858 2(3b+1) answered? 4 916 2005 difference between camry and lexus 4 971 2014 lexus is 250 price 4 973 2014 rzr 1000 weight 4 981 2015 Federal Bank Holidays 4 1079 2016 monster jam lineup 4 1094 2016 wonderlic test, nfl 4 1111 21 jump street cast 4 1137 24 carat gold price 4 1156 250g of chicken is how much protein 4 1195 3 characteristics of protozoa replication 4 1231 3 other systems that work with the respiratory system 4 1262 3 types of minerals in food 4 1274 3 year aggregate definition 4 1325 323 area code zip code 4 1381 4 of wands as what someone thinks 4 1393 4,146.70+700+11900 4 1579 613 585 area code 4 1658 803 area code zip code 4 1676 877 area code map 4 1691 9 ways to improve your memory 4 1819 A person with a pH below 7.35 is considered to be in 4 1876 Abc family Tuesday schedule 4 1915 Adelles address in Wheaton, IL 4 2123 Average cost of a brokn leg 4 2235 Bethel University was founded in what year 4 2253 Blood cells are produced by _____, which is stored _____. 4 2493 Chili Lime Fish Recipe 4 2618 DIY How to Repaint Old Furniture 4 2716 Destiny Cheat Codes for Weapons 4 2742 Different Kinds of Edible Seaweed 4 2962 Explain what a bone scan is and what it is used for. 4 3014 Fifth Harmony - BOSS BOSS Lyrics 4 3025 Finicio what country does the name originate from 4 3080 Freedom of Religion Amendment Definition 4 3135 Gemini syndrome eternity lyrics 4 3150 Gibby Gibson net worth 4 3156 Glaucoma is a condition caused by quizlet 4 3268 Han name meaning chinese 4 3369 How Long is Shania Twain Concert 4 3372 How Many Android Devices Exist 4 3387 How Much is Blanket Jackson Worth 4 3467 How do I find out where a package is through the post office 4 3495 How do falling prices affect supply? 4 3621 How is the dutch presence known today 4 3722 How long does it take to boil an egg 4 3854 How many points do the presidential candidates have 4 3934 How much does a professional Drummer charge 4 3947 How much does academy pay in oklahoma 4 4115 How old is Andy beer sac 4 4151 How to Customize Themes in Windows 10 4 4212 How to Replace Battery in Car Remote 4 4268 How to cut tomatoes for a pasta salad 4 4302 How to make cheese feel better? 4 4326 How to repair electri lounge chair 4 4341 How to tell difference between l99 and ls3 4 4344 How to tell if your refrigerator compressor is not pumping 4 4386 I can choo choo choose to read the scriptures 4 4437 IRA Contributions Rules excessive 4 4442 IS HIPER II GPS OUTDATED 4 4444 IS NESPORIN GOOD SCALP BURNS 4 4563 In what country was Elizabeth was she became queen 4 4600 Information Technology Turnover Rates 4 4642 Is Cait Jenner Dating Candis 4 4696 Is the Louisiana sales tax 4.75 4 4724 JEFF DILYARD OHIO PHONE 4 4750 Jeopardy All Time Money Winner 4 4757 Joan Hackett Cause of Death 4 4769 John of England he did which of the following quizlet 4 4831 LIST OF COUNTRIES IN THE LATAM REGION 4 4903 List of Chemical Plants in boiser 4 4929 Live share price dlf 4 4934 Loess deposits are typically composed of 4 4980 MSCE Pune Scholarship Result 4 5009 Maricopa County Zoning Codes 4 5169 Moto X Bluetooth Setup 4 5200 N8 Bus Schedule Nice 4 5226 NSW Lottery Results Latest 4 5230 NYC PARKS SIC CODE 4 5245 Names of Three Dimensional Shapes 4 5333 Nutritional Value of Corn Flakes 4 5355 Oldest BBQ Restaurant in Texas 4 5436 PPI withdrawal symptoms dizzy 4 5443 Pakistani Currency Exchange Rates 4 28 definition of Executive Officer defined by FRB 5 106 'com multi unit' zoning definition 5 158 +how to add differnt names on labels in word 5 222 +what does ca cells mean urine test 5 233 +what id justin bieber fav color 5 428 02 how long has my contract left to run 5 484 1 mile equals how many feet 5 506 1 to 1 1/8 shaft adapter ebay 5 514 1. define lymph and explain its function 5 584 104 bus time las Vegas 5 605 11050n autoranging multimeter how to change battery 5 726 17 nhl stars who fell from grace 5 801 1989 Corvette Blue Book Prices 5 810 1996 yamaha vmax 600 specs 5 852 2 teaspoons equals how many tablespoons 5 859 2,000,000 Swiss francs in GBP 5 899 2001vw 1.9 pump timing specs 5 900 2002 audi quattro gas mileage 5 917 2005 gmc yukon towing specifications 5 943 2007 va disability compensation rates 5 995 2015 average gas price in Nevada 5 1000 2015 blue jays best players 5 1014 2015 federal tax refund schedule 5 1021 2015 mississippi state football schedule 5 1038 2015 tax rates trust income 5 1051 2015-16 winter weather forecast 5 1073 2016 how long does the stomach bug last 5 1082 2016 outlook how to save view as default 5 1088 2016 sl550 roadster average cost 5 1091 2016 tn vols football schedule 5 1136 24 Hour eBay Phone Number 5 1157 25Kg BAG how much area will cover 5 1166 27000 annual salary per hour 5 1215 3 levels of government in canada and their responsibilities 5 1289 3/8 inch shank converted to mm 5 1310 30:1 ratio for 20 ounces 5 1424 401k roth qualified distribution rules 5 1447 48 hour holter monitor cost 5 1470 5 day weather forecast for york 5 1483 5 interesting things about chinua achebe 5 1697 90 ounces of water equals gallon 5 1831 A two year old goes to hospital 5 1842 AB positive autoimmune thyroid disease 5 1859 AIRLINES THAT FLY OUT OF RDU TO LAS VEGAS 5 1921 Advertising cost for opening day of a gym 5 1974 American Music Theatre 2015 Shows 5 2155 BATA SHARE DEPARTMENT CONTACT EMAIL 5 2215 Bellingham weather on August 1, 2015 5 2296 Brown Recluse Spider Bite Treatment 5 2305 Burj Al Arab Restaurant Prices 5 2423 Can you send certified USMail to a PO Box? 5 2429 Canned Peach Pie Filling Recipe 5 2619 DJ Paul Net Worth 2014 5 2645 DRBA David Hoppenjans contact information 5 2732 Did General custard wear a red bandanna 5 2835 Dr. Bhim Rao Ambedkar Result 5 2940 Everson WA Weather 10 Day 5 2972 Extending a line in word past bullets 5 3038 Flavones will turn red in the presence of an alkaline 5 3053 Foods to Feed A 10 Month Old 5 3114 Gaap on how long to keep records usa 5 3185 Greyhound bus schedules and ticket prices 5 3190 Guardian air conditioner unit prices 5 3207 HOW LONG DOES IT TAKE FOR TORN CARTLIDGE TO HEAL 5 3490 How do I thicken split pea soup in the rockpot 5 3513 How do you cook a pork joint in a Go chef 5 3521 How do you determine the age of inboard boat motor 5 3725 How long does it take to configure a zone transfer 5 3768 How long should cooked meatballs be reheated in a crock 5 3794 How many calories in a iced vanilla Americano 5 3879 How much IT is imported into Australia every year 5 4024 How much does the average american spend on fitness 5 4093 How much vitamin a/c is in an average banana 5 4116 How old is Coach Mike sure Jeske 5 4267 How to create or repair MBR in windows 10 5 4316 How to program wireless door on go control 5 4417 ICD 10 Cheat Code Printable 5 4671 Is apple pie more fattening or ice cream 5 4733 Jamie Lee Curtis Movie List 5 4784 KFC Fried Chicken Secret Recipe 5 4876 Lg firmware update how long does it take 5 4881 Like ethnicity and language, religion also is 5 4994 Maine Lobster Festival 2015 Dates 5 5328 Number One Cause of Death in America 5 5374 Once you take your dressing off your staples after surgery do you cover your staples 5 5449 Pangako Sa Iyo 2015 Cast 5 5457 Parvo distemper home tips cleaning 5 5496 Peter thomas roth customer service 5 5511 Photosynthetic protozoa that are motile by means of a flagellum 5 5634 RCA Gemstar Universal Remote Codes 5 5717 Rift how to get level 60 fast 5 5738 Rotator Cuff tears do they heal them shelves 5 5757 SAT and ACT test dates in FL 5 5765 SGOT normal range of 48 is good 5 5799 Samsung Galaxy Grand Prime Price 5 5844 Set Reminder sound on Windows 10 5 6031 Tacos Al Pastor Recipe Authentic Recipe 5 6159 The cells that secrete parathyroid hormone are called 5 6222 The inner lining of blood vessels is called 5 6266 The normal body temperature of a spider monkey is 5 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 6 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 6 413 .what is not the purpose of configuration control when configuration management is applied in a project? 6 509 1 us dollar equals how many euros 6 555 10 factors responsible for depletion of water table 6 667 14 oct 2015 mlb playoff schedule 6 699 15 straight line depreciation for leasehold improvements 6 807 1993 penthouse mark whalberg net worth 6 854 2 year olds knee hurts when he is sick 6 884 20 oz Post raisin bran price 6 891 2000 volkswagon cabriolet gas mileage and teliability 6 949 2008 subaru feraster washer moter how to install 6 1007 2015 college half term holiday dates 6 1034 2015 per diem rates for truck drivers 6 1099 2017 college football semi finals schedule 6 1329 331.3.3 Explain how disability may affect development 6 1477 5 food you should never eat men's health 6 1479 5 foot 10 inches in meters and centimeters 6 1692 9. how do the semicircular canals and vestibular sacs work 6 1706 92154 zip code plus 4 digit code 6 1901 According to your text, possessing an accurate view of yourself is called: 6 1956 All of the following resulted from the civil rights movement except 6 2059 Army service corps division train organisation 6 2098 At what temperature should you remove the chuck burgers from the grill in restaurants 6 2569 Consumer protection disputing a charge how long does the merchant have? 6 2593 Cranial nerve number 7 is also known as the 6 2671 December 30 2015 power ball number 6 2930 Estimating the cost of desalination plants using a cost database 6 3120 Game of Thrones season 6 episode names 6 3188 Gross wage 1800 per week salary 6 3450 How did the G.I bill affect the future since the 1950s 6 3451 How did the Huang River civilization develop differently from other river valley civilizations 6 3470 How do I get my keys on my laptop to work to increase the sound? 6 3569 How does Macbeth behave during the banquet, and what is the end result of his actions? 6 3877 How many years was Everybody hates chris was on television 6 3899 How much do diapers cost per month 2016? 6 4124 How old is the girl that plays Michelle in full house 6 4270 How to deactivate automatic screen lock on windows 10 6 4394 I have a swollen ball in the socket where my wisdom tooth was pulled 6 4464 If a person’s body has a pH of 7.4, why does his or her DNA not denature? 6 4539 In facebook program, how to change settings from spanish to english? 6 4549 In the 1990s, NATO lost much of its former purpose and: 6 4985 MTV Real World 2016 Cast 31 6 5526 Pittsburg Kansas region is located in which of the 11 physiographic regions 6 5612 Putting multiple CPUs on a single chip is defined as 6 5632 R.A.D lyrics a-z chris webby 6 5925 Sony PS-LX300USB how to connect to pc 6 5958 State Farm telephone number Baltimore Pike State Farm 6 5966 Steven Sanders poem, you are men who in your lives fought for 6 6083 The Civil Rights movement was based on which type of power 6 6486 Train timetable Melbourne to Wagga fridays and Sundays 6 6652 W1282X is a ________ mutation, which produces a premature stop codon 6 6750 Weather Iowa City Iowa 10 Day Forecast 6 6950 What channel can I hear the Eagle game tonight on what channel on the radio 6 7093 What does the Seventh Day Adventist church say about tithing 6 7375 What is the DNA formed by combining the DNA from 2 different species 6 7417 What is the average lifespan of red blood cells?​ quizlet 6 7695 What is the social definition of grandparent ? How does raising grandchildren challenge this definition 6 7791 What percentage of salary and wages can be put into roth ira 6 7875 What was the congress of Vienna and how did conservatives attempt to restore power 6 7968 When did the earthquake hit San Francisco during the World Series 6 8043 Where can I get T206 baseball cards graded and what price 6 8099 Where does the water involved in melting at subduction zones come from? 6 8276 Which of the following is the layer of blood cells where the PCV is read? 6 8326 Which term means pertaining to red blood cells? 6 8330 Which term refers to the amount of deviation from a project plan? 6 8759 ______ damages include damages for lost profits, as long as these damages are not too speculative 6 8937 _________ is the process of specifying (i.e., defining) what we mean by a term. 6 8992 __________ print film is the film of choice for most crime scene photography. 6 9046 ______________ is the dimension of color that is determined by the wave length of light 6 9208 a developing testes secrete hormones that does what to female structures 6 9528 a(n) ________ is a chamber that isolates the subject from the external environment. 6 9711 abandonment defense aider and abettor - legal definition 6 9888 about how early should i arrive at bwi to catch my flight to florida 6 9976 about how much lehman college spend on their daycare center 6 9978 about how much money do you get from a food poison lawsuit 6 10212 accelerated stability equals to how many normal months 6 10337 according to kohlberg's theory the 4th stage of development is 6 11021 actress who plays jo wilson on grey's anatomy 6 11261 adap rules for dispensing fees at contract pharmacies 6 11359 add password to open word document 2010 6 11381 add your phone number to new york times 6 11520 address of ye old tavern in manchester vt 6 11523 address to a van michael salon in roswell ga 6 11721 admission cost to rock and roll hall of fame 6 11979 advantages and disadvantages of using the internet to screen before employment 6 12021 advantages of pentane for gas detecrion in methane service 6 12066 adverse effects and nursing consideration for antithyroid medications 6 12538 after giving declearation can i edit pgcet application form 6 12565 after how many years a report falls off my credit score 6 12575 after installing windows 10 from usb system reboot 6 12601 after planting peas how long before the break to top of ground 6 13187 age limit for claiming dependent medical expenses 6 13728 age of josephine marie wagaman york pa 6 14039 age related or secondary epilepsy usually presents at what age 6 14056 age requirement for a concealed hand gun in texas 6 14782 air canada flight schedule calgary to vancouver 6 14834 air force ww2 plane crew list 6 14949 airport code nas jrb fort worth 6 15063 alabama central credit union routing number 6 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 7 169 +how many calories does a person burn in five minutes of standing 7 327 , does vinegar and water help. remove coffee stains in blouses 7 829 1x8 block with a sticker grand prix what set does it belong 7 1267 3 ways a log can move when bucking on side hill 7 1517 50% of the amount exceed 40 times minimum wage 7 1522 5000 iu of vitamin d causes leg pain 7 1977 American slavery first began in which of the following centuries? quizlet 7 2362 CURRENt rate for a color and cut hairstyle in denver co 7 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 7 2418 Can you place a house on HUD housing even if you still owe a mortgage 7 2616 DID the white fly flag half mast for whitney houston 7 2877 Early signs of alcohol abuse college students SAMHSA 7 3115 Gabon's largest river is the Ogooué. In which neighbouring country does this river originate? 7 3357 How Deming quality award is different than Malcom baldridge award and European quality award 7 3754 How long from the time you are exposed to influenza B do you get symptoms 7 4064 How much per hour does Bernie Sanders pay his staff? 7 4329 How to say im going on vacation next week in spanish 7 4398 I live in Florida and want to buy a car in Connecticut how do I register it 7 4520 In Tennessee how long before you can visit prison once you are no longer a corrections officer 7 4927 List two indicators that confirm that autism is an inherited disorder 7 5315 Non emergency police phone number Lincoln Nebraska 7 5320 Normal resting heart rate values for someone with T2DM 7 5325 Notice the differences between o'brien's home and winstons what does this signify 7 5373 Once it is determined a location was used to manufacture meth in NC, what is the procedure? 7 5520 Pierrot is a tragic clown character derived from European theater 7 5537 Please show where in Erfurt where the Culinary Olympics 2016 will be held 7 5559 Prader-Willi syndrome is usually caused by quizlet 7 5668 Reasons why the USSR was able to detonate the atomic bomb so soon 7 5699 Research on deindividuation suggests that when people are part of a crowd, they become 7 5961 Steinbock’s view of how sentience, desires and interests are related and then how harm is 7 6207 The first step in creating a personal plan for managing stress is to 7 6225 The invention of_________________ changed forever the way information could be transmitted 7 6482 Toyota tacoma pre runner vs Nissan frontier 7 7013 What does Jonathan suspect is in the bag the three wives are given in the courtyard? 7 7345 What is indicated by a clearing around bacterial growth in Tributyrin Agar? 7 7800 What products did Chinese people want to have in their country that we have in the United States 7 7856 What type of cloth should you use for fiberglassing a fishing boat floor 7 7921 What “homework” did James Madison do in preparation for the Philadelphia Convention? What were the consequences? 7 8431 Who wrote the Barry Manilow song looks like we made it 7 8439 Why am I having numbness in my lower leg since I had a total hip replacement 7 8691 _____ is an organization that is mainly devoted to compensation and performance management programs. 7 8996 __________ releases electrons when struck by light photons and _________ releases electrons when struck by x rays. 7 9279 a is a group of similar organisms that can ate with each other and produce fertile offspring 7 9436 a rod shaped structure that forms when a single dna molecule 7 9693 aba number for educational community credit union michigan 7 10339 according to sagan why should we reject assurances that a nuclear war will not occur 7 11379 add windows update icon to desktop Windows 2012 r2 7 11554 adelanto california average land price per acre 7 11759 adobe reader dc for windows 10 offline installer 7 12741 age and weight requirements for children to sit in the front seat 7 16463 america 2016 election total result until 6 april 2016 7 16675 american family ins madison wi phone number 7 17165 amount of co2 in atmosphere 3 billion years ago 7 17258 amount of money spent per capita on education in dc 7 18010 andrew irving born 1852 died 1921 scotland 7 18100 ane computer solutions llc federal id number 7 18990 another way to form who is covered and coverage start date 7 19671 ao smith water heater company phone number 7 20337 approximately how many different genes can be found in a human cell? 7 20671 are a category of sugars that contain either one or two molecules 7 20680 are abd pads good secondary dressing for packed wounds 7 21335 are broker fees considered real estate cost of sale? 7 22633 are genetic traits that give an advantage to an individual and can be passed to off spring 7 23148 are ira distributions counted as social security disability income 7 23638 are metis income tax exemption now due to new ruling 7 23916 are non-government organisations non-corporate entities? 7 23959 are often the initial sign of a heat-related emergency. 7 24137 are passporys necessary foe sonestic travel out of new mexico 7 25582 are there any self checks someone could do to detect cervical cancer 7 25703 are there limitations for a jury trial in a criminal matter in the united states 7 26986 article review on steps needed to develop advertising program 7 27809 at how many weeks does a fetus start to move inside the womb? 7 28145 at what age do black footed ferrets take age of sexual maturity 7 28806 at what point can you get supplemental oxygen for night time use 7 29163 at which job does percy earn the greater hourly wage? how much does percy earn each hour at this job? 7 30261 average SQ ft price for land grading michigan 7 31297 average bank teller pay rate urbandale ia 7 32435 average cost for rain gutter installation in deland florida 7 32694 average cost of a dental exam and cleaning in cranbrook b.c. 7 32731 average cost of a full redo of house drain system 7 33198 average cost of building 2000 square foot home 7 33246 average cost of carpeting an 1800 square feet home 7 33312 average cost of college tuition per credit hour 7 33499 average cost of electrician service call canton, sd 7 33832 average cost of house siding per foot colorado 7 33895 average cost of ip phone service per use 7 34014 average cost of machinist specialist in san jose, ca 7 34026 average cost of marble countertops per square foot 7 34386 average cost of replacing a headliner in a 4 door sedan 7 34770 average cost per foot for a chain link fence 7 34803 average cost per minute charge for a call center 7 35070 average cost to convert evaporative cooler to refrigerated air 7 35335 average cost to paint walls per square foot of wall 7 35564 average cost to tent a home for termites in broward county 7 36264 average employee size of sprouts farmers market store 7 37067 average help desk technician pay san diego 7 37312 average hourly rate for central ohio lawn service 7 37318 average hourly rate for entry level business analyst 7 37658 average income for clinical medical technician in huntsville, al 7 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 8 130 ). Puerto Rico says sick will head north without health-care fix 8 793 1981 Isuzu pup diesel average miles per gallon ? 8 814 1999 chevy suburban k2500 7.4l what to look for when buying 8 920 2006 GMC Yukon XL engine is off, but heater blower is still on 8 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 8 1353 39 x 29 foundation crawlspace takes how many cinder blocks 8 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 8 1577 61 km long and 3 km wide and average depth of 132m 8 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 8 1778 A feature of most autumns in the northeastern united states is a period of mild sunny weather 8 1887 According to Jean, how much money should Jean and her friend save to spend on next year’s supplies? 8 1962 Although the Civil Rights Act of 1964 was the most comprehensive civil rights legislation ever considered it does not 8 1993 An infection that is spread by the blood or lymphatic system to large portions of the body is called: 8 2127 Average cost of having electricity run to a rural acreage for builing site 8 2158 BEETHOVEN Symphony No. 3 in E-flat major Eroica why is it called eroica 8 2193 Based on the assigned readings, define “corporate welfare” and discuss its prevalence 8 2371 Calculate a long-run average cost curve for superconductors. 8 2412 Can you give an example of how a fibrous root system is used in woody plants and its benefits? 8 3049 Following the Supreme Court of the United States ruling on genetic testing, which of the following has been the largest driver of the increased amount of genes being tested for 8 3129 Gary made $6,975 working 25 hours a week for 18 weeks. How much did he earn per hour? 8 3333 Historians have often wondered why Mission San Jose was located as far east as it was 8 3526 How do you get a light colored machine oil out of a new cement patio 8 3618 How is it that the cells in different body tissues are able to perform different functions? quizlet 8 3827 How many hours does it take to get an undergraduate degree in accounting from Mercer University 8 3861 How many screws do you put in a 4 x 12 ft sheet of sheet rock? 8 4104 How much would it cost to drive from Chambersburg, PA to Ontario, Canada 8 4361 How would an increase in light intensity affect the carrying capacity of algae in a lake 8 4382 I am a molecular biologist who wants to become a computer programmer. how to become one? quora 8 4428 IE11 do you want to allow this website to open an app on your computer fix 8 4456 If Earth's remaining glaciers melt, how would the global sea level change? 8 4471 If a weather report gives the atmospheric pressure as 95.5 kPa, what is the pressure in torr 8 4490 If there are 710 mL of champagne in the bottle, how many milliliters of alcohol are present at 19% 8 4529 In a simplified model of a city, the zone where retail and office activities are clustered is the 8 4554 In the Stephen King and Peter Straub collaboration, 'Black House', the protagonist Jack Sawyer has a nickname which not only ties in with a place he has worked in but reflects his 'celebrity' status as a top (ex-)detective. What is this nickname? 8 4725 Jack drove at an average speed of 47.3 miles per hour for 2.5 hours. How far did he drive? 8 5195 My lap top says automatic repair then goes to blue screen 8 5399 Organizational leaders may use rewards to reduce new employees’ discomfort with immoral behaviors. This is called 8 5476 Pelvic inflammatory disease (PID) is an infection that is often caused by a sexually transmitted disease. 8 5663 Read the following two statements. Then, if possible, use the Law of Detachment to draw a conclusion.Driving is difficult if the weather is stormy. 8 6292 The powers granted to the national government in Articles I, II, and III of the Constitution are known as which of the following? 8 6301 The principle is 4096 rate is 12.5% time is 18 months find the compound interest 8 6304 The process in which information contained in RNA is used to manufacture proteins is called 8 6675 WHAT IS THE MAXIMUM HIEGHT RESTRICTION FOR BUILDING A HOME IN PONCE INLET, FL 8 7710 What is the theoretical yield of sodium chloride for the reaction of 36.0 g Na with 75.2 g Cl2? 8 7871 What was the Kansas-Nebraska Act? Why was it such a controversial measure? ap history 8 7953 When connecting to itunes store get a itunes has stopped working windows 7 8 7978 When does a new UPC need to be issued for a consumer electronics product when it changes 8 8206 Which eye disorder causes an opacity in the lens, which if left uncured, may blind you? 8 8265 Which of the following is an impact of increased ocean acidification as caused by global climate change 8 8305 Which process is catalyzed by rubisco during the light-independent reactions of photosynthesis? glycolysis 8 8332 Which two bones does the tendon of the long head of Biceps brachii run between? 8 8631 ____ is a process by which organisms with characteristics suited to the environment will survive and reproduce.a.natural selectionc.evolutionb.artificial selectiond.organic evolution 8 8684 _____ is a project performance measurement technique that integrates scope, time, and cost data 8 8701 _____ is the ability of cardiac pacemaker cells to spontaneously initiate an electrical impulse without being stimulated from another source, such as a nerve. 8 8714 _____ is the name used to refer to the era of legalized segregation in the united states 8 8735 _____ refers to an outbreak of a disease occurring over a large geographic area, possibly worldwide. 8 8801 _______ is an art form involving taking animals or objects and giving them human shapes and/or qualities. 8 8807 _______ is the earliest known epic hero and he shares many traits with later epic heroes, such as the anglo-saxon hero beowulf. 8 8845 ________ blocks aerobic metabolism by interfering with cell use of oxygen.. 8 8848 ________ connect sensory and motor neurons in neural pathways and their cell bodies are typically located in the central nervous system. 8 8854 ________ disparity refers to the slightly different view of the world that each eye receives.cyclopeanbinocularmonoculartrichromatic 8 8860 ________ is a lighting technique that uses deep gradations and subtle variations of lights and darks. 8 8926 ________ uses radio signals to provide high-speed internet and network connections. 8 9007 ___________ elevated the supreme court to a higher status, balancing the powers of the other branches. 8 9010 ___________ is a picture that includes the full face or another feature with much detail.a.portraiturec.montageb.photo essayd.close up 8 9044 ______________ is involved in voluntarily controlling muscles and is released during feelings of pleasure or reward. 8 9150 a car is moving at a constant velocity of 45 miles per hour 8 9168 a class of chemical compounds derived from vitamin a that are used in skin care 8 9278 a hypothetical disorder called G syndrome is an autosomal dominant disease characterized by visual 8 9424 a reflex that causes muscle relaxation and lengthening in response to muscle tension is called a ________. 8 10353 according to the five-factor model of personality people who score high in 8 12126 advocate obituary of mr harold broussard sr new orleans la 8 13963 age one can make any amount without social security limits 8 15200 alberta land registry remove mortgage from title without spouse 8 15988 alternating periods of economic growth and contraction in real gdp define 8 17352 amount of time it takes to return work after double knee replacement 8 17565 an appropriate number of people for a problem-solving and decision-making group is 8 17598 an error occurred in the vulnerability protection manager to vulnerability protection agent protocol: sockettimeoutexception: read timed out. 8 17635 an increase of adh causes the kidneys to ________ the reabsorption of water, causing them to produce _______ urine. 8 19590 anybody know where to buy parts for an air sander made by milwaukee pneumatic? 8 20423 apush in what ways did the treaty of versailles embody wilson's fourteen points 8 20647 are 10 year olds allowed to stay home alone in NJ 8 20788 are all the super heroes movies out these last years based on Marvel Comics? 8 24952 are skin rashes and hives on abdomen and thighs related to yeast infections 8 25200 are supplemental insurance policy exclude pre-existing conditions? 8 25266 are tax credits applicable to short-term limited duration policies 8 25807 are there women's shoes that aren't tennis shoes that can be worn with an orthopedic ankle brace and tie 8 28681 at what gestation age would a fetal death be counted as a delivery history 8 30910 average amount spent on groceries per month for a family of 2 canada 8 32092 average cost for Canadian rehab hospitals semi-private. 8 33257 average cost of cement driveway per square foot in So.Cal 8 33501 average cost of electricity and water per month in new delhi 8 34750 average cost per bed for dormitories in southeastern united states 8 34767 average cost per day at venice, fl hosp. for open heart surgery 8 34854 average cost per square foot install formica on countertops 8 34876 average cost per square foot to build a new home 8 35506 average cost to replace standard gas hot water heater 8 35543 average cost to ship a small car to las vegas ,nv. FROM CHICAGO IL 8 40985 average price per square foot for concrete flat work 8 timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/download.sh000077500000000000000000000053761525762047700265300ustar00rootroot00000000000000#!/bin/bash # Download MS MARCO Passage Ranking dataset # Full dataset: ~8.8M passages, ~500K queries # Source: https://microsoft.github.io/msmarco/ # # Usage: ./download.sh [size] # size: 100K, 500K, 1M, or full (default: full) set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATA_DIR="$SCRIPT_DIR/data" SIZE="${1:-full}" mkdir -p "$DATA_DIR" cd "$DATA_DIR" echo "=== MS MARCO Passage Ranking Dataset Download ===" echo "Requested size: $SIZE" echo "" # Collection (passages) if [ ! -f "collection.tsv" ]; then echo "Downloading passage collection..." if [ ! -f "collection.tar.gz" ]; then wget -q --show-progress \ https://msmarco.z22.web.core.windows.net/msmarcoranking/collection.tar.gz fi echo "Extracting collection..." tar -xzf collection.tar.gz rm collection.tar.gz echo "Collection extracted: $(wc -l < collection.tsv) passages" else echo "Collection already exists: $(wc -l < collection.tsv) passages" fi # Queries (dev set - 6980 queries with relevance judgments) if [ ! -f "queries.dev.small.tsv" ]; then echo "Downloading dev queries..." wget -q --show-progress \ https://msmarco.z22.web.core.windows.net/msmarcoranking/queries.tar.gz tar -xzf queries.tar.gz rm queries.tar.gz echo "Queries extracted" else echo "Queries already exist" fi # Relevance judgments (qrels) for dev set if [ ! -f "qrels.dev.small.tsv" ]; then echo "Downloading relevance judgments..." wget -q --show-progress \ https://msmarco.z22.web.core.windows.net/msmarcoranking/qrels.dev.small.tsv echo "Relevance judgments downloaded" else echo "Relevance judgments already exist" fi # Create subset if requested if [ "$SIZE" != "full" ]; then echo "" echo "=== Creating $SIZE passage subset ===" case "$SIZE" in 100K) LIMIT=100000 ;; 500K) LIMIT=500000 ;; 1M) LIMIT=1000000 ;; *) echo "Unknown size: $SIZE, using full dataset"; LIMIT=0 ;; esac if [ "$LIMIT" -gt 0 ]; then # Keep original as collection_full.tsv if [ ! -f "collection_full.tsv" ]; then mv collection.tsv collection_full.tsv fi # Create subset head -n "$LIMIT" collection_full.tsv > collection.tsv echo "Created subset: $(wc -l < collection.tsv) passages" # Clean up full version to save space rm -f collection_full.tsv fi fi echo "" echo "=== Download Complete ===" echo "Files in $DATA_DIR:" ls -lh "$DATA_DIR"/*.tsv 2>/dev/null | head -10 echo "" echo "Dataset statistics:" echo " Passages: $(wc -l < collection.tsv)" echo " Dev queries: $(wc -l < queries.dev.small.tsv 2>/dev/null || echo 'N/A')" echo " Relevance judgments: $(wc -l < qrels.dev.small.tsv 2>/dev/null || echo 'N/A')" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/generate_benchmark_queries.sh000077500000000000000000000052021525762047700322460ustar00rootroot00000000000000#!/bin/bash # Generate benchmark query samples bucketed by token count # # This script creates benchmark_queries.tsv with deterministically-sampled # queries from MS-MARCO, bucketed by the number of search tokens after # stop word removal and stemming (via to_tsvector). # # Output format: query_idquery_texttoken_count # # Usage: ./generate_benchmark_queries.sh # # Requires: PostgreSQL running locally set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATA_DIR="$SCRIPT_DIR/data" OUTPUT_FILE="$SCRIPT_DIR/benchmark_queries.tsv" INPUT_FILE="$DATA_DIR/queries.dev.tsv" TEMP_FILE=$(mktemp) trap "rm -f $TEMP_FILE" EXIT # Check if queries file exists if [ ! -f "$INPUT_FILE" ]; then echo "Error: $INPUT_FILE not found" echo "Run ./download.sh first to get the MS-MARCO dataset" exit 1 fi echo "Generating benchmark queries with 100 queries per token bucket..." echo "Loading and tokenizing $(wc -l < "$INPUT_FILE") queries..." # Preprocess: convert to CSV format to handle special characters echo "Preprocessing queries..." awk -F'\t' 'BEGIN {OFS=","} {gsub(/"/, "\"\"", $2); print $1, "\"" $2 "\""}' "$INPUT_FILE" > "$TEMP_FILE" # Run query in psql - output to stdout, redirect to file # Use -q to suppress notices, and filter out COPY/CREATE lines echo "Running tokenization and bucketing..." psql -U postgres -d postgres -p 5432 -t -A -F $' ' -q -v ON_ERROR_STOP=1 << EOSQL | grep -v "^CREATE\|^COPY" > "$OUTPUT_FILE" CREATE TEMP TABLE all_queries (query_id int, query_text text); \copy all_queries FROM '$TEMP_FILE' WITH (FORMAT csv) WITH tokenized AS ( SELECT query_id, query_text, array_length(string_to_array(to_tsvector('english', query_text)::text, ' '), 1) as token_count FROM all_queries ), bucketed AS ( SELECT query_id, query_text, CASE WHEN token_count >= 8 THEN 8 ELSE token_count END as token_bucket, row_number() OVER ( PARTITION BY CASE WHEN token_count >= 8 THEN 8 ELSE token_count END ORDER BY query_id ) as rn FROM tokenized WHERE token_count IS NOT NULL ) SELECT query_id, query_text, token_bucket FROM bucketed WHERE rn <= 100 ORDER BY token_bucket, query_id; EOSQL # Count results echo "" echo "Generated $OUTPUT_FILE" TOTAL=$(wc -l < "$OUTPUT_FILE") echo "Total queries: $TOTAL" echo "" echo "Distribution by token bucket:" cut -f3 "$OUTPUT_FILE" | sort | uniq -c | sort -k2 -n # Show sample queries from each bucket echo "" echo "Sample queries per bucket:" for bucket in 1 2 3 4 5 6 7 8; do echo " Bucket $bucket: $(awk -F'\t' -v b="$bucket" '$3==b {print $2; exit}' "$OUTPUT_FILE" | head -c 60)..." done timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/gin/000077500000000000000000000000001525762047700251245ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/gin/concurrent-setup.sql000066400000000000000000000034521525762047700311710ustar00rootroot00000000000000-- MS MARCO GIN+tsvector Baseline - Concurrent Insert Setup -- Loads passages into a staging table, creates empty target table -- with GIN index, and prepares a sequence for pgbench-driven -- concurrent inserts. -- -- Usage: -- DATA_DIR=/path/to/data psql -f gin/concurrent-setup.sql \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO GIN Concurrent Insert Setup ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_gin_passages CASCADE; DROP TABLE IF EXISTS msmarco_gin_staging CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create staging table (raw data, no tsv column) \echo 'Creating staging table...' CREATE TABLE msmarco_gin_staging ( staging_id SERIAL PRIMARY KEY, passage_id INTEGER, passage_text TEXT ); -- Load passages into staging \echo 'Loading passages into staging table...' \copy msmarco_gin_staging(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Create target passages table with GIN index \echo 'Creating target passages table...' CREATE TABLE msmarco_gin_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', passage_text) ) STORED ); \echo 'Creating GIN index on empty target table...' CREATE INDEX msmarco_gin_idx ON msmarco_gin_passages USING gin(tsv); -- Create sequence for pgbench inserts \echo 'Creating insert sequence...' CREATE SEQUENCE insert_seq; -- Report staging row count \echo '' \echo '=== Setup Verification ===' SELECT 'STAGING_ROW_COUNT:' as label, COUNT(*)::text as count FROM msmarco_gin_staging; \echo '' \echo '=== GIN Concurrent Insert Setup Complete ===' \echo 'Run pgbench with gin/pgbench-insert.sql' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/gin/insert-load.sql000066400000000000000000000033261525762047700300720ustar00rootroot00000000000000-- MS MARCO GIN+tsvector Baseline - Insert Loading -- Creates GIN index on EMPTY table, then loads data via COPY -- Measures insert performance into an already-indexed table -- -- Usage: -- DATA_DIR=/path/to/data psql -f gin/insert-load.sql \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO GIN Insert Benchmark - Data Loading ===' \echo 'Index created BEFORE loading passages' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_gin_passages CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_gin_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', passage_text) ) STORED ); -- Create GIN index on empty table \echo 'Creating GIN index on empty table...' CREATE INDEX msmarco_gin_idx ON msmarco_gin_passages USING gin(tsv); -- Load passages into the pre-indexed table \echo '' \echo 'INSERT_TIME:' \echo 'Loading passages into indexed table...' \copy msmarco_gin_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('msmarco_gin_idx') ) as index_size, pg_relation_size('msmarco_gin_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size('msmarco_gin_passages') ) as table_size, pg_total_relation_size('msmarco_gin_passages') as table_bytes; \echo '' \echo '=== MS MARCO GIN Insert Benchmark Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/gin/load.sql000066400000000000000000000030761525762047700265720ustar00rootroot00000000000000-- MS MARCO GIN+tsvector Baseline - Data Loading -- Loads data first, then creates GIN index on populated table -- Measures CREATE INDEX time and final sizes -- -- Usage: -- DATA_DIR=/path/to/data psql -f gin/load.sql \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO GIN Baseline - Data Loading ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_gin_passages CASCADE; -- Create passages table (without index) \echo 'Creating passages table...' CREATE TABLE msmarco_gin_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', passage_text) ) STORED ); -- Load passages \echo 'Loading passages...' \copy msmarco_gin_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Create GIN index on populated table (timed) \echo 'Building GIN index...' CREATE INDEX msmarco_gin_idx ON msmarco_gin_passages USING gin(tsv); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('msmarco_gin_idx') ) as index_size, pg_relation_size('msmarco_gin_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size('msmarco_gin_passages') ) as table_size, pg_total_relation_size('msmarco_gin_passages') as table_bytes; \echo '' \echo '=== MS MARCO GIN Load Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/gin/pgbench-insert.sql000066400000000000000000000003001525762047700305460ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO msmarco_gin_passages (passage_id, passage_text) SELECT passage_id, passage_text FROM msmarco_gin_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/ground_truth_pg17.tsv000066400000000000000000001636321525762047700305020ustar00rootroot00000000000000query_id query_text rank doc_id score 254 +what is cchaps 1 7373206 17.3859955932856 254 +what is cchaps 2 7373205 16.26544766572216 254 +what is cchaps 3 7373204 15.85665560139892 255 +what is centure 1 5337494 16.883701561739528 255 +what is centure 2 7629888 15.704311369973196 255 +what is centure 3 2332343 15.27743675372195 255 +what is centure 4 8672868 15.072585235978094 255 +what is centure 5 7942983 13.950251591592302 255 +what is centure 6 8740723 12.690323752809526 255 +what is centure 7 3534898 12.548656047129782 255 +what is centure 8 2624439 12.142015939139485 255 +what is centure 9 8080899 11.88525425587245 255 +what is centure 10 2280509 11.402986718300756 1724 < symbol 1 901289 10.804106157772686 1724 < symbol 2 7040513 10.712428538256683 1724 < symbol 3 330189 10.548631290745037 1724 < symbol 4 330186 10.548631290745037 1724 < symbol 5 330188 10.535487705230079 1724 < symbol 6 330187 10.513372759536452 1724 < symbol 7 3353273 10.431976378052354 1724 < symbol 8 1244169 10.414651870909909 1724 < symbol 9 7971836 10.391735608015605 1724 < symbol 10 1091451 10.383762129485397 4288 How to have sex 1 5601661 10.331804171254714 4288 How to have sex 2 8397394 10.285989280990554 4288 How to have sex 3 873941 10.255018968033507 4288 How to have sex 4 3527665 10.217052778348553 4288 How to have sex 5 3154655 10.204538294316578 4288 How to have sex 6 1314219 10.195084359771231 4288 How to have sex 7 4293186 10.179366669036224 4288 How to have sex 8 2806469 10.16691505099238 4288 How to have sex 9 309973 10.166827367335372 4288 How to have sex 10 4598378 10.138900978497944 7354 What is my ethnicity 1 8721023 13.009673757528194 7354 What is my ethnicity 2 4742620 12.96866283782917 7354 What is my ethnicity 3 4286243 12.814851993389619 7354 What is my ethnicity 4 4286246 12.77971622490698 7354 What is my ethnicity 5 4463943 12.77971622490698 7354 What is my ethnicity 6 6849222 12.712567180016881 7354 What is my ethnicity 7 4246109 12.657000242646157 7354 What is my ethnicity 8 301574 12.63691181577928 7354 What is my ethnicity 9 5315331 12.625779107687148 7354 What is my ethnicity 10 4655970 12.586968623509849 9340 a mutation is 1 1307633 13.208528855632112 9340 a mutation is 2 951868 13.109871118397523 9340 a mutation is 3 450025 13.101564037851844 9340 a mutation is 4 2917314 12.928864910483437 9340 a mutation is 5 1808727 12.926896861119054 9340 a mutation is 6 8799571 12.882317021697105 9340 a mutation is 7 1370509 12.872520752434916 9340 a mutation is 8 8612630 12.864618965895067 9340 a mutation is 9 8135964 12.82779778744428 9340 a mutation is 10 1307632 12.82779778744428 9351 a novel is a _____. 1 425159 11.701898483294718 9351 a novel is a _____. 2 8794243 11.666474801113688 9351 a novel is a _____. 3 7162299 11.617021565973623 9351 a novel is a _____. 4 3113367 11.429105550805302 9351 a novel is a _____. 5 3049840 11.410551910628133 9351 a novel is a _____. 6 4316285 11.361622218482658 9351 a novel is a _____. 7 2458138 11.361622218482658 9351 a novel is a _____. 8 8818382 11.361622218482658 9351 a novel is a _____. 9 7160625 11.317074323688349 9351 a novel is a _____. 10 5545053 11.317074323688349 16453 amendment you can be who you are 1 7309616 11.65336805883167 16453 amendment you can be who you are 2 234632 11.486854124762608 16453 amendment you can be who you are 3 6214128 11.308318750974435 16453 amendment you can be who you are 4 5339945 11.272075937624317 16453 amendment you can be who you are 5 5380203 11.261572285736511 16453 amendment you can be who you are 6 6419370 11.244016127677895 16453 amendment you can be who you are 7 4150098 11.244016127677895 16453 amendment you can be who you are 8 2516688 11.236064696294342 16453 amendment you can be who you are 9 8654254 11.209067515530448 16453 amendment you can be who you are 10 3258484 11.204966274200155 27302 ash who is ash. 1 1514389 27.534554993598974 27302 ash who is ash. 2 1514386 27.510113295520437 27302 ash who is ash. 3 5472249 27.31426159121804 27302 ash who is ash. 4 1514393 27.105202502589766 27302 ash who is ash. 5 6959114 26.892269744908873 27302 ash who is ash. 6 6959109 26.803128798351164 27302 ash who is ash. 7 8195223 26.792631292070062 27302 ash who is ash. 8 4817989 26.71203826948381 27302 ash who is ash. 9 5834313 26.688319029957043 27302 ash who is ash. 10 5734761 26.507713801975676 92357 churrascaria what is 1 7070979 21.943397473171135 92357 churrascaria what is 2 7070972 21.228118852174195 92357 churrascaria what is 3 7070975 20.19638923195742 92357 churrascaria what is 4 7070978 20.01022050652706 92357 churrascaria what is 5 1381758 18.635943248574485 92357 churrascaria what is 6 7978801 17.86893307432649 92357 churrascaria what is 7 1381754 15.599702892473939 92357 churrascaria what is 8 5233933 15.5630052473084 92357 churrascaria what is 9 4227481 15.378675362238436 92357 churrascaria what is 10 2143172 14.521892418519084 83 you are so dutch means what 1 278633 18.63486930341032 83 you are so dutch means what 2 1082844 18.36071149005347 83 you are so dutch means what 3 5756931 17.23359901871705 83 you are so dutch means what 4 3179538 17.063696992840548 83 you are so dutch means what 5 260525 17.009947531994158 83 you are so dutch means what 6 388763 16.989076607238854 83 you are so dutch means what 7 575717 16.87973495728966 83 you are so dutch means what 8 1979564 16.86194692877695 83 you are so dutch means what 9 848948 16.820502094282837 83 you are so dutch means what 10 7604175 16.80095637325342 102 % saturation do calculation 1 6976611 17.606777368773926 102 % saturation do calculation 2 1634347 17.291106643443552 102 % saturation do calculation 3 8345094 17.078388600749985 102 % saturation do calculation 4 6388328 16.44528073481093 102 % saturation do calculation 5 95458 16.37337341376454 102 % saturation do calculation 6 7396240 16.264696108437114 102 % saturation do calculation 7 4085778 16.208875558613272 102 % saturation do calculation 8 4048762 16.022272686388536 102 % saturation do calculation 9 4184833 15.741649370209636 102 % saturation do calculation 10 6287758 15.708441149571467 214 +what can mirna be used for 1 7994517 22.844992280376484 214 +what can mirna be used for 2 4860329 21.552856393288696 214 +what can mirna be used for 3 7994518 21.353715908170482 214 +what can mirna be used for 4 1258768 21.010890227117873 214 +what can mirna be used for 5 5244753 20.750001403303102 214 +what can mirna be used for 6 2080070 19.856954827088483 214 +what can mirna be used for 7 4767298 19.658679508215457 214 +what can mirna be used for 8 7575 19.358166267541016 214 +what can mirna be used for 9 3887922 19.211182584077132 214 +what can mirna be used for 10 7580 19.062375783262212 220 +what do pathogic organisms do 1 3806899 17.08961233754324 220 +what do pathogic organisms do 2 7073478 16.631311238813833 220 +what do pathogic organisms do 3 2557370 15.20422259349876 220 +what do pathogic organisms do 4 4091393 12.582222504964568 220 +what do pathogic organisms do 5 8841262 7.447426565136349 220 +what do pathogic organisms do 6 8149442 7.447426565136349 220 +what do pathogic organisms do 7 2085067 7.4266946158805975 220 +what do pathogic organisms do 8 2683335 7.416667519911943 220 +what do pathogic organisms do 9 6363761 7.413856361307537 220 +what do pathogic organisms do 10 6965787 7.391717839071723 244 +what is a lmu wall 1 4333527 20.401883705399683 244 +what is a lmu wall 2 7494723 19.88077006757744 244 +what is a lmu wall 3 7494731 19.15686557671884 244 +what is a lmu wall 4 6231730 18.438310216452273 244 +what is a lmu wall 5 3579599 18.373650742328554 244 +what is a lmu wall 6 4333528 18.109818339909076 244 +what is a lmu wall 7 6231734 17.914499506360404 244 +what is a lmu wall 8 7494724 17.914499506360404 244 +what is a lmu wall 9 4070455 17.897250009029424 244 +what is a lmu wall 10 3138100 17.604249627712797 317 +where is highlong park 1 7377457 16.559907589197838 317 +where is highlong park 2 2705703 9.3809150585561 317 +where is highlong park 3 7519913 9.323967387775658 317 +where is highlong park 4 2158849 9.320842304509085 317 +where is highlong park 5 6069576 9.302470424731894 317 +where is highlong park 6 7895309 9.292985120369359 317 +where is highlong park 7 1503702 9.292093416965889 317 +where is highlong park 8 1307622 9.291350461507008 317 +where is highlong park 9 2277045 9.291350461507008 317 +where is highlong park 10 5269566 9.28107235873653 370 . what is the institute for leaders? 1 7166129 16.988109851463015 370 . what is the institute for leaders? 2 8555674 15.78811835790328 370 . what is the institute for leaders? 3 8637982 15.517041235114295 370 . what is the institute for leaders? 4 6777654 15.484612491539355 370 . what is the institute for leaders? 5 8143487 15.415397564403088 370 . what is the institute for leaders? 6 8555673 15.384963694335479 370 . what is the institute for leaders? 7 7330372 15.328111349701821 370 . what is the institute for leaders? 8 5069946 15.30008198706799 370 . what is the institute for leaders? 9 186789 15.25511560519902 370 . what is the institute for leaders? 10 7272646 14.939862296429542 409 .vbs what it mean 1 7991064 22.23535622072094 409 .vbs what it mean 2 3761044 21.495604462548066 409 .vbs what it mean 3 7991068 20.521209122514495 409 .vbs what it mean 4 5434835 19.581291423571713 409 .vbs what it mean 5 8724474 19.540438332287774 409 .vbs what it mean 6 6402283 19.331490984559597 409 .vbs what it mean 7 2716893 18.796998065026994 409 .vbs what it mean 8 2716890 18.54527546058995 409 .vbs what it mean 9 7353652 18.368060362610205 409 .vbs what it mean 10 8240045 17.976631907273376 1288 3/5 of 60 1 7473138 20.968070410716972 1288 3/5 of 60 2 5938786 19.431639478446016 1288 3/5 of 60 3 904704 18.827788087472953 1288 3/5 of 60 4 6991390 18.294257477376625 1288 3/5 of 60 5 7473137 18.294257477376625 1288 3/5 of 60 6 7473136 18.294257477376625 1288 3/5 of 60 7 7473134 18.294257477376625 1288 3/5 of 60 8 7473140 18.038672909554606 1288 3/5 of 60 9 3391800 18.038672909554606 1288 3/5 of 60 10 904705 17.79013136357068 1504 5% of 329300 1 2605711 6.440680321095578 1504 5% of 329300 2 3611691 6.416183414982819 1504 5% of 329300 3 3611689 6.328722608005325 1504 5% of 329300 4 1563631 6.293144622023016 1504 5% of 329300 5 347397 6.275964618210015 1504 5% of 329300 6 841474 6.258636210352685 1504 5% of 329300 7 347392 6.209900442611933 1504 5% of 329300 8 2357255 6.204112670460299 1504 5% of 329300 9 5029515 6.193601149244308 1504 5% of 329300 10 4142129 6.1746721042938955 2 Androgen receptor define 1 1782337 27.030836121669747 2 Androgen receptor define 2 1001873 25.535860891938835 2 Androgen receptor define 3 3634076 25.23028775387614 2 Androgen receptor define 4 6285817 25.013592983006824 2 Androgen receptor define 5 4339075 24.99690568549568 2 Androgen receptor define 6 7496506 24.63450713364079 2 Androgen receptor define 7 6285819 24.36768511185484 2 Androgen receptor define 8 4339068 24.315933627909374 2 Androgen receptor define 9 4339072 24.168158216896632 2 Androgen receptor define 10 6920058 24.1065877119146 58 the knee is what type of joint 1 902245 23.18818212882956 58 the knee is what type of joint 2 1722750 22.329975776858028 58 the knee is what type of joint 3 7273917 21.892313939401667 58 the knee is what type of joint 4 5147996 21.824675820091713 58 the knee is what type of joint 5 164142 21.774009398055323 58 the knee is what type of joint 6 160408 21.710979050672055 58 the knee is what type of joint 7 8551492 21.487130838134735 58 the knee is what type of joint 8 6761719 21.304475314782177 58 the knee is what type of joint 9 5530374 21.250121624261006 58 the knee is what type of joint 10 160404 21.17447433374224 71 what nationality is the name Factor 1 2231513 16.49943987413744 71 what nationality is the name Factor 2 4359473 13.868920181585906 71 what nationality is the name Factor 3 4401632 13.775424636458977 71 what nationality is the name Factor 4 60130 13.013506286705582 71 what nationality is the name Factor 5 2231515 12.700068643348828 71 what nationality is the name Factor 6 7895387 12.623781356159956 71 what nationality is the name Factor 7 4027284 12.561578180984885 71 what nationality is the name Factor 8 274604 12.539702198833044 71 what nationality is the name Factor 9 7438179 12.485189063296785 71 what nationality is the name Factor 10 7895388 12.472064904918103 74 where are protists most abundant in humans 1 5500111 19.876657364710695 74 where are protists most abundant in humans 2 5421885 19.15042764598655 74 where are protists most abundant in humans 3 4754855 18.911145409238998 74 where are protists most abundant in humans 4 7066917 18.887955255557742 74 where are protists most abundant in humans 5 834070 18.585922405319508 74 where are protists most abundant in humans 6 5421887 18.537689011057022 74 where are protists most abundant in humans 7 2173116 18.27672294280887 74 where are protists most abundant in humans 8 2389895 18.094201288354448 74 where are protists most abundant in humans 9 4726269 18.094201288354448 74 where are protists most abundant in humans 10 7882916 18.01123634299811 184 +is biology a social science 1 7973103 24.323438824076135 184 +is biology a social science 2 5081059 23.413267610975463 184 +is biology a social science 3 7273446 23.34234900620375 184 +is biology a social science 4 4035289 23.200181037994966 184 +is biology a social science 5 7688042 23.112068317439977 184 +is biology a social science 6 5081055 23.067423310946765 184 +is biology a social science 7 6047595 22.85937884108674 184 +is biology a social science 8 8360978 22.728017084855736 184 +is biology a social science 9 8780186 22.668589039572122 184 +is biology a social science 10 8712432 22.30390530429262 321 +where is your siatic nerve located 1 7805997 23.690945911375323 321 +where is your siatic nerve located 2 114020 21.89708282571956 321 +where is your siatic nerve located 3 2225703 19.620828808880503 321 +where is your siatic nerve located 4 3155966 15.004016122642486 321 +where is your siatic nerve located 5 5730368 14.418741949628021 321 +where is your siatic nerve located 6 3521669 14.41355466284306 321 +where is your siatic nerve located 7 765376 14.313931017805475 321 +where is your siatic nerve located 8 1659971 14.237931663019278 321 +where is your siatic nerve located 9 3550718 14.23488966252652 321 +where is your siatic nerve located 10 7345758 14.142839645068916 361 . irritability medical definition 1 152235 20.89345608536463 361 . irritability medical definition 2 6625778 17.244430043869183 361 . irritability medical definition 3 2291302 16.693546017581376 361 . irritability medical definition 4 4544177 16.5441419186806 361 . irritability medical definition 5 4544186 16.5441419186806 361 . irritability medical definition 6 4351150 16.49005961149882 361 . irritability medical definition 7 992573 16.49005961149882 361 . irritability medical definition 8 1767633 16.253058890466935 361 . irritability medical definition 9 8025874 16.253058890466935 361 . irritability medical definition 10 7955280 16.167357926622444 401 .net culture code 1 7345855 17.68874896219404 401 .net culture code 2 7345856 16.410871963080627 401 .net culture code 3 7796505 16.15189752273769 401 .net culture code 4 5704124 15.840051224284798 401 .net culture code 5 2166029 15.76754930635455 401 .net culture code 6 5704132 15.747074715348841 401 .net culture code 7 7796498 15.351625489936742 401 .net culture code 8 5704131 15.335571819103746 401 .net culture code 9 8001902 15.328479923217632 401 .net culture code 10 5704130 15.25921858444103 537 1/2 cups in tbsp 1 8808551 28.74016902339512 537 1/2 cups in tbsp 2 835445 28.401993731555596 537 1/2 cups in tbsp 3 835446 28.401993731555596 537 1/2 cups in tbsp 4 5161044 28.064282266099355 537 1/2 cups in tbsp 5 5530333 27.881474110866257 537 1/2 cups in tbsp 6 941813 27.279124789524168 537 1/2 cups in tbsp 7 1504531 27.185345962881797 537 1/2 cups in tbsp 8 1357616 27.182960234984353 537 1/2 cups in tbsp 9 8256266 26.997352023410233 537 1/2 cups in tbsp 10 823339 26.944153369015673 548 10 am PST in AST 1 5108468 22.504509350799875 548 10 am PST in AST 2 8176303 21.850825956880445 548 10 am PST in AST 3 3755378 20.2286914039162 548 10 am PST in AST 4 5108466 19.3551306012928 548 10 am PST in AST 5 3490313 19.31679110007874 548 10 am PST in AST 6 1475660 19.244437587231968 548 10 am PST in AST 7 7070931 19.11477542210205 548 10 am PST in AST 8 6279960 18.99120157186563 548 10 am PST in AST 9 3331852 18.79039790072167 548 10 am PST in AST 10 3530702 18.762367516644517 34 guidelines for mothballing of process plants 1 7327168 34.091632547552166 34 guidelines for mothballing of process plants 2 7327165 33.36674708630404 34 guidelines for mothballing of process plants 3 7327163 31.999825403402546 34 guidelines for mothballing of process plants 4 7327169 26.05141891861436 34 guidelines for mothballing of process plants 5 7327164 25.22762711339406 34 guidelines for mothballing of process plants 6 7327167 24.906703638719932 34 guidelines for mothballing of process plants 7 7327166 24.594035763123006 34 guidelines for mothballing of process plants 8 551997 22.724744944737097 34 guidelines for mothballing of process plants 9 6913017 21.490946278379504 34 guidelines for mothballing of process plants 10 3481679 20.27616468782069 197 +micheal moore does he wear a wig 1 1680692 25.814448780992727 197 +micheal moore does he wear a wig 2 2900936 24.209146857390387 197 +micheal moore does he wear a wig 3 6255832 24.12459015545854 197 +micheal moore does he wear a wig 4 3906126 23.980063016314496 197 +micheal moore does he wear a wig 5 1743866 23.917727616682473 197 +micheal moore does he wear a wig 6 780059 23.660871650638832 197 +micheal moore does he wear a wig 7 7809022 23.599199896500444 197 +micheal moore does he wear a wig 8 5989391 22.692102848829364 197 +micheal moore does he wear a wig 9 5066833 22.533521442266338 197 +micheal moore does he wear a wig 10 8323383 22.49655672304469 215 +what causes bacteria inflamation of face 1 5486709 21.986998730839545 215 +what causes bacteria inflamation of face 2 3373799 21.367550276790425 215 +what causes bacteria inflamation of face 3 2595241 21.262970860350993 215 +what causes bacteria inflamation of face 4 3373800 21.181558954323307 215 +what causes bacteria inflamation of face 5 8591077 21.089664710729743 215 +what causes bacteria inflamation of face 6 5934220 20.463336823674247 215 +what causes bacteria inflamation of face 7 3556225 20.44173727360749 215 +what causes bacteria inflamation of face 8 7039861 20.34855500086173 215 +what causes bacteria inflamation of face 9 3864800 20.112045426394264 215 +what causes bacteria inflamation of face 10 8656951 20.062682298003416 232 +what happen when rome was invaded by germania 1 7859869 22.501824520542094 232 +what happen when rome was invaded by germania 2 8182253 21.31551439515339 232 +what happen when rome was invaded by germania 3 2374070 21.073917073427477 232 +what happen when rome was invaded by germania 4 3219581 20.682316836952623 232 +what happen when rome was invaded by germania 5 6581749 20.09872499890696 232 +what happen when rome was invaded by germania 6 8691039 19.557423015611143 232 +what happen when rome was invaded by germania 7 2374072 19.43879967583285 232 +what happen when rome was invaded by germania 8 8182247 19.36385880074681 232 +what happen when rome was invaded by germania 9 2905526 19.193912929139103 232 +what happen when rome was invaded by germania 10 8182252 19.183681513263288 357 . difference between isometric & dynamic exercises 1 102744 30.558196424218046 357 . difference between isometric & dynamic exercises 2 7101256 30.121658827016695 357 . difference between isometric & dynamic exercises 3 5212508 29.094555969792758 357 . difference between isometric & dynamic exercises 4 4834349 28.134326332582106 357 . difference between isometric & dynamic exercises 5 102743 27.87405486162618 357 . difference between isometric & dynamic exercises 6 7354165 27.215009948734878 357 . difference between isometric & dynamic exercises 7 663623 27.192822045453525 357 . difference between isometric & dynamic exercises 8 663614 26.429698724871024 357 . difference between isometric & dynamic exercises 9 3963910 26.342826038063393 357 . difference between isometric & dynamic exercises 10 7907124 26.27906024488727 362 . what are the president's main duties? explain each 1 7006160 20.462632062056052 362 . what are the president's main duties? explain each 2 908288 20.21835264231002 362 . what are the president's main duties? explain each 3 5087020 19.121213498099515 362 . what are the president's main duties? explain each 4 6510779 18.674628608702367 362 . what are the president's main duties? explain each 5 4181913 18.468404694657345 362 . what are the president's main duties? explain each 6 3728523 18.468404694657345 362 . what are the president's main duties? explain each 7 3948438 18.355407319294923 362 . what are the president's main duties? explain each 8 362544 18.039612100839914 362 . what are the president's main duties? explain each 9 4375132 17.80766477351738 362 . what are the president's main duties? explain each 10 3948440 17.615425015575624 364 . what does the c prefacing a date mean 1 2625347 19.021441639085428 364 . what does the c prefacing a date mean 2 4097272 18.969843120574055 364 . what does the c prefacing a date mean 3 3482104 18.759629699143122 364 . what does the c prefacing a date mean 4 6264008 18.387531450667474 364 . what does the c prefacing a date mean 5 6264010 18.287659383300472 364 . what does the c prefacing a date mean 6 6233756 17.600660803440576 364 . what does the c prefacing a date mean 7 994042 16.940601589766136 364 . what does the c prefacing a date mean 8 5077052 16.53436377298733 364 . what does the c prefacing a date mean 9 127211 16.41020199222509 364 . what does the c prefacing a date mean 10 5825848 16.109474424902956 427 02 Chevy 1500 Specs 1 7322779 26.33596320976399 427 02 Chevy 1500 Specs 2 7322780 24.92279238452173 427 02 Chevy 1500 Specs 3 1686422 24.533385195205664 427 02 Chevy 1500 Specs 4 8012343 24.219928899007815 427 02 Chevy 1500 Specs 5 4016576 24.179685641325264 427 02 Chevy 1500 Specs 6 3072032 23.588696207011175 427 02 Chevy 1500 Specs 7 3072024 23.175659979027188 427 02 Chevy 1500 Specs 8 5748529 22.14070271514423 427 02 Chevy 1500 Specs 9 8012346 22.006722547448852 427 02 Chevy 1500 Specs 10 1671601 21.990097282097405 579 100g chicken carb content 1 7362509 35.08779298096177 579 100g chicken carb content 2 7362513 32.90575278666931 579 100g chicken carb content 3 7362512 30.9498935411841 579 100g chicken carb content 4 7362510 29.52819592129119 579 100g chicken carb content 5 6904332 28.047848464234065 579 100g chicken carb content 6 4706542 27.74398905274984 579 100g chicken carb content 7 8522988 27.729485598866688 579 100g chicken carb content 8 5076354 27.180024340891386 579 100g chicken carb content 9 4706546 26.89622728557034 579 100g chicken carb content 10 8522995 26.749817392220702 609 115 days until baby is due 1 6792759 18.78317794046132 609 115 days until baby is due 2 3339863 18.57607649874825 609 115 days until baby is due 3 1792569 18.229797241830738 609 115 days until baby is due 4 1118391 17.936306870488277 609 115 days until baby is due 5 3335663 17.62956539020589 609 115 days until baby is due 6 3332304 17.254853782899822 609 115 days until baby is due 7 1127847 17.110798510409595 609 115 days until baby is due 8 1562632 17.01486996364119 609 115 days until baby is due 9 4015543 16.933389237692115 609 115 days until baby is due 10 3631185 16.615003536438145 28 definition of Executive Officer defined by FRB 1 2681573 23.5380572808867 28 definition of Executive Officer defined by FRB 2 5632946 23.32515412137452 28 definition of Executive Officer defined by FRB 3 1023224 20.268881961081952 28 definition of Executive Officer defined by FRB 4 612213 20.147402681144374 28 definition of Executive Officer defined by FRB 5 5802383 20.130206837087204 28 definition of Executive Officer defined by FRB 6 6683951 20.014119321930337 28 definition of Executive Officer defined by FRB 7 8430099 19.8931756911823 28 definition of Executive Officer defined by FRB 8 4983661 19.813811139941446 28 definition of Executive Officer defined by FRB 9 7996167 19.72328159717519 28 definition of Executive Officer defined by FRB 10 2424701 19.601701495970225 106 'com multi unit' zoning definition 1 4197946 18.952964146879854 106 'com multi unit' zoning definition 2 5336361 18.601182484377215 106 'com multi unit' zoning definition 3 5532730 18.521927002308633 106 'com multi unit' zoning definition 4 5225971 17.98597767883125 106 'com multi unit' zoning definition 5 7019617 17.722909533277168 106 'com multi unit' zoning definition 6 6346796 17.71600129450722 106 'com multi unit' zoning definition 7 6324045 17.537771495341623 106 'com multi unit' zoning definition 8 5245449 16.97789246668866 106 'com multi unit' zoning definition 9 3279255 16.84508757028482 106 'com multi unit' zoning definition 10 7961686 16.834650485275677 158 +how to add differnt names on labels in word 1 7377142 21.487613677133837 158 +how to add differnt names on labels in word 2 7377143 21.11802010986738 158 +how to add differnt names on labels in word 3 1838655 20.878154701081016 158 +how to add differnt names on labels in word 4 8449881 20.662994705657898 158 +how to add differnt names on labels in word 5 6419624 20.041593133777102 158 +how to add differnt names on labels in word 6 8242972 19.67117001429995 158 +how to add differnt names on labels in word 7 8228431 19.504682282531945 158 +how to add differnt names on labels in word 8 1399730 19.41426860773985 158 +how to add differnt names on labels in word 9 1435171 19.279129505563198 158 +how to add differnt names on labels in word 10 5653693 19.18918138624012 222 +what does ca cells mean urine test 1 267053 24.566283262165005 222 +what does ca cells mean urine test 2 267051 24.126297935219085 222 +what does ca cells mean urine test 3 554374 23.962997363215024 222 +what does ca cells mean urine test 4 4417815 23.705974727898308 222 +what does ca cells mean urine test 5 4417818 23.666715078527403 222 +what does ca cells mean urine test 6 1862820 23.46965928290136 222 +what does ca cells mean urine test 7 4935509 23.40606275319607 222 +what does ca cells mean urine test 8 4174523 23.30969430193029 222 +what does ca cells mean urine test 9 4174521 23.26506600675988 222 +what does ca cells mean urine test 10 5066554 23.109070067765963 233 +what id justin bieber fav color 1 2473952 44.787522005917644 233 +what id justin bieber fav color 2 2473945 37.84696638026389 233 +what id justin bieber fav color 3 23954 31.42172169725862 233 +what id justin bieber fav color 4 2473948 30.98679460248872 233 +what id justin bieber fav color 5 2473949 30.661495196122395 233 +what id justin bieber fav color 6 8546698 30.05619889264124 233 +what id justin bieber fav color 7 7247198 29.478520371585443 233 +what id justin bieber fav color 8 2442360 29.327578205236126 233 +what id justin bieber fav color 9 1346223 29.2613303095308 233 +what id justin bieber fav color 10 7247201 29.207263068259987 428 02 how long has my contract left to run 1 2628203 17.901413538404313 428 02 how long has my contract left to run 2 8703260 17.65132016185667 428 02 how long has my contract left to run 3 1722213 17.51450897424353 428 02 how long has my contract left to run 4 1722215 16.98837472712661 428 02 how long has my contract left to run 5 3274044 16.877629455967547 428 02 how long has my contract left to run 6 5807011 16.4614628587255 428 02 how long has my contract left to run 7 6974602 16.413779152226343 428 02 how long has my contract left to run 8 8359860 16.203263499030328 428 02 how long has my contract left to run 9 2963293 16.110385517406893 428 02 how long has my contract left to run 10 4730718 16.082535917249736 484 1 mile equals how many feet 1 3057824 29.3465610988622 484 1 mile equals how many feet 2 4770537 28.77176747737077 484 1 mile equals how many feet 3 7552264 26.092226794890383 484 1 mile equals how many feet 4 5605934 25.517772294694197 484 1 mile equals how many feet 5 6433661 25.517772294694197 484 1 mile equals how many feet 6 5974007 25.181020598789065 484 1 mile equals how many feet 7 7447821 24.999415585195983 484 1 mile equals how many feet 8 6433656 24.6348765764172 484 1 mile equals how many feet 9 4999103 24.393259595733298 484 1 mile equals how many feet 10 8117210 24.143364500864312 506 1 to 1 1/8 shaft adapter ebay 1 7956293 38.44070034674108 506 1 to 1 1/8 shaft adapter ebay 2 7956290 37.34995503071631 506 1 to 1 1/8 shaft adapter ebay 3 7956291 37.28036700940936 506 1 to 1 1/8 shaft adapter ebay 4 7956294 37.020351481892305 506 1 to 1 1/8 shaft adapter ebay 5 7956299 36.89248504548539 506 1 to 1 1/8 shaft adapter ebay 6 7956298 35.77374171918592 506 1 to 1 1/8 shaft adapter ebay 7 7956292 35.50468003978903 506 1 to 1 1/8 shaft adapter ebay 8 7956295 28.815725988618112 506 1 to 1 1/8 shaft adapter ebay 9 7956296 27.366313771047423 506 1 to 1 1/8 shaft adapter ebay 10 8566710 24.477097263366076 514 1. define lymph and explain its function 1 2268945 23.021630034421072 514 1. define lymph and explain its function 2 8098623 21.426008932604873 514 1. define lymph and explain its function 3 6414971 20.16334238237081 514 1. define lymph and explain its function 4 6286077 20.162468160872837 514 1. define lymph and explain its function 5 3675072 19.38132758988807 514 1. define lymph and explain its function 6 5237562 19.320053248742276 514 1. define lymph and explain its function 7 6954281 18.836750870683574 514 1. define lymph and explain its function 8 2605669 18.64447937283485 514 1. define lymph and explain its function 9 8073240 18.557502867185526 514 1. define lymph and explain its function 10 3736328 18.30635929998735 584 104 bus time las Vegas 1 7659686 33.931692779083704 584 104 bus time las Vegas 2 7659685 32.069596947932986 584 104 bus time las Vegas 3 7295902 31.511066522006004 584 104 bus time las Vegas 4 7659684 30.91362604474877 584 104 bus time las Vegas 5 3233966 30.482629027573324 584 104 bus time las Vegas 6 6730880 30.1100097748096 584 104 bus time las Vegas 7 5775088 29.847382198291847 584 104 bus time las Vegas 8 3252653 29.730703590619953 584 104 bus time las Vegas 9 8471727 29.722320636893716 584 104 bus time las Vegas 10 5690475 29.40327115026525 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 1 1404391 31.366278903595468 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 2 8008473 31.246871351918806 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 3 1404399 30.720000062584408 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 4 3141279 29.92283541326735 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 5 1787469 29.90844452541777 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 6 7193174 29.48445981903459 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 7 7193175 29.48445981903459 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 8 7193176 29.455727327850084 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 9 2844173 29.35914673957069 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 10 2028071 28.85288366438989 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 1 1353757 19.971387253842156 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 2 1520975 19.671607332278324 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 3 8591082 19.526392667870553 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 4 6411184 19.48844292584057 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 5 1345324 19.08027953909205 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 6 259491 18.92688497083616 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 7 6999639 18.830395827533195 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 8 1233123 18.751278749185147 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 9 5856140 18.66947459754372 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 10 6283600 18.6337413900196 413 .what is not the purpose of configuration control when configuration management is applied in a project? 1 7643197 37.516884875028786 413 .what is not the purpose of configuration control when configuration management is applied in a project? 2 2070964 37.13924437847116 413 .what is not the purpose of configuration control when configuration management is applied in a project? 3 7643204 36.63459159340182 413 .what is not the purpose of configuration control when configuration management is applied in a project? 4 7643201 36.448680930891044 413 .what is not the purpose of configuration control when configuration management is applied in a project? 5 7643202 36.448680930891044 413 .what is not the purpose of configuration control when configuration management is applied in a project? 6 5211469 34.51908893949319 413 .what is not the purpose of configuration control when configuration management is applied in a project? 7 2215330 34.50776328129989 413 .what is not the purpose of configuration control when configuration management is applied in a project? 8 1790670 33.66776574649802 413 .what is not the purpose of configuration control when configuration management is applied in a project? 9 4793205 33.53143889076198 413 .what is not the purpose of configuration control when configuration management is applied in a project? 10 840784 33.50723770086486 509 1 us dollar equals how many euros 1 149089 36.197120013945266 509 1 us dollar equals how many euros 2 149083 35.99486409567074 509 1 us dollar equals how many euros 3 7433164 35.116351082740316 509 1 us dollar equals how many euros 4 1804890 34.47626848226087 509 1 us dollar equals how many euros 5 8344829 34.3138607172956 509 1 us dollar equals how many euros 6 8344825 34.30017322162461 509 1 us dollar equals how many euros 7 8344823 32.8166615020934 509 1 us dollar equals how many euros 8 1489670 32.71203381866352 509 1 us dollar equals how many euros 9 7685087 32.45388152441442 509 1 us dollar equals how many euros 10 8344828 32.41744234848657 555 10 factors responsible for depletion of water table 1 7907043 33.85256312948023 555 10 factors responsible for depletion of water table 2 7907040 27.567921967968694 555 10 factors responsible for depletion of water table 3 7907044 21.34742220244792 555 10 factors responsible for depletion of water table 4 7907046 21.315953801328103 555 10 factors responsible for depletion of water table 5 7907049 20.857145129343934 555 10 factors responsible for depletion of water table 6 4142711 20.051188573216546 555 10 factors responsible for depletion of water table 7 7907047 19.974199032350796 555 10 factors responsible for depletion of water table 8 5380459 19.72911210888641 555 10 factors responsible for depletion of water table 9 7907042 19.37458545731268 555 10 factors responsible for depletion of water table 10 5801967 19.36791607897706 667 14 oct 2015 mlb playoff schedule 1 3623336 34.2461261530911 667 14 oct 2015 mlb playoff schedule 2 8634015 33.262314037779134 667 14 oct 2015 mlb playoff schedule 3 7584158 32.32180987225937 667 14 oct 2015 mlb playoff schedule 4 7880799 31.89651964500129 667 14 oct 2015 mlb playoff schedule 5 8601092 31.881587683760607 667 14 oct 2015 mlb playoff schedule 6 1123740 31.866226987003767 667 14 oct 2015 mlb playoff schedule 7 854155 31.03900212410375 667 14 oct 2015 mlb playoff schedule 8 6415043 30.419266924023308 667 14 oct 2015 mlb playoff schedule 9 1998856 30.401621020496407 667 14 oct 2015 mlb playoff schedule 10 2646384 29.457030471983895 699 15 straight line depreciation for leasehold improvements 1 7982767 46.85952835469048 699 15 straight line depreciation for leasehold improvements 2 7982766 46.85952835469048 699 15 straight line depreciation for leasehold improvements 3 2851447 46.08698791010585 699 15 straight line depreciation for leasehold improvements 4 7794540 45.76293412324243 699 15 straight line depreciation for leasehold improvements 5 7982763 44.39463047343927 699 15 straight line depreciation for leasehold improvements 6 7982762 43.7828146482192 699 15 straight line depreciation for leasehold improvements 7 7982760 42.432127954058856 699 15 straight line depreciation for leasehold improvements 8 7802559 42.313505422683676 699 15 straight line depreciation for leasehold improvements 9 7982761 40.47071649460148 699 15 straight line depreciation for leasehold improvements 10 2613365 39.30976392057837 807 1993 penthouse mark whalberg net worth 1 7340068 27.717143624084386 807 1993 penthouse mark whalberg net worth 2 689702 26.277249045573424 807 1993 penthouse mark whalberg net worth 3 6740404 25.90050895182418 807 1993 penthouse mark whalberg net worth 4 6786437 25.62863575670959 807 1993 penthouse mark whalberg net worth 5 1017821 25.4422237312338 807 1993 penthouse mark whalberg net worth 6 6740402 25.366207980367943 807 1993 penthouse mark whalberg net worth 7 1690754 24.93301821402405 807 1993 penthouse mark whalberg net worth 8 3355517 24.870300222152274 807 1993 penthouse mark whalberg net worth 9 8624845 24.856202207436244 807 1993 penthouse mark whalberg net worth 10 4801391 24.786343938911514 854 2 year olds knee hurts when he is sick 1 7959879 27.172466902457636 854 2 year olds knee hurts when he is sick 2 7959882 26.610847176453483 854 2 year olds knee hurts when he is sick 3 7959878 26.072930041023273 854 2 year olds knee hurts when he is sick 4 7959880 25.557183201236285 854 2 year olds knee hurts when he is sick 5 6761725 25.354644086375124 854 2 year olds knee hurts when he is sick 6 7959881 25.062208807961653 854 2 year olds knee hurts when he is sick 7 1482169 23.52896402459006 854 2 year olds knee hurts when he is sick 8 8343061 23.505871353899522 854 2 year olds knee hurts when he is sick 9 1136274 22.383941236962986 854 2 year olds knee hurts when he is sick 10 6463209 21.819033882423653 884 20 oz Post raisin bran price 1 3904427 36.11467295614103 884 20 oz Post raisin bran price 2 7296978 35.74443940099883 884 20 oz Post raisin bran price 3 2690665 35.5019951465861 884 20 oz Post raisin bran price 4 7296976 35.281816203793504 884 20 oz Post raisin bran price 5 7296979 34.99774442248569 884 20 oz Post raisin bran price 6 1960332 32.858574466247475 884 20 oz Post raisin bran price 7 2592023 29.044006099375174 884 20 oz Post raisin bran price 8 1055084 28.767339229960957 884 20 oz Post raisin bran price 9 2592019 28.165201970904022 884 20 oz Post raisin bran price 10 2592022 26.418667049301007 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 1 4132807 22.203229450405995 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 2 5229926 21.091690532688197 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 3 4636291 20.614253967597925 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 4 6195120 20.614253967597925 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 5 7914384 20.515380484875784 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 6 6567831 20.344916328705896 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 7 7038351 20.20077516361406 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 8 7925169 20.096756271071634 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 9 2530517 20.056594316286237 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 10 587355 20.026710067201375 169 +how many calories does a person burn in five minutes of standing 1 1225781 35.466533439189796 169 +how many calories does a person burn in five minutes of standing 2 7346456 32.04245534977608 169 +how many calories does a person burn in five minutes of standing 3 8352427 31.36585584632192 169 +how many calories does a person burn in five minutes of standing 4 6971376 31.357581730573624 169 +how many calories does a person burn in five minutes of standing 5 7994541 31.327482494549145 169 +how many calories does a person burn in five minutes of standing 6 8352434 31.007316264467843 169 +how many calories does a person burn in five minutes of standing 7 3599796 30.58800779442089 169 +how many calories does a person burn in five minutes of standing 8 2440389 30.422645518887585 169 +how many calories does a person burn in five minutes of standing 9 5796796 30.325457340604125 169 +how many calories does a person burn in five minutes of standing 10 2369117 30.080186050266597 327 , does vinegar and water help. remove coffee stains in blouses 1 6756165 36.38122187014534 327 , does vinegar and water help. remove coffee stains in blouses 2 4067845 34.60100286609976 327 , does vinegar and water help. remove coffee stains in blouses 3 7510742 34.15841412558086 327 , does vinegar and water help. remove coffee stains in blouses 4 724456 33.98996682610403 327 , does vinegar and water help. remove coffee stains in blouses 5 4067850 32.358901078270264 327 , does vinegar and water help. remove coffee stains in blouses 6 2588170 31.605561423570798 327 , does vinegar and water help. remove coffee stains in blouses 7 5592277 31.429701874119317 327 , does vinegar and water help. remove coffee stains in blouses 8 8255604 31.384168851368642 327 , does vinegar and water help. remove coffee stains in blouses 9 551278 30.74362208932428 327 , does vinegar and water help. remove coffee stains in blouses 10 5889570 30.578154477667937 829 1x8 block with a sticker grand prix what set does it belong 1 5640619 26.160242933421124 829 1x8 block with a sticker grand prix what set does it belong 2 3245940 26.003094989075215 829 1x8 block with a sticker grand prix what set does it belong 3 3392331 25.94436076399721 829 1x8 block with a sticker grand prix what set does it belong 4 4730916 25.772987060305137 829 1x8 block with a sticker grand prix what set does it belong 5 3245939 25.59008719676632 829 1x8 block with a sticker grand prix what set does it belong 6 4565492 25.382129460676815 829 1x8 block with a sticker grand prix what set does it belong 7 5677487 25.320878933435395 829 1x8 block with a sticker grand prix what set does it belong 8 5677486 25.171053574711205 829 1x8 block with a sticker grand prix what set does it belong 9 2802604 25.109549403323726 829 1x8 block with a sticker grand prix what set does it belong 10 3392333 25.096300730437363 1267 3 ways a log can move when bucking on side hill 1 7759479 27.939036461555474 1267 3 ways a log can move when bucking on side hill 2 7759478 23.595230835661262 1267 3 ways a log can move when bucking on side hill 3 3906880 23.037580234933024 1267 3 ways a log can move when bucking on side hill 4 7759484 21.580010586492687 1267 3 ways a log can move when bucking on side hill 5 1956260 20.29673453061869 1267 3 ways a log can move when bucking on side hill 6 6020772 19.9554093127961 1267 3 ways a log can move when bucking on side hill 7 1855619 19.58465715026421 1267 3 ways a log can move when bucking on side hill 8 5439052 19.516144816808275 1267 3 ways a log can move when bucking on side hill 9 2205571 18.846626232261464 1267 3 ways a log can move when bucking on side hill 10 398482 18.786004901855094 1517 50% of the amount exceed 40 times minimum wage 1 4840199 35.6995818953963 1517 50% of the amount exceed 40 times minimum wage 2 8820765 33.19464767053046 1517 50% of the amount exceed 40 times minimum wage 3 854861 32.50816984386711 1517 50% of the amount exceed 40 times minimum wage 4 718519 32.15001293293812 1517 50% of the amount exceed 40 times minimum wage 5 7396662 31.940582332284354 1517 50% of the amount exceed 40 times minimum wage 6 5898052 31.568192665592093 1517 50% of the amount exceed 40 times minimum wage 7 4840193 30.324879678713337 1517 50% of the amount exceed 40 times minimum wage 8 4840195 30.321298663425303 1517 50% of the amount exceed 40 times minimum wage 9 815441 30.14976987516211 1517 50% of the amount exceed 40 times minimum wage 10 3607745 30.06352661038376 1522 5000 iu of vitamin d causes leg pain 1 6453657 39.66437375792764 1522 5000 iu of vitamin d causes leg pain 2 1252625 38.112143331596755 1522 5000 iu of vitamin d causes leg pain 3 7699593 36.34473947744465 1522 5000 iu of vitamin d causes leg pain 4 8323752 36.18059372136971 1522 5000 iu of vitamin d causes leg pain 5 7445922 35.66945763302019 1522 5000 iu of vitamin d causes leg pain 6 1035425 35.10553628211443 1522 5000 iu of vitamin d causes leg pain 7 7445919 34.92875026097285 1522 5000 iu of vitamin d causes leg pain 8 1035430 34.911463657285935 1522 5000 iu of vitamin d causes leg pain 9 7331171 34.87133809976614 1522 5000 iu of vitamin d causes leg pain 10 5902280 34.35779910318627 1977 American slavery first began in which of the following centuries? quizlet 1 8542387 30.38089284167677 1977 American slavery first began in which of the following centuries? quizlet 2 3980526 27.867646604465612 1977 American slavery first began in which of the following centuries? quizlet 3 3571980 27.011948946198622 1977 American slavery first began in which of the following centuries? quizlet 4 5110015 26.848896251599868 1977 American slavery first began in which of the following centuries? quizlet 5 4503426 25.732398671389866 1977 American slavery first began in which of the following centuries? quizlet 6 3619305 25.534474137399975 1977 American slavery first began in which of the following centuries? quizlet 7 356898 25.24285503833015 1977 American slavery first began in which of the following centuries? quizlet 8 2562006 25.21046801334504 1977 American slavery first began in which of the following centuries? quizlet 9 3189687 25.205886899673644 1977 American slavery first began in which of the following centuries? quizlet 10 7171921 24.862006719794934 2362 CURRENt rate for a color and cut hairstyle in denver co 1 7352725 31.33945078364694 2362 CURRENt rate for a color and cut hairstyle in denver co 2 3286060 29.47651459412085 2362 CURRENt rate for a color and cut hairstyle in denver co 3 1541214 27.31992309357517 2362 CURRENt rate for a color and cut hairstyle in denver co 4 8153280 26.577524088940255 2362 CURRENt rate for a color and cut hairstyle in denver co 5 4988942 25.47752909725802 2362 CURRENt rate for a color and cut hairstyle in denver co 6 3339043 23.191953527031707 2362 CURRENt rate for a color and cut hairstyle in denver co 7 2246144 23.172286931333083 2362 CURRENt rate for a color and cut hairstyle in denver co 8 7484010 23.074931039108804 2362 CURRENt rate for a color and cut hairstyle in denver co 9 3264082 23.074931039108804 2362 CURRENt rate for a color and cut hairstyle in denver co 10 6087720 23.07067641129455 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 1 8113742 20.93932586740845 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 2 6585029 18.836723961717976 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 3 3331637 18.73436240071699 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 4 3000532 18.14426446116472 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 5 7291372 18.113869280704836 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 6 7132215 18.074370957085613 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 7 6585032 18.007437034482578 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 8 4173139 17.914600325118393 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 9 465595 17.81489642897058 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 10 3303071 17.48243709115495 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 1 398984 36.79778000553817 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 2 1864633 34.63349829681018 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 3 1091568 33.6649897233514 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 4 1778113 32.01852547544742 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 5 5608471 31.19227411768498 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 6 1091569 30.669777340650683 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 7 7466019 30.541657001509268 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 8 5806751 29.61739608216591 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 9 4783594 28.494912375804763 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 10 1868142 27.919438455876683 130 ). Puerto Rico says sick will head north without health-care fix 1 3111785 32.129176461466635 130 ). Puerto Rico says sick will head north without health-care fix 2 76454 31.483645825355737 130 ). Puerto Rico says sick will head north without health-care fix 3 1478979 30.808741027058762 130 ). Puerto Rico says sick will head north without health-care fix 4 1478972 30.594173711945608 130 ). Puerto Rico says sick will head north without health-care fix 5 5590104 30.54110646441982 130 ). Puerto Rico says sick will head north without health-care fix 6 834819 29.874291213111384 130 ). Puerto Rico says sick will head north without health-care fix 7 834820 29.265455153206666 130 ). Puerto Rico says sick will head north without health-care fix 8 5780442 29.15734173915525 130 ). Puerto Rico says sick will head north without health-care fix 9 512100 29.005941394782816 130 ). Puerto Rico says sick will head north without health-care fix 10 16626 28.88210516009398 793 1981 Isuzu pup diesel average miles per gallon ? 1 6303727 33.92232881654702 793 1981 Isuzu pup diesel average miles per gallon ? 2 6500912 32.92143214670872 793 1981 Isuzu pup diesel average miles per gallon ? 3 1194895 31.941530003209245 793 1981 Isuzu pup diesel average miles per gallon ? 4 5085499 31.69796281693597 793 1981 Isuzu pup diesel average miles per gallon ? 5 5797124 31.214179087585688 793 1981 Isuzu pup diesel average miles per gallon ? 6 1477834 31.005848575744597 793 1981 Isuzu pup diesel average miles per gallon ? 7 5326567 30.68594768560086 793 1981 Isuzu pup diesel average miles per gallon ? 8 5085504 30.648586766406524 793 1981 Isuzu pup diesel average miles per gallon ? 9 2212048 30.39665405528947 793 1981 Isuzu pup diesel average miles per gallon ? 10 98531 30.009645985249584 814 1999 chevy suburban k2500 7.4l what to look for when buying 1 8000886 46.2886883298325 814 1999 chevy suburban k2500 7.4l what to look for when buying 2 8000889 39.576130225546805 814 1999 chevy suburban k2500 7.4l what to look for when buying 3 7399878 37.73080515001264 814 1999 chevy suburban k2500 7.4l what to look for when buying 4 8000887 37.645283651131976 814 1999 chevy suburban k2500 7.4l what to look for when buying 5 8000881 36.03127982355054 814 1999 chevy suburban k2500 7.4l what to look for when buying 6 8000882 32.590487574423584 814 1999 chevy suburban k2500 7.4l what to look for when buying 7 7547621 25.917209639228123 814 1999 chevy suburban k2500 7.4l what to look for when buying 8 5789679 25.563811638464454 814 1999 chevy suburban k2500 7.4l what to look for when buying 9 5748421 25.336467119736817 814 1999 chevy suburban k2500 7.4l what to look for when buying 10 7403333 25.300228331250963 920 2006 GMC Yukon XL engine is off, but heater blower is still on 1 7068716 48.732993293919854 920 2006 GMC Yukon XL engine is off, but heater blower is still on 2 941524 44.04403647895222 920 2006 GMC Yukon XL engine is off, but heater blower is still on 3 941525 40.96875665000787 920 2006 GMC Yukon XL engine is off, but heater blower is still on 4 7980523 40.63241575100803 920 2006 GMC Yukon XL engine is off, but heater blower is still on 5 7068721 39.07577660260697 920 2006 GMC Yukon XL engine is off, but heater blower is still on 6 2357888 38.50348490134403 920 2006 GMC Yukon XL engine is off, but heater blower is still on 7 7394189 38.08770406252245 920 2006 GMC Yukon XL engine is off, but heater blower is still on 8 1845415 36.24566761000946 920 2006 GMC Yukon XL engine is off, but heater blower is still on 9 2782582 36.034173963655725 920 2006 GMC Yukon XL engine is off, but heater blower is still on 10 7980524 35.791786608839956 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 1 7327631 53.287062037407864 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 2 5323768 40.4139467449184 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 3 6098929 39.841990693952425 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 4 5972828 38.755824328726 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 5 8713377 38.518696268489975 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 6 8701758 38.37761838953604 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 7 8180256 38.080040227830686 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 8 6898829 37.68349060557658 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 9 7451300 37.38678616982831 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 10 3511084 37.03356506771942 1353 39 x 29 foundation crawlspace takes how many cinder blocks 1 950715 36.02259645677271 1353 39 x 29 foundation crawlspace takes how many cinder blocks 2 5521998 33.101573664572186 1353 39 x 29 foundation crawlspace takes how many cinder blocks 3 3752642 32.30139145472882 1353 39 x 29 foundation crawlspace takes how many cinder blocks 4 4788068 30.92029652923693 1353 39 x 29 foundation crawlspace takes how many cinder blocks 5 1581119 30.134961271819673 1353 39 x 29 foundation crawlspace takes how many cinder blocks 6 1581117 28.914429866429675 1353 39 x 29 foundation crawlspace takes how many cinder blocks 7 2202622 27.748716117633485 1353 39 x 29 foundation crawlspace takes how many cinder blocks 8 8180538 27.041425167564764 1353 39 x 29 foundation crawlspace takes how many cinder blocks 9 4788076 26.68487158145443 1353 39 x 29 foundation crawlspace takes how many cinder blocks 10 5581823 26.635014866506484 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 1 6872353 28.843585339150806 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 2 5771957 28.30776626746824 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 3 6872350 27.40833404389889 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 4 4412247 26.398909362821414 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 5 6941843 25.473746499755784 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 6 1039373 25.370038629577895 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 7 1014206 25.370038629577895 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 8 4412250 25.067531370917557 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 9 8470451 25.008626155401764 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 10 6872351 24.56005480414835 1577 61 km long and 3 km wide and average depth of 132m 1 6850832 41.49993894936622 1577 61 km long and 3 km wide and average depth of 132m 2 1805822 39.64841267001415 1577 61 km long and 3 km wide and average depth of 132m 3 7441296 36.91408657794906 1577 61 km long and 3 km wide and average depth of 132m 4 7156558 35.84154579108248 1577 61 km long and 3 km wide and average depth of 132m 5 106897 35.65172906307887 1577 61 km long and 3 km wide and average depth of 132m 6 6109186 34.85263315651749 1577 61 km long and 3 km wide and average depth of 132m 7 1805829 34.57486418568198 1577 61 km long and 3 km wide and average depth of 132m 8 4017985 33.43704308091363 1577 61 km long and 3 km wide and average depth of 132m 9 7857962 33.33692478146118 1577 61 km long and 3 km wide and average depth of 132m 10 2918511 33.04320512360779 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 1 650805 46.17489303526811 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 2 325212 44.13968270657331 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 3 7470096 42.81280159466686 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 4 325208 42.54173982768479 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 5 754910 41.52433620703032 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 6 4069010 40.35455640096317 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 7 754908 40.19475421162249 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 8 5398700 39.93973261210963 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 9 754915 39.31968641719142 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 10 7141027 39.225961681069705 timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/ground_truth_pg18.tsv000066400000000000000000001636331525762047700305040ustar00rootroot00000000000000query_id query_text rank doc_id score 254 +what is cchaps 1 7373206 17.3859955932856 254 +what is cchaps 2 7373205 16.26544766572216 254 +what is cchaps 3 7373204 15.85665560139892 255 +what is centure 1 5337494 16.883701561739528 255 +what is centure 2 7629888 15.704311369973196 255 +what is centure 3 2332343 15.27743675372195 255 +what is centure 4 8672868 15.072585235978094 255 +what is centure 5 7942983 13.950251591592302 255 +what is centure 6 8740723 12.690323752809526 255 +what is centure 7 3534898 12.548656047129782 255 +what is centure 8 2624439 12.142015939139485 255 +what is centure 9 8080899 11.88525425587245 255 +what is centure 10 2280509 11.402986718300756 1724 < symbol 1 901289 10.804106157772686 1724 < symbol 2 7040513 10.712428538256683 1724 < symbol 3 330189 10.548631290745037 1724 < symbol 4 330186 10.548631290745037 1724 < symbol 5 330188 10.535487705230079 1724 < symbol 6 330187 10.513372759536452 1724 < symbol 7 3353273 10.431976378052354 1724 < symbol 8 1244169 10.414651870909909 1724 < symbol 9 7971836 10.391735608015605 1724 < symbol 10 1091451 10.383762129485397 4288 How to have sex 1 5601661 10.331804171254714 4288 How to have sex 2 8397394 10.285989280990554 4288 How to have sex 3 873941 10.255018968033507 4288 How to have sex 4 3527665 10.217052778348553 4288 How to have sex 5 3154655 10.204538294316578 4288 How to have sex 6 1314219 10.195084359771231 4288 How to have sex 7 4293186 10.179366669036224 4288 How to have sex 8 2806469 10.16691505099238 4288 How to have sex 9 309973 10.166827367335372 4288 How to have sex 10 4598378 10.138900978497944 7354 What is my ethnicity 1 8721023 13.009673757528194 7354 What is my ethnicity 2 4742620 12.96866283782917 7354 What is my ethnicity 3 4286243 12.814851993389619 7354 What is my ethnicity 4 4286246 12.77971622490698 7354 What is my ethnicity 5 4463943 12.77971622490698 7354 What is my ethnicity 6 6849222 12.712567180016881 7354 What is my ethnicity 7 4246109 12.657000242646157 7354 What is my ethnicity 8 301574 12.63691181577928 7354 What is my ethnicity 9 5315331 12.625779107687148 7354 What is my ethnicity 10 4655970 12.586968623509849 9340 a mutation is 1 1307633 13.208528855632112 9340 a mutation is 2 951868 13.109871118397523 9340 a mutation is 3 450025 13.101564037851844 9340 a mutation is 4 2917314 12.928864910483437 9340 a mutation is 5 1808727 12.926896861119054 9340 a mutation is 6 8799571 12.882317021697105 9340 a mutation is 7 1370509 12.872520752434916 9340 a mutation is 8 8612630 12.864618965895067 9340 a mutation is 9 8135964 12.82779778744428 9340 a mutation is 10 1307632 12.82779778744428 9351 a novel is a _____. 1 425159 11.701898483294718 9351 a novel is a _____. 2 8794243 11.666474801113688 9351 a novel is a _____. 3 7162299 11.617021565973623 9351 a novel is a _____. 4 3113367 11.429105550805302 9351 a novel is a _____. 5 3049840 11.410551910628133 9351 a novel is a _____. 6 4316285 11.361622218482658 9351 a novel is a _____. 7 2458138 11.361622218482658 9351 a novel is a _____. 8 8818382 11.361622218482658 9351 a novel is a _____. 9 7160625 11.317074323688349 9351 a novel is a _____. 10 5545053 11.317074323688349 16453 amendment you can be who you are 1 7309616 11.65336805883167 16453 amendment you can be who you are 2 234632 11.486854124762608 16453 amendment you can be who you are 3 6214128 11.308318750974435 16453 amendment you can be who you are 4 5339945 11.272075937624317 16453 amendment you can be who you are 5 5380203 11.261572285736511 16453 amendment you can be who you are 6 6419370 11.244016127677895 16453 amendment you can be who you are 7 4150098 11.244016127677895 16453 amendment you can be who you are 8 2516688 11.236064696294342 16453 amendment you can be who you are 9 8654254 11.209067515530448 16453 amendment you can be who you are 10 3258484 11.204966274200155 27302 ash who is ash. 1 1514389 27.534554993598974 27302 ash who is ash. 2 1514386 27.510113295520437 27302 ash who is ash. 3 5472249 27.31426159121804 27302 ash who is ash. 4 1514393 27.105202502589766 27302 ash who is ash. 5 6959114 26.892269744908873 27302 ash who is ash. 6 6959109 26.803128798351164 27302 ash who is ash. 7 8195223 26.792631292070062 27302 ash who is ash. 8 4817989 26.71203826948381 27302 ash who is ash. 9 5834313 26.688319029957043 27302 ash who is ash. 10 5734761 26.507713801975676 92357 churrascaria what is 1 7070979 21.943397473171135 92357 churrascaria what is 2 7070972 21.228118852174195 92357 churrascaria what is 3 7070975 20.19638923195742 92357 churrascaria what is 4 7070978 20.01022050652706 92357 churrascaria what is 5 1381758 18.635943248574485 92357 churrascaria what is 6 7978801 17.86893307432649 92357 churrascaria what is 7 1381754 15.599702892473939 92357 churrascaria what is 8 5233933 15.5630052473084 92357 churrascaria what is 9 4227481 15.378675362238436 92357 churrascaria what is 10 2143172 14.521892418519084 83 you are so dutch means what 1 278633 18.63486930341032 83 you are so dutch means what 2 1082844 18.36071149005347 83 you are so dutch means what 3 5756931 17.23359901871705 83 you are so dutch means what 4 3179538 17.063696992840548 83 you are so dutch means what 5 260525 17.009947531994158 83 you are so dutch means what 6 388763 16.989076607238854 83 you are so dutch means what 7 575717 16.87973495728966 83 you are so dutch means what 8 1979564 16.86194692877695 83 you are so dutch means what 9 848948 16.820502094282837 83 you are so dutch means what 10 7604175 16.80095637325342 102 % saturation do calculation 1 6976611 17.606777368773926 102 % saturation do calculation 2 1634347 17.291106643443552 102 % saturation do calculation 3 8345094 17.078388600749985 102 % saturation do calculation 4 6388328 16.44528073481093 102 % saturation do calculation 5 95458 16.37337341376454 102 % saturation do calculation 6 7396240 16.264696108437114 102 % saturation do calculation 7 4085778 16.208875558613272 102 % saturation do calculation 8 4048762 16.022272686388536 102 % saturation do calculation 9 4184833 15.741649370209636 102 % saturation do calculation 10 6287758 15.708441149571467 214 +what can mirna be used for 1 7994517 22.844992280376484 214 +what can mirna be used for 2 4860329 21.552856393288696 214 +what can mirna be used for 3 7994518 21.353715908170482 214 +what can mirna be used for 4 1258768 21.010890227117873 214 +what can mirna be used for 5 5244753 20.750001403303102 214 +what can mirna be used for 6 2080070 19.856954827088483 214 +what can mirna be used for 7 4767298 19.658679508215457 214 +what can mirna be used for 8 7575 19.358166267541016 214 +what can mirna be used for 9 3887922 19.211182584077132 214 +what can mirna be used for 10 7580 19.062375783262212 220 +what do pathogic organisms do 1 3806899 17.08961233754324 220 +what do pathogic organisms do 2 7073478 16.631311238813833 220 +what do pathogic organisms do 3 2557370 15.20422259349876 220 +what do pathogic organisms do 4 4091393 12.582222504964568 220 +what do pathogic organisms do 5 8841262 7.447426565136349 220 +what do pathogic organisms do 6 8149442 7.447426565136349 220 +what do pathogic organisms do 7 2085067 7.4266946158805975 220 +what do pathogic organisms do 8 2683335 7.416667519911943 220 +what do pathogic organisms do 9 6363761 7.413856361307537 220 +what do pathogic organisms do 10 6965787 7.391717839071723 244 +what is a lmu wall 1 4333527 20.401883705399683 244 +what is a lmu wall 2 7494723 19.88077006757744 244 +what is a lmu wall 3 7494731 19.15686557671884 244 +what is a lmu wall 4 6231730 18.438310216452273 244 +what is a lmu wall 5 3579599 18.373650742328554 244 +what is a lmu wall 6 4333528 18.109818339909076 244 +what is a lmu wall 7 6231734 17.914499506360404 244 +what is a lmu wall 8 7494724 17.914499506360404 244 +what is a lmu wall 9 4070455 17.897250009029424 244 +what is a lmu wall 10 3138100 17.604249627712797 317 +where is highlong park 1 7377457 16.559907589197838 317 +where is highlong park 2 2705703 9.3809150585561 317 +where is highlong park 3 7519913 9.323967387775658 317 +where is highlong park 4 2158849 9.320842304509085 317 +where is highlong park 5 6069576 9.302470424731894 317 +where is highlong park 6 7895309 9.292985120369359 317 +where is highlong park 7 1503702 9.292093416965889 317 +where is highlong park 8 1307622 9.291350461507008 317 +where is highlong park 9 2277045 9.291350461507008 317 +where is highlong park 10 5269566 9.28107235873653 370 . what is the institute for leaders? 1 7166129 16.988109851463015 370 . what is the institute for leaders? 2 8555674 15.78811835790328 370 . what is the institute for leaders? 3 8637982 15.517041235114295 370 . what is the institute for leaders? 4 6777654 15.484612491539355 370 . what is the institute for leaders? 5 8143487 15.415397564403088 370 . what is the institute for leaders? 6 8555673 15.384963694335479 370 . what is the institute for leaders? 7 7330372 15.328111349701821 370 . what is the institute for leaders? 8 5069946 15.30008198706799 370 . what is the institute for leaders? 9 186789 15.25511560519902 370 . what is the institute for leaders? 10 7272646 14.939862296429542 409 .vbs what it mean 1 7991064 22.23535622072094 409 .vbs what it mean 2 3761044 21.495604462548066 409 .vbs what it mean 3 7991068 20.521209122514495 409 .vbs what it mean 4 5434835 19.581291423571713 409 .vbs what it mean 5 8724474 19.540438332287774 409 .vbs what it mean 6 6402283 19.331490984559597 409 .vbs what it mean 7 2716893 18.796998065026994 409 .vbs what it mean 8 2716890 18.54527546058995 409 .vbs what it mean 9 7353652 18.368060362610205 409 .vbs what it mean 10 8240045 17.976631907273376 1288 3/5 of 60 1 7473138 20.968070410716972 1288 3/5 of 60 2 5938786 19.431639478446016 1288 3/5 of 60 3 904704 18.827788087472953 1288 3/5 of 60 4 6991390 18.294257477376625 1288 3/5 of 60 5 7473137 18.294257477376625 1288 3/5 of 60 6 7473136 18.294257477376625 1288 3/5 of 60 7 7473134 18.294257477376625 1288 3/5 of 60 8 7473140 18.038672909554606 1288 3/5 of 60 9 3391800 18.038672909554606 1288 3/5 of 60 10 904705 17.79013136357068 1504 5% of 329300 1 2605711 6.440680321095578 1504 5% of 329300 2 3611691 6.416183414982819 1504 5% of 329300 3 3611689 6.328722608005325 1504 5% of 329300 4 1563631 6.293144622023016 1504 5% of 329300 5 347397 6.275964618210015 1504 5% of 329300 6 841474 6.258636210352685 1504 5% of 329300 7 347392 6.209900442611933 1504 5% of 329300 8 2357255 6.204112670460299 1504 5% of 329300 9 5029515 6.193601149244308 1504 5% of 329300 10 4142129 6.1746721042938955 2 Androgen receptor define 1 1782337 27.030836121669747 2 Androgen receptor define 2 1001873 25.535860891938835 2 Androgen receptor define 3 3634076 25.23028775387614 2 Androgen receptor define 4 6285817 25.013592983006824 2 Androgen receptor define 5 4339075 24.99690568549568 2 Androgen receptor define 6 7496506 24.63450713364079 2 Androgen receptor define 7 6285819 24.36768511185484 2 Androgen receptor define 8 4339068 24.315933627909374 2 Androgen receptor define 9 4339072 24.168158216896632 2 Androgen receptor define 10 6920058 24.1065877119146 58 the knee is what type of joint 1 902245 23.18818212882956 58 the knee is what type of joint 2 1722750 22.329975776858028 58 the knee is what type of joint 3 7273917 21.892313939401667 58 the knee is what type of joint 4 5147996 21.824675820091713 58 the knee is what type of joint 5 164142 21.774009398055323 58 the knee is what type of joint 6 160408 21.710979050672055 58 the knee is what type of joint 7 8551492 21.487130838134735 58 the knee is what type of joint 8 6761719 21.304475314782177 58 the knee is what type of joint 9 5530374 21.250121624261006 58 the knee is what type of joint 10 160404 21.17447433374224 71 what nationality is the name Factor 1 2231513 16.49943987413744 71 what nationality is the name Factor 2 4359473 13.868920181585906 71 what nationality is the name Factor 3 4401632 13.775424636458977 71 what nationality is the name Factor 4 60130 13.013506286705582 71 what nationality is the name Factor 5 2231515 12.700068643348828 71 what nationality is the name Factor 6 7895387 12.623781356159956 71 what nationality is the name Factor 7 4027284 12.561578180984885 71 what nationality is the name Factor 8 274604 12.539702198833044 71 what nationality is the name Factor 9 7438179 12.485189063296785 71 what nationality is the name Factor 10 7895388 12.472064904918103 74 where are protists most abundant in humans 1 5500111 19.876657364710695 74 where are protists most abundant in humans 2 5421885 19.15042764598655 74 where are protists most abundant in humans 3 4754855 18.911145409238998 74 where are protists most abundant in humans 4 7066917 18.887955255557742 74 where are protists most abundant in humans 5 834070 18.585922405319508 74 where are protists most abundant in humans 6 5421887 18.537689011057022 74 where are protists most abundant in humans 7 2173116 18.27672294280887 74 where are protists most abundant in humans 8 2389895 18.094201288354448 74 where are protists most abundant in humans 9 4726269 18.094201288354448 74 where are protists most abundant in humans 10 7882916 18.01123634299811 184 +is biology a social science 1 7973103 24.323438824076135 184 +is biology a social science 2 5081059 23.413267610975463 184 +is biology a social science 3 7273446 23.34234900620375 184 +is biology a social science 4 4035289 23.200181037994966 184 +is biology a social science 5 7688042 23.112068317439977 184 +is biology a social science 6 5081055 23.067423310946765 184 +is biology a social science 7 6047595 22.85937884108674 184 +is biology a social science 8 8360978 22.728017084855736 184 +is biology a social science 9 8780186 22.668589039572122 184 +is biology a social science 10 8712432 22.30390530429262 321 +where is your siatic nerve located 1 7805997 23.690945911375323 321 +where is your siatic nerve located 2 114020 21.89708282571956 321 +where is your siatic nerve located 3 2225703 19.620828808880503 321 +where is your siatic nerve located 4 3155966 15.004016122642486 321 +where is your siatic nerve located 5 5730368 14.418741949628021 321 +where is your siatic nerve located 6 3521669 14.41355466284306 321 +where is your siatic nerve located 7 765376 14.313931017805475 321 +where is your siatic nerve located 8 1659971 14.237931663019278 321 +where is your siatic nerve located 9 3550718 14.23488966252652 321 +where is your siatic nerve located 10 7345758 14.142839645068916 361 . irritability medical definition 1 152235 20.89345608536463 361 . irritability medical definition 2 6625778 17.244430043869183 361 . irritability medical definition 3 2291302 16.693546017581376 361 . irritability medical definition 4 4544177 16.5441419186806 361 . irritability medical definition 5 4544186 16.5441419186806 361 . irritability medical definition 6 4351150 16.49005961149882 361 . irritability medical definition 7 992573 16.49005961149882 361 . irritability medical definition 8 1767633 16.253058890466935 361 . irritability medical definition 9 8025874 16.253058890466935 361 . irritability medical definition 10 7955280 16.167357926622444 401 .net culture code 1 7345855 17.68874896219404 401 .net culture code 2 7345856 16.410871963080627 401 .net culture code 3 7796505 16.15189752273769 401 .net culture code 4 5704124 15.840051224284798 401 .net culture code 5 2166029 15.76754930635455 401 .net culture code 6 5704132 15.747074715348841 401 .net culture code 7 7796498 15.351625489936742 401 .net culture code 8 5704131 15.335571819103746 401 .net culture code 9 8001902 15.328479923217632 401 .net culture code 10 5704130 15.25921858444103 537 1/2 cups in tbsp 1 8808551 28.74016902339512 537 1/2 cups in tbsp 2 835445 28.401993731555596 537 1/2 cups in tbsp 3 835446 28.401993731555596 537 1/2 cups in tbsp 4 5161044 28.064282266099355 537 1/2 cups in tbsp 5 5530333 27.881474110866257 537 1/2 cups in tbsp 6 941813 27.279124789524168 537 1/2 cups in tbsp 7 1504531 27.185345962881797 537 1/2 cups in tbsp 8 1357616 27.182960234984353 537 1/2 cups in tbsp 9 8256266 26.997352023410233 537 1/2 cups in tbsp 10 823339 26.944153369015673 548 10 am PST in AST 1 5108468 22.504509350799875 548 10 am PST in AST 2 8176303 21.850825956880445 548 10 am PST in AST 3 3755378 20.2286914039162 548 10 am PST in AST 4 5108466 19.3551306012928 548 10 am PST in AST 5 3490313 19.31679110007874 548 10 am PST in AST 6 1475660 19.244437587231968 548 10 am PST in AST 7 7070931 19.11477542210205 548 10 am PST in AST 8 6279960 18.99120157186563 548 10 am PST in AST 9 3331852 18.79039790072167 548 10 am PST in AST 10 3530702 18.762367516644517 34 guidelines for mothballing of process plants 1 7327168 34.091632547552166 34 guidelines for mothballing of process plants 2 7327165 33.36674708630404 34 guidelines for mothballing of process plants 3 7327163 31.999825403402546 34 guidelines for mothballing of process plants 4 7327169 26.05141891861436 34 guidelines for mothballing of process plants 5 7327164 25.22762711339406 34 guidelines for mothballing of process plants 6 7327167 24.906703638719932 34 guidelines for mothballing of process plants 7 7327166 24.594035763123006 34 guidelines for mothballing of process plants 8 551997 22.724744944737097 34 guidelines for mothballing of process plants 9 6913017 21.490946278379504 34 guidelines for mothballing of process plants 10 3481679 20.27616468782069 197 +micheal moore does he wear a wig 1 1680692 25.814448780992727 197 +micheal moore does he wear a wig 2 2900936 24.209146857390387 197 +micheal moore does he wear a wig 3 6255832 24.12459015545854 197 +micheal moore does he wear a wig 4 3906126 23.980063016314496 197 +micheal moore does he wear a wig 5 1743866 23.917727616682473 197 +micheal moore does he wear a wig 6 780059 23.660871650638832 197 +micheal moore does he wear a wig 7 7809022 23.599199896500444 197 +micheal moore does he wear a wig 8 5989391 22.692102848829364 197 +micheal moore does he wear a wig 9 5066833 22.533521442266338 197 +micheal moore does he wear a wig 10 8323383 22.49655672304469 215 +what causes bacteria inflamation of face 1 5486709 21.986998730839545 215 +what causes bacteria inflamation of face 2 3373799 21.367550276790425 215 +what causes bacteria inflamation of face 3 2595241 21.262970860350993 215 +what causes bacteria inflamation of face 4 3373800 21.181558954323307 215 +what causes bacteria inflamation of face 5 8591077 21.089664710729743 215 +what causes bacteria inflamation of face 6 5934220 20.463336823674247 215 +what causes bacteria inflamation of face 7 3556225 20.44173727360749 215 +what causes bacteria inflamation of face 8 7039861 20.34855500086173 215 +what causes bacteria inflamation of face 9 3864800 20.112045426394264 215 +what causes bacteria inflamation of face 10 8656951 20.062682298003416 232 +what happen when rome was invaded by germania 1 7859869 22.501824520542094 232 +what happen when rome was invaded by germania 2 8182253 21.31551439515339 232 +what happen when rome was invaded by germania 3 2374070 21.073917073427477 232 +what happen when rome was invaded by germania 4 3219581 20.682316836952623 232 +what happen when rome was invaded by germania 5 6581749 20.09872499890696 232 +what happen when rome was invaded by germania 6 8691039 19.557423015611143 232 +what happen when rome was invaded by germania 7 2374072 19.43879967583285 232 +what happen when rome was invaded by germania 8 8182247 19.36385880074681 232 +what happen when rome was invaded by germania 9 2905526 19.193912929139103 232 +what happen when rome was invaded by germania 10 8182252 19.183681513263288 357 . difference between isometric & dynamic exercises 1 102744 30.558196424218046 357 . difference between isometric & dynamic exercises 2 7101256 30.121658827016695 357 . difference between isometric & dynamic exercises 3 5212508 29.094555969792758 357 . difference between isometric & dynamic exercises 4 4834349 28.134326332582106 357 . difference between isometric & dynamic exercises 5 102743 27.87405486162618 357 . difference between isometric & dynamic exercises 6 7354165 27.215009948734878 357 . difference between isometric & dynamic exercises 7 663623 27.192822045453525 357 . difference between isometric & dynamic exercises 8 663614 26.429698724871024 357 . difference between isometric & dynamic exercises 9 3963910 26.342826038063393 357 . difference between isometric & dynamic exercises 10 7907124 26.27906024488727 362 . what are the president's main duties? explain each 1 7006160 20.462632062056052 362 . what are the president's main duties? explain each 2 908288 20.21835264231002 362 . what are the president's main duties? explain each 3 5087020 19.121213498099515 362 . what are the president's main duties? explain each 4 6510779 18.674628608702367 362 . what are the president's main duties? explain each 5 4181913 18.468404694657345 362 . what are the president's main duties? explain each 6 3728523 18.468404694657345 362 . what are the president's main duties? explain each 7 3948438 18.355407319294923 362 . what are the president's main duties? explain each 8 362544 18.039612100839914 362 . what are the president's main duties? explain each 9 4375132 17.80766477351738 362 . what are the president's main duties? explain each 10 3948440 17.615425015575624 364 . what does the c prefacing a date mean 1 2625347 19.021441639085428 364 . what does the c prefacing a date mean 2 4097272 18.969843120574055 364 . what does the c prefacing a date mean 3 3482104 18.759629699143122 364 . what does the c prefacing a date mean 4 6264008 18.387531450667474 364 . what does the c prefacing a date mean 5 6264010 18.287659383300472 364 . what does the c prefacing a date mean 6 6233756 17.600660803440576 364 . what does the c prefacing a date mean 7 994042 16.940601589766136 364 . what does the c prefacing a date mean 8 5077052 16.53436377298733 364 . what does the c prefacing a date mean 9 127211 16.41020199222509 364 . what does the c prefacing a date mean 10 5825848 16.109474424902956 427 02 Chevy 1500 Specs 1 7322779 26.33596320976399 427 02 Chevy 1500 Specs 2 7322780 24.92279238452173 427 02 Chevy 1500 Specs 3 1686422 24.533385195205664 427 02 Chevy 1500 Specs 4 8012343 24.219928899007815 427 02 Chevy 1500 Specs 5 4016576 24.179685641325264 427 02 Chevy 1500 Specs 6 3072032 23.588696207011175 427 02 Chevy 1500 Specs 7 3072024 23.175659979027188 427 02 Chevy 1500 Specs 8 5748529 22.14070271514423 427 02 Chevy 1500 Specs 9 8012346 22.006722547448852 427 02 Chevy 1500 Specs 10 1671601 21.990097282097405 579 100g chicken carb content 1 7362509 35.08779298096177 579 100g chicken carb content 2 7362513 32.90575278666931 579 100g chicken carb content 3 7362512 30.9498935411841 579 100g chicken carb content 4 7362510 29.52819592129119 579 100g chicken carb content 5 6904332 28.047848464234065 579 100g chicken carb content 6 4706542 27.74398905274984 579 100g chicken carb content 7 8522988 27.729485598866688 579 100g chicken carb content 8 5076354 27.180024340891386 579 100g chicken carb content 9 4706546 26.89622728557034 579 100g chicken carb content 10 8522995 26.749817392220702 609 115 days until baby is due 1 6792759 18.78317794046132 609 115 days until baby is due 2 3339863 18.57607649874825 609 115 days until baby is due 3 1792569 18.229797241830738 609 115 days until baby is due 4 1118391 17.936306870488277 609 115 days until baby is due 5 3335663 17.62956539020589 609 115 days until baby is due 6 3332304 17.254853782899822 609 115 days until baby is due 7 1127847 17.110798510409595 609 115 days until baby is due 8 1562632 17.01486996364119 609 115 days until baby is due 9 4015543 16.933389237692115 609 115 days until baby is due 10 3631185 16.615003536438145 28 definition of Executive Officer defined by FRB 1 2681573 23.5380572808867 28 definition of Executive Officer defined by FRB 2 5632946 23.32515412137452 28 definition of Executive Officer defined by FRB 3 1023224 20.268881961081952 28 definition of Executive Officer defined by FRB 4 612213 20.147402681144374 28 definition of Executive Officer defined by FRB 5 5802383 20.130206837087204 28 definition of Executive Officer defined by FRB 6 6683951 20.014119321930337 28 definition of Executive Officer defined by FRB 7 8430099 19.8931756911823 28 definition of Executive Officer defined by FRB 8 4983661 19.813811139941446 28 definition of Executive Officer defined by FRB 9 7996167 19.72328159717519 28 definition of Executive Officer defined by FRB 10 2424701 19.601701495970225 106 'com multi unit' zoning definition 1 4197946 18.952964146879854 106 'com multi unit' zoning definition 2 5336361 18.601182484377215 106 'com multi unit' zoning definition 3 5532730 18.521927002308633 106 'com multi unit' zoning definition 4 5225971 17.98597767883125 106 'com multi unit' zoning definition 5 7019617 17.722909533277168 106 'com multi unit' zoning definition 6 6346796 17.71600129450722 106 'com multi unit' zoning definition 7 6324045 17.537771495341623 106 'com multi unit' zoning definition 8 5245449 16.97789246668866 106 'com multi unit' zoning definition 9 3279255 16.84508757028482 106 'com multi unit' zoning definition 10 7961686 16.834650485275677 158 +how to add differnt names on labels in word 1 7377142 20.918327159918128 158 +how to add differnt names on labels in word 2 7377143 20.491169530850495 158 +how to add differnt names on labels in word 3 1838655 20.341994083927382 158 +how to add differnt names on labels in word 4 6419624 20.041593133777102 158 +how to add differnt names on labels in word 5 8449881 19.96047147452756 158 +how to add differnt names on labels in word 6 1435171 19.279129505563198 158 +how to add differnt names on labels in word 7 8228431 19.063695447690776 158 +how to add differnt names on labels in word 8 8242972 19.05935864174691 158 +how to add differnt names on labels in word 9 1399730 18.97933941336675 158 +how to add differnt names on labels in word 10 1399729 18.695520185481627 222 +what does ca cells mean urine test 1 267053 24.566283262165005 222 +what does ca cells mean urine test 2 267051 24.126297935219085 222 +what does ca cells mean urine test 3 554374 23.962997363215024 222 +what does ca cells mean urine test 4 4417815 23.705974727898308 222 +what does ca cells mean urine test 5 4417818 23.666715078527403 222 +what does ca cells mean urine test 6 1862820 23.46965928290136 222 +what does ca cells mean urine test 7 4935509 23.40606275319607 222 +what does ca cells mean urine test 8 4174523 23.30969430193029 222 +what does ca cells mean urine test 9 4174521 23.26506600675988 222 +what does ca cells mean urine test 10 5066554 23.109070067765963 233 +what id justin bieber fav color 1 2473952 44.787522005917644 233 +what id justin bieber fav color 2 2473945 37.84696638026389 233 +what id justin bieber fav color 3 23954 31.42172169725862 233 +what id justin bieber fav color 4 2473948 30.98679460248872 233 +what id justin bieber fav color 5 2473949 30.661495196122395 233 +what id justin bieber fav color 6 8546698 30.05619889264124 233 +what id justin bieber fav color 7 7247198 29.478520371585443 233 +what id justin bieber fav color 8 2442360 29.327578205236126 233 +what id justin bieber fav color 9 1346223 29.2613303095308 233 +what id justin bieber fav color 10 7247201 29.207263068259987 428 02 how long has my contract left to run 1 2628203 17.901413538404313 428 02 how long has my contract left to run 2 8703260 17.65132016185667 428 02 how long has my contract left to run 3 1722213 17.51450897424353 428 02 how long has my contract left to run 4 1722215 16.98837472712661 428 02 how long has my contract left to run 5 3274044 16.877629455967547 428 02 how long has my contract left to run 6 5807011 16.4614628587255 428 02 how long has my contract left to run 7 6974602 16.413779152226343 428 02 how long has my contract left to run 8 8359860 16.203263499030328 428 02 how long has my contract left to run 9 2963293 16.110385517406893 428 02 how long has my contract left to run 10 4730718 16.082535917249736 484 1 mile equals how many feet 1 3057824 29.3465610988622 484 1 mile equals how many feet 2 4770537 28.77176747737077 484 1 mile equals how many feet 3 7552264 26.092226794890383 484 1 mile equals how many feet 4 5605934 25.517772294694197 484 1 mile equals how many feet 5 6433661 25.517772294694197 484 1 mile equals how many feet 6 5974007 25.181020598789065 484 1 mile equals how many feet 7 7447821 24.999415585195983 484 1 mile equals how many feet 8 6433656 24.6348765764172 484 1 mile equals how many feet 9 4999103 24.393259595733298 484 1 mile equals how many feet 10 8117210 24.143364500864312 506 1 to 1 1/8 shaft adapter ebay 1 7956293 38.44070034674108 506 1 to 1 1/8 shaft adapter ebay 2 7956290 37.34995503071631 506 1 to 1 1/8 shaft adapter ebay 3 7956291 37.28036700940936 506 1 to 1 1/8 shaft adapter ebay 4 7956294 37.020351481892305 506 1 to 1 1/8 shaft adapter ebay 5 7956299 36.89248504548539 506 1 to 1 1/8 shaft adapter ebay 6 7956298 35.77374171918592 506 1 to 1 1/8 shaft adapter ebay 7 7956292 35.50468003978903 506 1 to 1 1/8 shaft adapter ebay 8 7956295 28.815725988618112 506 1 to 1 1/8 shaft adapter ebay 9 7956296 27.366313771047423 506 1 to 1 1/8 shaft adapter ebay 10 8566710 24.477097263366076 514 1. define lymph and explain its function 1 2268945 23.021630034421072 514 1. define lymph and explain its function 2 8098623 21.426008932604873 514 1. define lymph and explain its function 3 6414971 20.16334238237081 514 1. define lymph and explain its function 4 6286077 20.162468160872837 514 1. define lymph and explain its function 5 3675072 19.38132758988807 514 1. define lymph and explain its function 6 5237562 19.320053248742276 514 1. define lymph and explain its function 7 6954281 18.836750870683574 514 1. define lymph and explain its function 8 2605669 18.64447937283485 514 1. define lymph and explain its function 9 8073240 18.557502867185526 514 1. define lymph and explain its function 10 3736328 18.30635929998735 584 104 bus time las Vegas 1 7659686 33.931692779083704 584 104 bus time las Vegas 2 7659685 32.069596947932986 584 104 bus time las Vegas 3 7295902 31.511066522006004 584 104 bus time las Vegas 4 7659684 30.91362604474877 584 104 bus time las Vegas 5 3233966 30.482629027573324 584 104 bus time las Vegas 6 6730880 30.1100097748096 584 104 bus time las Vegas 7 5775088 29.847382198291847 584 104 bus time las Vegas 8 3252653 29.730703590619953 584 104 bus time las Vegas 9 8471727 29.722320636893716 584 104 bus time las Vegas 10 5690475 29.40327115026525 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 1 1404391 31.366278903595468 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 2 8008473 31.246871351918806 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 3 1404399 30.720000062584408 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 4 3141279 29.92283541326735 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 5 1787469 29.90844452541777 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 6 7193174 29.48445981903459 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 7 7193175 29.48445981903459 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 8 7193176 29.455727327850084 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 9 2844173 29.35914673957069 128 ) what other tissue(s) might harbor streptococcus pyogenes? coursehero 10 2028071 28.85288366438989 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 1 1353757 19.971387253842156 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 2 1520975 19.671607332278324 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 3 8591082 19.526392667870553 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 4 6411184 19.48844292584057 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 5 1345324 19.08027953909205 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 6 259491 18.92688497083616 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 7 6999639 18.830395827533195 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 8 1233123 18.751278749185147 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 9 5856140 18.66947459754372 352 . What was the movement called humanism? How did it originate, and what were its main characteristics? 10 6283600 18.6337413900196 413 .what is not the purpose of configuration control when configuration management is applied in a project? 1 7643197 37.516884875028786 413 .what is not the purpose of configuration control when configuration management is applied in a project? 2 2070964 37.13924437847116 413 .what is not the purpose of configuration control when configuration management is applied in a project? 3 7643204 36.63459159340182 413 .what is not the purpose of configuration control when configuration management is applied in a project? 4 7643201 36.448680930891044 413 .what is not the purpose of configuration control when configuration management is applied in a project? 5 7643202 36.448680930891044 413 .what is not the purpose of configuration control when configuration management is applied in a project? 6 5211469 34.51908893949319 413 .what is not the purpose of configuration control when configuration management is applied in a project? 7 2215330 34.50776328129989 413 .what is not the purpose of configuration control when configuration management is applied in a project? 8 1790670 33.66776574649802 413 .what is not the purpose of configuration control when configuration management is applied in a project? 9 4793205 33.53143889076198 413 .what is not the purpose of configuration control when configuration management is applied in a project? 10 840784 33.50723770086486 509 1 us dollar equals how many euros 1 149089 36.197120013945266 509 1 us dollar equals how many euros 2 149083 35.99486409567074 509 1 us dollar equals how many euros 3 7433164 35.116351082740316 509 1 us dollar equals how many euros 4 1804890 34.47626848226087 509 1 us dollar equals how many euros 5 8344829 34.3138607172956 509 1 us dollar equals how many euros 6 8344825 34.30017322162461 509 1 us dollar equals how many euros 7 8344823 32.8166615020934 509 1 us dollar equals how many euros 8 1489670 32.71203381866352 509 1 us dollar equals how many euros 9 7685087 32.45388152441442 509 1 us dollar equals how many euros 10 8344828 32.41744234848657 555 10 factors responsible for depletion of water table 1 7907043 33.85256312948023 555 10 factors responsible for depletion of water table 2 7907040 27.567921967968694 555 10 factors responsible for depletion of water table 3 7907044 21.34742220244792 555 10 factors responsible for depletion of water table 4 7907046 21.315953801328103 555 10 factors responsible for depletion of water table 5 7907049 20.857145129343934 555 10 factors responsible for depletion of water table 6 4142711 20.051188573216546 555 10 factors responsible for depletion of water table 7 7907047 19.974199032350796 555 10 factors responsible for depletion of water table 8 5380459 19.72911210888641 555 10 factors responsible for depletion of water table 9 7907042 19.37458545731268 555 10 factors responsible for depletion of water table 10 5801967 19.36791607897706 667 14 oct 2015 mlb playoff schedule 1 3623336 34.2461261530911 667 14 oct 2015 mlb playoff schedule 2 8634015 33.262314037779134 667 14 oct 2015 mlb playoff schedule 3 7584158 32.32180987225937 667 14 oct 2015 mlb playoff schedule 4 7880799 31.89651964500129 667 14 oct 2015 mlb playoff schedule 5 8601092 31.881587683760607 667 14 oct 2015 mlb playoff schedule 6 1123740 31.866226987003767 667 14 oct 2015 mlb playoff schedule 7 854155 31.03900212410375 667 14 oct 2015 mlb playoff schedule 8 6415043 30.419266924023308 667 14 oct 2015 mlb playoff schedule 9 1998856 30.401621020496407 667 14 oct 2015 mlb playoff schedule 10 2646384 29.457030471983895 699 15 straight line depreciation for leasehold improvements 1 7982767 46.85952835469048 699 15 straight line depreciation for leasehold improvements 2 7982766 46.85952835469048 699 15 straight line depreciation for leasehold improvements 3 2851447 46.08698791010585 699 15 straight line depreciation for leasehold improvements 4 7794540 45.76293412324243 699 15 straight line depreciation for leasehold improvements 5 7982763 44.39463047343927 699 15 straight line depreciation for leasehold improvements 6 7982762 43.7828146482192 699 15 straight line depreciation for leasehold improvements 7 7982760 42.432127954058856 699 15 straight line depreciation for leasehold improvements 8 7802559 42.313505422683676 699 15 straight line depreciation for leasehold improvements 9 7982761 40.47071649460148 699 15 straight line depreciation for leasehold improvements 10 2613365 39.30976392057837 807 1993 penthouse mark whalberg net worth 1 7340068 27.717143624084386 807 1993 penthouse mark whalberg net worth 2 689702 26.277249045573424 807 1993 penthouse mark whalberg net worth 3 6740404 25.90050895182418 807 1993 penthouse mark whalberg net worth 4 6786437 25.62863575670959 807 1993 penthouse mark whalberg net worth 5 1017821 25.4422237312338 807 1993 penthouse mark whalberg net worth 6 6740402 25.366207980367943 807 1993 penthouse mark whalberg net worth 7 1690754 24.93301821402405 807 1993 penthouse mark whalberg net worth 8 3355517 24.870300222152274 807 1993 penthouse mark whalberg net worth 9 8624845 24.856202207436244 807 1993 penthouse mark whalberg net worth 10 4801391 24.786343938911514 854 2 year olds knee hurts when he is sick 1 7959879 27.172466902457636 854 2 year olds knee hurts when he is sick 2 7959882 26.610847176453483 854 2 year olds knee hurts when he is sick 3 7959878 26.072930041023273 854 2 year olds knee hurts when he is sick 4 7959880 25.557183201236285 854 2 year olds knee hurts when he is sick 5 6761725 25.354644086375124 854 2 year olds knee hurts when he is sick 6 7959881 25.062208807961653 854 2 year olds knee hurts when he is sick 7 1482169 23.52896402459006 854 2 year olds knee hurts when he is sick 8 8343061 23.505871353899522 854 2 year olds knee hurts when he is sick 9 1136274 22.383941236962986 854 2 year olds knee hurts when he is sick 10 6463209 21.819033882423653 884 20 oz Post raisin bran price 1 3904427 36.11467295614103 884 20 oz Post raisin bran price 2 7296978 35.74443940099883 884 20 oz Post raisin bran price 3 2690665 35.5019951465861 884 20 oz Post raisin bran price 4 7296976 35.281816203793504 884 20 oz Post raisin bran price 5 7296979 34.99774442248569 884 20 oz Post raisin bran price 6 1960332 32.858574466247475 884 20 oz Post raisin bran price 7 2592023 29.044006099375174 884 20 oz Post raisin bran price 8 1055084 28.767339229960957 884 20 oz Post raisin bran price 9 2592019 28.165201970904022 884 20 oz Post raisin bran price 10 2592022 26.418667049301007 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 1 4132807 22.203229450405995 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 2 5229926 21.091690532688197 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 3 4636291 20.614253967597925 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 4 6195120 20.614253967597925 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 5 7914384 20.515380484875784 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 6 6567831 20.344916328705896 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 7 7038351 20.20077516361406 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 8 7925169 20.096756271071634 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 9 2530517 20.056594316286237 127 ) what is the acceleration of the upper block? (indicate the direction with the sign of your answer.) 10 587355 20.026710067201375 169 +how many calories does a person burn in five minutes of standing 1 1225781 35.466533439189796 169 +how many calories does a person burn in five minutes of standing 2 7346456 32.04245534977608 169 +how many calories does a person burn in five minutes of standing 3 8352427 31.36585584632192 169 +how many calories does a person burn in five minutes of standing 4 6971376 31.357581730573624 169 +how many calories does a person burn in five minutes of standing 5 7994541 31.327482494549145 169 +how many calories does a person burn in five minutes of standing 6 8352434 31.007316264467843 169 +how many calories does a person burn in five minutes of standing 7 3599796 30.58800779442089 169 +how many calories does a person burn in five minutes of standing 8 2440389 30.422645518887585 169 +how many calories does a person burn in five minutes of standing 9 5796796 30.325457340604125 169 +how many calories does a person burn in five minutes of standing 10 2369117 30.080186050266597 327 , does vinegar and water help. remove coffee stains in blouses 1 6756165 36.38122187014534 327 , does vinegar and water help. remove coffee stains in blouses 2 4067845 34.60100286609976 327 , does vinegar and water help. remove coffee stains in blouses 3 7510742 34.15841412558086 327 , does vinegar and water help. remove coffee stains in blouses 4 724456 33.98996682610403 327 , does vinegar and water help. remove coffee stains in blouses 5 4067850 32.358901078270264 327 , does vinegar and water help. remove coffee stains in blouses 6 2588170 31.605561423570798 327 , does vinegar and water help. remove coffee stains in blouses 7 5592277 31.429701874119317 327 , does vinegar and water help. remove coffee stains in blouses 8 8255604 31.384168851368642 327 , does vinegar and water help. remove coffee stains in blouses 9 551278 30.74362208932428 327 , does vinegar and water help. remove coffee stains in blouses 10 5889570 30.578154477667937 829 1x8 block with a sticker grand prix what set does it belong 1 5640619 26.160242933421124 829 1x8 block with a sticker grand prix what set does it belong 2 3245940 26.003094989075215 829 1x8 block with a sticker grand prix what set does it belong 3 3392331 25.94436076399721 829 1x8 block with a sticker grand prix what set does it belong 4 4730916 25.772987060305137 829 1x8 block with a sticker grand prix what set does it belong 5 3245939 25.59008719676632 829 1x8 block with a sticker grand prix what set does it belong 6 4565492 25.382129460676815 829 1x8 block with a sticker grand prix what set does it belong 7 5677487 25.320878933435395 829 1x8 block with a sticker grand prix what set does it belong 8 5677486 25.171053574711205 829 1x8 block with a sticker grand prix what set does it belong 9 2802604 25.109549403323726 829 1x8 block with a sticker grand prix what set does it belong 10 3392333 25.096300730437363 1267 3 ways a log can move when bucking on side hill 1 7759479 27.939036461555474 1267 3 ways a log can move when bucking on side hill 2 7759478 23.595230835661262 1267 3 ways a log can move when bucking on side hill 3 3906880 23.037580234933024 1267 3 ways a log can move when bucking on side hill 4 7759484 21.580010586492687 1267 3 ways a log can move when bucking on side hill 5 1956260 20.29673453061869 1267 3 ways a log can move when bucking on side hill 6 6020772 19.9554093127961 1267 3 ways a log can move when bucking on side hill 7 1855619 19.58465715026421 1267 3 ways a log can move when bucking on side hill 8 5439052 19.516144816808275 1267 3 ways a log can move when bucking on side hill 9 2205571 18.846626232261464 1267 3 ways a log can move when bucking on side hill 10 398482 18.786004901855094 1517 50% of the amount exceed 40 times minimum wage 1 4840199 35.6995818953963 1517 50% of the amount exceed 40 times minimum wage 2 8820765 33.19464767053046 1517 50% of the amount exceed 40 times minimum wage 3 854861 32.50816984386711 1517 50% of the amount exceed 40 times minimum wage 4 718519 32.15001293293812 1517 50% of the amount exceed 40 times minimum wage 5 7396662 31.940582332284354 1517 50% of the amount exceed 40 times minimum wage 6 5898052 31.568192665592093 1517 50% of the amount exceed 40 times minimum wage 7 4840193 30.324879678713337 1517 50% of the amount exceed 40 times minimum wage 8 4840195 30.321298663425303 1517 50% of the amount exceed 40 times minimum wage 9 815441 30.14976987516211 1517 50% of the amount exceed 40 times minimum wage 10 3607745 30.06352661038376 1522 5000 iu of vitamin d causes leg pain 1 6453657 39.66437375792764 1522 5000 iu of vitamin d causes leg pain 2 1252625 38.112143331596755 1522 5000 iu of vitamin d causes leg pain 3 7699593 36.34473947744465 1522 5000 iu of vitamin d causes leg pain 4 8323752 36.18059372136971 1522 5000 iu of vitamin d causes leg pain 5 7445922 35.66945763302019 1522 5000 iu of vitamin d causes leg pain 6 1035425 35.10553628211443 1522 5000 iu of vitamin d causes leg pain 7 7445919 34.92875026097285 1522 5000 iu of vitamin d causes leg pain 8 1035430 34.911463657285935 1522 5000 iu of vitamin d causes leg pain 9 7331171 34.87133809976614 1522 5000 iu of vitamin d causes leg pain 10 5902280 34.35779910318627 1977 American slavery first began in which of the following centuries? quizlet 1 8542387 30.38089284167677 1977 American slavery first began in which of the following centuries? quizlet 2 3980526 27.867646604465612 1977 American slavery first began in which of the following centuries? quizlet 3 3571980 27.011948946198622 1977 American slavery first began in which of the following centuries? quizlet 4 5110015 26.848896251599868 1977 American slavery first began in which of the following centuries? quizlet 5 4503426 25.732398671389866 1977 American slavery first began in which of the following centuries? quizlet 6 3619305 25.534474137399975 1977 American slavery first began in which of the following centuries? quizlet 7 356898 25.24285503833015 1977 American slavery first began in which of the following centuries? quizlet 8 2562006 25.21046801334504 1977 American slavery first began in which of the following centuries? quizlet 9 3189687 25.205886899673644 1977 American slavery first began in which of the following centuries? quizlet 10 7171921 24.862006719794934 2362 CURRENt rate for a color and cut hairstyle in denver co 1 7352725 31.33945078364694 2362 CURRENt rate for a color and cut hairstyle in denver co 2 3286060 29.47651459412085 2362 CURRENt rate for a color and cut hairstyle in denver co 3 1541214 27.31992309357517 2362 CURRENt rate for a color and cut hairstyle in denver co 4 8153280 26.577524088940255 2362 CURRENt rate for a color and cut hairstyle in denver co 5 4988942 25.47752909725802 2362 CURRENt rate for a color and cut hairstyle in denver co 6 3339043 23.191953527031707 2362 CURRENt rate for a color and cut hairstyle in denver co 7 2246144 23.172286931333083 2362 CURRENt rate for a color and cut hairstyle in denver co 8 7484010 23.074931039108804 2362 CURRENt rate for a color and cut hairstyle in denver co 9 3264082 23.074931039108804 2362 CURRENt rate for a color and cut hairstyle in denver co 10 6087720 23.07067641129455 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 1 8113742 20.93932586740845 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 2 6585029 18.836723961717976 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 3 3331637 18.73436240071699 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 4 3000532 18.14426446116472 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 5 7291372 18.113869280704836 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 6 7132215 18.074370957085613 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 7 6585032 18.007437034482578 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 8 4173139 17.914600325118393 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 9 465595 17.81489642897058 2407 Can you drive to Reserva Nacional Radal Siete Tazas from Santiago 10 3303071 17.48243709115495 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 1 398984 36.79778000553817 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 2 1864633 34.63349829681018 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 3 1091568 33.6649897233514 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 4 1778113 32.01852547544742 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 5 5608471 31.19227411768498 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 6 1091569 30.669777340650683 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 7 7466019 30.541657001509268 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 8 5806751 29.61739608216591 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 9 4783594 28.494912375804763 52 pneumo is a prefix meaning air. Knowing this, explain why this condition is called pneumothorax 10 1868142 27.919438455876683 130 ). Puerto Rico says sick will head north without health-care fix 1 3111785 32.129176461466635 130 ). Puerto Rico says sick will head north without health-care fix 2 76454 31.483645825355737 130 ). Puerto Rico says sick will head north without health-care fix 3 1478979 30.808741027058762 130 ). Puerto Rico says sick will head north without health-care fix 4 1478972 30.594173711945608 130 ). Puerto Rico says sick will head north without health-care fix 5 5590104 30.54110646441982 130 ). Puerto Rico says sick will head north without health-care fix 6 834819 29.874291213111384 130 ). Puerto Rico says sick will head north without health-care fix 7 834820 29.265455153206666 130 ). Puerto Rico says sick will head north without health-care fix 8 5780442 29.15734173915525 130 ). Puerto Rico says sick will head north without health-care fix 9 512100 29.005941394782816 130 ). Puerto Rico says sick will head north without health-care fix 10 16626 28.88210516009398 793 1981 Isuzu pup diesel average miles per gallon ? 1 6303727 33.92232881654702 793 1981 Isuzu pup diesel average miles per gallon ? 2 6500912 32.92143214670872 793 1981 Isuzu pup diesel average miles per gallon ? 3 1194895 31.941530003209245 793 1981 Isuzu pup diesel average miles per gallon ? 4 5085499 31.69796281693597 793 1981 Isuzu pup diesel average miles per gallon ? 5 5797124 31.214179087585688 793 1981 Isuzu pup diesel average miles per gallon ? 6 1477834 31.005848575744597 793 1981 Isuzu pup diesel average miles per gallon ? 7 5326567 30.68594768560086 793 1981 Isuzu pup diesel average miles per gallon ? 8 5085504 30.648586766406524 793 1981 Isuzu pup diesel average miles per gallon ? 9 2212048 30.39665405528947 793 1981 Isuzu pup diesel average miles per gallon ? 10 98531 30.009645985249584 814 1999 chevy suburban k2500 7.4l what to look for when buying 1 8000886 46.2886883298325 814 1999 chevy suburban k2500 7.4l what to look for when buying 2 8000889 39.576130225546805 814 1999 chevy suburban k2500 7.4l what to look for when buying 3 7399878 37.73080515001264 814 1999 chevy suburban k2500 7.4l what to look for when buying 4 8000887 37.645283651131976 814 1999 chevy suburban k2500 7.4l what to look for when buying 5 8000881 36.03127982355054 814 1999 chevy suburban k2500 7.4l what to look for when buying 6 8000882 32.590487574423584 814 1999 chevy suburban k2500 7.4l what to look for when buying 7 7547621 25.917209639228123 814 1999 chevy suburban k2500 7.4l what to look for when buying 8 5789679 25.563811638464454 814 1999 chevy suburban k2500 7.4l what to look for when buying 9 5748421 25.336467119736817 814 1999 chevy suburban k2500 7.4l what to look for when buying 10 7403333 25.300228331250963 920 2006 GMC Yukon XL engine is off, but heater blower is still on 1 7068716 48.732993293919854 920 2006 GMC Yukon XL engine is off, but heater blower is still on 2 941524 44.04403647895222 920 2006 GMC Yukon XL engine is off, but heater blower is still on 3 941525 40.96875665000787 920 2006 GMC Yukon XL engine is off, but heater blower is still on 4 7980523 40.63241575100803 920 2006 GMC Yukon XL engine is off, but heater blower is still on 5 7068721 39.07577660260697 920 2006 GMC Yukon XL engine is off, but heater blower is still on 6 2357888 38.50348490134403 920 2006 GMC Yukon XL engine is off, but heater blower is still on 7 7394189 38.08770406252245 920 2006 GMC Yukon XL engine is off, but heater blower is still on 8 1845415 36.24566761000946 920 2006 GMC Yukon XL engine is off, but heater blower is still on 9 2782582 36.034173963655725 920 2006 GMC Yukon XL engine is off, but heater blower is still on 10 7980524 35.791786608839956 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 1 7327631 53.287062037407864 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 2 5323768 40.4139467449184 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 3 6098929 39.841990693952425 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 4 5972828 38.755824328726 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 5 8713377 38.518696268489975 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 6 8701758 38.37761838953604 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 7 8180256 38.080040227830686 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 8 6898829 37.68349060557658 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 9 7451300 37.38678616982831 1279 3. how might a nitrogen-oxygen atmosphere on the moon affect the range of temperatures on the moon 10 3511084 37.03356506771942 1353 39 x 29 foundation crawlspace takes how many cinder blocks 1 950715 36.02259645677271 1353 39 x 29 foundation crawlspace takes how many cinder blocks 2 5521998 33.101573664572186 1353 39 x 29 foundation crawlspace takes how many cinder blocks 3 3752642 32.30139145472882 1353 39 x 29 foundation crawlspace takes how many cinder blocks 4 4788068 30.92029652923693 1353 39 x 29 foundation crawlspace takes how many cinder blocks 5 1581119 30.134961271819673 1353 39 x 29 foundation crawlspace takes how many cinder blocks 6 1581117 28.914429866429675 1353 39 x 29 foundation crawlspace takes how many cinder blocks 7 2202622 27.748716117633485 1353 39 x 29 foundation crawlspace takes how many cinder blocks 8 8180538 27.041425167564764 1353 39 x 29 foundation crawlspace takes how many cinder blocks 9 4788076 26.68487158145443 1353 39 x 29 foundation crawlspace takes how many cinder blocks 10 5581823 26.635014866506484 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 1 6872353 28.843585339150806 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 2 5771957 28.30776626746824 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 3 6872350 27.40833404389889 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 4 4412247 26.398909362821414 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 5 6941843 25.473746499755784 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 6 1039373 25.370038629577895 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 7 1014206 25.370038629577895 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 8 4412250 25.067531370917557 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 9 8470451 25.008626155401764 1384 4 seasons tree service llc in breedsville michigan reviews on yelp 10 6872351 24.56005480414835 1577 61 km long and 3 km wide and average depth of 132m 1 6850832 41.49993894936622 1577 61 km long and 3 km wide and average depth of 132m 2 1805822 39.64841267001415 1577 61 km long and 3 km wide and average depth of 132m 3 7441296 36.91408657794906 1577 61 km long and 3 km wide and average depth of 132m 4 7156558 35.84154579108248 1577 61 km long and 3 km wide and average depth of 132m 5 106897 35.65172906307887 1577 61 km long and 3 km wide and average depth of 132m 6 6109186 34.85263315651749 1577 61 km long and 3 km wide and average depth of 132m 7 1805829 34.57486418568198 1577 61 km long and 3 km wide and average depth of 132m 8 4017985 33.43704308091363 1577 61 km long and 3 km wide and average depth of 132m 9 7857962 33.33692478146118 1577 61 km long and 3 km wide and average depth of 132m 10 2918511 33.04320512360779 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 1 650805 46.17489303526811 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 2 325212 44.13968270657331 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 3 7470096 42.81280159466686 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 4 325208 42.54173982768479 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 5 754910 41.52433620703032 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 6 4069010 40.35455640096317 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 7 754908 40.19475421162249 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 8 5398700 39.93973261210963 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 9 754915 39.31968641719142 1762 A bond is quoted as 6.5% coupon. The bondholder would receive _____ interest per year. 10 7141027 39.225961681069705 timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/insert/000077500000000000000000000000001525762047700256535ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/insert/concurrent-setup.sql000066400000000000000000000057001525762047700317160ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Concurrent Insert Setup -- Loads passages into a staging table, creates empty target table -- with BM25 index, and prepares a sequence for pgbench-driven -- concurrent inserts. -- -- Usage: -- DATA_DIR=/path/to/data psql -f insert/concurrent-setup.sql -- -- After running this script, use pgbench with pgbench-insert.sql -- to drive concurrent inserts from staging into the indexed table. \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO Concurrent Insert Setup ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages CASCADE; DROP TABLE IF EXISTS msmarco_queries CASCADE; DROP TABLE IF EXISTS msmarco_qrels CASCADE; DROP TABLE IF EXISTS msmarco_staging CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create staging table \echo 'Creating staging table...' CREATE TABLE msmarco_staging ( staging_id SERIAL PRIMARY KEY, passage_id INTEGER, passage_text TEXT ); -- Load passages into staging \echo 'Loading passages into staging table...' \copy msmarco_staging(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Create target passages table \echo 'Creating target passages table...' CREATE TABLE msmarco_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create BM25 index on empty target table \echo 'Creating BM25 index on empty target table...' CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_qrels ( query_id INTEGER NOT NULL, passage_id INTEGER NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load queries \echo 'Loading queries...' \copy msmarco_queries(query_id, query_text) FROM PROGRAM 'head -10000 "$DATA_DIR/queries.dev.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id INTEGER, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/qrels.dev.small.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_qrels (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Create sequence for pgbench to draw row IDs from \echo 'Creating insert sequence...' CREATE SEQUENCE insert_seq; -- Report staging row count \echo '' \echo '=== Setup Verification ===' SELECT 'STAGING_ROW_COUNT:' as label, COUNT(*)::text as count FROM msmarco_staging; \echo '' \echo '=== Concurrent Insert Setup Complete ===' \echo 'Run pgbench with pgbench-insert.sql to start inserts' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/insert/load.sql000066400000000000000000000061311525762047700273140ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Insert Benchmark (Single Transaction) -- Creates BM25 index on an empty table, then loads data via COPY -- to measure insert-path indexing performance. -- -- Usage: -- DATA_DIR=/path/to/data psql -f insert/load.sql -- -- The DATA_DIR environment variable should point to the directory -- containing: -- - collection.tsv (passage_id, passage_text) -- - queries.dev.tsv (query_id, query_text) -- - qrels.dev.small.tsv (query_id, 0, passage_id, relevance) \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO Insert Benchmark - Data Loading ===' \echo 'Index created BEFORE loading ~8.8M passages' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages CASCADE; DROP TABLE IF EXISTS msmarco_queries CASCADE; DROP TABLE IF EXISTS msmarco_qrels CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create BM25 index on the empty table \echo 'Creating BM25 index on empty table...' CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_qrels ( query_id INTEGER NOT NULL, passage_id INTEGER NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages into the pre-indexed table \echo '' \echo 'INSERT_TIME:' \echo 'Loading passages (this may take a while with live index)...' \copy msmarco_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries \echo 'Loading queries...' \copy msmarco_queries(query_id, query_text) FROM PROGRAM 'head -10000 "$DATA_DIR/queries.dev.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id INTEGER, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/qrels.dev.small.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_qrels (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_passages UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_queries UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_qrels ORDER BY metric; -- NOTE: spill + size + segment-stats moved to size_report.sql, run -- AFTER queries.sql, so the query phase exercises the realistic -- post-load state (in-flight memtable chain + cache + segments). \echo '' \echo '=== MS MARCO Insert Benchmark Load Complete ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/insert/pgbench-insert.sql000066400000000000000000000002701525762047700313030ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO msmarco_passages (passage_id, passage_text) SELECT passage_id, passage_text FROM msmarco_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/insert/run_concurrent_insert.sh000077500000000000000000000104361525762047700326500ustar00rootroot00000000000000#!/usr/bin/env bash # # Concurrent INSERT benchmark for MS MARCO dataset. # # Runs pgbench at multiple concurrency levels (1, 2, 4, 8, 16 clients) # for a configurable duration per level. Reports TPS and latency. # # The script does NOT reset between concurrency levels — the index # accumulates data across runs, which is more realistic (the memtable # grows, spills happen, segments compact under load). # # Prerequisites: # - Postgres running with the target extension installed # - msmarco staging data loaded via concurrent-setup.sql (or the # ParadeDB equivalent in paradedb/concurrent-setup.sql) # # Usage: # # pg_textsearch (default): # ./run_concurrent_insert.sh # # # ParadeDB: # ./run_concurrent_insert.sh --engine paradedb # # # Custom port/duration: # PGPORT=5434 DURATION=30 ./run_concurrent_insert.sh # # Environment variables: # PGPORT Postgres port (default: 5434) # PGDATABASE Database name (default: postgres) # DURATION Seconds per concurrency level (default: 60) # CLIENTS Space-separated client counts (default: "1 2 4 8 16") # SKIP_SETUP Set to 1 to skip setup phase (reuse existing staging) set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" MSMARCO_DIR="$(cd "$SCRIPT_DIR/.." && pwd)" DATA_DIR="${DATA_DIR:-$MSMARCO_DIR/data}" PGPORT="${PGPORT:-5434}" PGDATABASE="${PGDATABASE:-postgres}" DURATION="${DURATION:-60}" SKIP_SETUP="${SKIP_SETUP:-0}" # Default client counts: powers of 2 up to available CPUs NCPUS=$(sysctl -n hw.ncpu 2>/dev/null || nproc 2>/dev/null || echo 4) if [[ -z "${CLIENTS:-}" ]]; then CLIENTS="" for c in 1 2 4 8 16 32; do if [[ "$c" -le "$NCPUS" ]]; then CLIENTS="$CLIENTS $c" fi done CLIENTS="${CLIENTS# }" fi ENGINE="${1:-tapir}" # Strip leading -- from engine flag ENGINE="${ENGINE#--engine=}" ENGINE="${ENGINE#--engine }" if [[ "$ENGINE" == "--engine" ]]; then ENGINE="${2:-tapir}" fi # Resolve paths based on engine case "$ENGINE" in tapir|pg_textsearch) ENGINE_LABEL="pg_textsearch" SETUP_SQL="$SCRIPT_DIR/concurrent-setup.sql" PGBENCH_SQL="$SCRIPT_DIR/pgbench-insert.sql" COUNT_TABLE="msmarco_passages" STAGING_TABLE="msmarco_staging" ;; paradedb|pg_search) ENGINE_LABEL="ParadeDB" SETUP_SQL="$MSMARCO_DIR/paradedb/concurrent-setup.sql" PGBENCH_SQL="$MSMARCO_DIR/paradedb/pgbench-insert.sql" COUNT_TABLE="msmarco_passages_paradedb" STAGING_TABLE="msmarco_paradedb_staging" ;; *) echo "Unknown engine: $ENGINE" >&2 echo "Usage: $0 [--engine tapir|paradedb]" >&2 exit 1 ;; esac export PGPORT PGDATABASE echo "==============================================" echo " Concurrent INSERT Benchmark: $ENGINE_LABEL" echo "==============================================" echo "Port: $PGPORT" echo "Database: $PGDATABASE" echo "Duration: ${DURATION}s per level" echo "Clients: $CLIENTS" echo "Date: $(date '+%Y-%m-%d %H:%M:%S')" echo "" # --- Setup phase --- if [[ "$SKIP_SETUP" != "1" ]]; then echo "--- Setup: loading staging data ---" (cd "$MSMARCO_DIR" && DATA_DIR="$DATA_DIR" \ psql -f "$SETUP_SQL" 2>&1) | tail -5 echo "" fi # --- Benchmark runs --- echo "--- Results ---" printf "%-10s %10s %12s %12s %15s\n" \ "CLIENTS" "TPS" "AVG_LAT_MS" "ROWS_60S" "CUMULATIVE" for c in $CLIENTS; do # Check remaining rows before each level remaining=$(psql -qtAc \ "SELECT count(*) FROM $STAGING_TABLE s WHERE s.staging_id > (SELECT last_value FROM insert_seq);" \ 2>/dev/null || echo "999999") if [[ "$remaining" -lt 1000 ]]; then echo "STOPPING: only $remaining rows remaining in staging" break fi output=$(pgbench -n -c "$c" -j "$c" -T "$DURATION" \ -f "$PGBENCH_SQL" 2>&1) tps=$(echo "$output" | grep "^tps" | \ grep -oE '[0-9]+\.[0-9]+' | head -1) lat=$(echo "$output" | grep "latency average" | \ grep -oE '[0-9]+\.[0-9]+') txns=$(echo "$output" | \ grep "number of transactions actually processed" | \ grep -oE '[0-9]+' | head -1) cumulative=$(psql -qtAc \ "SELECT count(*) FROM $COUNT_TABLE;") printf "%-10s %10s %12s %12s %15s\n" \ "$c" "$tps" "$lat" "$txns" "$cumulative" done echo "" echo "--- Done ---" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/insert/size_report.sql000066400000000000000000000020511525762047700307370ustar00rootroot00000000000000-- MS MARCO insert benchmark: post-query size report. -- -- Runs AFTER queries.sql so the query phase exercises the realistic -- post-load state (in-flight memtable chain + cache + segments). -- Spilling here ensures pg_relation_size reflects all data on disk -- and bm25_summarize_index reports the final segment layout that -- downstream metric extractors expect. \echo '' \echo '=== MS MARCO Insert: Post-Query Size Report ===' -- Spill memtable to disk so pg_relation_size reflects actual index data SELECT bm25_spill_index('msmarco_bm25_idx'); \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('msmarco_bm25_idx') ) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size('msmarco_passages') ) as table_size, pg_total_relation_size('msmarco_passages') as table_bytes; \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('msmarco_bm25_idx'); timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/load.sql000066400000000000000000000076571525762047700260260ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Load and Index -- Loads the full 8.8M passage collection and creates BM25 index -- -- Usage: -- DATA_DIR=/path/to/data psql -f load.sql -- -- The DATA_DIR environment variable should point to the directory containing: -- - collection.tsv (passage_id, passage_text) -- - queries.dev.tsv (query_id, query_text) -- - qrels.dev.small.tsv (query_id, 0, passage_id, relevance) \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO Passage Ranking - Data Loading ===' \echo 'Loading ~8.8M passages from MS MARCO collection' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages CASCADE; DROP TABLE IF EXISTS msmarco_queries CASCADE; DROP TABLE IF EXISTS msmarco_qrels CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_qrels ( query_id INTEGER NOT NULL, passage_id INTEGER NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages (this is the big one - 8.8M rows) -- Convert TSV to CSV to avoid issues with \. sequences in text format \echo 'Loading passages (this may take several minutes)...' \copy msmarco_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries (use a subset for benchmarking - first 10K queries) \echo 'Loading queries...' \copy msmarco_queries(query_id, query_text) FROM PROGRAM 'head -10000 "$DATA_DIR/queries.dev.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments (qrels format: query_id, 0, passage_id, relevance) \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id INTEGER, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/qrels.dev.small.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_qrels (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_passages UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_queries UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_qrels ORDER BY metric; -- Show sample data \echo '' \echo 'Sample passages:' SELECT passage_id, LEFT(passage_text, 100) || '...' as passage_preview FROM msmarco_passages LIMIT 3; \echo '' \echo 'Sample queries:' SELECT query_id, query_text FROM msmarco_queries LIMIT 5; -- Create BM25 index \echo '' \echo '=== Building BM25 Index ===' \echo 'Creating BM25 index on ~8.8M passages (this will take a while)...' CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); -- Force-merge all segments into one so layout is deterministic -- regardless of parallel worker count. \echo 'Force-merging segments...' SELECT bm25_force_merge('msmarco_bm25_idx'); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_passages')) as table_size, pg_total_relation_size('msmarco_passages') as table_bytes; -- Segment statistics \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('msmarco_bm25_idx'); \echo '' \echo '=== MS MARCO Load Complete ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/load_data_only.sql000066400000000000000000000055411525762047700300460ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Data Loading Only (no index) -- Loads the full 8.8M passage collection without creating the index -- -- Usage: -- DATA_DIR=/path/to/data psql -f load_data_only.sql -- -- After loading, run parallel_scaling.sql to benchmark different worker counts \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO Passage Ranking - Data Loading ===' \echo 'Loading ~8.8M passages from MS MARCO collection (no index)' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages CASCADE; DROP TABLE IF EXISTS msmarco_queries CASCADE; DROP TABLE IF EXISTS msmarco_qrels CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_passages ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_qrels ( query_id INTEGER NOT NULL, passage_id INTEGER NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages (this is the big one - 8.8M rows) \echo 'Loading passages (this may take several minutes)...' \copy msmarco_passages(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries \echo 'Loading queries...' \copy msmarco_queries(query_id, query_text) FROM PROGRAM 'head -10000 "$DATA_DIR/queries.dev.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id INTEGER, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/qrels.dev.small.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_qrels (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Analyze for accurate statistics \echo 'Analyzing tables...' ANALYZE msmarco_passages; ANALYZE msmarco_queries; ANALYZE msmarco_qrels; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_passages UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_queries UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_qrels ORDER BY metric; -- Report table size \echo '' \echo '=== Table Size ===' SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_passages')) as table_size, pg_total_relation_size('msmarco_passages') as table_bytes; \echo '' \echo '=== Data Loading Complete ===' \echo 'Run parallel_scaling.sql to benchmark different worker counts' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/mixed-update-query/000077500000000000000000000000001525762047700301005ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/mixed-update-query/run.sh000077500000000000000000000365131525762047700312530ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail usage() { cat <<'EOF' Run concurrent top-10 BM25 queries and rate-limited indexed updates. The defaults target the MS MARCO v2 tables. Override TABLE, ID_COLUMN, TEXT_COLUMN, INDEX, QUERY_TABLE, and QUERY_COLUMN for another compatible corpus, including benchmarks/datasets/msmarco. Usage: run.sh [--dry-run] run.sh --help Connection environment: PGHOST, PGPORT, PGUSER, PGDATABASE Workload environment: TABLE=msmarco_v2_passages ID_COLUMN=passage_id TEXT_COLUMN=passage_text INDEX=msmarco_v2_bm25_idx QUERY_TABLE=msmarco_v2_queries QUERY_COLUMN=query_text DURATION=600 WARMUP=60 READ_CLIENTS=32 READ_THREADS=min(READ_CLIENTS, online CPUs) UPDATE_RATE=1000 UPDATE_TARGETS=100000 OUTPUT_DIR=benchmarks/results/mixed-update-query/ Options: --dry-run Validate configuration and print the resolved values. --help Show this help. EOF } die() { echo "ERROR: $*" >&2 exit 1 } validate_identifier() { local name="$1" local value="$2" [[ "$value" =~ ^[A-Za-z_][A-Za-z0-9_\$]*$ ]] || die "$name must be a simple SQL identifier: $value" } validate_relation_name() { local name="$1" local value="$2" [[ "$value" =~ ^[A-Za-z_][A-Za-z0-9_\$]*(\.[A-Za-z_][A-Za-z0-9_\$]*)?$ ]] || die "$name must be an optionally schema-qualified SQL name: $value" } validate_positive_integer() { local name="$1" local value="$2" [[ "$value" =~ ^[1-9][0-9]*$ ]] || die "$name must be a positive integer: $value" } extract_metric() { local pattern="$1" local file="$2" awk -v pattern="$pattern" ' $0 ~ pattern { for (i = 1; i <= NF; i++) if ($i ~ /^[0-9]+([.][0-9]+)?$/) value = $i } END { if (value != "") print value } ' "$file" } latency_p99_ms() { local log_prefix="$1" local latency_file latency_file="$(mktemp)" find "$(dirname "$log_prefix")" -maxdepth 1 -type f \ -name "$(basename "$log_prefix").*" -print0 | xargs -0 -r awk '$3 ~ /^[0-9]+$/ { print $3 / 1000.0 }' \ >"$latency_file" if [[ ! -s "$latency_file" ]]; then rm -f "$latency_file" echo "n/a" return fi sort -n "$latency_file" | awk ' { values[NR] = $1 } END { pos = int((NR * 99 + 99) / 100) printf "%.3f\n", values[pos] } ' rm -f "$latency_file" } wait_for_processes() { local first_pid="$1" local second_pid="$2" local monitor_file="$3" while kill -0 "$first_pid" 2>/dev/null || kill -0 "$second_pid" 2>/dev/null do if ! psql -X -qAt -F $'\t' -v ON_ERROR_STOP=1 \ -v reader_app="$READER_APP" \ -v writer_app="$WRITER_APP" \ >>"$monitor_file" <<'EOSQL' SELECT clock_timestamp(), application_name, COALESCE(wait_event_type, ''), COALESCE(wait_event, ''), COALESCE(array_to_string(pg_blocking_pids(pid), ','), '') FROM pg_stat_activity WHERE datname = current_database() AND application_name IN (:'reader_app', :'writer_app') ORDER BY application_name, pid; EOSQL then echo "wait sampler failed; wait-event results are incomplete" >&2 kill -TERM "$RUNNER_PID" return 1 fi sleep 1 done } cleanup_workload() { local status=$? local pid trap - EXIT INT TERM for pid in "${ACTIVE_PID:-}" "${READER_PID:-}" "${WRITER_PID:-}" \ "${MONITOR_PID:-}" do if [[ -n "$pid" ]] && kill -0 "$pid" 2>/dev/null; then kill "$pid" 2>/dev/null || true fi done for pid in "${ACTIVE_PID:-}" "${READER_PID:-}" "${WRITER_PID:-}" \ "${MONITOR_PID:-}" do if [[ -n "$pid" ]]; then wait "$pid" 2>/dev/null || true fi done if [[ "${HELPER_SCHEMA_CREATED:-0}" == 1 ]] && command -v psql >/dev/null then if ! psql -X -q -v ON_ERROR_STOP=1 \ -c "DROP SCHEMA IF EXISTS $HELPER_SCHEMA CASCADE" >/dev/null then echo "WARNING: could not remove benchmark helper schema" >&2 fi fi if [[ -n "${WORK_DIR:-}" && -d "$WORK_DIR" ]]; then rm -rf "$WORK_DIR" fi exit "$status" } run_reader() { local output_file="$1" local log_prefix="$2" exec env PGAPPNAME="$READER_APP" pgbench -n \ -c "$READ_CLIENTS" \ -j "$READ_THREADS" \ -T "$DURATION" \ -P 10 \ -l \ --log-prefix="$log_prefix" \ -f "$READER_SQL" \ >"$output_file" 2>&1 } TABLE="${TABLE:-msmarco_v2_passages}" ID_COLUMN="${ID_COLUMN:-passage_id}" TEXT_COLUMN="${TEXT_COLUMN:-passage_text}" INDEX="${INDEX:-msmarco_v2_bm25_idx}" QUERY_TABLE="${QUERY_TABLE:-msmarco_v2_queries}" QUERY_COLUMN="${QUERY_COLUMN:-query_text}" DURATION="${DURATION:-600}" WARMUP="${WARMUP:-60}" READ_CLIENTS="${READ_CLIENTS:-32}" UPDATE_RATE="${UPDATE_RATE:-1000}" UPDATE_TARGETS="${UPDATE_TARGETS:-100000}" validate_positive_integer "READ_CLIENTS" "$READ_CLIENTS" CPU_COUNT="$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 1)" if [[ ! "$CPU_COUNT" =~ ^[1-9][0-9]*$ ]]; then CPU_COUNT=1 fi if (( CPU_COUNT < READ_CLIENTS )); then DEFAULT_READ_THREADS="$CPU_COUNT" else DEFAULT_READ_THREADS="$READ_CLIENTS" fi READ_THREADS="${READ_THREADS:-$DEFAULT_READ_THREADS}" TIMESTAMP="$(date -u +%Y%m%dT%H%M%SZ)" ROOT_DIR="$(cd "$(dirname "$0")/../../../.." && pwd)" OUTPUT_DIR="${OUTPUT_DIR:-$ROOT_DIR/benchmarks/results/mixed-update-query/$TIMESTAMP}" RUN_NONCE="$(LC_ALL=C od -An -N8 -tx1 /dev/urandom | tr -d ' \n')" [[ "$RUN_NONCE" =~ ^[0-9a-f]{16}$ ]] || die "could not generate a benchmark run identifier" RUN_SUFFIX="${TIMESTAMP//[^A-Za-z0-9_]/_}_$$_$RUN_NONCE" HELPER_SCHEMA="pgts_benchmark_$RUN_SUFFIX" TARGET_TABLE="update_targets" QUERY_POOL_TABLE="query_pool" READER_APP="pgts-reader-$RUN_SUFFIX" WRITER_APP="pgts-writer-$RUN_SUFFIX" RUNNER_PID=$$ WORK_DIR="" ACTIVE_PID="" READER_PID="" WRITER_PID="" MONITOR_PID="" HELPER_SCHEMA_CREATED=0 case "${1:-}" in --help|-h) usage exit 0 ;; --dry-run) DRY_RUN=1 ;; "") DRY_RUN=0 ;; *) usage >&2 die "unknown option: $1" ;; esac for pair in \ "TABLE:$TABLE" \ "INDEX:$INDEX" \ "QUERY_TABLE:$QUERY_TABLE" do validate_relation_name "${pair%%:*}" "${pair#*:}" done for pair in \ "ID_COLUMN:$ID_COLUMN" \ "TEXT_COLUMN:$TEXT_COLUMN" \ "QUERY_COLUMN:$QUERY_COLUMN" do validate_identifier "${pair%%:*}" "${pair#*:}" done for pair in \ "DURATION:$DURATION" \ "WARMUP:$WARMUP" \ "READ_CLIENTS:$READ_CLIENTS" \ "READ_THREADS:$READ_THREADS" \ "UPDATE_RATE:$UPDATE_RATE" \ "UPDATE_TARGETS:$UPDATE_TARGETS" do validate_positive_integer "${pair%%:*}" "${pair#*:}" done cat </dev/null || die "psql is not available" command -v pgbench >/dev/null || die "pgbench is not available" mkdir -p "$(dirname "$OUTPUT_DIR")" if ! mkdir "$OUTPUT_DIR"; then if [[ -e "$OUTPUT_DIR" ]]; then die "output directory already exists: $OUTPUT_DIR" fi die "could not create output directory: $OUTPUT_DIR" fi WORK_DIR="$(mktemp -d "$OUTPUT_DIR/work.XXXXXX")" export PGHOST="${PGHOST:-127.0.0.1}" export PGPORT="${PGPORT:-5432}" export PGUSER="${PGUSER:-$USER}" export PGDATABASE="${PGDATABASE:-postgres}" trap 'exit 130' INT TERM trap cleanup_workload EXIT psql -X -qAt -v ON_ERROR_STOP=1 </dev/null SELECT '$TABLE'::regclass; SELECT '$INDEX'::regclass; SELECT '$QUERY_TABLE'::regclass; EOSQL SETUP_SQL="$WORK_DIR/setup.sql" READER_SQL="$WORK_DIR/reader.sql" WRITER_SQL="$WORK_DIR/writer.sql" cat >"$SETUP_SQL" < ''; ALTER TABLE $HELPER_SCHEMA.$QUERY_POOL_TABLE ADD PRIMARY KEY (query_id); ANALYZE $HELPER_SCHEMA.$TARGET_TABLE; ANALYZE $HELPER_SCHEMA.$QUERY_POOL_TABLE; COMMIT; EOSQL psql -X -q -v ON_ERROR_STOP=1 -f "$SETUP_SQL" & ACTIVE_PID=$! setup_status=0 if wait "$ACTIVE_PID"; then setup_status=0 else setup_status=$? fi ACTIVE_PID="" (( setup_status == 0 )) || die "benchmark setup failed; see the preceding psql error" HELPER_SCHEMA_CREATED=1 TARGET_COUNT="$( psql -X -qAt -v ON_ERROR_STOP=1 \ -c "SELECT count(*) FROM $HELPER_SCHEMA.$TARGET_TABLE" )" QUERY_COUNT="$( psql -X -qAt -v ON_ERROR_STOP=1 \ -c "SELECT count(*) FROM $HELPER_SCHEMA.$QUERY_POOL_TABLE" )" (( TARGET_COUNT > 0 )) || die "no update targets were selected" (( QUERY_COUNT > 0 )) || die "no benchmark queries were selected" cat >"$READER_SQL" < to_bm25query( (SELECT query_text FROM $HELPER_SCHEMA.$QUERY_POOL_TABLE WHERE query_id = :query_id), '$INDEX') LIMIT 10; EOSQL cat >"$WRITER_SQL" <"$OUTPUT_DIR/metadata.env" echo echo "Warming the query workload for ${WARMUP}s..." env PGAPPNAME="$READER_APP" pgbench -n -c 1 -j 1 -T "$WARMUP" \ -f "$READER_SQL" >"$OUTPUT_DIR/warmup.log" 2>&1 & ACTIVE_PID=$! warmup_status=0 if wait "$ACTIVE_PID"; then warmup_status=0 else warmup_status=$? fi ACTIVE_PID="" (( warmup_status == 0 )) || die "warmup pgbench failed; see $OUTPUT_DIR/warmup.log" echo "Running read-only phase for ${DURATION}s..." READ_ONLY_LOG_PREFIX="$OUTPUT_DIR/read_only_pgbench" run_reader "$OUTPUT_DIR/read_only.out" "$READ_ONLY_LOG_PREFIX" & ACTIVE_PID=$! read_only_status=0 if wait "$ACTIVE_PID"; then read_only_status=0 else read_only_status=$? fi ACTIVE_PID="" (( read_only_status == 0 )) || die "read-only pgbench failed; see $OUTPUT_DIR/read_only.out" READ_ONLY_QPS="$(extract_metric '^tps = ' "$OUTPUT_DIR/read_only.out")" READ_ONLY_AVG_MS="$( extract_metric '^latency average = ' "$OUTPUT_DIR/read_only.out" )" READ_ONLY_TXNS="$( extract_metric '^number of transactions actually processed:' \ "$OUTPUT_DIR/read_only.out" )" READ_ONLY_P99_MS="$(latency_p99_ms "$READ_ONLY_LOG_PREFIX")" echo "Running mixed read/update phase for ${DURATION}s..." MIXED_LOG_PREFIX="$OUTPUT_DIR/mixed_reader_pgbench" MIXED_WRITER_LOG_PREFIX="$OUTPUT_DIR/mixed_writer_pgbench" env PGAPPNAME="$READER_APP" pgbench -n \ -c "$READ_CLIENTS" \ -j "$READ_THREADS" \ -T "$DURATION" \ -P 10 \ -l \ --log-prefix="$MIXED_LOG_PREFIX" \ -f "$READER_SQL" \ >"$OUTPUT_DIR/mixed_reader.out" 2>&1 & READER_PID=$! env PGAPPNAME="$WRITER_APP" pgbench -n \ -c 1 \ -j 1 \ -T "$DURATION" \ -R "$UPDATE_RATE" \ -P 10 \ -l \ --log-prefix="$MIXED_WRITER_LOG_PREFIX" \ -f "$WRITER_SQL" \ >"$OUTPUT_DIR/mixed_writer.out" 2>&1 & WRITER_PID=$! : >"$OUTPUT_DIR/waits.tsv" wait_for_processes "$READER_PID" "$WRITER_PID" "$OUTPUT_DIR/waits.tsv" & MONITOR_PID=$! finished_status=0 finished_pid="" if wait -n -p finished_pid "$READER_PID" "$WRITER_PID"; then finished_status=0 else finished_status=$? fi if [[ "$finished_pid" == "$READER_PID" ]]; then finished_role="reader" reader_status=$finished_status remaining_pid="$WRITER_PID" remaining_role="writer" else finished_role="writer" writer_status=$finished_status remaining_pid="$READER_PID" remaining_role="reader" fi if (( finished_status != 0 )) && kill -0 "$remaining_pid" 2>/dev/null; then kill "$remaining_pid" fi remaining_status=0 if wait "$remaining_pid"; then remaining_status=0 else remaining_status=$? fi if [[ "$remaining_pid" == "$READER_PID" ]]; then reader_status=$remaining_status else writer_status=$remaining_status fi monitor_status=0 if wait "$MONITOR_PID"; then monitor_status=0 else monitor_status=$? fi READER_PID="" WRITER_PID="" MONITOR_PID="" if (( finished_status != 0 )); then die "$finished_role pgbench failed; see \ $OUTPUT_DIR/mixed_${finished_role}.out" fi if (( remaining_status != 0 )); then die "$remaining_role pgbench failed; see \ $OUTPUT_DIR/mixed_${remaining_role}.out" fi (( reader_status == 0 )) || die "reader pgbench failed; see $OUTPUT_DIR/mixed_reader.out" (( writer_status == 0 )) || die "writer pgbench failed; see $OUTPUT_DIR/mixed_writer.out" (( monitor_status == 0 )) || die "wait sampler failed; see $OUTPUT_DIR/waits.tsv" MIXED_QPS="$(extract_metric '^tps = ' "$OUTPUT_DIR/mixed_reader.out")" MIXED_AVG_MS="$( extract_metric '^latency average = ' "$OUTPUT_DIR/mixed_reader.out" )" MIXED_TXNS="$( extract_metric '^number of transactions actually processed:' \ "$OUTPUT_DIR/mixed_reader.out" )" MIXED_P99_MS="$(latency_p99_ms "$MIXED_LOG_PREFIX")" UPDATES="$( extract_metric '^number of transactions actually processed:' \ "$OUTPUT_DIR/mixed_writer.out" )" UPDATE_TPS="$(extract_metric '^tps = ' "$OUTPUT_DIR/mixed_writer.out")" UPDATE_AVG_MS="$( extract_metric '^latency average = ' "$OUTPUT_DIR/mixed_writer.out" )" cat >"$OUTPUT_DIR/summary.tsv" </dev/null || cat "$OUTPUT_DIR/summary.tsv" echo echo "Lock-wait samples:" { printf 'samples\tapplication_name\twait_event_type\twait_event\n' awk -F '\t' ' NF >= 4 { type = ($3 == "" ? "CPU" : $3) event = ($4 == "" ? "running" : $4) key = $2 "\t" type "\t" event count[key]++ } END { for (key in count) printf "%d\t%s\n", count[key], key } ' "$OUTPUT_DIR/waits.tsv" | sort -nr } >"$OUTPUT_DIR/wait_summary.tsv" if [[ "$(wc -l <"$OUTPUT_DIR/wait_summary.tsv")" -gt 1 ]]; then head -20 "$OUTPUT_DIR/wait_summary.tsv" else echo "(none sampled)" fi echo echo "Results: $OUTPUT_DIR" timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/paradedb/000077500000000000000000000000001525762047700261115ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/paradedb/concurrent-setup.sql000066400000000000000000000044601525762047700321560ustar00rootroot00000000000000-- MS MARCO Passage Ranking - ParadeDB Concurrent Insert Setup -- Loads passages into a staging table, creates empty target -- table with ParadeDB BM25 index, and prepares a sequence for -- pgbench-driven concurrent inserts. -- -- Usage: -- DATA_DIR=/path/to/data psql -f paradedb/concurrent-setup.sql \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO ParadeDB Concurrent Insert Setup ===' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_queries_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_qrels_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_paradedb_staging CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create staging table \echo 'Creating staging table...' CREATE TABLE msmarco_paradedb_staging ( staging_id SERIAL PRIMARY KEY, passage_id INTEGER, passage_text TEXT ); -- Load passages into staging \echo 'Loading passages into staging table...' \copy msmarco_paradedb_staging(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Create target passages table \echo 'Creating target passages table...' CREATE TABLE msmarco_passages_paradedb ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create ParadeDB BM25 index on empty target table \echo 'Creating ParadeDB BM25 index on empty table...' CREATE INDEX msmarco_paradedb_idx ON msmarco_passages_paradedb USING bm25 (passage_id, passage_text) WITH ( key_field = 'passage_id', text_fields = '{ "passage_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Create sequence for pgbench inserts \echo 'Creating insert sequence...' CREATE SEQUENCE insert_seq; -- Report staging row count \echo '' \echo '=== Setup Verification ===' SELECT 'STAGING_ROW_COUNT:' as label, COUNT(*)::text as count FROM msmarco_paradedb_staging; \echo '' \echo '=== ParadeDB Concurrent Insert Setup Complete ===' \echo 'Run pgbench with paradedb/pgbench-insert.sql' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/paradedb/insert-load.sql000066400000000000000000000074231525762047700310610ustar00rootroot00000000000000-- MS MARCO Passage Ranking - ParadeDB Insert Loading -- Creates ParadeDB BM25 index on EMPTY table, then loads data -- via COPY to measure insert-path indexing performance. -- -- Usage: -- DATA_DIR=/path/to/data psql -f paradedb/insert-load.sql \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO ParadeDB Insert Benchmark - Data Loading ===' \echo 'Index created BEFORE loading ~8.8M passages' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_queries_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_qrels_paradedb CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_passages_paradedb ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create ParadeDB BM25 index on the empty table \echo 'Creating ParadeDB BM25 index on empty table...' CREATE INDEX msmarco_paradedb_idx ON msmarco_passages_paradedb USING bm25 (passage_id, passage_text) WITH ( key_field = 'passage_id', text_fields = '{ "passage_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_queries_paradedb ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_qrels_paradedb ( query_id INTEGER NOT NULL, passage_id INTEGER NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages into the pre-indexed table \echo '' \echo 'INSERT_TIME:' \echo 'Loading passages (this may take a while)...' \copy msmarco_passages_paradedb(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries \echo 'Loading queries...' \copy msmarco_queries_paradedb(query_id, query_text) FROM PROGRAM 'head -10000 "$DATA_DIR/queries.dev.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id INTEGER, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/qrels.dev.small.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_qrels_paradedb (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Compact index segments (ParadeDB merges during VACUUM) \echo 'INDEX_VACUUM:' VACUUM msmarco_passages_paradedb; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_passages_paradedb UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_queries_paradedb UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_qrels_paradedb ORDER BY metric; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('msmarco_paradedb_idx') ) as index_size, pg_relation_size('msmarco_paradedb_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'msmarco_passages_paradedb') ) as table_size, pg_total_relation_size( 'msmarco_passages_paradedb') as table_bytes; \echo '' \echo '=== MS MARCO ParadeDB Insert Benchmark Complete ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/paradedb/load.sql000066400000000000000000000106521525762047700275550ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Load and Index (ParadeDB) -- Loads the full 8.8M passage collection and creates ParadeDB BM25 index -- -- Usage: -- DATA_DIR=/path/to/data psql -U postgres postgres -f load.sql -- -- The DATA_DIR environment variable should point to the directory containing: -- - collection.tsv (passage_id, passage_text) -- - queries.dev.tsv (query_id, query_text) -- - qrels.dev.small.tsv (query_id, 0, passage_id, relevance) \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO Passage Ranking - Data Loading (ParadeDB) ===' \echo 'Loading ~8.8M passages from MS MARCO collection' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS msmarco_passages_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_queries_paradedb CASCADE; DROP TABLE IF EXISTS msmarco_qrels_paradedb CASCADE; -- Create passages table \echo 'Creating passages table...' CREATE TABLE msmarco_passages_paradedb ( passage_id INTEGER PRIMARY KEY, passage_text TEXT NOT NULL ); -- Create queries table \echo 'Creating queries table...' CREATE TABLE msmarco_queries_paradedb ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Create relevance judgments table \echo 'Creating relevance judgments table...' CREATE TABLE msmarco_qrels_paradedb ( query_id INTEGER NOT NULL, passage_id INTEGER NOT NULL, relevance INTEGER NOT NULL, PRIMARY KEY (query_id, passage_id) ); -- Load passages (this is the big one - 8.8M rows) -- Convert TSV to CSV to avoid issues with \. sequences in text format \echo 'Loading passages (this may take several minutes)...' \copy msmarco_passages_paradedb(passage_id, passage_text) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); print \$1, \"\\\"\" \$2 \"\\\"\"}" "$DATA_DIR/collection.tsv"' WITH (FORMAT csv); -- Load queries (use a subset for benchmarking - first 10K queries) \echo 'Loading queries...' \copy msmarco_queries_paradedb(query_id, query_text) FROM PROGRAM 'head -10000 "$DATA_DIR/queries.dev.tsv"' WITH (FORMAT text, DELIMITER E'\t'); -- Load relevance judgments (qrels format: query_id, 0, passage_id, relevance) \echo 'Loading relevance judgments...' CREATE TEMP TABLE qrels_raw ( query_id INTEGER, zero INTEGER, passage_id INTEGER, relevance INTEGER ); \copy qrels_raw FROM PROGRAM 'cat "$DATA_DIR/qrels.dev.small.tsv"' WITH (FORMAT text, DELIMITER E'\t'); INSERT INTO msmarco_qrels_paradedb (query_id, passage_id, relevance) SELECT query_id, passage_id, relevance FROM qrels_raw; DROP TABLE qrels_raw; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Passages loaded:' as metric, COUNT(*)::text as count FROM msmarco_passages_paradedb UNION ALL SELECT 'Queries loaded:', COUNT(*)::text FROM msmarco_queries_paradedb UNION ALL SELECT 'Relevance judgments:', COUNT(*)::text FROM msmarco_qrels_paradedb ORDER BY metric; -- Show sample data \echo '' \echo 'Sample passages:' SELECT passage_id, LEFT(passage_text, 100) || '...' as passage_preview FROM msmarco_passages_paradedb LIMIT 3; \echo '' \echo 'Sample queries:' SELECT query_id, query_text FROM msmarco_queries_paradedb LIMIT 5; -- Create ParadeDB BM25 index with English tokenizer (stopwords + stemming) \echo '' \echo '=== Building ParadeDB BM25 Index ===' \echo 'Creating ParadeDB BM25 index on ~8.8M passages (this will take a while)...' CREATE INDEX msmarco_paradedb_idx ON msmarco_passages_paradedb USING bm25 (passage_id, passage_text) WITH ( key_field = 'passage_id', text_fields = '{ "passage_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Compact index segments (ParadeDB merges segments during VACUUM) \echo 'INDEX_VACUUM:' VACUUM msmarco_passages_paradedb; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_paradedb_idx')) as index_size, pg_relation_size('msmarco_paradedb_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_passages_paradedb')) as table_size, pg_total_relation_size('msmarco_passages_paradedb') as table_bytes; \echo '' \echo '=== MS MARCO Load Complete (ParadeDB) ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/paradedb/pgbench-insert.sql000066400000000000000000000003121525762047700315360ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO msmarco_passages_paradedb (passage_id, passage_text) SELECT passage_id, passage_text FROM msmarco_paradedb_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/paradedb/queries.sql000066400000000000000000000215441525762047700303150ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Query Benchmarks (ParadeDB) -- Runs various query workloads against the indexed MS MARCO collection using ParadeDB -- Outputs structured timing data for historical tracking \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO Query Benchmarks (ParadeDB) ===' \echo '' -- Load benchmark queries (pre-sampled by token count) \echo 'Loading benchmark queries...' DROP TABLE IF EXISTS benchmark_queries_paradedb; CREATE TABLE benchmark_queries_paradedb ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy benchmark_queries_paradedb FROM 'benchmarks/datasets/msmarco/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') -- Verify load SELECT 'Loaded ' || COUNT(*) || ' benchmark queries' as status FROM benchmark_queries_paradedb; SELECT token_bucket, COUNT(*) as count FROM benchmark_queries_paradedb GROUP BY token_bucket ORDER BY token_bucket; -- Warm up: run queries from each bucket to ensure index is cached \echo '' \echo 'Warming up index...' DO $$ DECLARE q record; BEGIN FOR q IN SELECT query_text FROM benchmark_queries_paradedb ORDER BY random() LIMIT 50 LOOP EXECUTE 'SELECT passage_id FROM msmarco_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10' USING q.query_text; END LOOP; END; $$; -- ============================================================ -- Benchmark 1: Query Latency by Token Count -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency by Token Count ===' \echo 'Running 100 queries per token bucket, reporting p50/p95/p99' \echo '' -- Function to benchmark a bucket and return percentiles -- Uses ParadeDB match() to handle arbitrary query strings CREATE OR REPLACE FUNCTION benchmark_bucket_paradedb(bucket int) RETURNS TABLE(p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint) AS $$ DECLARE q record; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; n int; result_count bigint; results_sum bigint := 0; BEGIN times := ARRAY[]::numeric[]; FOR q IN SELECT query_text FROM benchmark_queries_paradedb WHERE token_bucket = bucket ORDER BY query_id LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT COUNT(*) FROM (SELECT passage_id FROM msmarco_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10) t' INTO result_count USING q.query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); results_sum := results_sum + result_count; END LOOP; n := array_length(times, 1); SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; p50_ms := sorted_times[(n + 1) / 2]; p95_ms := sorted_times[(n * 95 + 99) / 100]; p99_ms := sorted_times[(n * 99 + 99) / 100]; avg_ms := (SELECT AVG(t) FROM unnest(times) t); num_queries := n; total_results := results_sum; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run benchmarks for each token bucket \echo 'Token bucket 1 (1 search token):' SELECT 'LATENCY_BUCKET_1: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(1); \echo '' \echo 'Token bucket 2 (2 search tokens):' SELECT 'LATENCY_BUCKET_2: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(2); \echo '' \echo 'Token bucket 3 (3 search tokens):' SELECT 'LATENCY_BUCKET_3: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(3); \echo '' \echo 'Token bucket 4 (4 search tokens):' SELECT 'LATENCY_BUCKET_4: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(4); \echo '' \echo 'Token bucket 5 (5 search tokens):' SELECT 'LATENCY_BUCKET_5: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(5); \echo '' \echo 'Token bucket 6 (6 search tokens):' SELECT 'LATENCY_BUCKET_6: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(6); \echo '' \echo 'Token bucket 7 (7 search tokens):' SELECT 'LATENCY_BUCKET_7: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(7); \echo '' \echo 'Token bucket 8 (8+ search tokens):' SELECT 'LATENCY_BUCKET_8: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM benchmark_bucket_paradedb(8); DROP FUNCTION benchmark_bucket_paradedb; -- ============================================================ -- Benchmark 2: Query Throughput (800 benchmark queries) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (800 queries, 3 iterations) ===' \echo 'Running all 800 benchmark queries with warmup' -- Helper function for throughput benchmark -- Uses ParadeDB match() to handle special characters in query strings CREATE OR REPLACE FUNCTION benchmark_throughput_paradedb(iterations int DEFAULT 3) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric, queries_run int) AS $$ DECLARE q record; i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; query_count int; BEGIN SELECT COUNT(*) INTO query_count FROM benchmark_queries_paradedb; -- Warmup: run all queries once FOR q IN SELECT query_text FROM benchmark_queries_paradedb ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10' USING q.query_text; END LOOP; -- Timed iterations times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); FOR q IN SELECT query_text FROM benchmark_queries_paradedb ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_passages_paradedb WHERE passage_text @@@ paradedb.match(''passage_text'', $1) ORDER BY paradedb.score(passage_id) DESC LIMIT 10' USING q.query_text; END LOOP; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; queries_run := query_count; RETURN NEXT; END; $$ LANGUAGE plpgsql; SELECT 'Execution Time: ' || round(median_ms / queries_run, 3) || ' ms (min=' || round(min_ms / queries_run, 3) || ', max=' || round(max_ms / queries_run, 3) || ')' as result, 'THROUGHPUT_RESULT: ' || queries_run || ' queries in ' || round(median_ms, 2) || ' ms (avg ' || round(median_ms / queries_run, 2) || ' ms/query)' as summary FROM benchmark_throughput_paradedb(); DROP FUNCTION benchmark_throughput_paradedb; -- ============================================================ -- Benchmark 3: Index Statistics -- ============================================================ \echo '' \echo '=== Benchmark 3: Index Statistics ===' SELECT 'msmarco_paradedb_idx' as index_name, pg_size_pretty(pg_relation_size('msmarco_paradedb_idx')) as index_size, pg_size_pretty(pg_relation_size('msmarco_passages_paradedb')) as table_size, (SELECT COUNT(*) FROM msmarco_passages_paradedb) as num_documents; -- Cleanup DROP TABLE benchmark_queries_paradedb; \echo '' \echo '=== MS MARCO Query Benchmarks Complete (ParadeDB) ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/parallel_scaling.sql000066400000000000000000000137461525762047700303770ustar00rootroot00000000000000-- MS MARCO Parallel Build Scaling Benchmark -- Tests index build performance with varying numbers of parallel workers -- -- Usage: -- DATA_DIR=/path/to/data psql -f parallel_scaling.sql -- -- Outputs: -- PARALLEL_BUILD_RESULT: workers=N, build_time_ms=X, query_avg_ms=Y -- -- Assumes data is already loaded (run load_data_only.sql first) \set ON_ERROR_STOP on \timing on \echo '=== Parallel Build Scaling Benchmark ===' \echo 'Testing index build performance with 0, 1, 2, 4 workers' \echo '' -- Verify data is loaded SELECT COUNT(*) as passage_count FROM msmarco_passages; \gset \if :passage_count < 100000 \echo 'ERROR: MS MARCO data not loaded or too small. Need at least 100K passages.' \echo 'Run load_data_only.sql or load.sql first.' \quit \endif \echo 'Found ' :passage_count ' passages' \echo '' -- Sample queries for quick performance test after each build CREATE TEMP TABLE bench_queries (query_text TEXT); INSERT INTO bench_queries VALUES ('what is machine learning'), ('how to bake a cake'), ('python programming tutorial'), ('best restaurants in new york'), ('climate change effects'), ('history of the internet'), ('how does gps work'), ('benefits of exercise'), ('quantum computing explained'), ('renewable energy sources'); -- Function to run quick query benchmark CREATE OR REPLACE FUNCTION bench_queries(index_name TEXT) RETURNS TABLE(avg_ms NUMERIC, min_ms NUMERIC, max_ms NUMERIC) AS $$ DECLARE start_ts TIMESTAMP; end_ts TIMESTAMP; total_ms NUMERIC := 0; min_time NUMERIC := 999999; max_time NUMERIC := 0; query_time NUMERIC; q TEXT; iter INT; BEGIN -- Warm up FOR q IN SELECT query_text FROM bench_queries LOOP PERFORM ctid FROM msmarco_passages ORDER BY passage_text <@> to_bm25query(q, index_name) LIMIT 10; END LOOP; -- Timed runs (3 iterations) FOR iter IN 1..3 LOOP FOR q IN SELECT query_text FROM bench_queries LOOP start_ts := clock_timestamp(); PERFORM ctid FROM msmarco_passages ORDER BY passage_text <@> to_bm25query(q, index_name) LIMIT 10; end_ts := clock_timestamp(); query_time := EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000; total_ms := total_ms + query_time; min_time := LEAST(min_time, query_time); max_time := GREATEST(max_time, query_time); END LOOP; END LOOP; avg_ms := total_ms / 30; -- 10 queries * 3 iterations min_ms := min_time; max_ms := max_time; RETURN NEXT; END; $$ LANGUAGE plpgsql; ---------------------------------------------------------------------- -- Test with 0 workers (serial build) ---------------------------------------------------------------------- \echo '' \echo '=== Testing with 0 workers (serial build) ===' DROP INDEX IF EXISTS msmarco_bm25_idx; SET max_parallel_maintenance_workers = 0; SHOW max_parallel_maintenance_workers; \timing on CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); \timing off -- Capture build time from psql timing \echo 'PARALLEL_BUILD_WORKERS: 0' -- Get index info SELECT bm25_summarize_index('msmarco_bm25_idx'); -- Query benchmark SELECT 'PARALLEL_QUERY_RESULT: workers=0, avg_ms=' || ROUND(avg_ms, 2) || ', min_ms=' || ROUND(min_ms, 2) || ', max_ms=' || ROUND(max_ms, 2) FROM bench_queries('msmarco_bm25_idx'); ---------------------------------------------------------------------- -- Test with 1 worker ---------------------------------------------------------------------- \echo '' \echo '=== Testing with 1 worker ===' DROP INDEX IF EXISTS msmarco_bm25_idx; SET max_parallel_maintenance_workers = 1; SHOW max_parallel_maintenance_workers; \timing on CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); \timing off \echo 'PARALLEL_BUILD_WORKERS: 1' SELECT bm25_summarize_index('msmarco_bm25_idx'); SELECT 'PARALLEL_QUERY_RESULT: workers=1, avg_ms=' || ROUND(avg_ms, 2) || ', min_ms=' || ROUND(min_ms, 2) || ', max_ms=' || ROUND(max_ms, 2) FROM bench_queries('msmarco_bm25_idx'); ---------------------------------------------------------------------- -- Test with 2 workers ---------------------------------------------------------------------- \echo '' \echo '=== Testing with 2 workers ===' DROP INDEX IF EXISTS msmarco_bm25_idx; SET max_parallel_maintenance_workers = 2; SHOW max_parallel_maintenance_workers; \timing on CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); \timing off \echo 'PARALLEL_BUILD_WORKERS: 2' SELECT bm25_summarize_index('msmarco_bm25_idx'); SELECT 'PARALLEL_QUERY_RESULT: workers=2, avg_ms=' || ROUND(avg_ms, 2) || ', min_ms=' || ROUND(min_ms, 2) || ', max_ms=' || ROUND(max_ms, 2) FROM bench_queries('msmarco_bm25_idx'); ---------------------------------------------------------------------- -- Test with 4 workers ---------------------------------------------------------------------- \echo '' \echo '=== Testing with 4 workers ===' DROP INDEX IF EXISTS msmarco_bm25_idx; SET max_parallel_maintenance_workers = 4; SHOW max_parallel_maintenance_workers; \timing on CREATE INDEX msmarco_bm25_idx ON msmarco_passages USING bm25(passage_text) WITH (text_config='english'); \timing off \echo 'PARALLEL_BUILD_WORKERS: 4' SELECT bm25_summarize_index('msmarco_bm25_idx'); SELECT 'PARALLEL_QUERY_RESULT: workers=4, avg_ms=' || ROUND(avg_ms, 2) || ', min_ms=' || ROUND(min_ms, 2) || ', max_ms=' || ROUND(max_ms, 2) FROM bench_queries('msmarco_bm25_idx'); ---------------------------------------------------------------------- -- Summary ---------------------------------------------------------------------- \echo '' \echo '=== Parallel Scaling Benchmark Complete ===' \echo 'Review PARALLEL_BUILD_WORKERS and PARALLEL_QUERY_RESULT lines above' \echo 'Build times shown in psql timing output after each CREATE INDEX' -- Cleanup DROP FUNCTION IF EXISTS bench_queries(TEXT); timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/precompute_ground_truth.sql000066400000000000000000000246301525762047700320640ustar00rootroot00000000000000-- MS-MARCO Ground Truth Precomputation -- Computes BM25 scores using the exact formula for validation purposes -- Output: ground_truth.tsv with (query_id, query_text, rank, doc_id, score) -- -- IMPORTANT: Different Postgres versions may produce different ground truth -- due to Snowball stemmer changes (e.g., PG17 vs PG18). After generating, -- rename the output to ground_truth_pgNN.tsv for the target version. -- -- Prerequisites: -- - msmarco_passages table loaded (index NOT required) -- - benchmark_queries table will be loaded -- -- Usage: -- psql -p PORT -f precompute_ground_truth.sql -- -- This is a ONE-TIME computation. Results are saved and reused for validation. -- Note: First precomputes document frequencies for all query terms (~6 min), -- then computes ground truth scores (~1 min per query). \set ON_ERROR_STOP on \timing on -- Ensure validation functions exist (for fieldnorm_quantize) \i test/sql/validation.sql -- Ensure benchmark_queries table exists \echo 'Loading benchmark queries...' DROP TABLE IF EXISTS benchmark_queries; CREATE TABLE benchmark_queries ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy benchmark_queries FROM 'benchmarks/datasets/msmarco/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') SELECT 'Loaded ' || COUNT(*) || ' benchmark queries' as status FROM benchmark_queries; -- Compute corpus statistics from raw data (no index dependency). -- This scans the full table to count tokens per document. -- Documents with empty tsvectors (no indexable terms) are excluded, -- matching the index behavior. \echo '' \echo 'Computing corpus statistics from raw data (slow, one-time)...' DROP TABLE IF EXISTS corpus_stats; CREATE TABLE corpus_stats AS SELECT COUNT(*)::bigint as total_docs, SUM(doc_len)::bigint as total_len, (SUM(doc_len)::float8 / COUNT(*)::float8) as avg_doc_len FROM ( SELECT passage_id, (SELECT SUM(array_length( string_to_array(t.positions::text, ','), 1)) FROM unnest(to_tsvector('english', passage_text)) as t(lexeme, positions, weights) ) as doc_len FROM msmarco_passages ) dl WHERE doc_len > 0; DO $$ DECLARE r record; BEGIN SELECT * INTO r FROM corpus_stats; RAISE NOTICE 'Corpus stats: total_docs=%, total_len=%, ' 'avg_doc_len=%', r.total_docs, r.total_len, round(r.avg_doc_len::numeric, 4); END; $$; -- Sample queries for validation (10 per token bucket) \echo '' \echo 'Selecting validation queries (10 per token bucket)...' DROP TABLE IF EXISTS validation_queries; CREATE TABLE validation_queries AS WITH ranked AS ( SELECT query_id, query_text, token_bucket, ROW_NUMBER() OVER (PARTITION BY token_bucket ORDER BY query_id) as rn FROM benchmark_queries ) SELECT query_id, query_text, token_bucket FROM ranked WHERE rn <= 10; SELECT 'Selected ' || COUNT(*) || ' queries for validation' as status FROM validation_queries; SELECT token_bucket, COUNT(*) as count FROM validation_queries GROUP BY token_bucket ORDER BY token_bucket; -- Step 1: Precompute document frequencies for all unique query terms -- This is the slow part (~1 sec/term) but only needs to run once \echo '' \echo 'Step 1: Precomputing document frequencies for all query terms...' DROP TABLE IF EXISTS term_doc_freq; CREATE TABLE term_doc_freq ( term text PRIMARY KEY, df bigint ); DO $$ DECLARE v_term record; v_count int := 0; v_total int; v_df bigint; v_start timestamp; v_elapsed interval; BEGIN -- Get unique terms from all validation queries CREATE TEMP TABLE unique_query_terms AS SELECT DISTINCT qt.lexeme::text as term FROM validation_queries vq, LATERAL unnest(to_tsvector('english', vq.query_text)) as qt(lexeme, positions, weights); SELECT COUNT(*) INTO v_total FROM unique_query_terms; v_start := clock_timestamp(); RAISE NOTICE 'Computing df for % validation query terms...', v_total; RAISE NOTICE 'Estimated time: ~% minutes', round(v_total * 1.0 / 60); FOR v_term IN SELECT term FROM unique_query_terms ORDER BY term LOOP v_count := v_count + 1; -- Count documents containing this term (use direct tsquery cast to -- avoid double-stemming since v_term.term is already stemmed) SELECT COUNT(*) INTO v_df FROM msmarco_passages WHERE to_tsvector('english', passage_text) @@ (v_term.term)::tsquery; INSERT INTO term_doc_freq (term, df) VALUES (v_term.term, v_df); IF v_count % 25 = 0 THEN v_elapsed := clock_timestamp() - v_start; RAISE NOTICE '[%/%] Elapsed: %, avg: %/term', v_count, v_total, v_elapsed, v_elapsed / v_count; END IF; END LOOP; v_elapsed := clock_timestamp() - v_start; RAISE NOTICE 'Completed % terms in %', v_count, v_elapsed; DROP TABLE unique_query_terms; END; $$; SELECT 'Computed df for ' || COUNT(*) || ' terms' as status FROM term_doc_freq; \echo '' \echo 'Most common terms:' SELECT term, df FROM term_doc_freq ORDER BY df DESC LIMIT 5; -- Step 2: Ground truth computation using precomputed df -- IMPORTANT: Must account for QUERY term frequencies (how many times a term -- appears in the query). E.g., "ash who is ash" has query_freq=2 for "ash". -- The score contribution for each term is multiplied by its query frequency. \echo '' \echo 'Step 2: Creating ground truth computation function...' CREATE OR REPLACE FUNCTION compute_ground_truth( p_query_id int, p_query_text text, p_k1 float DEFAULT 1.2, p_b float DEFAULT 0.75, p_limit int DEFAULT 10 ) RETURNS TABLE( query_id int, query_text text, rank int, doc_id int, score float8 ) AS $$ DECLARE v_total_docs bigint; v_avg_doc_len float8; v_query_terms text[]; BEGIN -- Get corpus stats SELECT cs.total_docs, cs.avg_doc_len INTO v_total_docs, v_avg_doc_len FROM corpus_stats cs; -- Get unique query terms for matching SELECT array_agg(lexeme::text) INTO v_query_terms FROM unnest(to_tsvector('english', p_query_text)); IF v_query_terms IS NULL OR array_length(v_query_terms, 1) = 0 THEN RETURN; END IF; RETURN QUERY WITH query_terms_with_freq AS ( -- Extract query terms with their frequencies -- This is CRITICAL: if "ash" appears twice in the query, query_freq=2 SELECT qt.lexeme::text as term, array_length(string_to_array(qt.positions::text, ','), 1) as query_freq FROM unnest(to_tsvector('english', p_query_text)) as qt(lexeme, positions, weights) ), matching_docs AS ( -- Find documents matching any query term SELECT passage_id, to_tsvector('english', passage_text) as doc_tsv FROM msmarco_passages WHERE to_tsvector('english', passage_text) @@ to_tsquery('english', array_to_string(v_query_terms, ' | ')) ), doc_scores AS ( SELECT m.passage_id, ( SELECT SUM( -- IDF from precomputed df ln(1.0 + (v_total_docs - tdf.df + 0.5) / (tdf.df + 0.5)) * -- TF component: tf*(k1+1) / (tf + k1*(1-b+b*dl/avgdl)) (doc_tf * (p_k1 + 1.0)) / (doc_tf + p_k1 * (1.0 - p_b + p_b * fieldnorm_quantize( (SELECT COALESCE(SUM(array_length(string_to_array(t2.positions::text, ','), 1)), 0)::int FROM unnest(m.doc_tsv) as t2(lexeme, positions, weights)) ) / v_avg_doc_len)) -- CRITICAL: Multiply by query term frequency for repeated terms * qtf.query_freq ) FROM query_terms_with_freq qtf JOIN term_doc_freq tdf ON tdf.term = qtf.term -- Get document term frequency for this query term CROSS JOIN LATERAL ( SELECT COALESCE( (SELECT array_length(string_to_array(t.positions::text, ','), 1) FROM unnest(m.doc_tsv) as t(lexeme, positions, weights) WHERE t.lexeme::text = qtf.term), 0 ) as doc_tf ) tf_lookup WHERE doc_tf > 0 ) as bm25_score FROM matching_docs m ), ranked AS ( SELECT passage_id, bm25_score, ROW_NUMBER() OVER (ORDER BY bm25_score DESC) as rn FROM doc_scores WHERE bm25_score > 0 ) SELECT p_query_id, p_query_text, rn::int, passage_id::int, bm25_score::float8 FROM ranked WHERE rn <= p_limit; END; $$ LANGUAGE plpgsql; -- Step 3: Compute ground truth for all validation queries \echo '' \echo 'Step 3: Computing ground truth for validation queries...' DROP TABLE IF EXISTS ground_truth; CREATE TABLE ground_truth ( query_id int, query_text text, rank int, doc_id int, score float8 ); DO $$ DECLARE q record; v_count int := 0; v_total int; v_start timestamp; v_elapsed interval; BEGIN SELECT COUNT(*) INTO v_total FROM validation_queries; v_start := clock_timestamp(); FOR q IN SELECT query_id, query_text, token_bucket FROM validation_queries ORDER BY token_bucket, query_id LOOP v_count := v_count + 1; RAISE NOTICE '[%/%] Bucket % - Query %: %...', v_count, v_total, q.token_bucket, q.query_id, left(q.query_text, 40); INSERT INTO ground_truth SELECT * FROM compute_ground_truth(q.query_id, q.query_text); IF v_count % 10 = 0 THEN v_elapsed := clock_timestamp() - v_start; RAISE NOTICE 'Progress: % queries in %, avg %/query', v_count, v_elapsed, v_elapsed / v_count; END IF; END LOOP; v_elapsed := clock_timestamp() - v_start; RAISE NOTICE 'Completed % queries in %', v_count, v_elapsed; END; $$; -- Export to TSV \echo '' \echo 'Exporting ground truth to TSV...' \copy ground_truth TO 'benchmarks/datasets/msmarco/ground_truth.tsv' WITH (FORMAT text, DELIMITER E'\t', HEADER true) SELECT 'Exported ' || COUNT(*) || ' rows for ' || COUNT(DISTINCT query_id) || ' queries' as status FROM ground_truth; -- Cleanup (keep term_doc_freq for potential debugging) DROP TABLE IF EXISTS validation_queries; DROP FUNCTION IF EXISTS compute_ground_truth; \echo '' \echo '=== Ground Truth Precomputation Complete ===' \echo 'Output: benchmarks/datasets/msmarco/ground_truth.tsv' \echo 'Note: term_doc_freq table retained for debugging' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/queries.sql000066400000000000000000000272741525762047700265610ustar00rootroot00000000000000-- MS MARCO Passage Ranking - Query Benchmarks -- Runs various query workloads against the indexed MS MARCO collection -- Outputs structured timing data for historical tracking \set ON_ERROR_STOP on \timing on -- Soft cap on per-statement runtime so a top-level psql query (e.g. the -- warmup DO block) can't hang forever. The benchmark loop functions -- (`benchmark_bucket`, `benchmark_throughput`) opt out of this cap -- locally so a slow-but-healthy run can complete and emit results; -- the workflow's step-level timeout-minutes is the real hang backstop. -- NOTE: statement_timeout cannot interrupt scans hung in pg_textsearch's -- BMW C-loop (no CHECK_FOR_INTERRUPTS) — for that case the workflow -- relies on a step-level timeout in benchmark.yml. SET statement_timeout = '5min'; \echo '=== MS MARCO Query Benchmarks ===' \echo '' -- Load benchmark queries (pre-sampled by token count) \echo 'Loading benchmark queries...' DROP TABLE IF EXISTS benchmark_queries; CREATE TABLE benchmark_queries ( query_id INTEGER, query_text TEXT, token_bucket INTEGER ); \copy benchmark_queries FROM 'benchmarks/datasets/msmarco/benchmark_queries.tsv' WITH (FORMAT text, DELIMITER E'\t') -- Verify load SELECT 'Loaded ' || COUNT(*) || ' benchmark queries' as status FROM benchmark_queries; SELECT token_bucket, COUNT(*) as count FROM benchmark_queries GROUP BY token_bucket ORDER BY token_bucket; -- Warm up: run queries from each bucket to ensure index is cached \echo '' \echo 'Warming up index...' DO $$ DECLARE q record; BEGIN FOR q IN SELECT query_text FROM benchmark_queries ORDER BY random() LIMIT 50 LOOP EXECUTE 'SELECT passage_id FROM msmarco_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; END; $$; -- ============================================================ -- Benchmark 1: Query Latency by Token Count -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency by Token Count ===' \echo 'Running 100 queries per token bucket, reporting p50/p95/p99' \echo '' -- Temp table to capture per-bucket results for weighted average CREATE TEMP TABLE bucket_results ( bucket int, p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint ); -- Function to benchmark a bucket and return percentiles CREATE OR REPLACE FUNCTION benchmark_bucket(bucket int) RETURNS TABLE(p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint) AS $$ DECLARE q record; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; n int; result_count bigint; results_sum bigint := 0; BEGIN -- The script-level statement_timeout caps a *single* outer -- statement; inside this function's loop it would cap the cumulative -- runtime of the whole bucket and abort partway through, throwing -- away every per-query timing already collected. Disable it locally -- so a slow-but-healthy bucket can finish; the workflow's -- step-level timeout-minutes is the real backstop for hangs. PERFORM set_config('statement_timeout', '0', true); times := ARRAY[]::numeric[]; FOR q IN SELECT query_text FROM benchmark_queries WHERE token_bucket = bucket ORDER BY query_id LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT COUNT(*) FROM (SELECT passage_id FROM msmarco_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_bm25_idx'') LIMIT 10) t' INTO result_count USING q.query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); results_sum := results_sum + result_count; END LOOP; n := array_length(times, 1); SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; p50_ms := sorted_times[(n + 1) / 2]; p95_ms := sorted_times[(n * 95 + 99) / 100]; p99_ms := sorted_times[(n * 99 + 99) / 100]; avg_ms := (SELECT AVG(t) FROM unnest(times) t); num_queries := n; total_results := results_sum; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run benchmarks for each token bucket (results saved for weighted avg) \echo 'Token bucket 1 (1 search token):' INSERT INTO bucket_results SELECT 1, * FROM benchmark_bucket(1); SELECT 'LATENCY_BUCKET_1: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 1; \echo '' \echo 'Token bucket 2 (2 search tokens):' INSERT INTO bucket_results SELECT 2, * FROM benchmark_bucket(2); SELECT 'LATENCY_BUCKET_2: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 2; \echo '' \echo 'Token bucket 3 (3 search tokens):' INSERT INTO bucket_results SELECT 3, * FROM benchmark_bucket(3); SELECT 'LATENCY_BUCKET_3: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 3; \echo '' \echo 'Token bucket 4 (4 search tokens):' INSERT INTO bucket_results SELECT 4, * FROM benchmark_bucket(4); SELECT 'LATENCY_BUCKET_4: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 4; \echo '' \echo 'Token bucket 5 (5 search tokens):' INSERT INTO bucket_results SELECT 5, * FROM benchmark_bucket(5); SELECT 'LATENCY_BUCKET_5: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 5; \echo '' \echo 'Token bucket 6 (6 search tokens):' INSERT INTO bucket_results SELECT 6, * FROM benchmark_bucket(6); SELECT 'LATENCY_BUCKET_6: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 6; \echo '' \echo 'Token bucket 7 (7 search tokens):' INSERT INTO bucket_results SELECT 7, * FROM benchmark_bucket(7); SELECT 'LATENCY_BUCKET_7: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 7; \echo '' \echo 'Token bucket 8 (8+ search tokens):' INSERT INTO bucket_results SELECT 8, * FROM benchmark_bucket(8); SELECT 'LATENCY_BUCKET_8: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 8; DROP FUNCTION benchmark_bucket; -- ============================================================ -- Weighted-Average Query Latency (MS-MARCO v1 distribution) -- ============================================================ -- Weights from observed lexeme distribution in 1,010,905 MS-MARCO -- v1 queries after to_tsvector: 1=3.5%, 2=16.3%, 3=30.2%, -- 4=26.1%, 5=14.2%, 6=5.9%, 7=2.2%, 8+=1.5% \echo '' \echo '=== Weighted-Average Query Latency & Throughput ===' SELECT 'WEIGHTED_LATENCY_RESULT: weighted_p50=' || round(SUM(b.p50_ms * w.weight) / SUM(w.weight), 2) || 'ms weighted_avg=' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || 'ms' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; SELECT 'WEIGHTED_THROUGHPUT_RESULT: ' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || ' ms/query' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; DROP TABLE bucket_results; -- ============================================================ -- Benchmark 2: Query Throughput (800 benchmark queries) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (800 queries, 3 iterations) ===' \echo 'Running all 800 benchmark queries with warmup' -- Helper function for throughput benchmark CREATE OR REPLACE FUNCTION benchmark_throughput(iterations int DEFAULT 3) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric, queries_run int) AS $$ DECLARE q record; i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; query_count int; BEGIN -- Throughput runs ~3,200 sub-queries inside one outer SELECT -- (800 warmup + 3 × 800 timed by default). The script-level -- statement_timeout would cap that cumulative runtime, aborting the -- benchmark before THROUGHPUT_RESULT is emitted — a regression -- that's slow-but-healthy still loses all signal. Opt out locally; -- the workflow's step-level timeout-minutes is the real backstop. PERFORM set_config('statement_timeout', '0', true); SELECT COUNT(*) INTO query_count FROM benchmark_queries; -- Warmup: run all queries once FOR q IN SELECT query_text FROM benchmark_queries ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; -- Timed iterations times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); FOR q IN SELECT query_text FROM benchmark_queries ORDER BY query_id LOOP EXECUTE 'SELECT passage_id FROM msmarco_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; queries_run := query_count; RETURN NEXT; END; $$ LANGUAGE plpgsql; SELECT 'Execution Time: ' || round(median_ms / queries_run, 3) || ' ms (min=' || round(min_ms / queries_run, 3) || ', max=' || round(max_ms / queries_run, 3) || ')' as result, 'THROUGHPUT_RESULT: ' || queries_run || ' queries in ' || round(median_ms, 2) || ' ms (avg ' || round(median_ms / queries_run, 2) || ' ms/query)' as summary FROM benchmark_throughput(); DROP FUNCTION benchmark_throughput; -- ============================================================ -- Benchmark 3: Index Statistics -- ============================================================ \echo '' \echo '=== Benchmark 3: Index Statistics ===' SELECT 'msmarco_bm25_idx' as index_name, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_size_pretty(pg_relation_size('msmarco_passages')) as table_size, (SELECT COUNT(*) FROM msmarco_passages) as num_documents; -- Cleanup DROP TABLE benchmark_queries; \echo '' \echo '=== MS MARCO Query Benchmarks Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/vacuum.sql000066400000000000000000000276041525762047700264010ustar00rootroot00000000000000-- MS MARCO Passage Ranking - VACUUM Performance Benchmark -- Measures partial vs full VACUUM overhead against a BM25 index -- with multiple segments. -- -- Prerequisites: run load.sql first (creates msmarco_passages -- table and msmarco_bm25_idx index). -- -- The standard load produces a single merged segment. To test -- partial vs full VACUUM we first create additional segments -- via batch inserts with controlled memtable spills, giving -- us a multi-segment index where dead tuples can be -- concentrated (partial) or spread everywhere (full). -- -- Scenarios: -- Setup: Insert batches → create multi-segment index -- A. Partial VACUUM — delete from one batch, few segments affected -- B. Full VACUUM — delete uniformly, all segments affected -- C. Full VACUUM — update uniformly, all segments affected \set ON_ERROR_STOP on \timing on \echo '=== MS MARCO VACUUM Performance Benchmark ===' \echo '' -- Disable autovacuum so only our explicit VACUUMs run ALTER TABLE msmarco_passages SET (autovacuum_enabled = false); -- ============================================================ -- Helper: run a batch of queries and report average latency -- ============================================================ CREATE OR REPLACE FUNCTION vacuum_benchmark_queries( num_queries int DEFAULT 100 ) RETURNS TABLE(avg_ms numeric, min_ms numeric, max_ms numeric, queries_run int) AS $$ DECLARE q record; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; n int; BEGIN times := ARRAY[]::numeric[]; FOR q IN SELECT query_text FROM msmarco_queries ORDER BY query_id LIMIT num_queries LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT passage_id FROM msmarco_passages ORDER BY passage_text <@> to_bm25query($1, ''msmarco_bm25_idx'') LIMIT 10' USING q.query_text; end_ts := clock_timestamp(); times := array_append( times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000 ); END LOOP; n := array_length(times, 1); IF n IS NULL OR n = 0 THEN avg_ms := 0; min_ms := 0; max_ms := 0; queries_run := 0; RETURN NEXT; RETURN; END IF; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; avg_ms := (SELECT AVG(t) FROM unnest(times) t); min_ms := sorted_times[1]; max_ms := sorted_times[n]; queries_run := n; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- ============================================================ -- Setup: Create a multi-segment index -- ============================================================ -- -- After load.sql the index has a single force-merged segment. -- We insert 4 batches of ~100K rows each (copied from existing -- data with offset passage_ids), forcing a memtable spill after -- each batch via bm25_spill_index. We suppress auto-spill by -- raising bulk_load_threshold so that each batch creates -- exactly one new segment. -- -- Result: 1 large original segment + 4 smaller L0 segments. \echo '=== Setup: Creating multi-segment index ===' \echo '' \echo 'Initial segment layout:' SELECT bm25_summarize_index('msmarco_bm25_idx'); -- Suppress auto-spill during batch inserts SET pg_textsearch.bulk_load_threshold = 2000000000; -- Batch 1: rows with passage_id offset by 10M \echo 'Inserting batch 1 (~100K rows)...' INSERT INTO msmarco_passages (passage_id, passage_text) SELECT passage_id + 10000000, passage_text FROM msmarco_passages WHERE passage_id < 100000; SELECT bm25_spill_index('msmarco_bm25_idx'); -- Batch 2: offset by 20M \echo 'Inserting batch 2 (~100K rows)...' INSERT INTO msmarco_passages (passage_id, passage_text) SELECT passage_id + 20000000, passage_text FROM msmarco_passages WHERE passage_id >= 100000 AND passage_id < 200000; SELECT bm25_spill_index('msmarco_bm25_idx'); -- Batch 3: offset by 30M \echo 'Inserting batch 3 (~100K rows)...' INSERT INTO msmarco_passages (passage_id, passage_text) SELECT passage_id + 30000000, passage_text FROM msmarco_passages WHERE passage_id >= 200000 AND passage_id < 300000; SELECT bm25_spill_index('msmarco_bm25_idx'); -- Batch 4: offset by 40M \echo 'Inserting batch 4 (~100K rows)...' INSERT INTO msmarco_passages (passage_id, passage_text) SELECT passage_id + 40000000, passage_text FROM msmarco_passages WHERE passage_id >= 300000 AND passage_id < 400000; SELECT bm25_spill_index('msmarco_bm25_idx'); -- Restore default threshold RESET pg_textsearch.bulk_load_threshold; \echo '' \echo 'Segment layout after setup:' SELECT bm25_summarize_index('msmarco_bm25_idx'); -- ============================================================ -- Baseline measurements -- ============================================================ \echo '' \echo '=== Baseline ===' SELECT 'VACUUM_TOTAL_ROWS:' as label, COUNT(*) as rows FROM msmarco_passages; SELECT 'VACUUM_BASELINE_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; SELECT 'VACUUM_BASELINE_TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('msmarco_passages')) as table_size, pg_total_relation_size('msmarco_passages') as table_bytes; -- Warm up \echo 'Warming up queries...' SELECT * FROM vacuum_benchmark_queries(20); \echo 'Baseline query throughput (100 queries):' SELECT 'VACUUM_BASELINE_QUERY: avg=' || round(avg_ms, 2) || 'ms min=' || round(min_ms, 2) || 'ms max=' || round(max_ms, 2) || 'ms (n=' || queries_run || ')' as result FROM vacuum_benchmark_queries(100); -- ============================================================ -- Scenario A: Partial VACUUM — concentrated delete -- ============================================================ -- Delete all ~100K rows from batch 1 (passage_id 10M..10.1M). -- Only the segment containing batch 1 data has dead tuples; -- the large original segment and other batch segments are -- unaffected. VACUUM should skip unaffected segments. \echo '' \echo '=== Scenario A: Partial VACUUM (concentrated delete) ===' \echo 'Deleting batch 1 rows (~100K, passage_id 10M..10.1M)...' \echo 'VACUUM_PARTIAL_DELETE_START:' DELETE FROM msmarco_passages WHERE passage_id >= 10000000 AND passage_id < 10100000; \echo 'VACUUM_PARTIAL_DELETE_END:' SELECT 'VACUUM_PARTIAL_ROWS_DELETED:' as label, (SELECT COUNT(*) FROM msmarco_passages) as remaining; SELECT 'VACUUM_PARTIAL_PRE_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; \echo 'Running VACUUM (partial — few segments affected)...' \echo 'VACUUM_PARTIAL_START:' VACUUM msmarco_passages; \echo 'VACUUM_PARTIAL_END:' SELECT 'VACUUM_PARTIAL_POST_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; \echo 'Post-partial-VACUUM query throughput (100 queries):' SELECT 'VACUUM_PARTIAL_QUERY: avg=' || round(avg_ms, 2) || 'ms min=' || round(min_ms, 2) || 'ms max=' || round(max_ms, 2) || 'ms (n=' || queries_run || ')' as result FROM vacuum_benchmark_queries(100); \echo '' \echo 'Segment layout after partial VACUUM:' SELECT bm25_summarize_index('msmarco_bm25_idx'); -- ============================================================ -- Scenario B: Full VACUUM — uniform delete -- ============================================================ -- Delete ~100K rows uniformly across the entire dataset using -- a hash function so dead tuples land in every segment. -- VACUUM must process all segments. \echo '' \echo '=== Scenario B: Full VACUUM (uniform delete) ===' \echo 'Deleting ~100K rows uniformly (hashint4 % N)...' -- Calculate modulus to delete roughly 100K of remaining rows DO $$ DECLARE total_rows bigint; target_deletes int := 100000; modulus int; BEGIN SELECT COUNT(*) INTO total_rows FROM msmarco_passages; modulus := greatest(total_rows / target_deletes, 2); RAISE NOTICE 'Total rows: %, modulus: % (target ~% deletes)', total_rows, modulus, total_rows / modulus; -- Store modulus for the DELETE below PERFORM set_config('vacuum_bench.modulus', modulus::text, false); END; $$; \echo 'VACUUM_FULL_DELETE_START:' DELETE FROM msmarco_passages WHERE hashint4(passage_id) % current_setting('vacuum_bench.modulus')::int = 0; \echo 'VACUUM_FULL_DELETE_END:' SELECT 'VACUUM_FULL_DELETE_ROWS_REMAINING:' as label, COUNT(*) as remaining FROM msmarco_passages; SELECT 'VACUUM_FULL_DELETE_PRE_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; \echo 'Running VACUUM (full — all segments affected)...' \echo 'VACUUM_FULL_START:' VACUUM msmarco_passages; \echo 'VACUUM_FULL_END:' SELECT 'VACUUM_FULL_POST_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; \echo 'Post-full-VACUUM query throughput (100 queries):' SELECT 'VACUUM_FULL_QUERY: avg=' || round(avg_ms, 2) || 'ms min=' || round(min_ms, 2) || 'ms max=' || round(max_ms, 2) || 'ms (n=' || queries_run || ')' as result FROM vacuum_benchmark_queries(100); \echo '' \echo 'Segment layout after full VACUUM:' SELECT bm25_summarize_index('msmarco_bm25_idx'); -- ============================================================ -- Scenario C: Full VACUUM — uniform update -- ============================================================ -- Update ~100K rows uniformly. Each update creates a new -- heap tuple version and dead-marks the old one. The BM25 -- index has entries for the old CTIDs that VACUUM must clean -- up across all segments. \echo '' \echo '=== Scenario C: Full VACUUM (uniform update) ===' \echo 'Updating ~100K rows uniformly...' \echo 'VACUUM_UPDATE_START:' UPDATE msmarco_passages SET passage_text = passage_text || ' (revised)' WHERE hashint4(passage_id) % current_setting('vacuum_bench.modulus')::int = 1; \echo 'VACUUM_UPDATE_END:' SELECT 'VACUUM_UPDATE_PRE_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; \echo 'Running VACUUM (full — all segments affected by update)...' \echo 'VACUUM_UPDATE_VACUUM_START:' VACUUM msmarco_passages; \echo 'VACUUM_UPDATE_VACUUM_END:' SELECT 'VACUUM_UPDATE_POST_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; \echo 'Post-update-VACUUM query throughput (100 queries):' SELECT 'VACUUM_UPDATE_QUERY: avg=' || round(avg_ms, 2) || 'ms min=' || round(min_ms, 2) || 'ms max=' || round(max_ms, 2) || 'ms (n=' || queries_run || ')' as result FROM vacuum_benchmark_queries(100); \echo '' \echo 'Segment layout after update VACUUM:' SELECT bm25_summarize_index('msmarco_bm25_idx'); -- ============================================================ -- Summary -- ============================================================ \echo '' \echo '=== VACUUM Benchmark Summary ===' SELECT 'VACUUM_FINAL_ROW_COUNT:' as label, COUNT(*) as rows FROM msmarco_passages; SELECT 'VACUUM_FINAL_INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('msmarco_bm25_idx')) as index_size, pg_relation_size('msmarco_bm25_idx') as index_bytes; -- Cleanup: re-enable autovacuum first in case later statements fail ALTER TABLE msmarco_passages SET (autovacuum_enabled = true); DROP FUNCTION IF EXISTS vacuum_benchmark_queries; \echo '' \echo '=== VACUUM Benchmark Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/msmarco/validate_queries.sql000066400000000000000000000222121525762047700304150ustar00rootroot00000000000000-- MS-MARCO Query Validation -- Compares Tapir query results against precomputed ground truth -- -- Requires: ground_truth.tsv in the same directory (created by CI or -- run_benchmark.sh from the versioned ground_truth_pgNN.tsv files) -- Usage: psql -p PORT -f validate_queries.sql -- -- Validates: the score at each rank (1..10) in the index's result -- matches the score at the same rank in ground truth, within absolute -- tolerance (0.001). -- -- Why per-rank score comparison rather than per-doc: -- -- BM25 scoring can produce *tied* docs (often clusters of 3+ docs at -- the rank-10 boundary on real corpora). Tie-break ordering is -- undefined and varies between: -- - single-writer COPY vs concurrent INSERT (different CTID order) -- - BMW top-k vs sequential scan -- - across pgbench runs (timing-dependent ordering) -- -- A naive doc-set comparison flags these tie-break differences as -- correctness failures. Per-rank score comparison treats them as the -- same result -- which they are, since BM25 ranking by score is what -- the index actually promises. -- -- For diagnostic purposes the doc set is still reported when scores -- diverge or as supplementary info, but pass/fail is decided on -- scores. \set ON_ERROR_STOP on \timing on \echo '=== MS-MARCO BM25 Validation ===' \echo '' -- Load ground truth data \echo 'Loading ground truth...' DROP TABLE IF EXISTS ground_truth; CREATE TABLE ground_truth ( query_id int, query_text text, rank int, doc_id int, score float8 ); \copy ground_truth FROM 'benchmarks/datasets/msmarco/ground_truth.tsv' WITH (FORMAT text, DELIMITER E'\t', HEADER true) SELECT 'Loaded ' || COUNT(DISTINCT query_id) || ' queries, ' || COUNT(*) || ' result rows' as status FROM ground_truth; -- Score-based validation: compare score at each rank between index -- and ground truth. CREATE OR REPLACE FUNCTION validate_single_query( p_query_id int, p_query_text text, p_tolerance float8 DEFAULT 0.001 ) RETURNS TABLE( query_id int, scores_match boolean, docs_match boolean, max_score_diff float8, worst_rank int, details text ) AS $$ DECLARE v_max_score_diff float8 := 0; v_worst_rank int := 0; v_all_match boolean := true; v_docs_match boolean; v_details text := ''; r record; BEGIN -- Get Tapir's top-10 results CREATE TEMP TABLE IF NOT EXISTS tapir_results ( rank int, doc_id int, score float8 ); TRUNCATE tapir_results; INSERT INTO tapir_results -- IMPORTANT: keep the LIMIT 10 in the inner subquery so it -- propagates to the BM25 index scan and BMW runs with K=10. -- If LIMIT 10 sits above a window function in the outer SELECT -- (the previous shape), the planner can't push it past the -- WindowAgg and the index scan falls back to -- pg_textsearch.default_limit (1000). BMW with K=1000 reports -- subtly-wrong scores for some docs on real corpora -- see -- https://github.com/timescale/pg_textsearch/issues/365. That -- false-positive misrouting was the source of the -- "concurrent-INSERT-only" validation failures repeatedly hit -- after #360 (queries 1267, 130, 1517, 222, 9351, 9340 across -- runs); they're not real correctness regressions, just BMW -- being asked to score 1000 docs where it should have been -- asked for 10. SELECT row_number() OVER ()::int as rank, t.passage_id, t.score FROM ( SELECT passage_id, -(passage_text <@> to_bm25query(p_query_text, 'msmarco_bm25_idx'))::float8 as score FROM msmarco_passages ORDER BY passage_text <@> to_bm25query(p_query_text, 'msmarco_bm25_idx') LIMIT 10 ) t; -- Compare score at each rank FOR r IN SELECT gt.rank, gt.score as gt_score, t.score as tapir_score, gt.doc_id as gt_doc, t.doc_id as tapir_doc, ABS(gt.score - t.score) as abs_diff FROM ground_truth gt JOIN tapir_results t ON t.rank = gt.rank WHERE gt.query_id = p_query_id ORDER BY gt.rank LOOP IF r.abs_diff > p_tolerance THEN v_all_match := false; IF r.abs_diff > v_max_score_diff THEN v_max_score_diff := r.abs_diff; v_worst_rank := r.rank; END IF; v_details := v_details || 'rank ' || r.rank || ': gt=' || round(r.gt_score::numeric, 4)::text || ' (doc ' || r.gt_doc || '), tapir=' || round(r.tapir_score::numeric, 4)::text || ' (doc ' || r.tapir_doc || '); '; ELSE v_max_score_diff := GREATEST(v_max_score_diff, r.abs_diff); END IF; END LOOP; -- Also check doc set as supplementary info (not part of pass/fail) SELECT COALESCE( (SELECT array_agg(gt.doc_id ORDER BY gt.doc_id) FROM ground_truth gt WHERE gt.query_id = p_query_id) = (SELECT array_agg(doc_id ORDER BY doc_id) FROM tapir_results), false ) INTO v_docs_match; query_id := p_query_id; scores_match := v_all_match; docs_match := v_docs_match; max_score_diff := v_max_score_diff; worst_rank := v_worst_rank; details := CASE WHEN v_details = '' THEN 'OK' ELSE trim(trailing '; ' from v_details) END; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run validation for all ground truth queries \echo '' \echo '=== Running Validation ===' DROP TABLE IF EXISTS validation_results; CREATE TABLE validation_results AS SELECT v.* FROM (SELECT DISTINCT gt.query_id, gt.query_text FROM ground_truth gt) q, LATERAL validate_single_query(q.query_id, q.query_text) v; -- Summary \echo '' \echo '=== Validation Summary ===' SELECT COUNT(*) as total_queries, SUM(CASE WHEN scores_match THEN 1 ELSE 0 END) as scores_match_count, SUM(CASE WHEN docs_match THEN 1 ELSE 0 END) as docs_match_count, round(100.0 * SUM(CASE WHEN scores_match THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 1) as scores_match_pct, round(100.0 * SUM(CASE WHEN docs_match THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 1) as docs_match_pct, round(MAX(max_score_diff)::numeric, 6) as worst_abs_diff FROM validation_results; -- Known-mismatch allowlist. -- Entries here have stable, pre-existing per-rank score discrepancies -- against ground_truth.tsv that are tracked separately and excluded -- from the pass/fail signal so the watchdog can still catch new -- regressions. Each entry must reference a tracking issue. DROP TABLE IF EXISTS known_mismatches; CREATE TABLE known_mismatches ( query_id int PRIMARY KEY, issue text NOT NULL, note text ); INSERT INTO known_mismatches (query_id, issue, note) VALUES -- See #361. Doc 1838655 contains "Adding" which stems to "ad" -- on current pg17 but stemmed to "add" on the pg17 minor that -- generated ground_truth_pg17.tsv in Feb 2026 (libstemmer/snowball -- rule drift). The index is correct against current stemmer -- rules; the committed GT file is stale. Fix is to regenerate -- ground_truth_pg17.tsv against current pg17 -- when that lands, -- remove this allowlist entry. (158, 'https://github.com/timescale/pg_textsearch/issues/361', '+how to add differnt names on labels in word: stale pg17 GT (stemmer drift on "Adding")'); -- Check for failures and report DO $$ DECLARE v_total int; v_failures int; v_known int; BEGIN SELECT COUNT(*) INTO v_total FROM validation_results; -- A query fails validation if any per-rank score differs beyond -- tolerance. Doc-set differences (tied-cluster ordering) do not -- fail validation as long as scores match. Allowlisted queries -- (known pre-existing score discrepancies) are counted -- separately and do NOT trigger VALIDATION FAILED. SELECT COUNT(*) INTO v_failures FROM validation_results vr WHERE NOT scores_match AND NOT EXISTS ( SELECT 1 FROM known_mismatches km WHERE km.query_id = vr.query_id ); SELECT COUNT(*) INTO v_known FROM validation_results vr JOIN known_mismatches km ON km.query_id = vr.query_id WHERE NOT scores_match; IF v_known > 0 THEN RAISE NOTICE 'VALIDATION: % allowlisted known mismatch(es) ignored (see known_mismatches table)', v_known; END IF; IF v_failures > 0 THEN RAISE NOTICE 'VALIDATION FAILED: % of % queries failed (per-rank scores differ beyond % tolerance, excluding allowlist)', v_failures, v_total, 0.001; ELSE RAISE NOTICE 'VALIDATION PASSED: All % queries match within tolerance (% allowlisted known mismatches)', v_total - v_known, v_known; END IF; END; $$; -- Failed queries detail \echo '' \echo '=== Validation Details (failures only) ===' SELECT query_id, scores_match, docs_match, round(max_score_diff::numeric, 6) as max_abs_diff, worst_rank, left(details, 300) as details FROM validation_results WHERE NOT scores_match ORDER BY max_score_diff DESC LIMIT 20; -- Cleanup DROP TABLE IF EXISTS ground_truth; DROP TABLE IF EXISTS validation_results; DROP TABLE IF EXISTS known_mismatches; DROP TABLE IF EXISTS tapir_results; DROP FUNCTION IF EXISTS validate_single_query; \echo '' \echo '=== Validation Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/000077500000000000000000000000001525762047700246545ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/download.sh000077500000000000000000000167371525762047700270400ustar00rootroot00000000000000#!/bin/bash # Download and process Wikipedia dump for benchmarking # Uses the Wikimedia Enterprise HTML dumps (cleaner than XML) # English Wikipedia: ~6.7M articles, ~21GB compressed # # For faster testing, we also support downloading a subset set -e is_py310() { "$1" <<'EOF' >/dev/null 2>&1 import sys sys.exit(0 if (sys.version_info.major, sys.version_info.minor) == (3, 10) else 1) EOF } PYTHON_BIN="" for candidate in python3.10 python3 python; do if is_py310 "$candidate"; then PYTHON_BIN="$candidate" break fi done if [ -z "$PYTHON_BIN" ]; then echo "Error: Python 3.10 is required. Install python3.10 and retry." exit 1 fi echo "Using Python interpreter: $PYTHON_BIN" SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" DATA_DIR="$SCRIPT_DIR/data" SUBSET_SIZE="${1:-full}" # "full", "1M", "100K", "10K" mkdir -p "$DATA_DIR" cd "$DATA_DIR" echo "=== Wikipedia Dataset Download ===" echo "Subset: $SUBSET_SIZE" echo "" # We'll use the Simple Wikipedia dump for a more manageable size, # or the BEIR Wikipedia subset which is pre-processed for IR benchmarks if [ "$SUBSET_SIZE" = "full" ]; then # Full English Wikipedia - articles only (no talk pages, etc.) # Latest dump from dumps.wikimedia.org DUMP_DATE="latest" DUMP_URL="https://dumps.wikimedia.org/enwiki/$DUMP_DATE/enwiki-$DUMP_DATE-pages-articles.xml.bz2" echo "Downloading full English Wikipedia dump..." echo "WARNING: This is ~21GB compressed, ~80GB uncompressed" echo "Consider using a subset for initial testing: ./download.sh 1M" echo "" if [ ! -f "enwiki-pages-articles.xml.bz2" ]; then # Get the actual latest dump date DUMP_DATE=$(curl -s https://dumps.wikimedia.org/enwiki/ | \ grep -oE '20[0-9]{6}' | sort -r | head -1) DUMP_URL="https://dumps.wikimedia.org/enwiki/$DUMP_DATE/enwiki-$DUMP_DATE-pages-articles.xml.bz2" echo "Latest dump: $DUMP_DATE" wget -q --show-progress -O enwiki-pages-articles.xml.bz2 "$DUMP_URL" fi echo "Processing Wikipedia XML dump..." echo "This requires WikiExtractor: python3.10 -m pip install 'wikiextractor==3.0.6'" # Always (re)install the pinned version. An importability check would # skip the pin on a persistent/self-hosted env that already has a # different (e.g. drift-inducing 3.0.8+) wikiextractor; pip with an # exact pin is idempotent when satisfied and downgrades otherwise. echo "Installing wikiextractor==3.0.6..." "$PYTHON_BIN" -m pip install 'wikiextractor==3.0.6' # Extract to JSON format if [ ! -d "extracted" ]; then "$PYTHON_BIN" -m wikiextractor.WikiExtractor \ --json \ --output extracted \ --compress \ enwiki-pages-articles.xml.bz2 fi # Convert to TSV format for COPY echo "Converting to TSV format..." "$PYTHON_BIN" << 'PYTHON_SCRIPT' import json import gzip import os from pathlib import Path output_file = open("wikipedia.tsv", "w") doc_id = 0 for subdir in sorted(Path("extracted").iterdir()): if not subdir.is_dir(): continue for wiki_file in sorted(subdir.iterdir()): if wiki_file.suffix == '.gz': opener = gzip.open else: opener = open with opener(wiki_file, 'rt', encoding='utf-8') as f: for line in f: doc = json.loads(line) title = doc.get('title', '').replace('\t', ' ').replace('\n', ' ') text = doc.get('text', '').replace('\t', ' ').replace('\n', ' ') if title and text and len(text) > 100: doc_id += 1 output_file.write(f"{doc_id}\t{title}\t{text}\n") if doc_id % 100000 == 0: print(f"Processed {doc_id} articles...") output_file.close() print(f"Total articles: {doc_id}") PYTHON_SCRIPT else # Use pre-processed subsets for faster testing # BEIR provides a Wikipedia subset, or we can use HuggingFace datasets echo "Downloading Wikipedia subset ($SUBSET_SIZE articles)..." # For subsets, we'll generate from Simple Wikipedia or use a seed + generate case "$SUBSET_SIZE" in "1M") NUM_ARTICLES=1000000 ;; "100K") NUM_ARTICLES=100000 ;; "10K") NUM_ARTICLES=10000 ;; *) echo "Unknown subset size: $SUBSET_SIZE" echo "Use: full, 1M, 100K, or 10K" exit 1 ;; esac # Download Simple English Wikipedia (smaller, ~200K articles) # and truncate to desired size. # # IMPORTANT: pin to a specific dump date so doc_ids assigned by # extraction order are stable across CI runs. The committed # ground_truth.tsv is keyed by these doc_ids; using "latest" # makes the ground truth go stale every time Wikimedia publishes # a new dump. When bumping SIMPLE_WIKI_DUMP_DATE, regenerate the # ground truth with precompute_ground_truth.sql in the same PR. # # For the same reason, wikiextractor is pinned to 3.0.6 everywhere # it is installed (here and in .github/workflows/benchmark.yml): # 3.0.8 (released 2026-07-23) changed extraction output, which # shifts doc_ids and BM25 scores and broke the committed # ground_truth.tsv. Bumping the wikiextractor pin likewise requires # regenerating the ground truth in the same PR. SIMPLE_WIKI_DUMP_DATE="${SIMPLE_WIKI_DUMP_DATE:-20260501}" DUMP_FILE="simplewiki-${SIMPLE_WIKI_DUMP_DATE}-pages-articles.xml.bz2" DUMP_URL="https://dumps.wikimedia.org/simplewiki/${SIMPLE_WIKI_DUMP_DATE}/${DUMP_FILE}" if [ ! -f "$DUMP_FILE" ]; then echo "Downloading Simple English Wikipedia (dump ${SIMPLE_WIKI_DUMP_DATE})..." wget -q --show-progress -O "$DUMP_FILE" "$DUMP_URL" fi # Always (re)install the pinned version (see note above); do not gate # on importability, which would skip the pin when a different version # is already present. echo "Installing wikiextractor==3.0.6..." "$PYTHON_BIN" -m pip install 'wikiextractor==3.0.6' if [ ! -d "simple_extracted" ]; then echo "Extracting Simple Wikipedia..." "$PYTHON_BIN" -m wikiextractor.WikiExtractor \ --json \ --output simple_extracted \ "$DUMP_FILE" fi echo "Converting to TSV (limiting to $NUM_ARTICLES articles)..." "$PYTHON_BIN" << PYTHON_SCRIPT import json import os from pathlib import Path output_file = open("wikipedia.tsv", "w") doc_id = 0 max_docs = $NUM_ARTICLES for subdir in sorted(Path("simple_extracted").iterdir()): if not subdir.is_dir(): continue for wiki_file in sorted(subdir.iterdir()): with open(wiki_file, 'r', encoding='utf-8') as f: for line in f: if doc_id >= max_docs: break doc = json.loads(line) title = doc.get('title', '').replace('\t', ' ').replace('\n', ' ') text = doc.get('text', '').replace('\t', ' ').replace('\n', ' ') if title and text and len(text) > 100: doc_id += 1 output_file.write(f"{doc_id}\t{title}\t{text}\n") if doc_id >= max_docs: break if doc_id >= max_docs: break output_file.close() print(f"Total articles: {doc_id}") PYTHON_SCRIPT fi echo "" echo "=== Download Complete ===" if [ -f "wikipedia.tsv" ]; then echo "Articles: $(wc -l < wikipedia.tsv)" echo "File size: $(du -h wikipedia.tsv | cut -f1)" fi timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/gin/000077500000000000000000000000001525762047700254315ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/gin/concurrent-setup.sql000066400000000000000000000036151525762047700314770ustar00rootroot00000000000000-- Wikipedia GIN+tsvector Baseline - Concurrent Insert Setup -- Loads articles into a staging table, creates empty target table -- with GIN index, and prepares a sequence for pgbench-driven -- concurrent inserts. -- -- Usage: -- DATA_DIR=/path/to/data psql -f gin/concurrent-setup.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia GIN Concurrent Insert Setup ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_gin_articles CASCADE; DROP TABLE IF EXISTS wikipedia_gin_staging CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create staging table (raw data, no tsv column) \echo 'Creating staging table...' CREATE TABLE wikipedia_gin_staging ( staging_id SERIAL PRIMARY KEY, article_id INTEGER, title TEXT, content TEXT ); -- Load articles into staging \echo 'Loading articles into staging table...' \copy wikipedia_gin_staging(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Create target articles table with GIN index \echo 'Creating target articles table...' CREATE TABLE wikipedia_gin_articles ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', content) ) STORED ); \echo 'Creating GIN index on empty target table...' CREATE INDEX wikipedia_gin_idx ON wikipedia_gin_articles USING gin(tsv); -- Create sequence for pgbench inserts \echo 'Creating insert sequence...' CREATE SEQUENCE insert_seq; -- Report staging row count \echo '' \echo '=== Setup Verification ===' SELECT 'STAGING_ROW_COUNT:' as label, COUNT(*)::text as count FROM wikipedia_gin_staging; \echo '' \echo '=== GIN Concurrent Insert Setup Complete ===' \echo 'Run pgbench with gin/pgbench-insert.sql' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/gin/insert-load.sql000066400000000000000000000034771525762047700304060ustar00rootroot00000000000000-- Wikipedia GIN+tsvector Baseline - Insert Loading -- Creates GIN index on EMPTY table, then loads data via COPY -- Measures insert performance into an already-indexed table -- -- Usage: -- DATA_DIR=/path/to/data psql -f gin/insert-load.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia GIN Insert Benchmark - Data Loading ===' \echo 'Index created BEFORE loading articles' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_gin_articles CASCADE; -- Create articles table \echo 'Creating articles table...' CREATE TABLE wikipedia_gin_articles ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', content) ) STORED ); -- Create GIN index on empty table \echo 'Creating GIN index on empty table...' CREATE INDEX wikipedia_gin_idx ON wikipedia_gin_articles USING gin(tsv); -- Load articles into the pre-indexed table \echo '' \echo 'INSERT_TIME:' \echo 'Loading articles into indexed table...' \copy wikipedia_gin_articles(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('wikipedia_gin_idx') ) as index_size, pg_relation_size('wikipedia_gin_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'wikipedia_gin_articles') ) as table_size, pg_total_relation_size( 'wikipedia_gin_articles') as table_bytes; \echo '' \echo '=== Wikipedia GIN Insert Benchmark Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/gin/load.sql000066400000000000000000000032611525762047700270730ustar00rootroot00000000000000-- Wikipedia GIN+tsvector Baseline - Data Loading -- Loads data first, then creates GIN index on populated table -- Measures CREATE INDEX time and final sizes -- -- Usage: -- DATA_DIR=/path/to/data psql -f gin/load.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia GIN Baseline - Data Loading ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_gin_articles CASCADE; -- Create articles table (without index) \echo 'Creating articles table...' CREATE TABLE wikipedia_gin_articles ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', content) ) STORED ); -- Load articles \echo 'Loading Wikipedia articles...' \copy wikipedia_gin_articles(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Create GIN index on populated table (timed) \echo 'Building GIN index...' CREATE INDEX wikipedia_gin_idx ON wikipedia_gin_articles USING gin(tsv); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('wikipedia_gin_idx') ) as index_size, pg_relation_size('wikipedia_gin_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'wikipedia_gin_articles') ) as table_size, pg_total_relation_size( 'wikipedia_gin_articles') as table_bytes; \echo '' \echo '=== Wikipedia GIN Load Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/gin/pgbench-insert.sql000066400000000000000000000003141525762047700310600ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO wikipedia_gin_articles (article_id, title, content) SELECT article_id, title, content FROM wikipedia_gin_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/ground_truth.tsv000066400000000000000000001472601525762047700301500ustar00rootroot00000000000000query_id query_text rank doc_id score 1 algorithm 1 40284 12.83112893332038 1 algorithm 2 76616 12.685531519354681 1 algorithm 3 40505 12.65424814278223 1 algorithm 4 39931 12.480282433190206 1 algorithm 5 9390 12.4622219756273 1 algorithm 6 11725 12.258081190473186 1 algorithm 7 58387 12.126362665938375 1 algorithm 8 74827 12.059459336182572 1 algorithm 9 5813 12.05797390405742 1 algorithm 10 30280 11.922906015966207 2 history 1 161 4.533737956822642 2 history 2 22410 4.49851828130276 2 history 3 87520 4.4792025850317225 2 history 4 21385 4.474443648886297 2 history 5 22671 4.455957159181547 2 history 6 172 4.4438665936825785 2 history 7 17685 4.431574976477312 2 history 8 69872 4.399706566859289 2 history 9 900 4.376031579526045 2 history 10 24933 4.348768270557561 3 science 1 22899 7.487448691808366 3 science 2 12957 7.479148843326338 3 science 3 1715 7.464960550461493 3 science 4 11301 7.279175193476345 3 science 5 11457 7.252277607212519 3 science 6 1419 7.248558707804247 3 science 7 77436 7.212868385245456 3 science 8 83721 7.139116389859332 3 science 9 28843 7.104451638356885 3 science 10 1938 7.053387615484497 4 computer 1 31303 7.8178464100762115 4 computer 2 9244 7.81545643399995 4 computer 3 46076 7.780212894807153 4 computer 4 30167 7.733286543106764 4 computer 5 6397 7.733152019639198 4 computer 6 57 7.697668052062501 4 computer 7 77317 7.6953099295279515 4 computer 8 89083 7.6953099295279515 4 computer 9 26696 7.688839411368012 4 computer 10 555 7.686178242005987 5 biology 1 8712 9.4841485871382 5 biology 2 2843 9.466273966702888 5 biology 3 14779 9.445660381444945 5 biology 4 16870 9.3537754998447 5 biology 5 62231 9.261290751528003 5 biology 6 91343 9.223027054504255 5 biology 7 75695 9.071285832740044 5 biology 8 12891 8.687689395061852 5 biology 9 53569 8.573084609805303 5 biology 10 99523 8.53444674845218 6 mathematics 1 44305 9.415303531286014 6 mathematics 2 44577 9.19503422640844 6 mathematics 3 64738 9.186243041971784 6 mathematics 4 64340 9.185707986556595 6 mathematics 5 29185 9.161988561919358 6 mathematics 6 40576 9.123177311747208 6 mathematics 7 3295 8.922324783522507 6 mathematics 8 44579 8.87401278557238 6 mathematics 9 229 8.85654201070714 6 mathematics 10 14235 8.715076391490403 7 physics 1 12185 8.031499448381563 7 physics 2 12971 7.900554481821919 7 physics 3 31089 7.88658072122865 7 physics 4 44305 7.8554489458712675 7 physics 5 24612 7.6969177559957975 7 physics 6 77300 7.6969177559957975 7 physics 7 22436 7.600492227392537 7 physics 8 293 7.515912433434138 7 physics 9 61412 7.508140374742202 7 physics 10 54600 7.4814540466818595 8 chemistry 1 62171 10.543895776044327 8 chemistry 2 64820 10.473463346687527 8 chemistry 3 53257 10.450787502354228 8 chemistry 4 64100 10.375111000237045 8 chemistry 5 61372 10.196797061203831 8 chemistry 6 61412 10.087154898274392 8 chemistry 7 54337 9.88969128963642 8 chemistry 8 63959 9.78614968500654 8 chemistry 9 62231 9.769484619110154 8 chemistry 10 90284 9.64958724729724 9 geography 1 78476 7.623569478828633 9 geography 2 145 7.612352702914163 9 geography 3 81625 7.555320702716882 9 geography 4 60651 6.943936514726062 9 geography 5 89395 6.939945657628698 9 geography 6 90013 6.8498219732218715 9 geography 7 74807 6.762009013815295 9 geography 8 36866 6.269255387884092 9 geography 9 37714 6.269255387884092 9 geography 10 20467 6.232218692061835 10 philosophy 1 75811 9.753395493424119 10 philosophy 2 292 9.700727561786914 10 philosophy 3 24046 9.643459390375423 10 philosophy 4 60799 9.624733663585726 10 philosophy 5 65546 9.581456016860365 10 philosophy 6 79199 9.469915883571943 10 philosophy 7 26330 9.182783639003201 10 philosophy 8 15910 9.161408965766453 10 philosophy 9 65119 8.954962123393239 10 philosophy 10 86521 8.838075602869148 101 machine learning 1 91925 15.177226004959646 101 machine learning 2 71937 12.80026800502575 101 machine learning 3 91923 12.56828965829239 101 machine learning 4 9135 11.774984861591976 101 machine learning 5 86062 11.569211854274496 101 machine learning 6 91921 11.263870518301202 101 machine learning 7 6397 10.878242852989914 101 machine learning 8 1716 10.876365495861618 101 machine learning 9 86519 10.821246261515206 101 machine learning 10 85045 10.39447874799719 102 world war 1 997 9.38567202553583 102 world war 2 9759 9.231962906604146 102 world war 3 12693 8.918113240563098 102 world war 4 81067 8.90346511530118 102 world war 5 48791 8.8344111978879 102 world war 6 42686 8.75613218370095 102 world war 7 84143 8.690143837312457 102 world war 8 77697 8.650497155640807 102 world war 9 75045 8.501320997267666 102 world war 10 75556 8.459302753387679 103 climate change 1 89456 12.75094599148225 103 climate change 2 45954 12.361214707721313 103 climate change 3 63467 12.135369502869395 103 climate change 4 442 12.087277982457879 103 climate change 5 32742 11.92330661603149 103 climate change 6 40125 11.906951338122834 103 climate change 7 7749 11.815514262465756 103 climate change 8 85739 11.691826353120174 103 climate change 9 87054 11.329326637795727 103 climate change 10 89386 11.299713631997381 104 solar system 1 22722 15.716817473674896 104 solar system 2 9294 14.649836683664653 104 solar system 3 12990 14.61299199643937 104 solar system 4 58237 14.612853143578103 104 solar system 5 13293 14.286968994787092 104 solar system 6 79208 14.238773201079734 104 solar system 7 97864 14.17835431968013 104 solar system 8 21839 14.114713441820738 104 solar system 9 311 13.925083212819743 104 solar system 10 17604 13.864139369440089 105 human body 1 173 12.671699240628485 105 human body 2 6338 11.818672946401364 105 human body 3 66074 11.577363787646597 105 human body 4 69917 11.528783189416366 105 human body 5 28529 11.24124984270669 105 human body 6 31468 11.034060915347949 105 human body 7 77203 10.966234943185682 105 human body 8 8399 10.82031277670914 105 human body 9 13090 10.730797905055216 105 human body 10 69644 10.727194357386539 106 ancient history 1 6752 11.213996295376699 106 ancient history 2 161 10.170750848809305 106 ancient history 3 15903 10.106437963410585 106 ancient history 4 2201 9.986944356762443 106 ancient history 5 57096 9.840677549740452 106 ancient history 6 31702 9.452076998631396 106 ancient history 7 18021 9.251052570469424 106 ancient history 8 63585 9.236231621418955 106 ancient history 9 95431 9.220681135242767 106 ancient history 10 6625 9.153107670849026 107 modern art 1 63639 12.802173250714631 107 modern art 2 54598 12.289657837258437 107 modern art 3 46521 12.158502405398739 107 modern art 4 20330 12.142648174118474 107 modern art 5 65324 11.869321879578722 107 modern art 6 70022 11.813769087685639 107 modern art 7 84958 11.569985252174794 107 modern art 8 54299 11.478217272877696 107 modern art 9 74657 11.283133574290297 107 modern art 10 42687 11.164541223642104 108 nuclear physics 1 57880 16.527307635730416 108 nuclear physics 2 28329 15.020080711871806 108 nuclear physics 3 44790 14.789756159603172 108 nuclear physics 4 18609 13.729659752520135 108 nuclear physics 5 7059 13.412656672701921 108 nuclear physics 6 86942 13.364333558635561 108 nuclear physics 7 85154 13.201177640181886 108 nuclear physics 8 7277 13.19793095695807 108 nuclear physics 9 90476 13.072649650451467 108 nuclear physics 10 28203 12.55133963969611 109 organic chemistry 1 62171 16.08913312910846 109 organic chemistry 2 53257 15.799317454400526 109 organic chemistry 3 64820 14.87790284481757 109 organic chemistry 4 17036 14.521256536425728 109 organic chemistry 5 2626 14.192793601956083 109 organic chemistry 6 64599 13.992555635297872 109 organic chemistry 7 99839 13.51939114851379 109 organic chemistry 8 54014 13.463368436332338 109 organic chemistry 9 61988 13.39020144205919 109 organic chemistry 10 61945 13.22217362440182 110 cell biology 1 42955 17.261258095211836 110 cell biology 2 16870 16.23302250401558 110 cell biology 3 2843 16.108318322232574 110 cell biology 4 94283 15.938604103634876 110 cell biology 5 46957 15.754776756810244 110 cell biology 6 57463 15.398834755893512 110 cell biology 7 75897 14.809445828569979 110 cell biology 8 11368 14.528959083985399 110 cell biology 9 59359 14.492506747230061 110 cell biology 10 68282 14.343848798755346 201 world war two 1 88020 11.310842930411013 201 world war two 2 997 11.288709171371101 201 world war two 3 77697 11.207707861832132 201 world war two 4 42686 10.903292135694375 201 world war two 5 15778 10.791454005764884 201 world war two 6 75229 10.649175193894015 201 world war two 7 9759 10.601237848886875 201 world war two 8 95014 10.492533195072495 201 world war two 9 88325 10.417535455506696 201 world war two 10 981 10.33101157418733 202 machine learning algorithms 1 39931 19.874765701073606 202 machine learning algorithms 2 30384 16.82081603251656 202 machine learning algorithms 3 50934 15.790787498525152 202 machine learning algorithms 4 91925 15.177226004959646 202 machine learning algorithms 5 40284 14.708393967942914 202 machine learning algorithms 6 15449 14.670584957945326 202 machine learning algorithms 7 49216 14.659252693730554 202 machine learning algorithms 8 30707 14.281310639998136 202 machine learning algorithms 9 71937 12.80026800502575 202 machine learning algorithms 10 76616 12.685531519354681 203 climate change effects 1 89456 16.884504448210617 203 climate change effects 2 40125 16.426334258305573 203 climate change effects 3 63342 16.301236663957084 203 climate change effects 4 99140 14.925973965453966 203 climate change effects 5 1964 13.870565288547956 203 climate change effects 6 63793 13.664746372665329 203 climate change effects 7 55468 13.442496802787915 203 climate change effects 8 2310 13.32255511800748 203 climate change effects 9 80016 13.118398134845858 203 climate change effects 10 23432 12.630004207235359 204 human immune system 1 193 20.08461383852788 204 human immune system 2 13020 19.893044399795897 204 human immune system 3 53915 19.066094054451707 204 human immune system 4 79737 18.765063813555063 204 human immune system 5 52572 17.65905302220905 204 human immune system 6 63636 17.632344406089356 204 human immune system 7 298 17.530849800113334 204 human immune system 8 44598 17.384371143170632 204 human immune system 9 57219 17.26171102609304 204 human immune system 10 72546 17.217883779917337 205 solar system planets 1 22722 23.95929661152069 205 solar system planets 2 79208 23.77447955884547 205 solar system planets 3 311 23.62280113984361 205 solar system planets 4 9294 23.262127746129117 205 solar system planets 5 43807 21.79716500536165 205 solar system planets 6 369 21.771241179213384 205 solar system planets 7 91344 21.652862948407616 205 solar system planets 8 67132 21.37471299730566 205 solar system planets 9 31116 21.25214632638599 205 solar system planets 10 43813 20.93710940764145 206 ancient greek philosophy 1 65546 21.804485550857024 206 ancient greek philosophy 2 43953 20.306461632651768 206 ancient greek philosophy 3 75811 18.170951388224545 206 ancient greek philosophy 4 29500 17.640480521418716 206 ancient greek philosophy 5 11623 17.42515132347137 206 ancient greek philosophy 6 10295 17.26711957047418 206 ancient greek philosophy 7 24046 17.162360244309 206 ancient greek philosophy 8 10975 17.08754483336716 206 ancient greek philosophy 9 32192 16.768641719621293 206 ancient greek philosophy 10 44879 16.67203808389754 207 modern art movements 1 63639 17.236771962107028 207 modern art movements 2 74505 17.001127477148206 207 modern art movements 3 42687 16.954206655139558 207 modern art movements 4 54646 15.699346784252779 207 modern art movements 5 54598 15.67949351335791 207 modern art movements 6 42181 15.464616110307055 207 modern art movements 7 84958 15.395880459601752 207 modern art movements 8 54299 14.888072648429123 207 modern art movements 9 70578 14.525363873797433 207 modern art movements 10 20329 14.367892486060892 208 nuclear power plants 1 21707 22.632404592507456 208 nuclear power plants 2 53451 22.36410881497936 208 nuclear power plants 3 23536 22.148002136595895 208 nuclear power plants 4 31069 21.221259843501016 208 nuclear power plants 5 23524 21.117209963178844 208 nuclear power plants 6 28231 21.111612254922946 208 nuclear power plants 7 66293 21.005730702938536 208 nuclear power plants 8 32873 20.715611604235143 208 nuclear power plants 9 10648 20.527189885297545 208 nuclear power plants 10 31007 20.520447220524524 209 organic chemistry compounds 1 53257 24.645759840981977 209 organic chemistry compounds 2 2626 21.624900521846204 209 organic chemistry compounds 3 61945 21.57261421084859 209 organic chemistry compounds 4 17036 20.347479737091156 209 organic chemistry compounds 5 61816 20.053928125660768 209 organic chemistry compounds 6 59483 19.674207806081583 209 organic chemistry compounds 7 62243 19.639839947822974 209 organic chemistry compounds 8 54014 19.39415657803099 209 organic chemistry compounds 9 78806 19.175168550020544 209 organic chemistry compounds 10 61382 19.1071380065981 210 cell division process 1 16582 20.756338242351504 210 cell division process 2 12716 18.732819106190554 210 cell division process 3 20000 18.02049608451698 210 cell division process 4 6559 17.881703131638872 210 cell division process 5 46955 17.158178741849753 210 cell division process 6 7664 16.807937113007107 210 cell division process 7 9965 16.728790459493393 210 cell division process 8 46793 16.724834003444293 210 cell division process 9 10683 16.112490834696917 210 cell division process 10 72406 16.0776701071197 301 second world war battles 1 59213 17.981783219239986 301 second world war battles 2 84853 17.959039459343764 301 second world war battles 3 3104 17.08314278610338 301 second world war battles 4 84597 17.031086121623826 301 second world war battles 5 21849 16.993927376731598 301 second world war battles 6 62047 16.183374941295657 301 second world war battles 7 58829 16.175105499575047 301 second world war battles 8 69903 15.61579297716984 301 second world war battles 9 16190 15.40036929094562 301 second world war battles 10 64655 15.370155532891438 302 deep learning neural networks 1 10408 28.847286487301552 302 deep learning neural networks 2 71630 26.37466151308683 302 deep learning neural networks 3 52123 15.041171276176865 302 deep learning neural networks 4 75861 14.815021799976805 302 deep learning neural networks 5 74197 14.374614003054337 302 deep learning neural networks 6 1716 13.794696954689423 302 deep learning neural networks 7 82013 12.87333060301891 302 deep learning neural networks 8 30385 12.468304231302273 302 deep learning neural networks 9 54649 12.088794444227815 302 deep learning neural networks 10 48018 12.087986693984057 303 global climate change impacts 1 32742 24.01426308688259 303 global climate change impacts 2 83139 20.44997017664933 303 global climate change impacts 3 32756 20.14098919693812 303 global climate change impacts 4 15440 19.82751087660175 303 global climate change impacts 5 40125 19.542780776735047 303 global climate change impacts 6 45954 19.477653901659522 303 global climate change impacts 7 3964 19.31114961891877 303 global climate change impacts 8 39823 17.516471866955207 303 global climate change impacts 9 1964 17.05601311245539 303 global climate change impacts 10 40149 16.730843887510687 304 human cardiovascular system function 1 27 23.585261834136467 304 human cardiovascular system function 2 310 16.14587099656464 304 human cardiovascular system function 3 59890 15.473201987431754 304 human cardiovascular system function 4 88734 14.435904168896071 304 human cardiovascular system function 5 173 14.335654021613948 304 human cardiovascular system function 6 3608 14.041891302959407 304 human cardiovascular system function 7 23290 14.006619173577736 304 human cardiovascular system function 8 3643 13.7338565221906 304 human cardiovascular system function 9 298 13.393025818802325 304 human cardiovascular system function 10 19939 13.369010968354546 305 outer solar system exploration 1 22722 23.681881950110963 305 outer solar system exploration 2 528 21.835601277985973 305 outer solar system exploration 3 18663 21.17044108994763 305 outer solar system exploration 4 7853 19.98964541258328 305 outer solar system exploration 5 67132 19.56659629873711 305 outer solar system exploration 6 6060 18.34686722916902 305 outer solar system exploration 7 9895 18.076765753027768 305 outer solar system exploration 8 369 17.753342764646927 305 outer solar system exploration 9 41002 16.96634601354414 305 outer solar system exploration 10 10188 16.931283028436194 306 ancient greek city states 1 30108 18.096786155778414 306 ancient greek city states 2 3111 17.049323376958633 306 ancient greek city states 3 3528 16.808162028526358 306 ancient greek city states 4 4240 16.096062656902607 306 ancient greek city states 5 76227 15.997440689443371 306 ancient greek city states 6 43153 15.531949945747286 306 ancient greek city states 7 24248 15.387838744254626 306 ancient greek city states 8 8100 15.333814015880098 306 ancient greek city states 9 29534 15.185651758760226 306 ancient greek city states 10 42394 15.168784017267733 307 abstract expressionist art movement 1 54646 30.298737820323346 307 abstract expressionist art movement 2 55064 28.076669579157617 307 abstract expressionist art movement 3 18527 24.581155463725295 307 abstract expressionist art movement 4 66446 23.083265653231194 307 abstract expressionist art movement 5 30211 21.931725062299215 307 abstract expressionist art movement 6 86373 21.77764771301119 307 abstract expressionist art movement 7 2253 21.58127202823354 307 abstract expressionist art movement 8 77191 21.15429973775301 307 abstract expressionist art movement 9 30736 19.44328785962209 307 abstract expressionist art movement 10 63645 18.96613459132653 308 nuclear fusion energy research 1 5081 30.56981870577558 308 nuclear fusion energy research 2 66293 28.22273093537947 308 nuclear fusion energy research 3 7080 27.246180669976113 308 nuclear fusion energy research 4 10331 23.482920516620258 308 nuclear fusion energy research 5 10648 23.3673356479149 308 nuclear fusion energy research 6 30059 23.188889175120416 308 nuclear fusion energy research 7 12602 23.076395207150618 308 nuclear fusion energy research 8 69285 22.509693891203465 308 nuclear fusion energy research 9 24087 21.29762106633203 308 nuclear fusion energy research 10 52079 20.88894366828326 309 aromatic organic chemistry reactions 1 78806 30.958618060633505 309 aromatic organic chemistry reactions 2 2626 26.21798675238378 309 aromatic organic chemistry reactions 3 20069 23.03526112689763 309 aromatic organic chemistry reactions 4 61382 22.96518650085912 309 aromatic organic chemistry reactions 5 62033 21.61379719724948 309 aromatic organic chemistry reactions 6 61284 21.321427827902145 309 aromatic organic chemistry reactions 7 61338 20.939581731378333 309 aromatic organic chemistry reactions 8 77171 20.724281516422664 309 aromatic organic chemistry reactions 9 62243 19.86031745534969 309 aromatic organic chemistry reactions 10 61945 19.583639273451936 310 stem cell research applications 1 73218 29.261546169831696 310 stem cell research applications 2 41592 23.95038286243355 310 stem cell research applications 3 92591 23.328889007883035 310 stem cell research applications 4 73202 18.864905268817854 310 stem cell research applications 5 78791 17.91969351354464 310 stem cell research applications 6 9440 17.903240447047946 310 stem cell research applications 7 57460 17.876145067062858 310 stem cell research applications 8 56944 17.764068307500054 310 stem cell research applications 9 526 17.3086951706847 310 stem cell research applications 10 41589 17.156577339554975 401 battle of stalingrad world war 1 45148 27.504883200518854 401 battle of stalingrad world war 2 42753 24.127677280599812 401 battle of stalingrad world war 3 93964 24.07840130064461 401 battle of stalingrad world war 4 42868 23.745375790250066 401 battle of stalingrad world war 5 40983 22.185675711635202 401 battle of stalingrad world war 6 85085 20.57760920369618 401 battle of stalingrad world war 7 77409 20.19795703947472 401 battle of stalingrad world war 8 22242 19.950532598934164 401 battle of stalingrad world war 9 30033 19.862522319030024 401 battle of stalingrad world war 10 85754 18.616393252877174 402 transformer architecture deep learning models 1 99533 18.948793942723952 402 transformer architecture deep learning models 2 52566 16.6349164805996 402 transformer architecture deep learning models 3 29301 15.067844896999809 402 transformer architecture deep learning models 4 90229 14.471639210846481 402 transformer architecture deep learning models 5 31282 14.415341924689429 402 transformer architecture deep learning models 6 65180 13.997694300339408 402 transformer architecture deep learning models 7 72055 13.490069376817491 402 transformer architecture deep learning models 8 20027 13.301476290188521 402 transformer architecture deep learning models 9 98787 13.155138302061669 402 transformer architecture deep learning models 10 96934 13.14299712659108 403 paris agreement climate change targets 1 87054 18.520115092177587 403 paris agreement climate change targets 2 40125 18.18197145356475 403 paris agreement climate change targets 3 90576 15.842820000533031 403 paris agreement climate change targets 4 48198 13.648866318182566 403 paris agreement climate change targets 5 8075 13.295871947290074 403 paris agreement climate change targets 6 15630 13.276655919122678 403 paris agreement climate change targets 7 47112 13.230795967850225 403 paris agreement climate change targets 8 5952 13.174370631610685 403 paris agreement climate change targets 9 23023 13.111129846225285 403 paris agreement climate change targets 10 8433 12.793032739825865 404 human heart blood circulation system 1 3532 31.30591816199625 404 human heart blood circulation system 2 10144 27.42663592666976 404 human heart blood circulation system 3 12620 22.06814881130878 404 human heart blood circulation system 4 3530 21.983956132365453 404 human heart blood circulation system 5 5711 21.857305260504276 404 human heart blood circulation system 6 52572 21.690410711979627 404 human heart blood circulation system 7 18165 20.611805173334076 404 human heart blood circulation system 8 29777 20.583459374990714 404 human heart blood circulation system 9 3521 19.8419660772852 404 human heart blood circulation system 10 4482 19.47592427714768 405 voyager probe outer solar system 1 18663 37.60863850183168 405 voyager probe outer solar system 2 15573 33.616367572255726 405 voyager probe outer solar system 3 60993 31.83343627691962 405 voyager probe outer solar system 4 6060 30.79349692748741 405 voyager probe outer solar system 5 10880 27.692527210719124 405 voyager probe outer solar system 6 31530 24.554282869612276 405 voyager probe outer solar system 7 22722 23.681881950110963 405 voyager probe outer solar system 8 370 23.043795499475486 405 voyager probe outer solar system 9 200 21.87659080087528 405 voyager probe outer solar system 10 18272 21.44926270212664 406 athenian democracy ancient greek politics 1 15774 25.080084582438523 406 athenian democracy ancient greek politics 2 20163 23.027386841626694 406 athenian democracy ancient greek politics 3 3528 22.738025394809302 406 athenian democracy ancient greek politics 4 83608 22.36330055411412 406 athenian democracy ancient greek politics 5 84397 22.189875440408933 406 athenian democracy ancient greek politics 6 20360 21.428595295239873 406 athenian democracy ancient greek politics 7 26608 21.14868845428867 406 athenian democracy ancient greek politics 8 29148 20.843207001038756 406 athenian democracy ancient greek politics 9 76039 20.616431038587418 406 athenian democracy ancient greek politics 10 53299 20.260066089549795 407 jackson pollock abstract expressionist paintings 1 54646 40.44500653898618 407 jackson pollock abstract expressionist paintings 2 4092 35.48589876773657 407 jackson pollock abstract expressionist paintings 3 55064 34.44443568077572 407 jackson pollock abstract expressionist paintings 4 93604 31.354763235598927 407 jackson pollock abstract expressionist paintings 5 2253 28.07898623717402 407 jackson pollock abstract expressionist paintings 6 66446 23.764772559345012 407 jackson pollock abstract expressionist paintings 7 86373 23.027727579436252 407 jackson pollock abstract expressionist paintings 8 54598 19.50461317154182 407 jackson pollock abstract expressionist paintings 9 63645 19.4487045446296 407 jackson pollock abstract expressionist paintings 10 18527 19.327612488940346 408 iter tokamak nuclear fusion reactor 1 66293 47.51880296972212 408 iter tokamak nuclear fusion reactor 2 57880 26.59739469052873 408 iter tokamak nuclear fusion reactor 3 10648 26.385529261920674 408 iter tokamak nuclear fusion reactor 4 12602 25.767470057270906 408 iter tokamak nuclear fusion reactor 5 7632 22.971781307962658 408 iter tokamak nuclear fusion reactor 6 21707 22.74440911532725 408 iter tokamak nuclear fusion reactor 7 7772 22.142899897161243 408 iter tokamak nuclear fusion reactor 8 6077 21.697805193152103 408 iter tokamak nuclear fusion reactor 9 65041 21.53980952537485 408 iter tokamak nuclear fusion reactor 10 14440 21.364369392461338 409 benzene ring aromatic compound chemistry 1 78806 46.05825288329221 409 benzene ring aromatic compound chemistry 2 62033 43.1436216305957 409 benzene ring aromatic compound chemistry 3 7066 41.67330078422733 409 benzene ring aromatic compound chemistry 4 62051 39.01397632616984 409 benzene ring aromatic compound chemistry 5 31825 36.327928831464114 409 benzene ring aromatic compound chemistry 6 20069 34.922987373139854 409 benzene ring aromatic compound chemistry 7 1990 34.48305907490565 409 benzene ring aromatic compound chemistry 8 76935 32.757828757084184 409 benzene ring aromatic compound chemistry 9 2626 28.92059187768075 409 benzene ring aromatic compound chemistry 10 62579 25.468301366996414 410 induced pluripotent stem cell research 1 73218 40.43518072813797 410 induced pluripotent stem cell research 2 73202 28.438906189632633 410 induced pluripotent stem cell research 3 46956 25.28297279388378 410 induced pluripotent stem cell research 4 41592 23.95038286243355 410 induced pluripotent stem cell research 5 92591 23.328889007883035 410 induced pluripotent stem cell research 6 526 22.07830586246234 410 induced pluripotent stem cell research 7 46957 18.86589322051838 410 induced pluripotent stem cell research 8 78791 17.91969351354464 410 induced pluripotent stem cell research 9 9440 17.903240447047946 410 induced pluripotent stem cell research 10 57460 17.876145067062858 501 operation barbarossa german invasion soviet union 1 71243 41.823093670220146 501 operation barbarossa german invasion soviet union 2 40983 36.455741784053394 501 operation barbarossa german invasion soviet union 3 6296 36.42451891673274 501 operation barbarossa german invasion soviet union 4 69314 32.92362348808891 501 operation barbarossa german invasion soviet union 5 58828 28.27436798350394 501 operation barbarossa german invasion soviet union 6 4133 23.91744943507559 501 operation barbarossa german invasion soviet union 7 84771 23.49043038240819 501 operation barbarossa german invasion soviet union 8 39729 22.814832213161672 501 operation barbarossa german invasion soviet union 9 12352 22.154189707658 501 operation barbarossa german invasion soviet union 10 22242 21.839153488335768 502 attention mechanism transformer neural network architecture 1 71630 21.84794303805718 502 attention mechanism transformer neural network architecture 2 10408 20.544754700734636 502 attention mechanism transformer neural network architecture 3 88241 16.43219389974631 502 attention mechanism transformer neural network architecture 4 15493 16.065912322041065 502 attention mechanism transformer neural network architecture 5 99533 14.976124876884121 502 attention mechanism transformer neural network architecture 6 30570 14.823991999192137 502 attention mechanism transformer neural network architecture 7 5622 14.227763988638694 502 attention mechanism transformer neural network architecture 8 31282 13.912416428937284 502 attention mechanism transformer neural network architecture 9 99149 13.881426718893803 502 attention mechanism transformer neural network architecture 10 22489 13.420118439082657 503 intergovernmental panel climate change ipcc reports 1 22240 40.213278901834876 503 intergovernmental panel climate change ipcc reports 2 40120 30.214754285072956 503 intergovernmental panel climate change ipcc reports 3 15440 20.493266060974726 503 intergovernmental panel climate change ipcc reports 4 1964 20.187929592693305 503 intergovernmental panel climate change ipcc reports 5 40125 17.848606632429497 503 intergovernmental panel climate change ipcc reports 6 4380 17.35380847914885 503 intergovernmental panel climate change ipcc reports 7 47262 16.664357191074775 503 intergovernmental panel climate change ipcc reports 8 64104 16.57828570594334 503 intergovernmental panel climate change ipcc reports 9 53916 15.25028892260622 503 intergovernmental panel climate change ipcc reports 10 43781 14.498542816489554 504 sinoatrial node heart electrical conduction system 1 74588 32.671712124661354 504 sinoatrial node heart electrical conduction system 2 3521 29.835439933138332 504 sinoatrial node heart electrical conduction system 3 40274 18.298958482180325 504 sinoatrial node heart electrical conduction system 4 31821 17.882660454679957 504 sinoatrial node heart electrical conduction system 5 12620 17.67330468777459 504 sinoatrial node heart electrical conduction system 6 3340 17.237050244132114 504 sinoatrial node heart electrical conduction system 7 7022 17.01102466203414 504 sinoatrial node heart electrical conduction system 8 54508 16.827085280038304 504 sinoatrial node heart electrical conduction system 9 45557 16.60380304592529 504 sinoatrial node heart electrical conduction system 10 80364 16.501096418964025 505 new horizons pluto kuiper belt flyby 1 20931 49.06759596408127 505 new horizons pluto kuiper belt flyby 2 71068 36.8804362172003 505 new horizons pluto kuiper belt flyby 3 9891 34.33836275928452 505 new horizons pluto kuiper belt flyby 4 18955 34.14580104746021 505 new horizons pluto kuiper belt flyby 5 8562 32.271828896552435 505 new horizons pluto kuiper belt flyby 6 93779 27.572445422126517 505 new horizons pluto kuiper belt flyby 7 40105 26.851823121671877 505 new horizons pluto kuiper belt flyby 8 50484 22.935236631445868 505 new horizons pluto kuiper belt flyby 9 9214 22.326515286612167 505 new horizons pluto kuiper belt flyby 10 26494 22.267877060395673 506 pericles funeral oration athenian democracy speech 1 51230 59.470965116478425 506 pericles funeral oration athenian democracy speech 2 15774 40.24185468146163 506 pericles funeral oration athenian democracy speech 3 26608 30.0187803754565 506 pericles funeral oration athenian democracy speech 4 16092 19.118485139730176 506 pericles funeral oration athenian democracy speech 5 55089 19.016058820994502 506 pericles funeral oration athenian democracy speech 6 62040 18.103906332633823 506 pericles funeral oration athenian democracy speech 7 12484 17.42692312280825 506 pericles funeral oration athenian democracy speech 8 20163 17.36029435177679 506 pericles funeral oration athenian democracy speech 9 20360 17.228259181938242 506 pericles funeral oration athenian democracy speech 10 40040 16.633322048022535 507 number one jackson pollock drip painting 1 4092 38.27157366033974 507 number one jackson pollock drip painting 2 54646 27.349208375508987 507 number one jackson pollock drip painting 3 57020 19.89487916420191 507 number one jackson pollock drip painting 4 93604 19.346089892945027 507 number one jackson pollock drip painting 5 20330 18.25463531213042 507 number one jackson pollock drip painting 6 55064 17.96054747310471 507 number one jackson pollock drip painting 7 1615 17.16624942172565 507 number one jackson pollock drip painting 8 31120 16.219123816881602 507 number one jackson pollock drip painting 9 43439 15.609909949184736 507 number one jackson pollock drip painting 10 54598 15.186740172145374 508 national ignition facility laser fusion experiment 1 68156 20.939555911058868 508 national ignition facility laser fusion experiment 2 66293 19.463862970338774 508 national ignition facility laser fusion experiment 3 64532 18.518774462557058 508 national ignition facility laser fusion experiment 4 40452 16.44233713032262 508 national ignition facility laser fusion experiment 5 64339 16.14720295516298 508 national ignition facility laser fusion experiment 6 50707 16.05572850099353 508 national ignition facility laser fusion experiment 7 5081 15.18368932037241 508 national ignition facility laser fusion experiment 8 1082 13.73222880098465 508 national ignition facility laser fusion experiment 9 64545 12.996878659138975 508 national ignition facility laser fusion experiment 10 9988 12.955974050557957 509 nucleophilic aromatic substitution organic chemistry mechanism 1 2626 39.97453351594669 509 nucleophilic aromatic substitution organic chemistry mechanism 2 77171 35.55118703843031 509 nucleophilic aromatic substitution organic chemistry mechanism 3 62015 35.2954400959807 509 nucleophilic aromatic substitution organic chemistry mechanism 4 75925 33.269587802636366 509 nucleophilic aromatic substitution organic chemistry mechanism 5 62016 28.898017026122922 509 nucleophilic aromatic substitution organic chemistry mechanism 6 8379 27.084285349994936 509 nucleophilic aromatic substitution organic chemistry mechanism 7 75841 24.472093486750666 509 nucleophilic aromatic substitution organic chemistry mechanism 8 76161 24.10254711384553 509 nucleophilic aromatic substitution organic chemistry mechanism 9 78806 23.375588455223856 509 nucleophilic aromatic substitution organic chemistry mechanism 10 75998 23.289127255172602 510 shinya yamanaka induced pluripotent stem cells 1 73218 55.82567677813995 510 shinya yamanaka induced pluripotent stem cells 2 73202 34.41154840476033 510 shinya yamanaka induced pluripotent stem cells 3 46956 25.28297279388378 510 shinya yamanaka induced pluripotent stem cells 4 8377 23.40209904504175 510 shinya yamanaka induced pluripotent stem cells 5 41592 20.2852556620589 510 shinya yamanaka induced pluripotent stem cells 6 526 19.33625881945356 510 shinya yamanaka induced pluripotent stem cells 7 46957 18.86589322051838 510 shinya yamanaka induced pluripotent stem cells 8 9440 17.903240447047946 510 shinya yamanaka induced pluripotent stem cells 9 41589 17.156577339554975 510 shinya yamanaka induced pluripotent stem cells 10 41167 17.108174913842966 601 operation barbarossa largest military invasion in history 1 71243 36.12907110799549 601 operation barbarossa largest military invasion in history 2 5243 23.79710680678129 601 operation barbarossa largest military invasion in history 3 43949 23.40303169026172 601 operation barbarossa largest military invasion in history 4 40983 20.822450425127975 601 operation barbarossa largest military invasion in history 5 6296 17.847411306174514 601 operation barbarossa largest military invasion in history 6 58826 17.391270181254768 601 operation barbarossa largest military invasion in history 7 22206 16.814909764135464 601 operation barbarossa largest military invasion in history 8 94682 16.109040193691637 601 operation barbarossa largest military invasion in history 9 94095 16.057477333427343 601 operation barbarossa largest military invasion in history 10 49021 15.939910143519054 602 multi head self attention transformer encoder decoder 1 11881 24.613087714510563 602 multi head self attention transformer encoder decoder 2 63010 20.331080181394142 602 multi head self attention transformer encoder decoder 3 45884 19.50438654790874 602 multi head self attention transformer encoder decoder 4 22217 19.24592139438813 602 multi head self attention transformer encoder decoder 5 41278 18.76132166153807 602 multi head self attention transformer encoder decoder 6 28278 18.2773096893603 602 multi head self attention transformer encoder decoder 7 19818 17.610489465725994 602 multi head self attention transformer encoder decoder 8 57165 16.974991488838747 602 multi head self attention transformer encoder decoder 9 58403 15.851442367567714 602 multi head self attention transformer encoder decoder 10 18366 15.534168263064984 603 ipcc sixth assessment report climate change impacts 1 22240 43.36712673763958 603 ipcc sixth assessment report climate change impacts 2 53020 21.301609504762165 603 ipcc sixth assessment report climate change impacts 3 32742 18.027040720567534 603 ipcc sixth assessment report climate change impacts 4 15440 17.99184001055729 603 ipcc sixth assessment report climate change impacts 5 40125 17.848606632429497 603 ipcc sixth assessment report climate change impacts 6 63749 16.27534168745709 603 ipcc sixth assessment report climate change impacts 7 40120 15.910057825451402 603 ipcc sixth assessment report climate change impacts 8 3964 15.541422237215325 603 ipcc sixth assessment report climate change impacts 9 4003 14.665548974238018 603 ipcc sixth assessment report climate change impacts 10 81393 14.567688849234472 604 purkinje fibers heart ventricular electrical conduction system 1 3521 42.48558435547877 604 purkinje fibers heart ventricular electrical conduction system 2 84179 25.587497149600118 604 purkinje fibers heart ventricular electrical conduction system 3 11486 25.166642938767147 604 purkinje fibers heart ventricular electrical conduction system 4 11483 20.626126332436566 604 purkinje fibers heart ventricular electrical conduction system 5 9451 20.158280476654397 604 purkinje fibers heart ventricular electrical conduction system 6 3340 17.237050244132114 604 purkinje fibers heart ventricular electrical conduction system 7 7022 17.01102466203414 604 purkinje fibers heart ventricular electrical conduction system 8 54508 16.827085280038304 604 purkinje fibers heart ventricular electrical conduction system 9 45557 16.60380304592529 604 purkinje fibers heart ventricular electrical conduction system 10 59142 16.319734225053995 605 voyager golden record sounds of earth message 1 63354 31.86292551614013 605 voyager golden record sounds of earth message 2 15573 28.305568073294904 605 voyager golden record sounds of earth message 3 43041 20.75456753324762 605 voyager golden record sounds of earth message 4 10880 17.612903796694034 605 voyager golden record sounds of earth message 5 15589 17.562911984776136 605 voyager golden record sounds of earth message 6 72098 15.047388676681233 605 voyager golden record sounds of earth message 7 1776 14.99163819550161 605 voyager golden record sounds of earth message 8 78036 14.5687853694384 605 voyager golden record sounds of earth message 9 15646 14.55613356909469 605 voyager golden record sounds of earth message 10 18663 14.060405482033431 606 delian league athenian empire tribute collection system 1 15774 25.225929299696798 606 delian league athenian empire tribute collection system 2 12584 21.376370765318953 606 delian league athenian empire tribute collection system 3 48627 20.024855705430358 606 delian league athenian empire tribute collection system 4 3528 19.71892720706321 606 delian league athenian empire tribute collection system 5 11232 17.717992153203298 606 delian league athenian empire tribute collection system 6 20163 16.57438329551176 606 delian league athenian empire tribute collection system 7 76039 15.769533344920493 606 delian league athenian empire tribute collection system 8 10702 13.450303098152165 606 delian league athenian empire tribute collection system 9 9369 13.191291046198048 606 delian league athenian empire tribute collection system 10 55089 13.127666974977625 607 autumn rhythm number 30 jackson pollock moma 1 20330 31.282593396014036 607 autumn rhythm number 30 jackson pollock moma 2 6676 18.633734422520522 607 autumn rhythm number 30 jackson pollock moma 3 4092 18.53062781085533 607 autumn rhythm number 30 jackson pollock moma 4 32286 16.082250906846483 607 autumn rhythm number 30 jackson pollock moma 5 97003 16.058629123097838 607 autumn rhythm number 30 jackson pollock moma 6 97001 15.799090698167454 607 autumn rhythm number 30 jackson pollock moma 7 84882 15.549999260244016 607 autumn rhythm number 30 jackson pollock moma 8 84920 15.24008904025414 607 autumn rhythm number 30 jackson pollock moma 9 31120 14.540380115616742 607 autumn rhythm number 30 jackson pollock moma 10 54646 14.33938738957755 608 lawrence livermore national ignition facility fusion breakthrough 1 88770 29.268914179729677 608 lawrence livermore national ignition facility fusion breakthrough 2 85536 23.519489243179287 608 lawrence livermore national ignition facility fusion breakthrough 3 27003 22.705502314561247 608 lawrence livermore national ignition facility fusion breakthrough 4 16437 20.180945270885587 608 lawrence livermore national ignition facility fusion breakthrough 5 50187 19.34656512338639 608 lawrence livermore national ignition facility fusion breakthrough 6 10363 17.00943634045197 608 lawrence livermore national ignition facility fusion breakthrough 7 36659 15.869151251791134 608 lawrence livermore national ignition facility fusion breakthrough 8 75528 15.804559646100241 608 lawrence livermore national ignition facility fusion breakthrough 9 64365 14.943721101302607 608 lawrence livermore national ignition facility fusion breakthrough 10 23536 14.862790318216817 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 1 62015 37.65293831170252 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 2 76935 27.14284877878984 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 3 64174 26.018762526682234 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 4 62016 24.789778495880118 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 5 2626 24.748122954929364 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 6 77171 24.630275322586193 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 7 62008 22.461552894335572 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 8 62051 19.705659368367797 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 9 93323 18.330351545676717 609 electrophilic aromatic substitution friedel crafts alkylation mechanism 10 61343 17.7104933027265 610 takahashi shinya yamanaka nobel prize stem cells 1 73218 51.21043629185016 610 takahashi shinya yamanaka nobel prize stem cells 2 73202 35.31766712292938 610 takahashi shinya yamanaka nobel prize stem cells 3 8377 25.824661936397035 610 takahashi shinya yamanaka nobel prize stem cells 4 57468 25.76137338234198 610 takahashi shinya yamanaka nobel prize stem cells 5 56944 22.87499048203442 610 takahashi shinya yamanaka nobel prize stem cells 6 54524 22.72767955563861 610 takahashi shinya yamanaka nobel prize stem cells 7 62526 20.69843516533233 610 takahashi shinya yamanaka nobel prize stem cells 8 41592 20.2852556620589 610 takahashi shinya yamanaka nobel prize stem cells 9 62527 20.07893670578158 610 takahashi shinya yamanaka nobel prize stem cells 10 96225 19.925573528768883 701 operation barbarossa june 1941 german invasion of soviet union 1 71243 48.18673249483159 701 operation barbarossa june 1941 german invasion of soviet union 2 6296 42.95657956803483 701 operation barbarossa june 1941 german invasion of soviet union 3 40983 39.63502861357084 701 operation barbarossa june 1941 german invasion of soviet union 4 69314 36.27700137377383 701 operation barbarossa june 1941 german invasion of soviet union 5 58828 33.82112871202272 701 operation barbarossa june 1941 german invasion of soviet union 6 6297 31.650538710499223 701 operation barbarossa june 1941 german invasion of soviet union 7 39729 29.797507682738974 701 operation barbarossa june 1941 german invasion of soviet union 8 4133 29.36802033198252 701 operation barbarossa june 1941 german invasion of soviet union 9 40549 27.466959516276717 701 operation barbarossa june 1941 german invasion of soviet union 10 47726 27.240041046467763 702 scaled dot product attention multi head self attention mechanism 1 44322 20.96916905257727 702 scaled dot product attention multi head self attention mechanism 2 27919 20.8890038895842 702 scaled dot product attention multi head self attention mechanism 3 90849 18.98939814443515 702 scaled dot product attention multi head self attention mechanism 4 77918 18.81125118672342 702 scaled dot product attention multi head self attention mechanism 5 55177 18.171840650824294 702 scaled dot product attention multi head self attention mechanism 6 815 17.873413665406645 702 scaled dot product attention multi head self attention mechanism 7 77185 17.775833677817303 702 scaled dot product attention multi head self attention mechanism 8 31087 17.321223595643175 702 scaled dot product attention multi head self attention mechanism 9 239 17.31460032031591 702 scaled dot product attention multi head self attention mechanism 10 12413 16.79848813524599 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 1 22240 34.36473379660542 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 2 3964 20.669163749831597 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 3 40125 19.358664313791664 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 4 1203 17.663138034242728 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 5 1964 17.076319756889674 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 6 15440 16.576022880267345 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 7 30262 16.126187162330478 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 8 84150 16.03248466213946 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 9 50395 15.621214218794307 703 ipcc ar6 synthesis report limiting global warming 1.5 degrees 10 71135 15.621214218794307 704 sinoatrial node atrioventricular node bundle of his conduction 1 3521 33.6019548196289 704 sinoatrial node atrioventricular node bundle of his conduction 2 89082 28.128365261618885 704 sinoatrial node atrioventricular node bundle of his conduction 3 58255 27.839104337604763 704 sinoatrial node atrioventricular node bundle of his conduction 4 77123 27.69918791965967 704 sinoatrial node atrioventricular node bundle of his conduction 5 40274 26.90266070447189 704 sinoatrial node atrioventricular node bundle of his conduction 6 74588 26.658594275100036 704 sinoatrial node atrioventricular node bundle of his conduction 7 17620 25.021175235999827 704 sinoatrial node atrioventricular node bundle of his conduction 8 59595 24.92618232109624 704 sinoatrial node atrioventricular node bundle of his conduction 9 26207 24.071060333742526 704 sinoatrial node atrioventricular node bundle of his conduction 10 159 23.75556506483658 705 voyager 1 golden record sounds of earth carl sagan 1 63354 48.643927187884806 705 voyager 1 golden record sounds of earth carl sagan 2 15573 43.74726778113284 705 voyager 1 golden record sounds of earth carl sagan 3 43041 30.115103533371283 705 voyager 1 golden record sounds of earth carl sagan 4 15569 29.78595809433194 705 voyager 1 golden record sounds of earth carl sagan 5 1115 28.493664012733756 705 voyager 1 golden record sounds of earth carl sagan 6 63390 23.26021457844397 705 voyager 1 golden record sounds of earth carl sagan 7 10880 22.804443117602986 705 voyager 1 golden record sounds of earth carl sagan 8 26446 18.509632448437117 705 voyager 1 golden record sounds of earth carl sagan 9 63353 17.247044936106114 705 voyager 1 golden record sounds of earth carl sagan 10 18663 17.156390961611887 706 delian league treasury moved from delos to athens tribute 1 48627 27.29044636096034 706 delian league treasury moved from delos to athens tribute 2 12584 24.10264614118498 706 delian league treasury moved from delos to athens tribute 3 15774 20.321561718484077 706 delian league treasury moved from delos to athens tribute 4 11232 19.73887060969448 706 delian league treasury moved from delos to athens tribute 5 20163 19.601016471582298 706 delian league treasury moved from delos to athens tribute 6 42394 17.367040274645895 706 delian league treasury moved from delos to athens tribute 7 3528 17.34593717425529 706 delian league treasury moved from delos to athens tribute 8 16770 15.792639859585517 706 delian league treasury moved from delos to athens tribute 9 31429 14.575300209003153 706 delian league treasury moved from delos to athens tribute 10 45012 13.824147638842618 707 autumn rhythm number 30 1950 jackson pollock moma collection 1 20330 31.282593396014036 707 autumn rhythm number 30 1950 jackson pollock moma collection 2 6676 18.633734422520522 707 autumn rhythm number 30 1950 jackson pollock moma collection 3 4092 18.53062781085533 707 autumn rhythm number 30 1950 jackson pollock moma collection 4 63186 17.112622973824685 707 autumn rhythm number 30 1950 jackson pollock moma collection 5 76713 16.44218083208175 707 autumn rhythm number 30 1950 jackson pollock moma collection 6 32286 16.082250906846483 707 autumn rhythm number 30 1950 jackson pollock moma collection 7 97003 16.058629123097838 707 autumn rhythm number 30 1950 jackson pollock moma collection 8 97001 15.799090698167454 707 autumn rhythm number 30 1950 jackson pollock moma collection 9 84882 15.549999260244016 707 autumn rhythm number 30 1950 jackson pollock moma collection 10 84920 15.24008904025414 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 1 88770 29.268914179729677 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 2 85536 23.519489243179287 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 3 27003 22.705502314561247 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 4 16437 20.180945270885587 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 5 10363 17.00943634045197 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 6 36659 15.869151251791134 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 7 75528 15.804559646100241 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 8 64365 14.943721101302607 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 9 23536 14.862790318216817 708 national ignition facility lawrence livermore 3.15 megajoule fusion yield 10 61954 13.802320528668789 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 1 76935 44.1883671360937 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 2 77171 43.12537996207849 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 3 2626 39.475217716977596 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 4 62015 36.68780938414012 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 5 78806 30.555677974956545 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 6 62033 29.67126181898177 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 7 62051 29.237597080122722 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 8 20069 25.72747169518914 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 9 7066 25.67319255098233 709 electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction 10 62008 24.606011435070197 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 1 73218 40.38106951994731 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 2 73202 24.04833644015195 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 3 87585 23.458524518800488 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 4 46956 16.389822508549983 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 5 69696 14.761954139960025 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 6 8377 13.19713642703599 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 7 32101 12.051562609610817 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 8 16339 11.836267572627133 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 9 3632 11.762920750319097 710 yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem 10 14126 11.742047601603232 timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/insert/000077500000000000000000000000001525762047700261605ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/insert/concurrent-setup.sql000066400000000000000000000037131525762047700322250ustar00rootroot00000000000000-- Wikipedia Dataset - Concurrent Insert Setup -- Loads articles into a staging table, creates empty target table -- with BM25 index, and prepares a sequence for pgbench-driven -- concurrent inserts. -- -- Usage: -- DATA_DIR=/path/to/data psql -f insert/concurrent-setup.sql -- -- After running this script, use pgbench with pgbench-insert.sql -- to drive concurrent inserts from staging into the indexed table. \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia Concurrent Insert Setup ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_articles CASCADE; DROP TABLE IF EXISTS wikipedia_staging CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create staging table \echo 'Creating staging table...' CREATE TABLE wikipedia_staging ( staging_id SERIAL PRIMARY KEY, article_id INTEGER, title TEXT, content TEXT ); -- Load articles into staging \echo 'Loading articles into staging table...' \copy wikipedia_staging(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Create target articles table \echo 'Creating target articles table...' CREATE TABLE wikipedia_articles ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL ); -- Create BM25 index on empty target table \echo 'Creating BM25 index on empty target table...' CREATE INDEX wikipedia_bm25_idx ON wikipedia_articles USING bm25(content) WITH (text_config='english'); -- Create sequence for pgbench to draw row IDs from \echo 'Creating insert sequence...' CREATE SEQUENCE insert_seq; -- Report staging row count \echo '' \echo '=== Setup Verification ===' SELECT 'STAGING_ROW_COUNT:' as label, COUNT(*)::text as count FROM wikipedia_staging; \echo '' \echo '=== Concurrent Insert Setup Complete ===' \echo 'Run pgbench with pgbench-insert.sql to start inserts' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/insert/load.sql000066400000000000000000000040541525762047700276230ustar00rootroot00000000000000-- Wikipedia Dataset - Insert Benchmark (Single Transaction) -- Creates BM25 index on an empty table, then loads data via COPY -- to measure insert-path indexing performance. -- -- Usage: -- DATA_DIR=/path/to/data psql -f insert/load.sql -- -- The DATA_DIR environment variable should point to the directory -- containing: -- - wikipedia.tsv (article_id, title, content) \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia Insert Benchmark - Data Loading ===' \echo 'Index created BEFORE loading articles' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_articles CASCADE; -- Create articles table \echo 'Creating articles table...' CREATE TABLE wikipedia_articles ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL ); -- Create BM25 index on the empty table \echo 'Creating BM25 index on empty table...' CREATE INDEX wikipedia_bm25_idx ON wikipedia_articles USING bm25(content) WITH (text_config='english'); -- Load articles into the pre-indexed table \echo '' \echo 'INSERT_TIME:' \echo 'Loading articles (this may take a while with live index)...' \copy wikipedia_articles(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Articles loaded' as metric, COUNT(*)::text as value FROM wikipedia_articles UNION ALL SELECT 'Avg content length (chars)', ROUND(AVG(LENGTH(content)))::text FROM wikipedia_articles UNION ALL SELECT 'Total text size (MB)', ROUND(SUM(LENGTH(content)) / 1024.0 / 1024.0, 1)::text FROM wikipedia_articles; -- NOTE: spill + size + segment-stats moved to size_report.sql, run -- AFTER queries.sql, so the query phase exercises the realistic -- post-load state (in-flight memtable chain + cache + segments). \echo '' \echo '=== Wikipedia Insert Benchmark Load Complete ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/insert/pgbench-insert.sql000066400000000000000000000003001525762047700316020ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO wikipedia_articles (article_id, title, content) SELECT article_id, title, content FROM wikipedia_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/insert/size_report.sql000066400000000000000000000020671525762047700312530ustar00rootroot00000000000000-- Wikipedia insert benchmark: post-query size report. -- -- Runs AFTER queries.sql so the query phase exercises the realistic -- post-load state (in-flight memtable chain + cache + segments). -- Spilling here ensures pg_relation_size reflects all data on disk -- and bm25_summarize_index reports the final segment layout that -- downstream metric extractors expect. \echo '' \echo '=== Wikipedia Insert: Post-Query Size Report ===' -- Spill memtable to disk so pg_relation_size reflects actual index data SELECT bm25_spill_index('wikipedia_bm25_idx'); \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('wikipedia_bm25_idx') ) as index_size, pg_relation_size('wikipedia_bm25_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size('wikipedia_articles') ) as table_size, pg_total_relation_size('wikipedia_articles') as table_bytes; \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('wikipedia_bm25_idx'); timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/load.sql000066400000000000000000000044101525762047700263130ustar00rootroot00000000000000-- Wikipedia Dataset - Load and Index -- Loads Wikipedia articles and creates BM25 index -- -- Usage: -- DATA_DIR=/path/to/data psql -f load.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia Dataset - Data Loading ===' \echo '' -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_articles CASCADE; -- Create articles table \echo 'Creating articles table...' CREATE TABLE wikipedia_articles ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL ); -- Load articles from TSV (convert to CSV to avoid \. escaping issues) \echo 'Loading Wikipedia articles (this may take several minutes)...' \copy wikipedia_articles(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Articles loaded' as metric, COUNT(*)::text as value FROM wikipedia_articles UNION ALL SELECT 'Avg content length (chars)', ROUND(AVG(LENGTH(content)))::text FROM wikipedia_articles UNION ALL SELECT 'Total text size (MB)', ROUND(SUM(LENGTH(content)) / 1024.0 / 1024.0, 1)::text FROM wikipedia_articles; -- Show sample data \echo '' \echo 'Sample articles:' SELECT article_id, title, LEFT(content, 100) || '...' as content_preview FROM wikipedia_articles ORDER BY article_id LIMIT 5; -- Create BM25 index \echo '' \echo '=== Building BM25 Index ===' \echo 'Creating BM25 index on Wikipedia articles...' CREATE INDEX wikipedia_bm25_idx ON wikipedia_articles USING bm25(content) WITH (text_config='english'); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('wikipedia_bm25_idx')) as index_size, pg_relation_size('wikipedia_bm25_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('wikipedia_articles')) as table_size, pg_total_relation_size('wikipedia_articles') as table_bytes; -- Segment statistics \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('wikipedia_bm25_idx'); \echo '' \echo '=== Wikipedia Load Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/paradedb/000077500000000000000000000000001525762047700264165ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/paradedb/concurrent-setup.sql000066400000000000000000000044361525762047700324660ustar00rootroot00000000000000-- Wikipedia Dataset - ParadeDB Concurrent Insert Setup -- Loads articles into a staging table, creates empty target -- table with ParadeDB BM25 index, and prepares a sequence for -- pgbench-driven concurrent inserts. -- -- Usage: -- DATA_DIR=/path/to/data psql -f paradedb/concurrent-setup.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia ParadeDB Concurrent Insert Setup ===' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_articles_paradedb CASCADE; DROP TABLE IF EXISTS wikipedia_paradedb_staging CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create staging table \echo 'Creating staging table...' CREATE TABLE wikipedia_paradedb_staging ( staging_id SERIAL PRIMARY KEY, article_id INTEGER, title TEXT, content TEXT ); -- Load articles into staging \echo 'Loading articles into staging table...' \copy wikipedia_paradedb_staging(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Create target articles table \echo 'Creating target articles table...' CREATE TABLE wikipedia_articles_paradedb ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL ); -- Create ParadeDB BM25 index on empty target table \echo 'Creating ParadeDB BM25 index on empty table...' CREATE INDEX wikipedia_paradedb_idx ON wikipedia_articles_paradedb USING bm25 (article_id, content) WITH ( key_field = 'article_id', text_fields = '{ "content": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Create sequence for pgbench inserts \echo 'Creating insert sequence...' CREATE SEQUENCE insert_seq; -- Report staging row count \echo '' \echo '=== Setup Verification ===' SELECT 'STAGING_ROW_COUNT:' as label, COUNT(*)::text as count FROM wikipedia_paradedb_staging; \echo '' \echo '=== ParadeDB Concurrent Insert Setup Complete ===' \echo 'Run pgbench with paradedb/pgbench-insert.sql' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/paradedb/insert-load.sql000066400000000000000000000054641525762047700313710ustar00rootroot00000000000000-- Wikipedia Dataset - ParadeDB Insert Loading -- Creates ParadeDB BM25 index on EMPTY table, then loads data -- via COPY to measure insert-path indexing performance. -- -- Usage: -- DATA_DIR=/path/to/data psql -f paradedb/insert-load.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia ParadeDB Insert Benchmark - Data Loading ===' \echo 'Index created BEFORE loading articles' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_articles_paradedb CASCADE; -- Create articles table \echo 'Creating articles table...' CREATE TABLE wikipedia_articles_paradedb ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL ); -- Create ParadeDB BM25 index on the empty table \echo 'Creating ParadeDB BM25 index on empty table...' CREATE INDEX wikipedia_paradedb_idx ON wikipedia_articles_paradedb USING bm25 (article_id, content) WITH ( key_field = 'article_id', text_fields = '{ "content": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Load articles into the pre-indexed table \echo '' \echo 'INSERT_TIME:' \echo 'Loading articles (this may take a while)...' \copy wikipedia_articles_paradedb(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Compact index segments (ParadeDB merges during VACUUM) \echo 'INDEX_VACUUM:' VACUUM wikipedia_articles_paradedb; -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Articles loaded' as metric, COUNT(*)::text as value FROM wikipedia_articles_paradedb UNION ALL SELECT 'Avg content length (chars)', ROUND(AVG(LENGTH(content)))::text FROM wikipedia_articles_paradedb UNION ALL SELECT 'Total text size (MB)', ROUND( SUM(LENGTH(content)) / 1024.0 / 1024.0, 1 )::text FROM wikipedia_articles_paradedb; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('wikipedia_paradedb_idx') ) as index_size, pg_relation_size('wikipedia_paradedb_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'wikipedia_articles_paradedb') ) as table_size, pg_total_relation_size( 'wikipedia_articles_paradedb') as table_bytes; \echo '' \echo '=== Wikipedia ParadeDB Insert Benchmark Complete ===' \echo 'Ready for query benchmarks' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/paradedb/load.sql000066400000000000000000000053241525762047700300620ustar00rootroot00000000000000-- Wikipedia Dataset - Load and Index (ParadeDB) -- Loads Wikipedia articles and creates ParadeDB BM25 index -- -- Usage: -- psql -v data_dir="'/path/to/data'" -f load.sql \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia Dataset - Data Loading (ParadeDB) ===' \echo '' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up existing tables DROP TABLE IF EXISTS wikipedia_articles_paradedb CASCADE; -- Create articles table \echo 'Creating articles table...' CREATE TABLE wikipedia_articles_paradedb ( article_id INTEGER PRIMARY KEY, title TEXT NOT NULL, content TEXT NOT NULL ); -- Load articles from TSV (convert to CSV to avoid \. escaping issues) \echo 'Loading Wikipedia articles (this may take several minutes)...' \copy wikipedia_articles_paradedb(article_id, title, content) FROM PROGRAM 'awk -F"\t" "{OFS=\",\"; gsub(/\"/, \"\\\"\\\"\", \$2); gsub(/\"/, \"\\\"\\\"\", \$3); print \$1, \"\\\"\" \$2 \"\\\"\", \"\\\"\" \$3 \"\\\"\"}" "$DATA_DIR/wikipedia.tsv"' WITH (FORMAT csv); -- Verify data loading \echo '' \echo '=== Data Loading Verification ===' SELECT 'Articles loaded' as metric, COUNT(*)::text as value FROM wikipedia_articles_paradedb UNION ALL SELECT 'Avg content length (chars)', ROUND(AVG(LENGTH(content)))::text FROM wikipedia_articles_paradedb UNION ALL SELECT 'Total text size (MB)', ROUND(SUM(LENGTH(content)) / 1024.0 / 1024.0, 1)::text FROM wikipedia_articles_paradedb; -- Show sample data \echo '' \echo 'Sample articles:' SELECT article_id, title, LEFT(content, 100) || '...' as content_preview FROM wikipedia_articles_paradedb ORDER BY article_id LIMIT 5; -- Create ParadeDB BM25 index with English tokenizer \echo '' \echo '=== Building ParadeDB BM25 Index ===' \echo 'Creating ParadeDB BM25 index on Wikipedia articles...' CREATE INDEX wikipedia_paradedb_idx ON wikipedia_articles_paradedb USING bm25 (article_id, content) WITH ( key_field = 'article_id', text_fields = '{ "content": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('wikipedia_paradedb_idx')) as index_size, pg_relation_size('wikipedia_paradedb_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('wikipedia_articles_paradedb')) as table_size, pg_total_relation_size('wikipedia_articles_paradedb') as table_bytes; \echo '' \echo '=== Wikipedia Load Complete (ParadeDB) ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/paradedb/pgbench-insert.sql000066400000000000000000000003261525762047700320500ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO wikipedia_articles_paradedb (article_id, title, content) SELECT article_id, title, content FROM wikipedia_paradedb_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/paradedb/queries.sql000066400000000000000000000125231525762047700306170ustar00rootroot00000000000000-- Wikipedia Dataset - Query Benchmarks (ParadeDB) -- Runs various query workloads against the indexed Wikipedia collection \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia Query Benchmarks (ParadeDB) ===' \echo '' -- Get dataset size SELECT 'Dataset size: ' || COUNT(*) || ' articles' as info FROM wikipedia_articles_paradedb; -- Warm up \echo 'Warming up index...' SELECT article_id FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'test') ORDER BY paradedb.score(article_id) DESC LIMIT 10; -- Benchmark 1: Single-word queries \echo '' \echo '=== Benchmark 1: Single-Word Queries ===' \echo 'Query: "algorithm"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'algorithm') ORDER BY score DESC LIMIT 10; \echo '' \echo 'Query: "history"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'history') ORDER BY score DESC LIMIT 10; \echo '' \echo 'Query: "science"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'science') ORDER BY score DESC LIMIT 10; -- Benchmark 2: Multi-word queries \echo '' \echo '=== Benchmark 2: Multi-Word Queries ===' \echo 'Query: "machine learning"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'machine learning') ORDER BY score DESC LIMIT 10; \echo '' \echo 'Query: "world war history"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'world war history') ORDER BY score DESC LIMIT 10; \echo '' \echo 'Query: "climate change effects environment"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'climate change effects environment') ORDER BY score DESC LIMIT 10; -- Benchmark 3: Question-style queries \echo '' \echo '=== Benchmark 3: Question-Style Queries ===' \echo 'Query: "how does photosynthesis work"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'how does photosynthesis work') ORDER BY score DESC LIMIT 10; \echo '' \echo 'Query: "what causes earthquakes"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'what causes earthquakes') ORDER BY score DESC LIMIT 10; -- Benchmark 4: Rare term queries \echo '' \echo '=== Benchmark 4: Rare Term Queries ===' \echo 'Query: "mitochondria"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'mitochondria') ORDER BY score DESC LIMIT 10; \echo '' \echo 'Query: "Constantinople Byzantine Empire"' EXPLAIN ANALYZE SELECT article_id, title, paradedb.score(article_id) as score FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'Constantinople Byzantine Empire') ORDER BY score DESC LIMIT 10; -- Benchmark 5: Batch query processing \echo '' \echo '=== Benchmark 5: Batch Query Processing ===' \echo 'Running 20 different queries sequentially...' DO $$ DECLARE queries text[] := ARRAY[ 'computer', 'biology', 'mathematics', 'physics', 'chemistry', 'geography', 'philosophy', 'economics', 'psychology', 'medicine', 'engineering', 'astronomy', 'geology', 'linguistics', 'anthropology', 'artificial intelligence', 'quantum mechanics', 'natural selection', 'industrial revolution', 'renaissance art' ]; q text; start_time timestamp; total_ms numeric := 0; result_count integer; BEGIN start_time := clock_timestamp(); FOREACH q IN ARRAY queries LOOP EXECUTE format( 'SELECT COUNT(*) FROM ( SELECT article_id FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match(''content'', %L) ORDER BY paradedb.score(article_id) DESC LIMIT 10 ) t', q ) INTO result_count; END LOOP; total_ms := EXTRACT(EPOCH FROM (clock_timestamp() - start_time)) * 1000; RAISE NOTICE 'Total time for 20 queries: % ms', round(total_ms, 1); RAISE NOTICE 'Average query time: % ms', round(total_ms / 20, 1); END $$; -- Benchmark 6: Large LIMIT queries \echo '' \echo '=== Benchmark 6: Large LIMIT Queries ===' \echo 'Fetching more results per query' \echo 'Query: "history" LIMIT 100' EXPLAIN ANALYZE SELECT article_id, title FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'history') ORDER BY paradedb.score(article_id) DESC LIMIT 100; \echo '' \echo 'Query: "science technology" LIMIT 100' EXPLAIN ANALYZE SELECT article_id, title FROM wikipedia_articles_paradedb WHERE content @@@ paradedb.match('content', 'science technology') ORDER BY paradedb.score(article_id) DESC LIMIT 100; \echo '' \echo '=== Wikipedia Query Benchmarks Complete (ParadeDB) ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/precompute_ground_truth.sql000066400000000000000000000166251525762047700323760ustar00rootroot00000000000000-- Wikipedia Ground Truth Precomputation -- Computes BM25 scores using the exact BM25 formula (pure SQL, no -- pg_textsearch index involvement) and writes ground_truth.tsv for -- the validator (validate_queries.sql) to compare index output -- against. -- -- Prerequisites: -- - wikipedia_articles table loaded (from load.sql) -- - benchmark_queries table (from queries.sql; do NOT run queries.sql -- to completion -- it drops the table at the end) -- -- Usage: -- psql -p PORT -f precompute_ground_truth.sql -- -- Implementation notes: -- The original version of this script iterated over query terms one -- at a time, computing document frequency via a full to_tsvector() -- scan of wikipedia_articles per term. On 100K Simple Wikipedia -- this is hopelessly slow (still in Step 1 after 10+ minutes). -- -- This version materializes (article_id, term, tf) once via a -- single tsvector unnest pass, indexes the result on term, then -- computes corpus stats, df, and per-query top-10 scores via -- cheap index lookups. Total runtime on 100K Simple Wikipedia is -- ~30s end-to-end. -- -- No reference to wikipedia_bm25_idx anywhere -- this is pure SQL -- ground truth, which is exactly the independent reference the -- validator needs to compare the index against. \set ON_ERROR_STOP on \timing on -- Pull in fieldnorm_quantize (matches the index's length quantization) \i test/sql/validation.sql \echo '' \echo 'Step 1: Materializing (article_id, term, tf) with a single tsvector pass...' DROP TABLE IF EXISTS doc_terms_tmp; CREATE UNLOGGED TABLE doc_terms_tmp AS SELECT a.article_id, t.lexeme::text AS term, array_length(string_to_array(t.positions::text, ','), 1)::int AS tf FROM wikipedia_articles a, LATERAL unnest(to_tsvector('english', a.content)) AS t(lexeme, positions, weights); CREATE INDEX ON doc_terms_tmp (term); CREATE INDEX ON doc_terms_tmp (article_id); ANALYZE doc_terms_tmp; SELECT COUNT(*) AS doc_term_pairs FROM doc_terms_tmp; \echo '' \echo 'Step 2: Materializing per-doc lengths...' DROP TABLE IF EXISTS doc_lengths_tmp; CREATE UNLOGGED TABLE doc_lengths_tmp AS SELECT article_id, SUM(tf)::int AS dl FROM doc_terms_tmp GROUP BY article_id; CREATE UNIQUE INDEX ON doc_lengths_tmp (article_id); ANALYZE doc_lengths_tmp; \echo '' \echo 'Step 3: Computing corpus statistics...' -- Matches the index behavior: documents with empty tsvector contribute -- nothing (they're absent from doc_lengths_tmp since the LATERAL unnest -- in step 1 produced zero rows for them). DROP TABLE IF EXISTS corpus_stats; CREATE TABLE corpus_stats AS SELECT COUNT(*)::bigint AS total_docs, SUM(dl)::bigint AS total_len, (SUM(dl)::float8 / COUNT(*)::float8) AS avg_doc_len FROM doc_lengths_tmp; SELECT * FROM corpus_stats; \echo '' \echo 'Step 4: Selecting validation queries (10 per token bucket)...' DROP TABLE IF EXISTS validation_queries; CREATE TABLE validation_queries AS WITH ranked AS ( SELECT query_id, query_text, token_bucket, ROW_NUMBER() OVER (PARTITION BY token_bucket ORDER BY query_id) AS rn FROM benchmark_queries ) SELECT query_id, query_text, token_bucket FROM ranked WHERE rn <= 10; SELECT COUNT(*) AS validation_query_count FROM validation_queries; \echo '' \echo 'Step 5: Computing per-term document frequency...' DROP TABLE IF EXISTS term_doc_freq; CREATE TABLE term_doc_freq AS WITH query_terms AS ( SELECT DISTINCT qt.lexeme::text AS term FROM validation_queries vq, LATERAL unnest(to_tsvector('english', vq.query_text)) AS qt(lexeme, positions, weights) ) SELECT qt.term, COALESCE((SELECT COUNT(DISTINCT dt.article_id) FROM doc_terms_tmp dt WHERE dt.term = qt.term), 0)::bigint AS df FROM query_terms qt; CREATE UNIQUE INDEX ON term_doc_freq (term); ANALYZE term_doc_freq; SELECT COUNT(*) AS term_count FROM term_doc_freq; \echo '' \echo 'Step 6: Computing top-10 ground truth per validation query...' DROP TABLE IF EXISTS ground_truth; CREATE TABLE ground_truth ( query_id int, query_text text, rank int, doc_id int, score float8 ); DO $$ DECLARE q record; v_total_docs bigint; v_avg_doc_len float8; v_k1 float8 := 1.2; v_b float8 := 0.75; v_count int := 0; v_total int; v_start timestamp; BEGIN SELECT cs.total_docs, cs.avg_doc_len INTO v_total_docs, v_avg_doc_len FROM corpus_stats cs; SELECT COUNT(*) INTO v_total FROM validation_queries; v_start := clock_timestamp(); FOR q IN SELECT query_id, query_text FROM validation_queries ORDER BY query_id LOOP v_count := v_count + 1; INSERT INTO ground_truth WITH qt AS ( SELECT t.lexeme::text AS term, array_length(string_to_array(t.positions::text, ','), 1)::int AS qf FROM unnest(to_tsvector('english', q.query_text)) AS t(lexeme, positions, weights) ), per_doc_term AS ( -- One row per (matching doc, query term) SELECT dt.article_id, qt.term, dt.tf, qt.qf, tdf.df FROM qt JOIN term_doc_freq tdf ON tdf.term = qt.term JOIN doc_terms_tmp dt ON dt.term = qt.term WHERE tdf.df > 0 ), scored AS ( SELECT pdt.article_id, SUM( -- IDF: ln(1 + (N - df + 0.5) / (df + 0.5)) ln(1.0 + (v_total_docs - pdt.df + 0.5) / (pdt.df + 0.5)) * -- TF normalization with quantized field length (pdt.tf * (v_k1 + 1.0)) / (pdt.tf + v_k1 * (1.0 - v_b + v_b * fieldnorm_quantize(dl.dl) / v_avg_doc_len)) * -- Multiply by query-term frequency for repeated terms pdt.qf )::float8 AS bm25_score FROM per_doc_term pdt JOIN doc_lengths_tmp dl ON dl.article_id = pdt.article_id GROUP BY pdt.article_id ), ranked AS ( SELECT article_id, bm25_score, ROW_NUMBER() OVER (ORDER BY bm25_score DESC, article_id) AS rn FROM scored WHERE bm25_score > 0 ) SELECT q.query_id, q.query_text, rn::int, article_id::int, bm25_score FROM ranked WHERE rn <= 10; IF v_count % 10 = 0 THEN RAISE NOTICE '[%/%] elapsed=%', v_count, v_total, clock_timestamp() - v_start; END IF; END LOOP; RAISE NOTICE 'Completed % queries in %', v_count, clock_timestamp() - v_start; END; $$; \echo '' \echo 'Step 7: Exporting ground truth...' \copy ground_truth TO 'benchmarks/datasets/wikipedia/ground_truth.tsv' WITH (FORMAT text, DELIMITER E'\t', HEADER true) SELECT 'Exported ' || COUNT(*) || ' rows for ' || COUNT(DISTINCT query_id) || ' queries' AS status FROM ground_truth; -- Cleanup (keep corpus_stats and term_doc_freq for potential debugging) DROP TABLE IF EXISTS doc_terms_tmp; DROP TABLE IF EXISTS doc_lengths_tmp; DROP TABLE IF EXISTS validation_queries; \echo '' \echo '=== Ground Truth Precomputation Complete ===' \echo 'Output: benchmarks/datasets/wikipedia/ground_truth.tsv' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/queries.sql000066400000000000000000001260721525762047700270620ustar00rootroot00000000000000-- Wikipedia Dataset - Query Benchmarks -- Runs various query workloads against the indexed Wikipedia collection -- Outputs structured timing data for historical tracking \set ON_ERROR_STOP on \timing on -- Soft cap on per-statement runtime so a pathological query fails fast -- instead of hanging the benchmark. See msmarco/queries.sql for context. SET statement_timeout = '5min'; \echo '=== Wikipedia Query Benchmarks ===' \echo '' -- Get dataset size SELECT 'Dataset size: ' || COUNT(*) || ' articles' as info FROM wikipedia_articles; -- Create benchmark queries table with token buckets (matching MS MARCO format) -- Using representative Wikipedia queries across different token counts \echo 'Creating benchmark queries...' DROP TABLE IF EXISTS benchmark_queries; CREATE TABLE benchmark_queries ( query_id SERIAL, query_text TEXT, token_bucket INTEGER ); -- Insert 100 queries per bucket (800 total) for comprehensive benchmarking -- Bucket 1: 1 token INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'algorithm', 'history', 'science', 'computer', 'biology', 'mathematics', 'physics', 'chemistry', 'geography', 'philosophy', 'economics', 'psychology', 'medicine', 'engineering', 'astronomy', 'geology', 'linguistics', 'anthropology', 'sociology', 'ecology', 'democracy', 'evolution', 'electricity', 'architecture', 'literature', 'music', 'art', 'religion', 'politics', 'technology', 'agriculture', 'industry', 'commerce', 'military', 'education', 'language', 'culture', 'society', 'nature', 'universe', 'atom', 'molecule', 'cell', 'gene', 'protein', 'bacteria', 'virus', 'plant', 'animal', 'human', 'brain', 'heart', 'blood', 'bone', 'muscle', 'water', 'air', 'earth', 'fire', 'metal', 'energy', 'force', 'mass', 'velocity', 'temperature', 'pressure', 'volume', 'density', 'gravity', 'magnetism', 'radiation', 'wave', 'particle', 'quantum', 'relativity', 'carbon', 'oxygen', 'hydrogen', 'nitrogen', 'iron', 'gold', 'silver', 'copper', 'lead', 'zinc', 'ocean', 'river', 'mountain', 'desert', 'forest', 'island', 'continent', 'volcano', 'earthquake', 'climate', 'weather', 'season', 'rain', 'snow', 'wind' ]), 1; -- Bucket 2: 2 tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'machine learning', 'world war', 'climate change', 'solar system', 'human body', 'ancient history', 'modern art', 'nuclear physics', 'organic chemistry', 'cell biology', 'quantum mechanics', 'natural selection', 'genetic engineering', 'artificial intelligence', 'computer science', 'space exploration', 'ocean current', 'mountain range', 'river delta', 'tropical forest', 'political science', 'economic theory', 'social psychology', 'cultural anthropology', 'medical research', 'civil war', 'cold war', 'industrial revolution', 'french revolution', 'american revolution', 'roman empire', 'british empire', 'ottoman empire', 'mongol empire', 'persian empire', 'greek philosophy', 'eastern philosophy', 'religious history', 'art history', 'music theory', 'literary criticism', 'film history', 'theater arts', 'dance history', 'architecture design', 'urban planning', 'environmental science', 'marine biology', 'plant science', 'animal behavior', 'human evolution', 'primate research', 'fossil record', 'geological time', 'plate tectonics', 'volcanic activity', 'seismic waves', 'atmospheric science', 'weather patterns', 'ocean temperature', 'global warming', 'carbon dioxide', 'renewable energy', 'nuclear energy', 'solar power', 'wind energy', 'water cycle', 'nitrogen cycle', 'carbon cycle', 'food chain', 'ecosystem dynamics', 'biodiversity loss', 'species extinction', 'habitat destruction', 'conservation biology', 'population growth', 'demographic transition', 'migration patterns', 'urban growth', 'rural development', 'agricultural practices', 'crop production', 'livestock farming', 'food security', 'water resources', 'mineral resources', 'energy resources', 'forest resources', 'fishery resources', 'land use', 'transportation systems', 'communication technology', 'information systems', 'network infrastructure', 'digital technology', 'data science', 'cyber security', 'cloud computing', 'internet protocol', 'mobile technology' ]), 2; -- Bucket 3: 3 tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'world war two', 'machine learning algorithms', 'climate change effects', 'human immune system', 'solar system planets', 'ancient greek philosophy', 'modern art movements', 'nuclear power plants', 'organic chemistry compounds', 'cell division process', 'quantum computing applications', 'natural language processing', 'genetic modification techniques', 'artificial neural networks', 'computer network protocols', 'space shuttle missions', 'deep ocean exploration', 'mountain formation processes', 'river flood control', 'tropical rainforest biodiversity', 'political party systems', 'economic growth factors', 'social media impact', 'cultural heritage preservation', 'medical imaging technology', 'american civil war', 'european cold war', 'industrial revolution effects', 'french revolution causes', 'russian revolution history', 'roman empire decline', 'british colonial history', 'ottoman empire expansion', 'mongol invasion europe', 'persian empire culture', 'greek mythology stories', 'eastern religious traditions', 'medieval church history', 'renaissance art masters', 'classical music composers', 'shakespearean theater plays', 'hollywood film industry', 'modern dance pioneers', 'gothic architecture style', 'sustainable building design', 'smart city planning', 'marine ecosystem health', 'plant photosynthesis process', 'animal migration routes', 'human brain function', 'primate social behavior', 'dinosaur fossil discoveries', 'ice age periods', 'continental drift theory', 'volcanic eruption types', 'earthquake prediction methods', 'hurricane formation process', 'drought impact agriculture', 'flood prevention measures', 'wildfire management strategies', 'greenhouse gas emissions', 'ozone layer depletion', 'solar panel efficiency', 'wind turbine technology', 'hydroelectric power generation', 'electric vehicle development', 'hydrogen fuel cells', 'battery storage systems', 'smart grid technology', 'energy efficiency standards', 'endangered species protection', 'wildlife habitat restoration', 'marine protected areas', 'forest conservation efforts', 'wetland ecosystem services', 'world population trends', 'aging population challenges', 'international migration flows', 'megacity development patterns', 'rural poverty reduction', 'sustainable agriculture methods', 'precision farming technology', 'organic food production', 'global food distribution', 'clean water access', 'rare earth minerals', 'offshore oil drilling', 'coal mining impacts', 'timber harvesting practices', 'sustainable fishing methods', 'deep sea mining', 'arctic ice melting', 'ocean acidification effects', 'coral reef bleaching', 'mangrove forest destruction', 'amazon rainforest fires', 'african savanna wildlife', 'australian outback ecosystem', 'siberian permafrost thawing', 'antarctic ice shelf' ]), 3; -- Bucket 4: 4 tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'second world war battles', 'deep learning neural networks', 'global climate change impacts', 'human cardiovascular system function', 'outer solar system exploration', 'ancient greek city states', 'abstract expressionist art movement', 'nuclear fusion energy research', 'aromatic organic chemistry reactions', 'stem cell research applications', 'quantum entanglement physics experiments', 'large language model training', 'crispr gene editing technology', 'convolutional neural network architectures', 'tcp ip protocol stack', 'international space station missions', 'mariana trench deep exploration', 'himalayan mountain range formation', 'amazon river basin ecosystem', 'congo rainforest wildlife diversity', 'parliamentary democracy political systems', 'keynesian economic growth theory', 'social network analysis methods', 'indigenous cultural preservation efforts', 'magnetic resonance imaging technology', 'american civil war reconstruction', 'berlin wall cold war', 'steam engine industrial revolution', 'bastille storming french revolution', 'bolshevik october revolution russia', 'fall of roman empire', 'east india company colonialism', 'siege of constantinople ottoman', 'genghis khan mongol conquests', 'cyrus the great persia', 'olympian gods greek mythology', 'buddhist meditation eastern traditions', 'catholic church medieval europe', 'sistine chapel renaissance masterpiece', 'beethoven symphony classical music', 'hamlet shakespeare tragedy play', 'golden age hollywood cinema', 'martha graham modern dance', 'notre dame gothic cathedral', 'leed certified building design', 'singapore smart city initiative', 'coral reef marine ecosystem', 'c4 plant photosynthesis pathway', 'arctic tern migration route', 'prefrontal cortex brain function', 'great ape social behavior', 'tyrannosaurus rex fossil discovery', 'pleistocene ice age mammals', 'wegener continental drift hypothesis', 'mount vesuvius eruption pompeii', 'san andreas fault earthquakes', 'atlantic hurricane season patterns', 'sahel region drought impacts', 'mississippi river flood history', 'california wildfire prevention strategies', 'methane greenhouse gas emissions', 'antarctic ozone hole recovery', 'perovskite solar cell efficiency', 'offshore wind farm development', 'three gorges dam hydroelectric', 'tesla electric vehicle technology', 'toyota mirai hydrogen fuel', 'lithium ion battery advances', 'european smart grid rollout', 'passive house energy standards', 'giant panda conservation success', 'yellowstone wolf reintroduction program', 'great barrier reef protection', 'amazon deforestation prevention efforts', 'chesapeake bay wetland restoration', 'sub saharan africa population', 'japan aging society challenges', 'syrian refugee migration crisis', 'tokyo metropolitan area growth', 'appalachian poverty reduction programs', 'vertical farming urban agriculture', 'john deere precision farming', 'usda organic certification standards', 'world food programme distribution', 'flint michigan water crisis', 'democratic republic congo minerals', 'deepwater horizon oil spill', 'appalachian coal mining decline', 'canadian boreal forest logging', 'north sea fishing quotas', 'mediterranean sea plastic pollution', 'indian monsoon climate patterns', 'sahara desert solar potential', 'amazon basin water resources', 'ganges river pollution levels', 'california drought water management', 'great lakes ecosystem health', 'colorado river water rights', 'nile river dam construction', 'yangtze river flood control' ]), 4; -- Bucket 5: 5 tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'battle of stalingrad world war', 'transformer architecture deep learning models', 'paris agreement climate change targets', 'human heart blood circulation system', 'voyager probe outer solar system', 'athenian democracy ancient greek politics', 'jackson pollock abstract expressionist paintings', 'iter tokamak nuclear fusion reactor', 'benzene ring aromatic compound chemistry', 'induced pluripotent stem cell research', 'double slit quantum interference experiment', 'gpt transformer language model architecture', 'cas9 crispr gene editing mechanism', 'resnet deep convolutional neural network', 'border gateway protocol internet routing', 'hubble space telescope deep field', 'challenger deep mariana trench dive', 'mount everest himalayan peak climbing', 'amazon river dolphin endangered species', 'bonobo great ape social structure', 'westminster parliamentary democracy united kingdom', 'phillips curve inflation unemployment tradeoff', 'facebook social network growth analysis', 'maori cultural heritage new zealand', 'functional mri brain imaging technology', 'freedmens bureau civil war reconstruction', 'checkpoint charlie berlin wall crossing', 'spinning jenny textile industrial revolution', 'tennis court oath french revolution', 'winter palace storming october revolution', 'diocletian reforms late roman empire', 'robert clive east india company', 'mehmed conquest of constantinople fall', 'battle of kalka river mongols', 'achaemenid royal road persian empire', 'twelve labors of hercules mythology', 'eightfold path buddhist enlightenment practice', 'great schism eastern western church', 'michelangelo david renaissance sculpture masterpiece', 'ninth symphony beethoven choral finale', 'to be or not hamlet', 'citizen kane orson welles masterpiece', 'appalachian spring martha graham ballet', 'flying buttress notre dame cathedral', 'passive house institute energy standard', 'autonomous vehicle smart city integration', 'great pacific garbage patch marine', 'rubisco enzyme c3 plant photosynthesis', 'bar tailed godwit record migration', 'broca area speech language processing', 'jane goodall chimpanzee behavior research', 'sue tyrannosaurus rex field museum', 'woolly mammoth ice age extinction', 'alfred wegener pangaea supercontinent theory', 'pyroclastic flow mount vesuvius eruption', 'usgs earthquake early warning system', 'saffir simpson hurricane intensity scale', 'dust bowl great plains drought', 'levee failure hurricane katrina flooding', 'cal fire wildfire suppression operations', 'permafrost thaw methane release arctic', 'montreal protocol ozone layer protection', 'perovskite tandem solar cell record', 'hornsea offshore wind farm uk', 'itaipu dam hydroelectric power output', 'tesla model s electric range', 'hyundai nexo hydrogen fuel efficiency', 'solid state battery energy density', 'nord pool smart grid trading', 'net zero energy building design', 'chengdu giant panda breeding center', 'yellowstone grizzly bear population recovery', 'great barrier reef bleaching events', 'brazilian amazon soy moratorium deforestation', 'chesapeake bay oyster reef restoration', 'mount everest climbing expedition routes', 'mariana trench deep sea creatures', 'amazon river basin indigenous peoples', 'sahara desert ancient trade routes', 'antarctic research station operations', 'mediterranean diet health benefits study', 'japanese bullet train technology development', 'german autobahn speed limit debate', 'swiss alps tunnel construction projects', 'panama canal expansion modernization project', 'suez canal maritime shipping traffic', 'gibraltar strait underwater tunnel proposal', 'bering strait land bridge theory', 'english channel tunnel rail service', 'golden gate bridge construction history', 'brooklyn bridge engineering innovation design', 'sydney harbour bridge construction workers', 'tower bridge london opening mechanism', 'london underground metro system history', 'new york subway expansion plans', 'paris metro art nouveau stations', 'moscow metro palatial underground architecture', 'tokyo metro busiest railway system', 'beijing subway rapid expansion growth', 'shanghai maglev train airport connection' ]), 5; -- Bucket 6: 6 tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'operation barbarossa german invasion soviet union', 'attention mechanism transformer neural network architecture', 'intergovernmental panel climate change ipcc reports', 'sinoatrial node heart electrical conduction system', 'new horizons pluto kuiper belt flyby', 'pericles funeral oration athenian democracy speech', 'number one jackson pollock drip painting', 'national ignition facility laser fusion experiment', 'nucleophilic aromatic substitution organic chemistry mechanism', 'shinya yamanaka induced pluripotent stem cells', 'delayed choice quantum eraser experiment results', 'bidirectional encoder representations transformers bert model', 'jennifer doudna emmanuelle charpentier crispr discovery', 'imagenet large scale visual recognition challenge', 'autonomous system number bgp internet routing', 'james webb space telescope first images', 'don walsh jacques piccard trieste bathyscaphe', 'tenzing norgay edmund hillary everest summit', 'amazon river basin deforestation satellite monitoring', 'bonobos matriarchal society female social dominance', 'house of commons parliamentary procedure rules', 'natural rate of unemployment nairu concept', 'cambridge analytica facebook data privacy scandal', 'te papa tongarewa maori taonga collection', 'diffusion tensor imaging white matter tracts', 'fourteenth amendment equal protection clause reconstruction', 'tear down this wall reagan speech', 'richard arkwright water frame spinning machine', 'declaration rights of man citizen france', 'vladimir lenin april theses bolshevik revolution', 'constantine edict of milan christian legalization', 'battle of plassey british india conquest', 'hagia sophia ottoman mosque christian cathedral', 'battle of liegnitz mongol polish encounter', 'cyrus cylinder human rights ancient declaration', 'perseus slaying medusa greek hero mythology', 'four noble truths buddhist suffering doctrine', 'papal infallibility first vatican council doctrine', 'sistine chapel ceiling creation of adam', 'ode to joy ninth symphony finale', 'yorick skull gravedigger scene hamlet soliloquy', 'rosebud sled ending citizen kane mystery', 'night journey martha graham cave heart', 'chartres cathedral labyrinth medieval pilgrimage symbol', 'passivhaus institut darmstadt germany certification standard', 'waymo autonomous vehicle phoenix arizona deployment', 'microplastics great pacific garbage patch accumulation', 'photorespiration rubisco oxygenase c3 plant inefficiency', 'e7 bar tailed godwit alaska migration', 'wernicke area language comprehension temporal lobe', 'gombe stream national park chimpanzee research', 'sue tyrannosaurus rex chicago field museum', 'la brea tar pits ice age', 'south atlantic seafloor spreading mid ocean', 'herculaneum preservation mount vesuvius pyroclastic surge', 'shakealert earthquake early warning los angeles', 'hurricane maria puerto rico category five', 'grapes of wrath dust bowl migration', 'hurricane katrina lower ninth ward flooding', 'paradise california camp fire town destruction', 'siberian permafrost crater methane explosion events', 'kigali amendment hfc phase down protocol', 'oxford pv perovskite silicon tandem efficiency', 'dogger bank offshore wind farm north', 'three gorges dam yangtze river capacity', 'tesla supercharger network electric vehicle charging', 'hyundai nexo 380 mile hydrogen range', 'quantumscape solid state lithium metal battery', 'entsoe european smart grid frequency regulation', 'bullitt center seattle net zero building', 'wolong giant panda nature reserve china', 'northern rocky mountain wolf recovery program', 'coral bleaching great barrier reef australia', 'amazon soy moratorium cattle ranching deforestation', 'harris creek oyster restoration chesapeake bay', 'mount kilimanjaro glacier retreat climate change', 'lake baikal freshwater ecosystem biodiversity russia', 'galapagos islands darwin finch evolution study', 'madagascar lemur species endangered habitat loss', 'borneo orangutan rainforest conservation efforts indonesia', 'serengeti wildebeest migration annual cycle tanzania', 'everglades wetland restoration project florida usa', 'dead sea shrinking water level jordan', 'caspian sea sturgeon caviar fishing industry', 'aral sea environmental disaster soviet irrigation', 'venice flood barrier mose project construction', 'netherlands sea level rise delta works', 'new orleans hurricane levee protection system', 'tokyo bay land reclamation expansion project', 'hong kong airport artificial island construction', 'dubai palm islands artificial archipelago development', 'singapore marina bay land reclamation project', 'shanghai pudong skyline rapid urban development', 'sydney opera house architectural design competition', 'bilbao guggenheim museum urban regeneration catalyst', 'louvre museum paris glass pyramid renovation', 'british museum london elgin marbles controversy', 'smithsonian institution washington dc national mall', 'metropolitan museum art new york city', 'hermitage museum saint petersburg winter palace' ]), 6; -- Bucket 7: 7 tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'operation barbarossa largest military invasion in history', 'multi head self attention transformer encoder decoder', 'ipcc sixth assessment report climate change impacts', 'purkinje fibers heart ventricular electrical conduction system', 'voyager golden record sounds of earth message', 'delian league athenian empire tribute collection system', 'autumn rhythm number 30 jackson pollock moma', 'lawrence livermore national ignition facility fusion breakthrough', 'electrophilic aromatic substitution friedel crafts alkylation mechanism', 'takahashi shinya yamanaka nobel prize stem cells', 'which way delayed choice quantum eraser interference', 'google bert bidirectional transformer pretraining language model', 'crispr cas9 double strand break dna repair', 'alexnet imagenet classification deep convolutional neural network', 'border gateway protocol path vector routing algorithm', 'jwst deep field first galaxies early universe', 'mariana trench hadal zone challenger deep exploration', 'hillary step mount everest summit death zone', 'modis satellite amazon deforestation monitoring terra aqua', 'bonobo make love not war conflict resolution', 'speaker of the house parliamentary chamber rules', 'friedman phelps natural rate hypothesis inflation expectations', 'cambridge analytica facebook fifty million user breach', 'te papa tongarewa museum wellington new zealand', 'dti tractography neural pathway white matter imaging', 'brown v board of education fourteenth amendment', 'mr gorbachev tear down this wall berlin', 'spinning mule samuel crompton cotton textile invention', 'rights of man thomas paine french revolution', 'sealed train journey lenin switzerland russia revolution', 'council of nicaea constantine christian church doctrine', 'black hole of calcutta east india company', 'theodora justinian hagia sophia byzantine mosaic art', 'subutai mongol general european campaign battle tactics', 'cyrus the great cylinder british museum babylon', 'athena birth from head of zeus mythology', 'middle way buddha enlightenment bodhi tree meditation', 'pope pius ix first vatican council infallibility', 'god touching adam sistine chapel ceiling fresco', 'schiller ode to joy beethoven ninth symphony', 'alas poor yorick hamlet gravedigger skull scene', 'citizen kane rosebud sled childhood snow globe', 'martha graham cave of the heart medea', 'amiens cathedral gothic architecture flying buttress height', 'zero carbon passivhaus standard blower door airtightness', 'waymo one robotaxi service phoenix metro area', 'five gyres institute microplastic pollution ocean research', 'c4 cam photosynthesis kranz anatomy bundle sheath', 'e7 alaska new zealand bar tailed godwit', 'broca wernicke arcuate fasciculus language processing circuit', 'goodall gombe chimpanzee tool use termite fishing', 'sue tyrannosaurus rex skeleton chicago natural history', 'rancho la brea tar pits pleistocene fossils', 'mid atlantic ridge seafloor spreading magnetic striping', 'herculaneum scrolls villa papyri vesuvius carbonization preservation', 'shakealert myshake earthquake early warning system california', 'hurricane maria puerto rico power grid total collapse', 'dust bowl migration grapes of wrath steinbeck', 'lower ninth ward levee failure hurricane katrina', 'camp fire paradise california deadliest wildfire history', 'batagaika crater siberian permafrost thermokarst methane release', 'montreal protocol kigali amendment hfc refrigerant phasedown', 'perovskite silicon tandem solar cell 29 percent', 'dogger bank wind farm north sea capacity', 'three gorges dam largest hydroelectric power station', 'tesla model 3 standard range plus efficiency', 'toyota mirai hydrogen fuel cell electric vehicle', 'quantumscape solid state battery ev range improvement', 'european network transmission system operators smart grid', 'bullitt center seattle living building challenge certified', 'mount everest base camp altitude sickness prevention', 'pacific crest trail thru hike completion rate', 'appalachian trail longest hiking path eastern united', 'great wall china tourist visitor number statistics', 'machu picchu inca citadel archaeological site peru', 'angkor wat temple complex cambodia world heritage', 'petra ancient city jordan rose red sandstone', 'colosseum rome gladiator arena seating capacity history', 'parthenon athens acropolis greek temple marble columns', 'stonehenge prehistoric monument wiltshire england summer solstice', 'pyramids giza egypt pharaoh tomb construction mystery', 'taj mahal india mughal architecture white marble', 'eiffel tower paris iron lattice gustave construction', 'statue liberty new york copper neoclassical sculpture', 'big ben london clock tower westminster palace', 'leaning tower pisa italy bell tower inclination', 'christ redeemer rio janeiro brazil art deco', 'mount rushmore presidential sculpture black hills south', 'golden gate bridge san francisco suspension span', 'brooklyn bridge new york east river gothic', 'sydney harbour bridge australia steel arch design', 'tower bridge london victorian gothic bascule drawbridge', 'burj khalifa dubai tallest building world skyscraper', 'empire state building new york art deco', 'petronas towers kuala lumpur twin skyscraper malaysia', 'cn tower toronto canada observation deck height', 'sears tower chicago willis tower rename controversy', 'one world trade center new york memorial', 'taipei 101 tower taiwan tuned mass damper', 'shanghai tower china twisted form sustainable design' ]), 7; -- Bucket 8: 8+ tokens INSERT INTO benchmark_queries (query_text, token_bucket) SELECT unnest(ARRAY[ 'operation barbarossa june 1941 german invasion of soviet union', 'scaled dot product attention multi head self attention mechanism', 'ipcc ar6 synthesis report limiting global warming 1.5 degrees', 'sinoatrial node atrioventricular node bundle of his conduction', 'voyager 1 golden record sounds of earth carl sagan', 'delian league treasury moved from delos to athens tribute', 'autumn rhythm number 30 1950 jackson pollock moma collection', 'national ignition facility lawrence livermore 3.15 megajoule fusion yield', 'electrophilic aromatic substitution mechanism benzene friedel crafts acylation reaction', 'yamanaka factors oct4 sox2 klf4 myc induced pluripotent stem', 'wheeler delayed choice quantum eraser experiment wave particle duality', 'bert masked language model next sentence prediction pretraining task', 'crispr cas9 guide rna pam sequence double strand break', 'alexnet krizhevsky sutskever hinton imagenet 2012 deep learning breakthrough', 'bgp path vector protocol autonomous system number prefix announcement', 'james webb space telescope first deep field image release', 'challenger deep mariana trench 10994 meters deepest ocean point', 'tenzing norgay edmund hillary may 29 1953 everest summit', 'amazon basin deforestation monitoring modis landsat satellite time series', 'bonobo pan paniscus great ape female dominance sexual behavior', 'speaker of the house of commons parliamentary procedure mace', 'phillips curve friedman phelps expectations augmented natural rate hypothesis', 'cambridge analytica alexander nix facebook user data trump campaign', 'te papa tongarewa national museum new zealand wellington waterfront', 'diffusion tensor imaging fractional anisotropy white matter integrity measure', 'brown v board of education 1954 separate but equal overturned', 'reagan tear down this wall brandenburg gate berlin speech', 'samuel crompton spinning mule 1779 cotton textile industrial revolution', 'declaration of the rights of man and citizen 1789', 'vladimir lenin sealed train journey petrograd finland station april', 'council of nicaea 325 ad nicene creed constantine christianity', 'battle of plassey 1757 robert clive east india company bengal', 'hagia sophia 537 justinian theodora byzantine mosaic orthodox mosque', 'battle of mohi 1241 mongol invasion hungary batu khan europe', 'cyrus cylinder 539 bc persian empire human rights declaration babylon', 'athena goddess of wisdom born from head of zeus', 'siddhartha gautama buddha four noble truths eightfold path enlightenment', 'first vatican council 1870 pope pius ix papal infallibility dogma', 'michelangelo sistine chapel ceiling creation of adam god fresco', 'ludwig van beethoven ninth symphony ode to joy choral finale', 'to be or not to be hamlet act 3', 'citizen kane 1941 orson welles rosebud sled childhood mystery', 'martha graham cave of the heart medea greek tragedy dance', 'chartres cathedral gothic architecture labyrinth pilgrimage rose window stained glass', 'passivhaus standard airtightness heat recovery ventilation thermal bridge free construction', 'waymo autonomous vehicle level 4 self driving robotaxi phoenix', 'great pacific garbage patch north pacific subtropical convergence zone microplastics', 'c4 photosynthesis kranz anatomy bundle sheath mesophyll cell co2 concentration', 'bar tailed godwit e7 alaska new zealand 11000 km nonstop', 'broca wernicke model arcuate fasciculus language network dual stream processing', 'jane goodall gombe stream chimpanzee tool use termite fishing observation', 'sue tyrannosaurus rex field museum chicago largest most complete skeleton', 'rancho la brea tar pits los angeles pleistocene ice age', 'mid atlantic ridge seafloor spreading paleomagnetism vine matthews morley hypothesis', 'herculaneum papyrus scrolls villa of papyri vesuvius 79 ad carbonized', 'shakealert earthquake early warning system california usgs seismic network alert', 'hurricane maria 2017 puerto rico category 5 power grid collapse', 'dust bowl 1930s great plains drought soil erosion okies migration', 'hurricane katrina 2005 new orleans levee failure lower ninth ward', 'camp fire 2018 paradise california deadliest most destructive wildfire history', 'batagaika megaslump siberia permafrost thaw thermokarst crater methane emissions climate', 'montreal protocol ozone layer cfcs hfcs kigali amendment phasedown schedule', 'oxford pv perovskite silicon tandem solar cell 29.5 percent efficiency', 'dogger bank offshore wind farm north sea 3.6 gw capacity', 'three gorges dam yangtze river 22500 mw largest hydroelectric station', 'tesla model 3 long range dual motor all wheel drive', 'toyota mirai second generation hydrogen fuel cell 402 mile range', 'quantumscape solid state lithium metal anode battery energy density range', 'european network transmission system operators entsoe frequency containment reserve balancing', 'bullitt center seattle six story office living building challenge net zero', 'wolong national nature reserve giant panda conservation breeding chengdu sichuan', 'yellowstone to yukon conservation initiative grizzly bear wolf corridor connectivity', 'great barrier reef marine park authority coral bleaching mass mortality', 'amazon soy moratorium 2006 cattle ranching deforestation cerrado biome expansion', 'harris creek oyster reef restoration project chesapeake bay largest maryland', 'mount everest summit expedition 1953 tenzing norgay edmund hillary first ascent', 'apollo 11 moon landing 1969 neil armstrong buzz aldrin lunar module', 'wright brothers first powered flight 1903 kitty hawk north carolina flyer', 'charles lindbergh transatlantic solo flight 1927 spirit of st louis paris', 'amelia earhart circumnavigation attempt 1937 disappearance pacific ocean lockheed electra', 'titanic sinking 1912 north atlantic iceberg collision lifeboats passengers crew', 'hindenburg disaster 1937 lakehurst new jersey hydrogen airship fire explosion', 'challenger space shuttle disaster 1986 o ring failure solid rocket booster', 'columbia space shuttle disaster 2003 foam insulation thermal protection system damage', 'chernobyl nuclear disaster 1986 reactor explosion radiation contamination exclusion zone', 'fukushima daiichi nuclear disaster 2011 tsunami reactor meltdown evacuation zone', 'deepwater horizon oil spill 2010 gulf mexico blowout environmental damage', 'exxon valdez oil spill 1989 prince william sound alaska tanker grounding', 'bhopal gas tragedy 1984 union carbide methyl isocyanate leak india', 'great smog london 1952 air pollution coal burning respiratory deaths', 'spanish flu pandemic 1918 h1n1 influenza virus global death toll', 'black death bubonic plague 1347 europe yersinia pestis mortality rate', 'covid 19 pandemic 2020 sars cov 2 coronavirus global lockdown measures', 'world war i 1914 1918 trench warfare western front armistice', 'world war ii 1939 1945 nazi germany allied powers axis powers', 'vietnam war 1955 1975 american military involvement viet cong guerrilla', 'korean war 1950 1953 armistice 38th parallel dmz north south', 'gulf war 1990 1991 operation desert storm iraq kuwait invasion', 'american civil war 1861 1865 union confederate slavery abolition lincoln', 'french revolution 1789 1799 reign of terror napoleon rise power' ]), 8; SELECT 'Loaded ' || COUNT(*) || ' benchmark queries' as status FROM benchmark_queries; SELECT token_bucket, COUNT(*) as count FROM benchmark_queries GROUP BY token_bucket ORDER BY token_bucket; -- Warm up: run queries from each bucket to ensure index is cached \echo '' \echo 'Warming up index...' DO $$ DECLARE q record; BEGIN FOR q IN SELECT query_text FROM benchmark_queries ORDER BY random() LIMIT 50 LOOP EXECUTE 'SELECT article_id FROM wikipedia_articles ORDER BY content <@> to_bm25query($1, ''wikipedia_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; END; $$; -- ============================================================ -- Benchmark 1: Query Latency by Token Count -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency by Token Count ===' \echo 'Running 100 queries per token bucket, reporting p50/p95/p99' \echo '' -- Temp table to capture per-bucket results for weighted average CREATE TEMP TABLE bucket_results ( bucket int, p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint ); -- Function to benchmark a bucket and return percentiles CREATE OR REPLACE FUNCTION benchmark_bucket(bucket int) RETURNS TABLE(p50_ms numeric, p95_ms numeric, p99_ms numeric, avg_ms numeric, num_queries int, total_results bigint) AS $$ DECLARE q record; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; n int; result_count bigint; results_sum bigint := 0; BEGIN -- Disable the script-level statement_timeout for the duration of -- this function so the cap can't abort partway through and discard -- the per-query timings. See msmarco/queries.sql for context. PERFORM set_config('statement_timeout', '0', true); times := ARRAY[]::numeric[]; FOR q IN SELECT query_text FROM benchmark_queries WHERE token_bucket = bucket ORDER BY query_id LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT COUNT(*) FROM (SELECT article_id FROM wikipedia_articles ORDER BY content <@> to_bm25query($1, ''wikipedia_bm25_idx'') LIMIT 10) t' INTO result_count USING q.query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); results_sum := results_sum + result_count; END LOOP; n := array_length(times, 1); SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; p50_ms := sorted_times[(n + 1) / 2]; p95_ms := sorted_times[(n * 95 + 99) / 100]; p99_ms := sorted_times[(n * 99 + 99) / 100]; avg_ms := (SELECT AVG(t) FROM unnest(times) t); num_queries := n; total_results := results_sum; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run benchmarks for each token bucket (results saved for weighted avg) \echo 'Token bucket 1 (1 search token):' INSERT INTO bucket_results SELECT 1, * FROM benchmark_bucket(1); SELECT 'LATENCY_BUCKET_1: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 1; \echo '' \echo 'Token bucket 2 (2 search tokens):' INSERT INTO bucket_results SELECT 2, * FROM benchmark_bucket(2); SELECT 'LATENCY_BUCKET_2: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 2; \echo '' \echo 'Token bucket 3 (3 search tokens):' INSERT INTO bucket_results SELECT 3, * FROM benchmark_bucket(3); SELECT 'LATENCY_BUCKET_3: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 3; \echo '' \echo 'Token bucket 4 (4 search tokens):' INSERT INTO bucket_results SELECT 4, * FROM benchmark_bucket(4); SELECT 'LATENCY_BUCKET_4: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 4; \echo '' \echo 'Token bucket 5 (5 search tokens):' INSERT INTO bucket_results SELECT 5, * FROM benchmark_bucket(5); SELECT 'LATENCY_BUCKET_5: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 5; \echo '' \echo 'Token bucket 6 (6 search tokens):' INSERT INTO bucket_results SELECT 6, * FROM benchmark_bucket(6); SELECT 'LATENCY_BUCKET_6: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 6; \echo '' \echo 'Token bucket 7 (7 search tokens):' INSERT INTO bucket_results SELECT 7, * FROM benchmark_bucket(7); SELECT 'LATENCY_BUCKET_7: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 7; \echo '' \echo 'Token bucket 8 (8+ search tokens):' INSERT INTO bucket_results SELECT 8, * FROM benchmark_bucket(8); SELECT 'LATENCY_BUCKET_8: p50=' || round(p50_ms, 2) || 'ms p95=' || round(p95_ms, 2) || 'ms p99=' || round(p99_ms, 2) || 'ms avg=' || round(avg_ms, 2) || 'ms (n=' || num_queries || ', results=' || total_results || ')' as result FROM bucket_results WHERE bucket = 8; DROP FUNCTION benchmark_bucket; -- ============================================================ -- Weighted-Average Query Latency (MS-MARCO v1 distribution) -- ============================================================ -- Weights from observed lexeme distribution in 1,010,905 MS-MARCO -- v1 queries after to_tsvector: 1=3.5%, 2=16.3%, 3=30.2%, -- 4=26.1%, 5=14.2%, 6=5.9%, 7=2.2%, 8+=1.5% \echo '' \echo '=== Weighted-Average Query Latency & Throughput ===' SELECT 'WEIGHTED_LATENCY_RESULT: weighted_p50=' || round(SUM(b.p50_ms * w.weight) / SUM(w.weight), 2) || 'ms weighted_avg=' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || 'ms' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; SELECT 'WEIGHTED_THROUGHPUT_RESULT: ' || round(SUM(b.avg_ms * w.weight) / SUM(w.weight), 2) || ' ms/query' as result FROM bucket_results b JOIN (VALUES (1, 35638), (2, 165033), (3, 304887), (4, 264177), (5, 143765), (6, 59558), (7, 22595), (8, 15252) ) AS w(bucket, weight) ON b.bucket = w.bucket; DROP TABLE bucket_results; -- ============================================================ -- Benchmark 2: Query Throughput (800 queries, 3 iterations) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (800 queries, 3 iterations) ===' \echo 'Running all 800 benchmark queries with warmup' -- Helper function for throughput benchmark CREATE OR REPLACE FUNCTION benchmark_throughput(iterations int DEFAULT 3) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric, queries_run int) AS $$ DECLARE q record; i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; query_count int; BEGIN SELECT COUNT(*) INTO query_count FROM benchmark_queries; -- Warmup: run all queries once FOR q IN SELECT query_text FROM benchmark_queries ORDER BY query_id LOOP EXECUTE 'SELECT article_id FROM wikipedia_articles ORDER BY content <@> to_bm25query($1, ''wikipedia_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; -- Timed iterations times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); FOR q IN SELECT query_text FROM benchmark_queries ORDER BY query_id LOOP EXECUTE 'SELECT article_id FROM wikipedia_articles ORDER BY content <@> to_bm25query($1, ''wikipedia_bm25_idx'') LIMIT 10' USING q.query_text; END LOOP; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; queries_run := query_count; RETURN NEXT; END; $$ LANGUAGE plpgsql; SELECT 'Execution Time: ' || round(median_ms / queries_run, 3) || ' ms (min=' || round(min_ms / queries_run, 3) || ', max=' || round(max_ms / queries_run, 3) || ')' as result, 'THROUGHPUT_RESULT: ' || queries_run || ' queries in ' || round(median_ms, 2) || ' ms (avg ' || round(median_ms / queries_run, 2) || ' ms/query)' as summary FROM benchmark_throughput(); DROP FUNCTION benchmark_throughput; -- ============================================================ -- Benchmark 3: Index Statistics -- ============================================================ \echo '' \echo '=== Benchmark 3: Index Statistics ===' SELECT 'wikipedia_bm25_idx' as index_name, pg_size_pretty(pg_relation_size('wikipedia_bm25_idx')) as index_size, pg_size_pretty(pg_relation_size('wikipedia_articles')) as table_size, (SELECT COUNT(*) FROM wikipedia_articles) as num_documents; -- Cleanup DROP TABLE benchmark_queries; \echo '' \echo '=== Wikipedia Query Benchmarks Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/datasets/wikipedia/validate_queries.sql000066400000000000000000000205131525762047700307240ustar00rootroot00000000000000-- Wikipedia Query Validation -- Compares Tapir query results against precomputed ground truth -- -- Requires: ground_truth.tsv in the same directory (created by CI or -- run_benchmark.sh from the versioned ground_truth_pgNN.tsv files) -- Usage: psql -p PORT -f validate_queries.sql -- -- Validates: the score at each rank (1..10) in the index's result -- matches the score at the same rank in ground truth, within absolute -- tolerance (0.001). -- -- Why per-rank score comparison rather than per-doc: -- -- BM25 scoring can produce *tied* docs (often clusters of 3+ docs at -- the rank-10 boundary on real corpora). Tie-break ordering is -- undefined and varies between: -- - single-writer COPY vs concurrent INSERT (different CTID order) -- - BMW top-k vs sequential scan -- - across pgbench runs (timing-dependent ordering) -- -- A naive doc-set comparison flags these tie-break differences as -- correctness failures. Per-rank score comparison treats them as the -- same result -- which they are, since BM25 ranking by score is what -- the index actually promises. -- -- For diagnostic purposes the doc set is still reported when scores -- diverge or as supplementary info, but pass/fail is decided on -- scores. -- -- This script mirrors benchmarks/datasets/msmarco/validate_queries.sql; -- keep them structurally aligned when making changes. \set ON_ERROR_STOP on \timing on \echo '=== Wikipedia BM25 Validation ===' \echo '' -- Load ground truth data \echo 'Loading ground truth...' DROP TABLE IF EXISTS ground_truth; CREATE TABLE ground_truth ( query_id int, query_text text, rank int, doc_id int, score float8 ); \copy ground_truth FROM 'benchmarks/datasets/wikipedia/ground_truth.tsv' WITH (FORMAT text, DELIMITER E'\t', HEADER true) SELECT 'Loaded ' || COUNT(DISTINCT query_id) || ' queries, ' || COUNT(*) || ' result rows' as status FROM ground_truth; -- Score-based validation: compare score at each rank between index -- and ground truth. CREATE OR REPLACE FUNCTION validate_single_query( p_query_id int, p_query_text text, p_tolerance float8 DEFAULT 0.001 ) RETURNS TABLE( query_id int, scores_match boolean, docs_match boolean, max_score_diff float8, worst_rank int, details text ) AS $$ DECLARE v_max_score_diff float8 := 0; v_worst_rank int := 0; v_all_match boolean := true; v_docs_match boolean; v_details text := ''; r record; BEGIN -- Get Tapir's top-10 results CREATE TEMP TABLE IF NOT EXISTS tapir_results ( rank int, doc_id int, score float8 ); TRUNCATE tapir_results; INSERT INTO tapir_results -- IMPORTANT: keep the LIMIT 10 in the inner subquery so it -- propagates to the BM25 index scan and BMW runs with K=10. -- If LIMIT 10 sits above a window function in the outer SELECT, -- the planner can't push it past the WindowAgg and the index -- scan falls back to pg_textsearch.default_limit (1000). BMW -- with K=1000 reports subtly-wrong scores for some docs on real -- corpora -- see #365. SELECT row_number() OVER ()::int as rank, t.article_id, t.score FROM ( SELECT article_id, -(content <@> to_bm25query(p_query_text, 'wikipedia_bm25_idx'))::float8 as score FROM wikipedia_articles ORDER BY content <@> to_bm25query(p_query_text, 'wikipedia_bm25_idx') LIMIT 10 ) t; -- Compare score at each rank FOR r IN SELECT gt.rank, gt.score as gt_score, t.score as tapir_score, gt.doc_id as gt_doc, t.doc_id as tapir_doc, ABS(gt.score - t.score) as abs_diff FROM ground_truth gt JOIN tapir_results t ON t.rank = gt.rank WHERE gt.query_id = p_query_id ORDER BY gt.rank LOOP IF r.abs_diff > p_tolerance THEN v_all_match := false; IF r.abs_diff > v_max_score_diff THEN v_max_score_diff := r.abs_diff; v_worst_rank := r.rank; END IF; v_details := v_details || 'rank ' || r.rank || ': gt=' || round(r.gt_score::numeric, 4)::text || ' (doc ' || r.gt_doc || '), tapir=' || round(r.tapir_score::numeric, 4)::text || ' (doc ' || r.tapir_doc || '); '; ELSE v_max_score_diff := GREATEST(v_max_score_diff, r.abs_diff); END IF; END LOOP; -- Also check doc set as supplementary info (not part of pass/fail) SELECT COALESCE( (SELECT array_agg(gt.doc_id ORDER BY gt.doc_id) FROM ground_truth gt WHERE gt.query_id = p_query_id) = (SELECT array_agg(doc_id ORDER BY doc_id) FROM tapir_results), false ) INTO v_docs_match; query_id := p_query_id; scores_match := v_all_match; docs_match := v_docs_match; max_score_diff := v_max_score_diff; worst_rank := v_worst_rank; details := CASE WHEN v_details = '' THEN 'OK' ELSE trim(trailing '; ' from v_details) END; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Run validation for all ground truth queries \echo '' \echo '=== Running Validation ===' DROP TABLE IF EXISTS validation_results; CREATE TABLE validation_results AS SELECT v.* FROM (SELECT DISTINCT gt.query_id, gt.query_text FROM ground_truth gt) q, LATERAL validate_single_query(q.query_id, q.query_text) v; -- Summary \echo '' \echo '=== Validation Summary ===' SELECT COUNT(*) as total_queries, SUM(CASE WHEN scores_match THEN 1 ELSE 0 END) as scores_match_count, SUM(CASE WHEN docs_match THEN 1 ELSE 0 END) as docs_match_count, round(100.0 * SUM(CASE WHEN scores_match THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 1) as scores_match_pct, round(100.0 * SUM(CASE WHEN docs_match THEN 1 ELSE 0 END) / NULLIF(COUNT(*), 0), 1) as docs_match_pct, round(MAX(max_score_diff)::numeric, 6) as worst_abs_diff FROM validation_results; -- Known-mismatch allowlist. -- Entries here have stable, pre-existing per-rank score discrepancies -- against ground_truth.tsv that are tracked separately and excluded -- from the pass/fail signal so the watchdog can still catch new -- regressions. Each entry must reference a tracking issue. DROP TABLE IF EXISTS known_mismatches; CREATE TABLE known_mismatches ( query_id int PRIMARY KEY, issue text NOT NULL, note text ); -- (No allowlist entries currently. Add as needed when a stable -- pre-existing score divergence is investigated and tracked.) -- Check for failures and report DO $$ DECLARE v_total int; v_failures int; v_known int; BEGIN SELECT COUNT(*) INTO v_total FROM validation_results; -- A query fails validation if any per-rank score differs beyond -- tolerance. Doc-set differences (tied-cluster ordering) do not -- fail validation as long as scores match. Allowlisted queries -- (known pre-existing score discrepancies) are counted -- separately and do NOT trigger VALIDATION FAILED. SELECT COUNT(*) INTO v_failures FROM validation_results vr WHERE NOT scores_match AND NOT EXISTS ( SELECT 1 FROM known_mismatches km WHERE km.query_id = vr.query_id ); SELECT COUNT(*) INTO v_known FROM validation_results vr JOIN known_mismatches km ON km.query_id = vr.query_id WHERE NOT scores_match; IF v_known > 0 THEN RAISE NOTICE 'VALIDATION: % allowlisted known mismatch(es) ignored (see known_mismatches table)', v_known; END IF; IF v_failures > 0 THEN RAISE NOTICE 'VALIDATION FAILED: % of % queries failed (per-rank scores differ beyond % tolerance, excluding allowlist)', v_failures, v_total, 0.001; ELSE RAISE NOTICE 'VALIDATION PASSED: All % queries match within tolerance (% allowlisted known mismatches)', v_total - v_known, v_known; END IF; END; $$; -- Failed queries detail \echo '' \echo '=== Validation Details (failures only) ===' SELECT query_id, scores_match, docs_match, round(max_score_diff::numeric, 6) as max_abs_diff, worst_rank, left(details, 300) as details FROM validation_results WHERE NOT scores_match ORDER BY max_score_diff DESC LIMIT 20; -- Cleanup DROP TABLE IF EXISTS ground_truth; DROP TABLE IF EXISTS validation_results; DROP TABLE IF EXISTS known_mismatches; DROP TABLE IF EXISTS tapir_results; DROP FUNCTION IF EXISTS validate_single_query; \echo '' \echo '=== Validation Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/expected/000077500000000000000000000000001525762047700226775ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/expected/memory_stress_micro.out000066400000000000000000000056711525762047700275450ustar00rootroot00000000000000psql:sql/memory_stress_micro.sql:15: NOTICE: table "stress_docs_micro" does not exist, skipping psql:sql/memory_stress_micro.sql:68: NOTICE: Tapir index build started for relation stress_micro_idx psql:sql/memory_stress_micro.sql:68: NOTICE: Using text search configuration: english psql:sql/memory_stress_micro.sql:68: NOTICE: Using index options: k1=1.20, b=0.75 psql:sql/memory_stress_micro.sql:68: NOTICE: Tapir index build completed: 1000 documents, avg_length=19.05, text_config='english' (k1=1.20, b=0.75) psql:sql/memory_stress_micro.sql:135: NOTICE: Tapir index called without ORDER BY clause - providing high cost Timing is on. Starting Tapir micro memory stress benchmark... DROP TABLE Time: 2.605 ms CREATE TABLE Time: 16.553 ms Generating micro document dataset (1K documents)... INSERT 0 1000 Time: 11.109 ms Document generation complete. Dataset summary: total_docs | avg_content_length | max_content_length | min_content_length ------------+----------------------+--------------------+-------------------- 1000 | 161.6930000000000000 | 189 | 150 (1 row) Time: 4.344 ms Current Tapir configuration: tapir.shared_memory_size -------------------------- 64 (1 row) Time: 0.072 ms tapir.default_limit --------------------- 1000 (1 row) Time: 0.045 ms Creating Tapir index... CREATE INDEX Time: 817.227 ms Index created successfully! Running performance tests... === Performance Test Results === Test 1: Technical terms search matches --------- 0 (1 row) Time: 52.697 ms Test 2: AI/ML search matches --------- 0 (1 row) Time: 71.801 ms Test 3: Database search matches --------- 0 (1 row) Time: 56.136 ms Test 4: Top results with scores title | score -------+------- (0 rows) Time: 65.885 ms Test 5: Category-based analysis category | total_docs | programming_matches -------------+------------+--------------------- Business | 334 | 0 Engineering | 333 | 0 Technology | 333 | 0 (3 rows) Time: 65.993 ms === Memory Pressure Tests === Test 6: Common terms (should find many matches) matches --------- 0 (1 row) Time: 104.978 ms Test 7: Specific terms (should find fewer matches) matches --------- 0 (1 row) Time: 22.652 ms Test 8: Complex multi-term query matches --------- 0 (1 row) Time: 251.223 ms === Benchmark Summary === status | total_documents | index_status | query_status ---------------------------+-----------------+----------------------------+------------------------------- Micro benchmark completed | 1000 | Index created successfully | Queries executed successfully (1 row) Time: 0.091 ms Cleaning up test resources... DROP TABLE Time: 7.166 ms Micro memory stress benchmark completed. This establishes baseline functionality with 1K documents. For true stress testing, try larger datasets or reduced memory settings. timescale-pg_textsearch-b63fe58/benchmarks/gh-pages/000077500000000000000000000000001525762047700225715ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/gh-pages/comparison.html000066400000000000000000000753771525762047700256540ustar00rootroot00000000000000 pg_textsearch vs ParadeDB - Development Progress
Dashboard Methodology CPU Profiles

pg_textsearch vs ParadeDB

Development Progress Report

Snapshot in time: This comparison reflects pg_textsearch as of March 10, 2026. The project is under active development with performance improvements shipping regularly. Check the benchmark dashboard for the latest numbers.

Dataset: MS MARCO 8.8M passages | Date: 2026-03-10 | Commit: f31d1af | ParadeDB: v0.21.6

Current Status

pg_textsearch today

  • 3.1x faster overall query throughput
  • Faster on all query lengths (1-8+ tokens)
  • Smaller index (no positions stored)*
  • Parallel index build (4 workers)
  • Native Postgres integration

ParadeDB v0.21.6

  • Faster index build (1.7x)
  • Phrase queries supported
  • Larger feature set (facets, etc.)

Recent Improvements

  • BMW cache optimizations - Cached skip entries and reusable decompression buffers reduce per-block overhead by 20–25% (PR #274)
  • SIMD-accelerated decoding - Bitpack decoding with SIMD intrinsics (PR #250)
  • Stack-allocated decode buffers - Reduced allocation overhead (PR #253)
  • BMW term state optimization - Pointer indirection for ordering (PR #249)
  • Arena allocator - Rewritten index build with parallel page pool (PR #231)
  • Overall throughput - pg_textsearch now 3.1x faster than ParadeDB on 8.8M dataset (up from 2.8x in February)

Index Size & Build Time

Metric pg_textsearch ParadeDB Difference
Index Size 1,215 MB 1,503 MB -19%
Build Time 233.5 sec 140.1 sec +67%
Documents 8,841,823 -
Index size caveat: pg_textsearch does not store term positions, so it cannot support phrase queries like "quick brown fox". ParadeDB stores positions by default, which adds significant overhead but enables phrase search. This accounts for most of the index size difference—it's a feature tradeoff, not a compression advantage.
Build time improvement: With the arena allocator rewrite (PR #231) and leader-only merge (PR #244), build time dropped from 270s to 234s. ParadeDB also improved its build time from 151s to 140s in this run.

Query Latency (p50)

Median latency in milliseconds. Lower is better.

Query Tokens pg_textsearch ParadeDB Difference
1 token 0.70 ms 18.84 ms -96%
2 tokens 1.53 ms 18.40 ms -92%
3 tokens 3.00 ms 24.74 ms -88%
4 tokens 4.52 ms 26.71 ms -83%
5 tokens 7.63 ms 29.16 ms -74%
6 tokens 11.17 ms 35.57 ms -69%
7 tokens 16.74 ms 36.68 ms -54%
8+ tokens 26.32 ms 44.87 ms -41%

Query Latency (p95)

95th percentile latency in milliseconds. Lower is better.

Query Tokens pg_textsearch ParadeDB Difference
1 token 1.56 ms 27.41 ms -94%
2 tokens 4.63 ms 30.99 ms -85%
3 tokens 9.22 ms 37.48 ms -75%
4 tokens 14.85 ms 36.61 ms -59%
5 tokens 22.78 ms 39.81 ms -43%
6 tokens 27.15 ms 58.66 ms -54%
7 tokens 42.51 ms 67.78 ms -37%
8+ tokens 57.00 ms 70.37 ms -19%

Throughput

Total time to execute 800 test queries sequentially.

Metric pg_textsearch ParadeDB Difference
Total time 8.03 sec 25.07 sec -68%
Avg ms/query 10.04 ms 31.33 ms -68%

Analysis

Query latency: pg_textsearch faster across all token counts

pg_textsearch is faster on all 8 token buckets at p50, ranging from 27x faster on single-token queries to 1.7x faster on 8+ token queries. SIMD-accelerated bitpack decoding (PR #250) and stack-allocated decode buffers (PR #253) improved segment read performance, while BMW term state pointer indirection (PR #249) reduced overhead in the query scoring path.

Overall throughput: pg_textsearch 3.1x faster

pg_textsearch completes 800 queries in 8.0s vs 25.1s for ParadeDB, a 3.1x throughput advantage. This is up from 2.8x in the February 9 comparison, driven by segment decoding and scoring path optimizations.

Index build: gap narrowing

ParadeDB builds its index in 140s vs 234s for pg_textsearch (1.7x faster). The arena allocator rewrite (PR #231) and leader-only merge (PR #244) cut build time from 270s to 234s, narrowing the gap from 2.0x to 1.7x.

Methodology

Both extensions benchmarked on identical GitHub Actions runners with the same Postgres configuration. See full methodology for details.

Note: Both extensions use default configurations. Results are from a single run; expect ~10% variance between runs. This page will be updated as optimizations land.

MS-MARCO v2 — 138M Passages

Large-Scale Benchmark

New experiment: This section uses the full MS-MARCO v2 passage collection (138M documents, ~16x larger than v1 above). The hardware and configuration differ from the 8.8M experiment — see details below. Numbers between the two sections are not directly comparable.

Dataset: MS MARCO v2 — 138,364,158 passages | Date: 2026-03-10 | pg_textsearch: v1.0.0-dev (main @ fb3b3b1) | ParadeDB: v0.21.6

Environment

Component Specification
CPU Intel Xeon Platinum 8375C @ 2.90 GHz, 8 cores / 16 threads
RAM 123 GB
Storage NVMe SSD (885 GB)
Postgres 17.7, shared_buffers = 31 GB, data on NVMe
Table size 47 GB (87 GB with TOAST)

Current Status (138M)

pg_textsearch

  • 2.3x faster weighted p50 query latency
  • 4.7x higher concurrent throughput (16 clients)
  • Faster on all 8 token buckets at p50
  • 26% smaller index on disk
  • Block-Max WAND with cached skip entries
  • SIMD-accelerated bitpack decoding

ParadeDB v0.21.6

  • 1.9x faster index build
  • Phrase queries supported
  • Larger feature set (facets, etc.)

Index Build (138M)

Metric pg_textsearch ParadeDB Difference
Build time 17 min 37 s 8 min 55 s 1.9x slower
Parallel workers 15 14 -
Index size 17 GB 23 GB -26%
Documents 138,364,158 -
Unique terms 17,373,764 - -
Build time note: pg_textsearch's parallel build has two phases: scan/tokenize (parallel) and merge (single-threaded I/O-bound). The merge phase dominates at this scale. ParadeDB uses a Tantivy-based backend with a different merge strategy.

Single-Client Query Latency (138M)

Top-10 results (LIMIT 10), BMW optimization enabled. 691 queries sampled across 8 token-count buckets.

Median Latency (p50)

Query Tokens pg_textsearch ParadeDB Speedup
1 token 5.11 ms 59.83 ms 11.7x
2 tokens 9.14 ms 59.65 ms 6.5x
3 tokens 20.04 ms 77.62 ms 3.9x
4 tokens 41.92 ms 98.89 ms 2.4x
5 tokens 67.76 ms 125.38 ms 1.9x
6 tokens 102.82 ms 148.78 ms 1.4x
7 tokens 159.37 ms 169.65 ms 1.1x
8+ tokens 177.95 ms 190.47 ms 1.1x

95th Percentile Latency (p95)

Query Tokens pg_textsearch ParadeDB Speedup
1 token 6.43 ms 68.34 ms 10.6x
2 tokens 32.63 ms 103.17 ms 3.2x
3 tokens 51.51 ms 114.79 ms 2.2x
4 tokens 124.17 ms 147.32 ms 1.2x
5 tokens 167.05 ms 190.07 ms 1.1x
6 tokens 262.07 ms 201.76 ms 0.77x
7 tokens 311.58 ms 291.09 ms 0.94x
8+ tokens 404.95 ms 310.68 ms 0.77x

Weighted-Average Latency

Weighted by observed query-length distribution from 1,010,916 MS-MARCO v1 Bing queries after English stopword removal and stemming (mean 3.7 lexemes, mode 3).

Query length distribution (click to expand) 
MS-MARCO Query Lexeme Count Distribution (1,010,916 queries)
  Lexemes = distinct stems after English stopword removal

  lexemes   queries      %  distribution
  ───────  ────────  ─────  ──────────────────────────────────────────────────
        0        11   0.0%  ▏
        1    35,638   3.5%  ▓▓▓▓▓
        2   165,033  16.3%  ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓
        3   304,887  30.2%  ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓
        4   264,177  26.1%  ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓
        5   143,765  14.2%  ▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓▓
        6    59,558   5.9%  ▓▓▓▓▓▓▓▓▓
        7    22,595   2.2%  ▓▓▓
        8     8,627   0.9%  ▓
        9     3,395   0.3%  ▏
       10     1,555   0.2%  ▏
       11       721   0.1%  ▏
       12       402   0.0%  ▏
       13       235   0.0%  ▏
       14       123   0.0%  ▏
       15+      193   0.0%  ▏

    Total: 1,010,916 queries
     Mean: 3.7 lexemes
     Mode: 3 lexemes (30.2%)

  72.6% of queries have 2-4 lexemes.
  96.2% of queries have 1-6 lexemes.

Benchmark buckets 1–7 contain 100 queries each; bucket 8+ contains 38 queries covering all lengths ≥8. Weights applied to each bucket match the distribution above.

Metric pg_textsearch ParadeDB Speedup
Weighted p50 40.61 ms 94.36 ms 2.3x
Weighted avg 46.69 ms 101.66 ms 2.2x

Throughput (138M)

Single-Client Sequential

691 queries run 3 times; median iteration reported.

Metric pg_textsearch ParadeDB Speedup
Avg ms/query 62.92 ms 106.53 ms 1.7x
Total (691 queries) 43.5 s 73.6 s 1.7x

Concurrent (pgbench, 16 clients, 60 s)

Metric pg_textsearch ParadeDB Ratio
Transactions/sec (TPS) 91.4 19.4 4.7x
Avg latency 175 ms 823 ms 4.7x
Transactions (60 s) 5,526 1,180 4.7x

Analysis (138M)

Query latency: pg_textsearch faster across all token counts

pg_textsearch is faster on all 8 token buckets at p50, ranging from 11.7x faster on single-token queries to 1.1x on 8+ token queries. Cached skip entries and reusable decompression buffers (PR #274) reduced per-block overhead in the WAND inner loop by 20–25%, closing the gap on high-token queries. The weighted p50 advantage is 2.3x.

Tail latency: improved but mixed at p95

pg_textsearch has tighter tail latency on 1–5 token queries at p95. On 6–8+ token queries, ParadeDB still has tighter tails. The p95 gap narrowed significantly with the cache optimizations (e.g., 5-token p95 went from 200ms to 167ms, now faster than ParadeDB's 190ms). Further tail latency optimization on long queries remains an active area of work.

Concurrent throughput: pg_textsearch 4.7x higher TPS

Under 16-client concurrent load, pg_textsearch achieves 91.4 TPS vs 19.4 TPS for ParadeDB — a 4.7x advantage. This is significantly wider than the 1.5x single-client gap, indicating that pg_textsearch scales much better under concurrency. pg_textsearch uses native Postgres buffer management and shared memory, avoiding the external process coordination overhead present in ParadeDB's architecture.

Index build: ParadeDB 1.9x faster

ParadeDB builds its index in 8 min 55 s vs 17 min 37 s for pg_textsearch (1.9x faster). pg_textsearch's parallel build uses 15 workers for the scan phase, but the subsequent merge phase is single-threaded and I/O-bound, accounting for the majority of the build time. Despite the slower build, pg_textsearch produces a 26% smaller index (17 GB vs 23 GB).

Methodology (138M)

Both extensions benchmarked on the same dedicated EC2 instance (c6i.4xlarge), same Postgres 17.7 installation, same dataset. The table was loaded once; each extension built its index from scratch with page cache dropped before each build. Query benchmarks include warmup passes. The pgbench power test uses -M prepared mode with random query selection from 691 benchmark queries.

Note: Both extensions use default configurations. Results are from a single run on dedicated hardware; expect ~5-10% variance between runs.
timescale-pg_textsearch-b63fe58/benchmarks/gh-pages/index.html000066400000000000000000000317311525762047700245730ustar00rootroot00000000000000 pg_textsearch Benchmarks

pg_textsearch Benchmarks

Performance tracking for pg_textsearch. Click any data point to view the commit.
● main branch ✕ feature branch
ParadeDB Comparison | Methodology | CPU Profiles | Concurrent INSERT (pg_textsearch) | Concurrent INSERT (ParadeDB)

timescale-pg_textsearch-b63fe58/benchmarks/gh-pages/methodology.html000066400000000000000000000217661525762047700260250ustar00rootroot00000000000000 Benchmark Methodology - pg_textsearch
Dashboard ParadeDB Comparison CPU Profiles

Benchmark Methodology

Overview

pg_textsearch benchmarks measure full-text search performance using real-world datasets. Benchmarks run nightly on the main branch and on-demand for feature branches via GitHub Actions.

Datasets

MS MARCO (Primary)

The MS MARCO passage ranking dataset is our primary benchmark. It contains 8.8 million passages from web documents with real search queries from Bing users.

MetricValue
Documents8,841,823
Average document length~35 tokens
Test queries800 (100 per token bucket)
Query token buckets1, 2, 3, 4, 5, 6, 7, 8+

MS MARCO v2

The MS MARCO v2 passage ranking dataset scales the primary benchmark to 138 million passages for large-corpus testing.

MetricValue
Documents138,364,198
Test queries691 (sampled from dev set)
Query token buckets1, 2, 3, 4, 5, 6, 7, 8+

Wikipedia

Wikipedia article abstracts provide longer documents for testing scalability. Available in 10K, 100K, 1M, and full (~6M) configurations.

Cranfield

The classic Cranfield collection (1,400 documents) is used for quick regression testing.

Environment

GitHub Actions Runner

ComponentSpecification
PlatformUbuntu 24.04 (ubuntu-latest)
CPU2-core AMD EPYC (GitHub-hosted)
Memory7 GB RAM
Storage~30 GB available after cleanup
PostgresPostgreSQL 17

Postgres Configuration

shared_buffers = 4GB
maintenance_work_mem = 1GB
work_mem = 256MB
effective_cache_size = 6GB
random_page_cost = 1.1

Metrics Collected

Index Build

  • Build time: Wall-clock time to create the BM25 index
  • Index size: On-disk size of the index (from pg_relation_size)

Query Latency

Queries are grouped by token count (1-8+). For each bucket, we run 100 queries and report:

  • p50: Median latency
  • p95: 95th percentile latency
  • p99: 99th percentile latency
  • avg: Mean latency

Throughput

Total time to execute all test queries sequentially, reported as average milliseconds per query.

Weighted-Average Latency

A single summary metric that weights per-bucket p50 latencies by the observed query-length distribution from MS MARCO v1 (1,010,905 queries after to_tsvector). This reflects realistic workload performance more accurately than an equal-weight average across buckets.

Token BucketQueriesWeight
135,6383.5%
2165,03316.3%
3304,88730.2%
4264,17726.1%
5143,76514.2%
659,5585.9%
722,5952.2%
8+15,2521.5%

Formula: weighted_p50 = Σ(bucket_p50 × weight) / Σ(weight)

Benchmark Procedure

  1. Setup: Start fresh Postgres instance, create extension
  2. Load: Bulk load dataset using \copy
  3. Index: Create BM25 index, force spill to segments
  4. Warmup: Run each query once to warm caches
  5. Measure: Run timed queries, collect statistics
  6. Report: Extract metrics, publish to dashboard
Note: All queries use LIMIT 10 to simulate typical search result pages. The Block-Max WAND optimization is enabled by default.

ParadeDB Comparison

We run identical benchmarks against ParadeDB (a competitive Postgres BM25 extension) to provide context for our performance numbers. Both extensions:

  • Run on the same GitHub Actions runner
  • Use identical Postgres configuration
  • Process the same dataset and queries
  • Use their default configurations

See the detailed comparison for latest results.

Reproducibility

All benchmark code is in the benchmarks/ directory. To run locally: 

# MS MARCO benchmark
cd benchmarks/datasets/msmarco
./download.sh full
psql -f load.sql

# Or use the runner script
./benchmarks/runner/run_benchmark.sh msmarco

Limitations

  • GitHub Actions runners have variable performance; expect ~10% variance
  • Single-threaded query execution (no concurrent load testing)
  • Cold start not measured (cache warmup before timing)
  • Network latency not included (local socket connection)
timescale-pg_textsearch-b63fe58/benchmarks/run_build_memory.py000066400000000000000000000451761525762047700250400ustar00rootroot00000000000000#!/usr/bin/env python3 """Measure BM25 build memory on a disposable local PostgreSQL cluster.""" import argparse import csv from datetime import datetime, timezone import json import math import os from pathlib import Path import re import shutil import subprocess import sys import tempfile import time CASES = ("repeated", "half", "unique", "double", "low-budget", "serial") FIELDS = ( "seconds", "phase", "leader_pid", "workers", "leader_rss", "leader_private", "worker_private", "build_pss", "temp_bytes", "cgroup_current", "cgroup_anon", "cgroup_file", ) def read_memory(pid, proc=Path("/proc")): try: text = (proc / str(pid) / "smaps_rollup").read_text() except (FileNotFoundError, ProcessLookupError): return None values = {} for line in text.splitlines(): parts = line.split() if len(parts) == 3 and parts[2] == "kB": values[parts[0].rstrip(":")] = int(parts[1]) * 1024 required = ("Rss", "Pss", "Private_Clean", "Private_Dirty") if any(key not in values for key in required): raise ValueError(f"Incomplete smaps_rollup for PID {pid}") return { "rss": values["Rss"], "pss": values["Pss"], "private": values["Private_Clean"] + values["Private_Dirty"], } def read_cgroup(path): def pairs(name): return { key: int(value) for key, value in ( line.split() for line in (path / name).read_text().splitlines() ) } stats = pairs("memory.stat") return { "current": int((path / "memory.current").read_text()), "max": (path / "memory.max").read_text().strip(), "anon": stats["anon"], "file": stats["file"], "events": pairs("memory.events"), } def summarize_samples(samples): def peak(rows, field): values = [row[field] for row in rows if row[field] is not None] return max(values) if values else None result = { "samples": len(samples), "incomplete_samples": sum(s["build_pss"] is None for s in samples), "peak_build_pss_bytes": peak(samples, "build_pss"), "phases": {}, } for phase in sorted({s["phase"] for s in samples}): rows = [s for s in samples if s["phase"] == phase] result["phases"][phase] = { "peak_build_pss_bytes": peak(rows, "build_pss"), "peak_leader_private_bytes": peak(rows, "leader_private"), "peak_worker_private_bytes": peak(rows, "worker_private"), } return result def temporary_bytes(data): total = 0 for path in (data / "base/pgsql_tmp").rglob("*"): try: if path.is_file(): total += path.stat().st_size except FileNotFoundError: continue # A worker can unlink its files between samples. return total class Cluster: def __init__(self, args, output): self.args = args self.output = output self.data = output / "data" self.process = None self.socket = None self.log = None self.env = { key: value for key, value in os.environ.items() if not key.startswith("PG") } self.env["LC_ALL"] = "C" def start(self): self.bin = Path(subprocess.check_output( [self.args.pg_config, "--bindir"], text=True ).strip()) self.socket = tempfile.TemporaryDirectory(prefix="pgts-build-memory-") self.log = (self.output / "server.log").open("w") with (self.output / "initdb.log").open("w") as log: subprocess.run( [str(self.bin / "initdb"), "-D", str(self.data), "-A", "trust", "--no-locale", "--encoding=UTF8"], env=self.env, stdout=log, stderr=subprocess.STDOUT, check=True, ) self.process = subprocess.Popen( [str(self.bin / "postgres"), "-D", str(self.data), "-k", self.socket.name, "-p", "5432", "-c", "listen_addresses=", "-c", "shared_preload_libraries=pg_textsearch", "-c", "shared_buffers=32MB", "-c", "max_connections=12", "-c", f"max_worker_processes={max(8, self.args.workers + 2)}", "-c", f"max_parallel_workers={self.args.workers}", "-c", "pg_textsearch.memory_limit=16MB", "-c", "max_wal_size=256MB", "-c", "min_wal_size=80MB", "-c", "log_temp_files=0", "-c", "log_line_prefix=%m [%p] "], env=self.env, stdout=self.log, stderr=subprocess.STDOUT, ) deadline = time.monotonic() + 30 while time.monotonic() < deadline: if self.process.poll() is not None: raise RuntimeError("PostgreSQL exited; see server.log") ready = subprocess.run( [str(self.bin / "pg_isready"), "-h", self.socket.name, "-p", "5432", "-d", "postgres"], env=self.env, capture_output=True, ) if ready.returncode == 0: self.execute("CREATE EXTENSION pg_textsearch") return time.sleep(0.1) raise RuntimeError("PostgreSQL startup timed out; see server.log") def command(self, statement): return [ str(self.bin / "psql"), "-X", "-qAt", "-h", self.socket.name, "-p", "5432", "-d", "postgres", "-v", "ON_ERROR_STOP=1", "-c", statement, ] def execute(self, statement): result = subprocess.run( self.command(statement), env=self.env, capture_output=True, text=True, timeout=self.args.timeout, ) if result.returncode: raise RuntimeError(result.stderr.strip()) return result.stdout.strip() def query(self, statement): return json.loads(self.execute(statement)) def stop(self): if self.process is not None and self.process.poll() is None: # Do not wait for crash recovery after an intentional OOM. result = subprocess.run( [str(self.bin / "pg_ctl"), "-D", str(self.data), "stop", "-m", "immediate", "-w", "-t", "30"], env=self.env, capture_output=True, text=True, ) if result.returncode: raise RuntimeError( f"Could not stop benchmark PostgreSQL: {result.stderr}" f"\nData retained at {self.data}" ) self.process.wait(timeout=30) if self.log is not None: self.log.close() if self.socket is not None: self.socket.cleanup() if self.data.exists(): shutil.rmtree(self.data) def sample(cluster, start, cgroup): state = cluster.query(""" SELECT coalesce(( SELECT json_build_object( 'pid', a.pid, 'phase', coalesce(p.phase, 'starting/finishing'), 'workers', (SELECT coalesce(json_agg(w.pid), '[]'::json) FROM pg_stat_activity w WHERE w.leader_pid=a.pid)) FROM pg_stat_activity a LEFT JOIN pg_stat_progress_create_index p ON p.pid=a.pid WHERE a.application_name='build_memory_build' AND a.leader_pid IS NULL ), 'null'::json) """) if state is None: return None leader = read_memory(state["pid"]) workers = [read_memory(pid) for pid in state["workers"]] complete = leader is not None and all(m is not None for m in workers) group = read_cgroup(cgroup) if cgroup else None return { "seconds": round(time.monotonic() - start, 6), "phase": state["phase"], "leader_pid": state["pid"], "workers": len(workers), "leader_rss": leader["rss"] if leader else None, "leader_private": leader["private"] if leader else None, "worker_private": ( sum(m["private"] for m in workers) if all(m is not None for m in workers) else None ), "build_pss": ( leader["pss"] + sum(m["pss"] for m in workers) if complete else None ), "temp_bytes": temporary_bytes(cluster.data), "cgroup_current": group["current"] if group else None, "cgroup_anon": group["anon"] if group else None, "cgroup_file": group["file"] if group else None, } def run_case(cluster, name, args, output, cgroup, result): terms = args.terms if name == "half": terms = max(1, terms // 2) elif name == "double": terms *= 2 vocabulary = min(1000, args.rows * terms) if name == "repeated" else 0 workers = 0 if name == "serial" else args.workers budget = ( "16MB" if name in ("serial", "low-budget") else args.maintenance_work_mem ) result.update(case=name, rows=args.rows, terms_per_doc=terms, vocabulary=vocabulary, workers_requested=workers, maintenance_work_mem=budget, status="running") token = f"((i::bigint - 1) * {terms} + j)" if vocabulary: token = f"({token} % {vocabulary})" width = max(9, len(str(args.rows * args.terms * 2))) started = time.monotonic() cluster.execute(f""" DROP TABLE IF EXISTS docs; CREATE TABLE docs AS SELECT i AS id, 'anchor ' || ( SELECT string_agg('t' || lpad({token}::text, {width}, '0'), ' ' ORDER BY j) FROM generate_series(1, {terms}) j ) AS body FROM generate_series(1, {args.rows}) i; ALTER TABLE docs SET (parallel_workers={workers}); ANALYZE docs; """) result["load_seconds"] = time.monotonic() - started result.update(cluster.query(""" SELECT json_build_object( 'text_bytes', sum(octet_length(body)), 'heap_bytes', pg_relation_size('docs')) FROM docs """)) cluster.execute("CHECKPOINT") result["cgroup_before"] = read_cgroup(cgroup) if cgroup else None samples = [] log_path = output / f"{name}.log" csv_path = output / f"{name}.csv" with log_path.open("w") as log, csv_path.open("w") as csvfile: writer = csv.DictWriter(csvfile, fieldnames=FIELDS) writer.writeheader() started = time.monotonic() process = subprocess.Popen( cluster.command(f""" SET maintenance_work_mem='{budget}'; SET max_parallel_maintenance_workers={workers}; SET statement_timeout={int(args.timeout * 1000)}; """) + ["-c", "\\timing on", "-c", """ CREATE INDEX probe_idx ON docs USING bm25(body) WITH (text_config='simple'); """], env={**cluster.env, "PGAPPNAME": "build_memory_build"}, stdout=log, stderr=subprocess.STDOUT, ) try: while process.poll() is None: if time.monotonic() - started > args.timeout: raise RuntimeError(f"{name}: build timed out") row = sample(cluster, started, cgroup) if row is not None: samples.append(row) writer.writerow(row) csvfile.flush() time.sleep(args.interval) finally: result["observed_seconds"] = time.monotonic() - started if process.poll() is None: process.terminate() result["exit_code"] = process.wait(timeout=30) result.update(summarize_samples(samples)) if cgroup: try: result["cgroup_after"] = read_cgroup(cgroup) except FileNotFoundError: result["cgroup_after"] = None result["cgroup_error"] = "Cgroup disappeared after exit" if process.returncode: raise RuntimeError(f"{name}: CREATE INDEX failed; see {log_path}") build_log = log_path.read_text() timing = re.search(r"^Time: ([0-9.]+) ms", build_log, re.MULTILINE) if timing is None: raise RuntimeError(f"{name}: missing CREATE INDEX timing") result["build_seconds"] = float(timing[1]) / 1000 if result["peak_build_pss_bytes"] is None: raise RuntimeError( f"{name}: no complete memory samples; use more rows" ) launched = re.search( r"launched (\d+) of \d+ requested workers", build_log ) result["workers_launched"] = int(launched[1]) if launched else 0 if result["workers_launched"] != workers: raise RuntimeError(f"{name}: requested {workers} workers but launched " f"{result['workers_launched']}") result.update(cluster.query(f""" SET enable_seqscan=off; SELECT json_build_object( 'index_bytes', pg_relation_size('probe_idx'), 'matching_docs', (SELECT count(*) FROM ( SELECT id FROM docs ORDER BY body <@> to_bm25query('anchor', 'probe_idx') LIMIT {args.rows}) q)) """)) if result["matching_docs"] != args.rows: raise RuntimeError(f"{name}: anchor query lost documents") if not vocabulary: last = f"t{args.rows * terms:0{width}d}" matches = cluster.query(f""" SELECT coalesce(json_agg(id), '[]'::json) FROM ( SELECT id FROM docs ORDER BY body <@> to_bm25query('{last}', 'probe_idx') LIMIT 10) q """) if matches != [args.rows]: raise RuntimeError(f"{name}: last-token query returned {matches}") result["status"] = "ok" def parse_args(): parser = argparse.ArgumentParser(description=__doc__) parser.add_argument("--pg-config", default=os.environ.get("PG_CONFIG", "pg_config")) parser.add_argument("--case", choices=CASES, action="append", dest="cases", help="repeat to select cases; default: all six") parser.add_argument("--rows", type=int, default=100000) parser.add_argument("--terms", type=int, default=10) parser.add_argument("--workers", type=int, default=2) parser.add_argument("--maintenance-work-mem", default="64MB") parser.add_argument("--interval", type=float, default=0.1, help="delay between samples, seconds") parser.add_argument("--timeout", type=float, default=300, help="per-command/build timeout, seconds") parser.add_argument("--output", type=Path, help="new results directory; must not already exist") parser.add_argument("--cgroup", action="store_true", help="record cgroup v2 metrics; use a private scope") args = parser.parse_args() if sys.platform != "linux": parser.error("memory sampling requires Linux /proc") if args.rows <= 0 or args.terms <= 0 or args.workers < 0: parser.error("rows/terms must be positive and workers nonnegative") args.cases = list(dict.fromkeys(args.cases or CASES)) if (args.workers > 0 and args.rows < 100000 and any(case != "serial" for case in args.cases)): parser.error("parallel cases require at least 100000 rows") if any(not math.isfinite(v) or v <= 0 for v in (args.interval, args.timeout)): parser.error("interval and timeout must be positive finite seconds") if not re.fullmatch(r"[1-9][0-9]*(?:kB|MB|GB)", args.maintenance_work_mem): parser.error("maintenance-work-mem must be a size such as 64MB") return args def main(): args = parse_args() timestamp = datetime.now(timezone.utc).strftime("%Y%m%dT%H%M%S_%fZ") output = (args.output or Path(__file__).resolve().parent / "results" / f"build_memory_{timestamp}").resolve() output.mkdir(parents=True, exist_ok=False) summary = { "status": "running", "timestamp": timestamp, "options": {k: str(v) if isinstance(v, Path) else v for k, v in vars(args).items()}, "cases": [], } cluster = Cluster(args, output) try: cgroup = None if args.cgroup: entries = Path("/proc/self/cgroup").read_text().splitlines() unified = [line[3:] for line in entries if line.startswith("0::")] if len(unified) != 1: raise RuntimeError("--cgroup requires a unified cgroup v2") cgroup = Path("/sys/fs/cgroup") / unified[0].lstrip("/") summary["cgroup_path"] = str(cgroup) read_cgroup(cgroup) cluster.start() summary["server"] = cluster.query(""" SELECT json_build_object( 'postgres_version', version(), 'extension_version', (SELECT extversion FROM pg_extension WHERE extname='pg_textsearch'), 'settings', (SELECT json_object_agg(name, setting) FROM pg_settings WHERE name IN ( 'shared_buffers', 'block_size', 'fsync', 'max_parallel_workers', 'max_worker_processes', 'pg_textsearch.memory_limit', 'pg_textsearch.compress_segments'))) """) print(f"Results: {output}", flush=True) print("case seconds peak build PSS MiB index MiB", flush=True) for name in args.cases: result = {} summary["cases"].append(result) try: run_case(cluster, name, args, output, cgroup, result) except (OSError, RuntimeError, ValueError, subprocess.SubprocessError, KeyboardInterrupt) as exc: result.update(status="error", error=str(exc) or "Interrupted") raise finally: (output / "summary.json").write_text( json.dumps(summary, indent=2) + "\n" ) print(f"{name:16} {result['build_seconds']:7.2f} " f"{result['peak_build_pss_bytes'] / 2**20:20.1f} " f"{result['index_bytes'] / 2**20:11.2f}", flush=True) summary["status"] = "ok" except (OSError, RuntimeError, ValueError, subprocess.SubprocessError, KeyboardInterrupt) as exc: summary.update(status="error", error=str(exc) or "Interrupted") print(f"ERROR: {summary['error']}", file=sys.stderr) finally: try: cluster.stop() except (OSError, RuntimeError, subprocess.SubprocessError) as exc: summary.update(status="error", cleanup_error=str(exc)) print(f"ERROR: {exc}", file=sys.stderr) (output / "summary.json").write_text( json.dumps(summary, indent=2) + "\n" ) return 0 if summary["status"] == "ok" else 1 if __name__ == "__main__": sys.exit(main()) timescale-pg_textsearch-b63fe58/benchmarks/run_cranfield.sh000077500000000000000000000050521525762047700242520ustar00rootroot00000000000000#!/bin/bash # Cranfield Collection BM25 benchmark runner for pg_textsearch # This script runs the complete Cranfield IR benchmark (1400 documents, 225 queries) set -e echo "=== pg_textsearch Cranfield Collection Benchmark ===" echo "Standard Information Retrieval benchmark with 1400 aerodynamics abstracts" echo "Starting at: $(date)" echo "" # Set up environment export PGPORT=${PGPORT:-5432} export PGHOST=${PGHOST:-localhost} export PGUSER=${PGUSER:-$(whoami)} export PGDATABASE=${PGDATABASE:-postgres} # Navigate to Cranfield benchmark directory SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" CRANFIELD_DIR="$SCRIPT_DIR/sql/cranfield" if [ ! -d "$CRANFIELD_DIR" ]; then echo "ERROR: Cranfield benchmark directory not found at $CRANFIELD_DIR" exit 1 fi cd "$CRANFIELD_DIR" # Check if pg_textsearch extension is available echo "Checking pg_textsearch extension availability..." psql -c "CREATE EXTENSION IF NOT EXISTS pg_textsearch;" || { echo "ERROR: pg_textsearch extension not available. Please install it first with 'make install'" exit 1 } echo "pg_textsearch extension loaded successfully." echo "" # Show initial configuration (skip if parameters not available) echo "=== Initial Configuration ===" psql -c "SHOW pg_textsearch.bulk_load_threshold;" 2>/dev/null || echo "pg_textsearch.bulk_load_threshold: (parameter not visible)" psql -c "SHOW pg_textsearch.default_limit;" 2>/dev/null || echo "pg_textsearch.default_limit: (parameter not visible)" echo "" # Phase 1: Load Cranfield dataset echo "=== Phase 1: Data Loading ===" echo "Loading 1400 Cranfield collection documents..." echo "" START_TIME=$(date +%s) psql -f 01-load.sql || { echo "ERROR: Failed to load Cranfield dataset" exit 1 } LOAD_TIME=$(($(date +%s) - START_TIME)) echo "" echo "Data loading completed in ${LOAD_TIME} seconds" echo "" # Phase 2: Run query benchmarks echo "=== Phase 2: Query Benchmarks ===" echo "Running 225 Cranfield queries with BM25 scoring..." echo "" START_TIME=$(date +%s) psql -f 02-queries.sql || { echo "ERROR: Failed to run query benchmarks" exit 1 } QUERY_TIME=$(($(date +%s) - START_TIME)) echo "" echo "Query benchmarks completed in ${QUERY_TIME} seconds" echo "" # Summary TOTAL_TIME=$((LOAD_TIME + QUERY_TIME)) echo "=== Benchmark Summary ===" echo "Data Loading Time: ${LOAD_TIME} seconds" echo "Query Execution: ${QUERY_TIME} seconds" echo "Total Runtime: ${TOTAL_TIME} seconds" echo "Dataset Size: 1400 documents, 225 queries" echo "Completed at: $(date)" echo "" echo "Benchmark completed successfully!" timescale-pg_textsearch-b63fe58/benchmarks/run_filtered_seed.sh000077500000000000000000000065271525762047700251310ustar00rootroot00000000000000#!/bin/bash # Run the filtered-seed top-K benchmark (issues #434, #435) # # Measures what pg_textsearch.filtered_seed is worth on filtered top-k # queries -- WHERE ORDER BY LIMIT k -- by toggling the # GUC over identical data and queries. Metrics are latency and scoring # passes; result parity is covered by the filtered_seed regression. # # The union2/union3 shapes are the #435 case: several BM25 scans of one # index in one statement. To see what #435 bought, run this on main and # again on the branch: the on_passes column drops to one pass per arm # only when the seed is bound per scan. # # Usage: # ./run_filtered_seed.sh [ndocs] [-- psql args...] # # Arguments: # ndocs: corpus size. Default 200000. Below ~100k the planner may # prefer a seq scan and sort. Cells without a BM25 scan in # every arm are dropped with a warning. # # Examples: # ./run_filtered_seed.sh # ./run_filtered_seed.sh 1000000 # ./run_filtered_seed.sh 200000 -- -p 55433 -d contrib_regression # # Outputs: # - Console output with the results table # - results/filtered_seed_.txt with raw output # - results/filtered_seed_.json with parsed metrics set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" NDOCS="${1:-200000}" shift || true if [ "${1:-}" = "--" ]; then shift; fi TS="$(date +%Y%m%d_%H%M%S)" RESULTS_DIR="$SCRIPT_DIR/results" OUTPUT_FILE="$RESULTS_DIR/filtered_seed_${TS}.txt" METRICS_FILE="$RESULTS_DIR/filtered_seed_${TS}.json" mkdir -p "$RESULTS_DIR" echo "=== Filtered-Seed Top-K Benchmark ===" echo "Corpus size: $NDOCS documents" echo "" psql -v ndocs="$NDOCS" "$@" \ -f "$SCRIPT_DIR/sql/filtered_seed.sql" 2>&1 | tee "$OUTPUT_FILE" if ! grep -q '^FSB_RESULT:' "$OUTPUT_FILE"; then echo "" echo "ERROR: no results produced. See $OUTPUT_FILE" >&2 exit 1 fi # Warn loudly rather than let a dropped cell pass unnoticed. if grep -q 'BM25 scan count mismatch for shape=' "$OUTPUT_FILE"; then echo "" echo "WARNING: some cells were dropped because the planner chose a" echo " seq scan instead of the BM25 index scan. Try a larger" echo " corpus size." fi # FSB_RESULT: shape=X sel=Y limit=Z off_ms=.. off_passes=.. on_ms=.. # on_passes=.. speedup=.. field() { sed -n "s/.*[[:space:]]$2=\([^[:space:]]*\).*/\1/p" <<< "$1"; } { echo "{" echo " \"timestamp\": \"$(date -u +%Y-%m-%dT%H:%M:%SZ)\"," echo " \"ndocs\": $NDOCS," echo " \"cells\": [" first=1 while IFS= read -r line; do [ $first -eq 1 ] || echo "," first=0 printf ' {"shape": "%s", "selectivity": "%s", "limit": %s, ' \ "$(field "$line" shape)" \ "$(field "$line" sel)" \ "$(field "$line" limit)" printf '"bm25_scans": %s, ' "$(field "$line" scans)" printf '"seed_off_ms": %s, "seed_off_passes": %s, ' \ "$(field "$line" off_ms)" \ "$(field "$line" off_passes)" printf '"seed_on_ms": %s, "seed_on_passes": %s, "speedup": %s}' \ "$(field "$line" on_ms)" \ "$(field "$line" on_passes)" \ "$(field "$line" speedup)" done < <(grep '^FSB_RESULT:' "$OUTPUT_FILE") echo "" echo " ]" echo "}" } > "$METRICS_FILE" echo "" echo "Results written to:" echo " - $OUTPUT_FILE (raw output)" echo " - $METRICS_FILE (parsed metrics)" timescale-pg_textsearch-b63fe58/benchmarks/run_memory_stress.sh000077500000000000000000000063701525762047700252420ustar00rootroot00000000000000#!/bin/bash # Memory stress benchmark runner for pg_textsearch # This script runs the memory stress test and captures output set -e echo "=== pg_textsearch Memory Stress Benchmark ===" echo "Starting at: $(date)" echo "" # Set up environment export PGPORT=${PGPORT:-5432} export PGHOST=${PGHOST:-localhost} export PGUSER=${PGUSER:-$(whoami)} export PGDATABASE=${PGDATABASE:-postgres} # Check if pg_textsearch extension is available echo "Checking pg_textsearch extension availability..." psql -c "CREATE EXTENSION IF NOT EXISTS pg_textsearch;" || { echo "ERROR: pg_textsearch extension not available. Please install it first with 'make install'" exit 1 } echo "pg_textsearch extension loaded successfully." echo "" # Show initial configuration (skip if parameters not available) echo "=== Initial Configuration ===" psql -c "SHOW pg_textsearch.bulk_load_threshold;" 2>/dev/null || echo "pg_textsearch.bulk_load_threshold: (parameter not visible)" psql -c "SHOW pg_textsearch.memtable_pages_threshold;" 2>/dev/null || echo "pg_textsearch.memtable_pages_threshold: (parameter not visible)" psql -c "SHOW pg_textsearch.memory_limit;" 2>/dev/null || echo "pg_textsearch.memory_limit: (parameter not visible)" echo "" # Choose benchmark size based on argument BENCHMARK_SIZE="${1:-micro}" BENCHMARK_FILE="" case "$BENCHMARK_SIZE" in "micro") BENCHMARK_FILE="sql/memory_stress_micro.sql" echo "Running MICRO memory stress benchmark (1K documents, ~1 minute)" ;; "small") BENCHMARK_FILE="sql/memory_stress_small.sql" echo "Running SMALL memory stress benchmark (25K documents, ~5-10 minutes)" ;; "large"|"full") BENCHMARK_FILE="sql/memory_stress.sql" echo "Running FULL memory stress benchmark (100K documents, may timeout)" ;; *) echo "Usage: $0 [micro|small|large]" echo " micro: 1K documents (default, quick test)" echo " small: 25K documents (moderate stress)" echo " large: 100K documents (high stress, may fail)" exit 1 ;; esac # Run the memory stress benchmark echo "=== Running Memory Stress Benchmark ($BENCHMARK_SIZE) ===" echo "This demonstrates extension behavior under different dataset sizes..." echo "" # Run the benchmark and capture both stdout and stderr if psql -f "$(dirname "$0")/$BENCHMARK_FILE" 2>&1; then echo "" echo "=== Benchmark Results ===" echo "Benchmark completed successfully at: $(date)" echo "" case "$BENCHMARK_SIZE" in "micro") echo "Micro benchmark establishes baseline functionality." echo "Try 'small' or 'large' sizes for stress testing." ;; "small"|"large") echo "To exercise on-disk-chain fallback, try:" echo "- Lowering pg_textsearch.memory_limit in postgresql.conf" echo " and reloading the server configuration" echo "- Running 'large' size for maximum stress" ;; esac else echo "" echo "=== Benchmark Results ===" echo "Benchmark failed at: $(date)" echo "" echo "Check the errors above. The pg_textsearch.memory_limit cache cap" echo "should cause on-disk-chain fallback rather than query failure." fi echo "" echo "=== Memory Stress Benchmark Complete ===" timescale-pg_textsearch-b63fe58/benchmarks/run_parallel_scaling.sh000077500000000000000000000071411525762047700256200ustar00rootroot00000000000000#!/bin/bash # Run parallel build scaling benchmark on MS MARCO # # Usage: # ./run_parallel_scaling.sh [msmarco_size] # # Arguments: # msmarco_size: "full" (8.8M) or "sample" (100K). Default: full # # Outputs: # - Console output with benchmark results # - parallel_scaling_results.txt with raw output # - parallel_scaling_metrics.json with parsed metrics set -e SCRIPT_DIR="$(cd "$(dirname "$0")" && pwd)" MSMARCO_SIZE="${1:-full}" DATA_DIR="$SCRIPT_DIR/datasets/msmarco/data" OUTPUT_FILE="parallel_scaling_results.txt" METRICS_FILE="parallel_scaling_metrics.json" echo "=== Parallel Build Scaling Benchmark ===" echo "MS MARCO size: $MSMARCO_SIZE" echo "" # Check if data exists if [ ! -f "$DATA_DIR/collection.tsv" ]; then echo "Downloading MS MARCO dataset..." cd "$SCRIPT_DIR/datasets/msmarco" ./download.sh "$MSMARCO_SIZE" cd "$SCRIPT_DIR" fi # Load data (without index) echo "" echo "Loading MS MARCO data..." DATA_DIR="$DATA_DIR" psql -f "$SCRIPT_DIR/datasets/msmarco/load_data_only.sql" 2>&1 | tee load_output.txt # Run parallel scaling benchmark echo "" echo "Running parallel scaling benchmark..." psql -f "$SCRIPT_DIR/datasets/msmarco/parallel_scaling.sql" 2>&1 | tee "$OUTPUT_FILE" # Extract metrics from output echo "" echo "Extracting metrics..." # Parse build times from psql timing output # Format: "Time: 12345.678 ms" extract_build_time() { local workers=$1 # Find the CREATE INDEX timing after the workers=$workers section awk -v w="$workers" ' /Testing with '"$workers"' worker/ { found=1 } found && /CREATE INDEX/ { getline; if (/Time:/) { gsub(/[^0-9.]/, "", $2); print $2; exit } } ' "$OUTPUT_FILE" } # Parse query results # Format: "PARALLEL_QUERY_RESULT: workers=N, avg_ms=X, min_ms=Y, max_ms=Z" extract_query_metrics() { local workers=$1 grep "PARALLEL_QUERY_RESULT: workers=$workers" "$OUTPUT_FILE" | \ sed 's/.*avg_ms=\([0-9.]*\).*/\1/' } # Build JSON output BUILD_0=$(extract_build_time 0) BUILD_1=$(extract_build_time 1) BUILD_2=$(extract_build_time 2) BUILD_4=$(extract_build_time 4) QUERY_0=$(extract_query_metrics 0) QUERY_1=$(extract_query_metrics 1) QUERY_2=$(extract_query_metrics 2) QUERY_4=$(extract_query_metrics 4) # Generate JSON metrics file cat > "$METRICS_FILE" << EOF { "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", "dataset": "msmarco-$MSMARCO_SIZE", "parallel_scaling": { "workers_0": { "build_time_ms": ${BUILD_0:-null}, "query_avg_ms": ${QUERY_0:-null} }, "workers_1": { "build_time_ms": ${BUILD_1:-null}, "query_avg_ms": ${QUERY_1:-null} }, "workers_2": { "build_time_ms": ${BUILD_2:-null}, "query_avg_ms": ${QUERY_2:-null} }, "workers_4": { "build_time_ms": ${BUILD_4:-null}, "query_avg_ms": ${QUERY_4:-null} } } } EOF echo "" echo "=== Parallel Scaling Results ===" echo "" printf "%-10s %15s %15s\n" "Workers" "Build Time (s)" "Query Avg (ms)" printf "%-10s %15s %15s\n" "-------" "--------------" "--------------" printf "%-10s %15.1f %15.2f\n" "0 (serial)" "${BUILD_0:-0}" "${QUERY_0:-0}" printf "%-10s %15.1f %15.2f\n" "1" "${BUILD_1:-0}" "${QUERY_1:-0}" printf "%-10s %15.1f %15.2f\n" "2" "${BUILD_2:-0}" "${QUERY_2:-0}" printf "%-10s %15.1f %15.2f\n" "4" "${BUILD_4:-0}" "${QUERY_4:-0}" echo "" # Calculate speedups if we have valid build times if [ -n "$BUILD_0" ] && [ -n "$BUILD_4" ]; then SPEEDUP=$(echo "scale=2; $BUILD_0 / $BUILD_4" | bc 2>/dev/null || echo "N/A") echo "Speedup (4 workers vs serial): ${SPEEDUP}x" fi echo "" echo "Results written to:" echo " - $OUTPUT_FILE (raw output)" echo " - $METRICS_FILE (parsed metrics)" timescale-pg_textsearch-b63fe58/benchmarks/runner/000077500000000000000000000000001525762047700224075ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/runner/extract_metrics.sh000077500000000000000000000261421525762047700261530ustar00rootroot00000000000000#!/bin/bash # Extract benchmark metrics from log output and create JSON summary # # Usage: ./extract_metrics.sh [dataset_name] [section] # # If section is specified (e.g., "Cranfield", "MS MARCO", "Wikipedia"), # only extracts metrics from that section of the log file. # # Parses benchmark output to extract: # - Index build time # - Query latencies by token bucket (p50/p95/p99) # - Throughput metrics # - Index/table sizes set -e LOG_FILE="${1:-benchmark_results.txt}" OUTPUT_FILE="${2:-benchmark_metrics.json}" DATASET_NAME="${3:-unknown}" SECTION="${4:-}" # If section specified, extract only that section from log if [ -n "$SECTION" ]; then TEMP_LOG=$(mktemp) awk -v section="$SECTION" ' /^=== .* Benchmark ===/ { if (index($0, section) > 0) { capture = 1 } else if (capture) { exit } } capture { print } ' "$LOG_FILE" > "$TEMP_LOG" LOG_FILE="$TEMP_LOG" trap "rm -f $TEMP_LOG" EXIT fi # Helper to output number or null num_or_null() { if [ -n "$1" ] && [[ "$1" =~ ^[0-9]+\.?[0-9]*$ ]]; then echo "$1" else echo "null" fi } # Extract index build time (from CREATE INDEX timing) INDEX_BUILD_MS=$(grep -E "CREATE INDEX" "$LOG_FILE" -A 1 2>/dev/null | \ grep -oE "Time: [0-9]+\.[0-9]+ ms" | head -1 | grep -oE "[0-9]+\.[0-9]+" || echo "") # Add post-build VACUUM time if present (ParadeDB segment compaction) INDEX_VACUUM_MS=$(grep -E "^INDEX_VACUUM:" "$LOG_FILE" -A 2 2>/dev/null | \ grep -oE "Time: [0-9]+\.[0-9]+ ms" | head -1 | grep -oE "[0-9]+\.[0-9]+" || echo "") if [ -n "$INDEX_VACUUM_MS" ] && [ -n "$INDEX_BUILD_MS" ]; then INDEX_BUILD_MS=$(echo "$INDEX_BUILD_MS + $INDEX_VACUUM_MS" | bc) fi # Extract data load time (COPY statements) LOAD_TIME_MS=$(grep -E "^COPY [0-9]+" "$LOG_FILE" -A 1 2>/dev/null | \ grep -oE "Time: [0-9]+\.[0-9]+ ms" | head -1 | grep -oE "[0-9]+\.[0-9]+" || echo "") # Extract insert time (for insert-based benchmarks) # INSERT_TIME: marker appears before the COPY/INSERT block INSERT_TIME_MS=$(grep -E "INSERT_TIME:" "$LOG_FILE" -A 10 2>/dev/null | \ grep -oE "Time: [0-9]+\.[0-9]+ ms" | head -1 | \ grep -oE "[0-9]+\.[0-9]+" || echo "") # Extract concurrent insert time (wall-clock from pgbench) # Format: CONCURRENT_INSERT_TIME: XXXms CONCURRENT_INSERT_TIME_MS=$(grep -E "CONCURRENT_INSERT_TIME:" \ "$LOG_FILE" 2>/dev/null | \ grep -oE "[0-9]+\.[0-9]+" | head -1 || echo "") # Extract document count # Try pg_textsearch format first NUM_DOCUMENTS=$(grep -E "BM25 index build completed:" "$LOG_FILE" 2>/dev/null | \ grep -oE "[0-9]+ documents" | grep -oE "[0-9]+" || echo "") # Fallback: look for various "X loaded:" patterns in SQL output (Passages, Documents, Articles) if [ -z "$NUM_DOCUMENTS" ]; then NUM_DOCUMENTS=$(grep -E "(Passages|Documents|Articles) loaded" "$LOG_FILE" 2>/dev/null | \ grep -oE "\| +[0-9]+" | grep -oE "[0-9]+" | head -1 || echo "") fi # Extract index and table sizes INDEX_SIZE=$(grep -E "INDEX_SIZE:" "$LOG_FILE" 2>/dev/null | \ grep -oE "[0-9]+ [kMGT]?B" | head -1 || echo "") INDEX_SIZE_BYTES=$(grep -E "INDEX_SIZE:" "$LOG_FILE" 2>/dev/null | \ awk '{print $NF}' | grep -E "^[0-9]+$" | head -1 || echo "") TABLE_SIZE=$(grep -E "TABLE_SIZE:" "$LOG_FILE" 2>/dev/null | \ grep -oE "[0-9]+ [kMGT]?B" | head -1 || echo "") TABLE_SIZE_BYTES=$(grep -E "TABLE_SIZE:" "$LOG_FILE" 2>/dev/null | \ awk '{print $NF}' | grep -E "^[0-9]+$" | head -1 || echo "") # Extract latency buckets (new format) # Format: LATENCY_BUCKET_N: p50=Xms p95=Yms p99=Zms avg=Wms (n=100) extract_bucket() { local bucket=$1 # Validate bucket is an integer between 1 and 8 if ! [[ "$bucket" =~ ^[1-8]$ ]]; then echo "null" return 0 fi local line=$(grep -E "LATENCY_BUCKET_${bucket}:" "$LOG_FILE" 2>/dev/null || echo "") if [ -n "$line" ]; then local p50=$(echo "$line" | grep -oE "p50=[0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") local p95=$(echo "$line" | grep -oE "p95=[0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") local p99=$(echo "$line" | grep -oE "p99=[0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") local avg=$(echo "$line" | grep -oE "avg=[0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") echo "{\"p50\": $(num_or_null "$p50"), \"p95\": $(num_or_null "$p95"), \"p99\": $(num_or_null "$p99"), \"avg\": $(num_or_null "$avg")}" else echo "null" fi } BUCKET_1=$(extract_bucket 1) BUCKET_2=$(extract_bucket 2) BUCKET_3=$(extract_bucket 3) BUCKET_4=$(extract_bucket 4) BUCKET_5=$(extract_bucket 5) BUCKET_6=$(extract_bucket 6) BUCKET_7=$(extract_bucket 7) BUCKET_8=$(extract_bucket 8) # Extract throughput result # Format: "THROUGHPUT_RESULT: N queries in XXXX.XX ms (avg YY.YY ms/query)" THROUGHPUT_LINE=$(grep -E "THROUGHPUT_RESULT:" "$LOG_FILE" 2>/dev/null | head -1 || echo "") THROUGHPUT_TOTAL_MS="" THROUGHPUT_AVG_MS="" THROUGHPUT_NUM_QUERIES="" if [ -n "$THROUGHPUT_LINE" ]; then THROUGHPUT_NUM_QUERIES=$(echo "$THROUGHPUT_LINE" | grep -oE "[0-9]+ queries" | grep -oE "[0-9]+" || echo "") THROUGHPUT_TOTAL_MS=$(echo "$THROUGHPUT_LINE" | grep -oE "in [0-9]+\.[0-9]+ ms" | grep -oE "[0-9]+\.[0-9]+" || echo "") THROUGHPUT_AVG_MS=$(echo "$THROUGHPUT_LINE" | grep -oE "avg [0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") fi # Extract weighted-average latency # Format: "WEIGHTED_LATENCY_RESULT: weighted_p50=XX.XXms weighted_avg=YY.YYms" WEIGHTED_LINE=$(grep -E "WEIGHTED_LATENCY_RESULT:" "$LOG_FILE" 2>/dev/null | head -1 || echo "") WEIGHTED_P50_MS="" WEIGHTED_AVG_MS="" if [ -n "$WEIGHTED_LINE" ]; then WEIGHTED_P50_MS=$(echo "$WEIGHTED_LINE" | grep -oE "weighted_p50=[0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") WEIGHTED_AVG_MS=$(echo "$WEIGHTED_LINE" | grep -oE "weighted_avg=[0-9]+\.[0-9]+" | grep -oE "[0-9]+\.[0-9]+" || echo "") fi # Extract weighted throughput # Format: "WEIGHTED_THROUGHPUT_RESULT: XX.XX ms/query" WEIGHTED_THROUGHPUT_LINE=$(grep -E "WEIGHTED_THROUGHPUT_RESULT:" "$LOG_FILE" 2>/dev/null | head -1 || echo "") WEIGHTED_THROUGHPUT_MS="" if [ -n "$WEIGHTED_THROUGHPUT_LINE" ]; then WEIGHTED_THROUGHPUT_MS=$(echo "$WEIGHTED_THROUGHPUT_LINE" | grep -oE "[0-9]+\.[0-9]+ ms/query" | grep -oE "[0-9]+\.[0-9]+" || echo "") fi # ============================================================ # VACUUM benchmark metrics (from vacuum.sql output) # Scenarios: partial (concentrated), full (uniform), update # ============================================================ # Extract timing between START/END markers using \timing output extract_marker_time() { local start_marker="$1" local end_marker="$2" local time_val time_val=$(sed -n "/${start_marker}/,/${end_marker}/p" "$LOG_FILE" 2>/dev/null | \ grep -oE "Time: [0-9]+\.[0-9]+ ms" | tail -1 | \ grep -oE "[0-9]+\.[0-9]+" || echo "") echo "$time_val" } # Scenario A: Partial VACUUM (concentrated delete) VACUUM_PARTIAL_DELETE_MS=$(extract_marker_time "VACUUM_PARTIAL_DELETE_START:" "VACUUM_PARTIAL_DELETE_END:") VACUUM_PARTIAL_MS=$(extract_marker_time "VACUUM_PARTIAL_START:" "VACUUM_PARTIAL_END:") # Scenario B: Full VACUUM (uniform delete) VACUUM_FULL_DELETE_MS=$(extract_marker_time "VACUUM_FULL_DELETE_START:" "VACUUM_FULL_DELETE_END:") VACUUM_FULL_MS=$(extract_marker_time "VACUUM_FULL_START:" "VACUUM_FULL_END:") # Scenario C: Full VACUUM (uniform update) VACUUM_UPDATE_MS=$(extract_marker_time "VACUUM_UPDATE_START:" "VACUUM_UPDATE_END:") VACUUM_UPDATE_VACUUM_MS=$(extract_marker_time "VACUUM_UPDATE_VACUUM_START:" "VACUUM_UPDATE_VACUUM_END:") # Extract index sizes (bytes from last column of psql output) extract_index_bytes() { local marker="$1" grep -E "${marker}" "$LOG_FILE" 2>/dev/null | \ awk '{print $NF}' | grep -E "^[0-9]+$" | head -1 || echo "" } VACUUM_BASELINE_INDEX_BYTES=$(extract_index_bytes "VACUUM_BASELINE_INDEX_SIZE:") VACUUM_PARTIAL_POST_INDEX_BYTES=$(extract_index_bytes "VACUUM_PARTIAL_POST_INDEX_SIZE:") VACUUM_FULL_POST_INDEX_BYTES=$(extract_index_bytes "VACUUM_FULL_POST_INDEX_SIZE:") VACUUM_UPDATE_POST_INDEX_BYTES=$(extract_index_bytes "VACUUM_UPDATE_POST_INDEX_SIZE:") # Extract query latency averages per scenario # Format: VACUUM_*_QUERY: avg=Xms min=Yms max=Zms (n=N) extract_vacuum_query_avg() { local marker="$1" grep -E "${marker}" "$LOG_FILE" 2>/dev/null | \ grep -oE "avg=[0-9]+\.[0-9]+" | \ grep -oE "[0-9]+\.[0-9]+" | head -1 || echo "" } VACUUM_BASELINE_QUERY_AVG=$(extract_vacuum_query_avg "VACUUM_BASELINE_QUERY:") VACUUM_PARTIAL_QUERY_AVG=$(extract_vacuum_query_avg "VACUUM_PARTIAL_QUERY:") VACUUM_FULL_QUERY_AVG=$(extract_vacuum_query_avg "VACUUM_FULL_QUERY:") VACUUM_UPDATE_QUERY_AVG=$(extract_vacuum_query_avg "VACUUM_UPDATE_QUERY:") # Build JSON output cat > "$OUTPUT_FILE" << EOJSON { "timestamp": "$(date -u +%Y-%m-%dT%H:%M:%SZ)", "commit": "${GITHUB_SHA:-unknown}", "dataset": "$DATASET_NAME", "metrics": { "load_time_ms": $(num_or_null "$LOAD_TIME_MS"), "index_build_time_ms": $(num_or_null "$INDEX_BUILD_MS"), "insert_time_ms": $(num_or_null "$INSERT_TIME_MS"), "concurrent_insert_time_ms": $(num_or_null "$CONCURRENT_INSERT_TIME_MS"), "num_documents": $(num_or_null "$NUM_DOCUMENTS"), "index_size": "${INDEX_SIZE:-unknown}", "index_size_bytes": $(num_or_null "$INDEX_SIZE_BYTES"), "table_size": "${TABLE_SIZE:-unknown}", "table_size_bytes": $(num_or_null "$TABLE_SIZE_BYTES"), "latency_by_tokens": { "bucket_1": $BUCKET_1, "bucket_2": $BUCKET_2, "bucket_3": $BUCKET_3, "bucket_4": $BUCKET_4, "bucket_5": $BUCKET_5, "bucket_6": $BUCKET_6, "bucket_7": $BUCKET_7, "bucket_8": $BUCKET_8 }, "throughput": { "num_queries": $(num_or_null "$THROUGHPUT_NUM_QUERIES"), "total_ms": $(num_or_null "$THROUGHPUT_TOTAL_MS"), "avg_ms_per_query": $(num_or_null "$THROUGHPUT_AVG_MS") }, "weighted_latency": { "weighted_p50_ms": $(num_or_null "$WEIGHTED_P50_MS"), "weighted_avg_ms": $(num_or_null "$WEIGHTED_AVG_MS") }, "weighted_throughput": { "avg_ms_per_query": $(num_or_null "$WEIGHTED_THROUGHPUT_MS") }, "vacuum": { "partial_delete_ms": $(num_or_null "$VACUUM_PARTIAL_DELETE_MS"), "partial_vacuum_ms": $(num_or_null "$VACUUM_PARTIAL_MS"), "full_delete_ms": $(num_or_null "$VACUUM_FULL_DELETE_MS"), "full_vacuum_ms": $(num_or_null "$VACUUM_FULL_MS"), "update_ms": $(num_or_null "$VACUUM_UPDATE_MS"), "update_vacuum_ms": $(num_or_null "$VACUUM_UPDATE_VACUUM_MS"), "baseline_index_bytes": $(num_or_null "$VACUUM_BASELINE_INDEX_BYTES"), "partial_post_index_bytes": $(num_or_null "$VACUUM_PARTIAL_POST_INDEX_BYTES"), "full_post_index_bytes": $(num_or_null "$VACUUM_FULL_POST_INDEX_BYTES"), "update_post_index_bytes": $(num_or_null "$VACUUM_UPDATE_POST_INDEX_BYTES"), "baseline_query_avg_ms": $(num_or_null "$VACUUM_BASELINE_QUERY_AVG"), "partial_query_avg_ms": $(num_or_null "$VACUUM_PARTIAL_QUERY_AVG"), "full_query_avg_ms": $(num_or_null "$VACUUM_FULL_QUERY_AVG"), "update_query_avg_ms": $(num_or_null "$VACUUM_UPDATE_QUERY_AVG") } } } EOJSON echo "Metrics extracted to $OUTPUT_FILE" cat "$OUTPUT_FILE" timescale-pg_textsearch-b63fe58/benchmarks/runner/format_for_action.sh000077500000000000000000000203151525762047700264420ustar00rootroot00000000000000#!/bin/bash # Convert benchmark_metrics.json to github-action-benchmark format # # Usage: ./format_for_action.sh # # Input format (our benchmark_metrics.json): # { "metrics": { "index_build_time_ms": 547000, ... } } # # Output format (github-action-benchmark customSmallerIsBetter): # [ {"name": "...", "unit": "ms", "value": 547000}, ... ] set -e INPUT_FILE="${1:-benchmark_metrics.json}" OUTPUT_FILE="${2:-benchmark_action.json}" if [ ! -f "$INPUT_FILE" ]; then echo "Error: Input file $INPUT_FILE not found" exit 1 fi # Extract dataset name and document count from input DATASET=$(jq -r '.dataset // "Benchmark"' "$INPUT_FILE") NUM_DOCS=$(jq -r '.metrics.num_documents // empty' "$INPUT_FILE") # Format document count (e.g., 1400 -> "1.4K", 8841823 -> "8.8M") format_docs() { local n=$1 if [ -z "$n" ] || [ "$n" = "null" ]; then echo "" return fi if [ "$n" -ge 1000000 ]; then printf "%.1fM" "$(echo "scale=1; $n / 1000000" | bc)" elif [ "$n" -ge 1000 ]; then printf "%.1fK" "$(echo "scale=1; $n / 1000" | bc)" else echo "$n" fi } DOCS_LABEL=$(format_docs "$NUM_DOCS") if [ -n "$DOCS_LABEL" ]; then DATASET_LABEL="$DATASET ($DOCS_LABEL docs)" else DATASET_LABEL="$DATASET" fi # Build the output array using jq RESULT=$(jq --arg dataset "$DATASET_LABEL" '[ # Index build time (if .metrics.index_build_time_ms != null then { name: "\($dataset) - Index Build Time", unit: "ms", value: .metrics.index_build_time_ms } else empty end), # Insert time (for insert-based benchmarks) (if .metrics.insert_time_ms != null then { name: "\($dataset) - Insert Time", unit: "ms", value: .metrics.insert_time_ms } else empty end), # Concurrent insert time (for pgbench benchmarks) (if .metrics.concurrent_insert_time_ms != null then { name: "\($dataset) - Concurrent Insert Time", unit: "ms", value: .metrics.concurrent_insert_time_ms } else empty end), # Latency by token count (p50 values) (if .metrics.latency_by_tokens.bucket_1.p50 != null then { name: "\($dataset) - 1 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_1.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_2.p50 != null then { name: "\($dataset) - 2 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_2.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_3.p50 != null then { name: "\($dataset) - 3 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_3.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_4.p50 != null then { name: "\($dataset) - 4 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_4.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_5.p50 != null then { name: "\($dataset) - 5 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_5.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_6.p50 != null then { name: "\($dataset) - 6 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_6.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_7.p50 != null then { name: "\($dataset) - 7 Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_7.p50 } else empty end), (if .metrics.latency_by_tokens.bucket_8.p50 != null then { name: "\($dataset) - 8+ Token Query (p50)", unit: "ms", value: .metrics.latency_by_tokens.bucket_8.p50 } else empty end), # Throughput (average latency across all queries) (if .metrics.throughput.avg_ms_per_query != null then { name: "\($dataset) - Throughput (avg ms/query)", unit: "ms", value: .metrics.throughput.avg_ms_per_query } else empty end), # Weighted-average latency (p50, MS-MARCO distribution) (if .metrics.weighted_latency.weighted_p50_ms != null then { name: "\($dataset) - Weighted Latency (p50, ms)", unit: "ms", value: .metrics.weighted_latency.weighted_p50_ms } else empty end), # Weighted throughput (avg ms/query, MS-MARCO distribution) (if .metrics.weighted_throughput.avg_ms_per_query != null then { name: "\($dataset) - Weighted Throughput (avg ms/query)", unit: "ms", value: .metrics.weighted_throughput.avg_ms_per_query } else empty end), # Index size (in MB for readability) (if .metrics.index_size_bytes != null then { name: "\($dataset) - Index Size", unit: "MB", value: ((.metrics.index_size_bytes / 1048576 * 100 | floor) / 100) } else empty end), # VACUUM benchmark metrics: partial vs full (if .metrics.vacuum.partial_vacuum_ms != null then { name: "\($dataset) - Partial VACUUM (concentrated delete)", unit: "ms", value: .metrics.vacuum.partial_vacuum_ms } else empty end), (if .metrics.vacuum.full_vacuum_ms != null then { name: "\($dataset) - Full VACUUM (uniform delete)", unit: "ms", value: .metrics.vacuum.full_vacuum_ms } else empty end), (if .metrics.vacuum.update_vacuum_ms != null then { name: "\($dataset) - Full VACUUM (uniform update)", unit: "ms", value: .metrics.vacuum.update_vacuum_ms } else empty end), (if .metrics.vacuum.partial_post_index_bytes != null then { name: "\($dataset) - Index Size After Partial VACUUM", unit: "MB", value: ((.metrics.vacuum.partial_post_index_bytes / 1048576 * 100 | floor) / 100) } else empty end), (if .metrics.vacuum.full_post_index_bytes != null then { name: "\($dataset) - Index Size After Full VACUUM", unit: "MB", value: ((.metrics.vacuum.full_post_index_bytes / 1048576 * 100 | floor) / 100) } else empty end), (if .metrics.vacuum.partial_query_avg_ms != null then { name: "\($dataset) - Query Latency After Partial VACUUM", unit: "ms", value: .metrics.vacuum.partial_query_avg_ms } else empty end), (if .metrics.vacuum.full_query_avg_ms != null then { name: "\($dataset) - Query Latency After Full VACUUM", unit: "ms", value: .metrics.vacuum.full_query_avg_ms } else empty end), (if .metrics.vacuum.update_post_index_bytes != null then { name: "\($dataset) - Index Size After Update VACUUM", unit: "MB", value: ((.metrics.vacuum.update_post_index_bytes / 1048576 * 100 | floor) / 100) } else empty end), (if .metrics.vacuum.update_query_avg_ms != null then { name: "\($dataset) - Query Latency After Update VACUUM", unit: "ms", value: .metrics.vacuum.update_query_avg_ms } else empty end) ]' "$INPUT_FILE") # github-action-benchmark errors out on an empty array ("Benchmark output # was '[]'"). When no metrics were parsed -- e.g. an upstream benchmark # step was skipped or produced no output -- do not write an output file at # all. The publish steps gate on hashFiles('..._action.json') != '', so a # missing file is skipped cleanly instead of failing the job. if [ "$(printf '%s' "$RESULT" | jq 'length')" -gt 0 ]; then printf '%s\n' "$RESULT" > "$OUTPUT_FILE" echo "Converted $INPUT_FILE to github-action-benchmark format:" cat "$OUTPUT_FILE" else echo "No metrics parsed from $INPUT_FILE; not writing $OUTPUT_FILE (nothing to publish)" rm -f "$OUTPUT_FILE" fi timescale-pg_textsearch-b63fe58/benchmarks/runner/index_comparison.html000066400000000000000000000241131525762047700266370ustar00rootroot00000000000000 pg_textsearch Benchmarks

pg_textsearch Benchmarks

Performance tracking for pg_textsearch. Click any data point to view the commit.
● main branch ✕ feature branch
CPU Profiles & Flamegraphs

timescale-pg_textsearch-b63fe58/benchmarks/runner/run_benchmark.sh000077500000000000000000000276471525762047700256040ustar00rootroot00000000000000#!/bin/bash # pg_textsearch Benchmark Runner # Runs benchmarks and generates a markdown report # # Usage: # ./run_benchmark.sh [dataset] [options] # # Datasets: # msmarco - MS MARCO Passage Ranking (8.8M passages) # msmarco-v2 - MS MARCO v2 Passage Ranking (138M passages) # wikipedia - Wikipedia articles # cranfield - Cranfield collection (1,400 docs) - for quick validation # all - Run all benchmarks # # Options: # --download - Download dataset if not present # --load - Load data and create index (drops existing) # --query - Run query benchmarks only # --vacuum - Run VACUUM benchmark (destructive; run after --query) # --skip-validate - Skip validation (not recommended) # --report - Generate markdown report # --port PORT - Postgres port (default: 5433 for release build) # # Validation: # Validation runs automatically after queries if ground truth exists. # Uses version-specific ground truth (ground_truth_pg17.tsv, etc.) # falling back to ground_truth.tsv if no versioned file is found. # Generate ground truth with: psql -f /precompute_ground_truth.sql set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" BENCHMARK_DIR="$(dirname "$SCRIPT_DIR")" RESULTS_DIR="$BENCHMARK_DIR/results" # Defaults - use release build for benchmarks PGPORT="${PGPORT:-5433}" PGHOST="${PGHOST:-localhost}" PGUSER="${PGUSER:-$(whoami)}" PGDATABASE="${PGDATABASE:-postgres}" DATASET="" DO_DOWNLOAD=false DO_LOAD=false DO_QUERY=false DO_VACUUM=false SKIP_VALIDATE=false DO_REPORT=false VALIDATION_FAILED=false # Parse arguments while [[ $# -gt 0 ]]; do case $1 in msmarco|msmarco-v2|wikipedia|cranfield|all) DATASET="$1" shift ;; --download) DO_DOWNLOAD=true shift ;; --load) DO_LOAD=true shift ;; --query) DO_QUERY=true shift ;; --vacuum) DO_VACUUM=true shift ;; --skip-validate) SKIP_VALIDATE=true shift ;; --report) DO_REPORT=true shift ;; --port) PGPORT="$2" shift 2 ;; -h|--help) echo "Usage: $0 [dataset] [options]" echo "" echo "Datasets:" echo " msmarco - MS MARCO Passage Ranking (8.8M passages)" echo " msmarco-v2 - MS MARCO v2 Passage Ranking (138M passages)" echo " wikipedia - Wikipedia articles" echo " cranfield - Cranfield collection (1,400 docs)" echo " all - Run all benchmarks" echo "" echo "Options:" echo " --download - Download dataset if not present" echo " --load - Load data and create index" echo " --query - Run query benchmarks" echo " --vacuum - Run VACUUM benchmark (destructive)" echo " --skip-validate - Skip validation (not recommended)" echo " --report - Generate markdown report" echo " --port PORT - Postgres port (default: 5433)" echo "" echo "Validation runs automatically after queries. Fails if scores don't match to 4 decimal places." exit 0 ;; *) echo "Unknown option: $1" exit 1 ;; esac done # Default: run everything if no specific action requested if ! $DO_DOWNLOAD && ! $DO_LOAD && ! $DO_QUERY && ! $DO_VACUUM && ! $DO_REPORT; then DO_DOWNLOAD=true DO_LOAD=true DO_QUERY=true DO_REPORT=true fi if [ -z "$DATASET" ]; then echo "Error: No dataset specified" echo "Usage: $0 [msmarco|wikipedia|cranfield|all] [options]" exit 1 fi export PGPORT PGHOST PGUSER PGDATABASE mkdir -p "$RESULTS_DIR" TIMESTAMP=$(date +%Y%m%d_%H%M%S) REPORT_FILE="$RESULTS_DIR/benchmark_${DATASET}_${TIMESTAMP}.md" echo "=== pg_textsearch Benchmark Runner ===" echo "Dataset: $DATASET" echo "Postgres: $PGHOST:$PGPORT/$PGDATABASE" echo "Timestamp: $TIMESTAMP" echo "" # Function to run a single dataset benchmark run_benchmark() { local dataset="$1" local dataset_dir="$BENCHMARK_DIR/datasets/$dataset" local data_dir="$dataset_dir/data" echo "" echo "==========================================" echo "Running benchmark: $dataset" echo "==========================================" # Download if $DO_DOWNLOAD; then if [ -f "$dataset_dir/download.sh" ]; then echo "" echo "--- Downloading $dataset dataset ---" chmod +x "$dataset_dir/download.sh" (cd "$dataset_dir" && ./download.sh) fi fi # Load if $DO_LOAD; then if [ -f "$dataset_dir/load.sql" ]; then echo "" echo "--- Loading $dataset dataset ---" local load_log="$RESULTS_DIR/${dataset}_load_${TIMESTAMP}.log" # Capture timing local start_time=$(date +%s.%N) # Export DATA_DIR for load.sql scripts that use it export DATA_DIR="$data_dir" psql -f "$dataset_dir/load.sql" 2>&1 | tee "$load_log" local end_time=$(date +%s.%N) local load_time=$(echo "$end_time - $start_time" | bc) echo "" echo "Load time: ${load_time}s" echo "LOAD_TIME_${dataset}=${load_time}" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" fi fi # Query benchmarks if $DO_QUERY; then if [ -f "$dataset_dir/queries.sql" ]; then echo "" echo "--- Running $dataset query benchmarks ---" local query_log="$RESULTS_DIR/${dataset}_queries_${TIMESTAMP}.log" local start_time=$(date +%s.%N) psql -f "$dataset_dir/queries.sql" 2>&1 | tee "$query_log" local end_time=$(date +%s.%N) local query_time=$(echo "$end_time - $start_time" | bc) echo "" echo "Query benchmark time: ${query_time}s" echo "QUERY_TIME_${dataset}=${query_time}" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" fi fi # Validation (runs automatically after queries unless --skip-validate) if $DO_QUERY && ! $SKIP_VALIDATE; then local ground_truth_file="$dataset_dir/ground_truth.tsv" local validate_script="$dataset_dir/validate_queries.sql" # Select version-specific ground truth if available if [ -f "$validate_script" ]; then local pg_major pg_major=$(pg_config --version | sed 's/PostgreSQL //' \ | cut -d. -f1) local versioned_gt="$dataset_dir/ground_truth_pg${pg_major}.tsv" if [ -f "$versioned_gt" ]; then echo "Using ground truth for PG${pg_major}" cp "$versioned_gt" "$ground_truth_file" fi if [ -f "$ground_truth_file" ]; then echo "" echo "--- Validating $dataset query results ---" local validate_log="$RESULTS_DIR/${dataset}_validation_${TIMESTAMP}.log" psql -f "$validate_script" 2>&1 | tee "$validate_log" # Check for validation failures (scores must match to 4 decimal places) if grep -q "VALIDATION FAILED" "$validate_log"; then echo "" echo "ERROR: Validation FAILED for $dataset!" echo "Scores don't match to 4 decimal places - indicates BM25 scoring bugs." echo "VALIDATION_${dataset}=FAILED" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" VALIDATION_FAILED=true else echo "" echo "Validation PASSED for $dataset" echo "VALIDATION_${dataset}=PASSED" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" fi else echo "" echo "--- Skipping validation: ground_truth.tsv not found ---" echo "Generate ground truth with: psql -f $dataset_dir/precompute_ground_truth.sql" echo "VALIDATION_${dataset}=SKIPPED" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" fi else echo "" echo "--- Skipping validation: validate_queries.sql not found ---" echo "VALIDATION_${dataset}=SKIPPED" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" fi fi # VACUUM benchmark (runs after validation; destructive to data) if $DO_VACUUM; then if [ -f "$dataset_dir/vacuum.sql" ]; then echo "" echo "--- Running $dataset VACUUM benchmark ---" local vacuum_log="$RESULTS_DIR/${dataset}_vacuum_${TIMESTAMP}.log" local start_time=$(date +%s.%N) psql -f "$dataset_dir/vacuum.sql" 2>&1 | tee "$vacuum_log" local end_time=$(date +%s.%N) local vacuum_time=$(echo "$end_time - $start_time" | bc) echo "" echo "VACUUM benchmark time: ${vacuum_time}s" echo "VACUUM_TIME_${dataset}=${vacuum_time}" >> "$RESULTS_DIR/metrics_${TIMESTAMP}.env" else echo "" echo "--- No vacuum.sql found for $dataset, skipping ---" fi fi } # Function to generate report generate_report() { echo "" echo "--- Generating Report ---" cat > "$REPORT_FILE" << EOF # pg_textsearch Benchmark Report **Date:** $(date -u +"%Y-%m-%d %H:%M:%S UTC") **Dataset:** $DATASET **Postgres:** $PGHOST:$PGPORT ## System Information - **OS:** $(uname -s) $(uname -r) - **CPU:** $(sysctl -n machdep.cpu.brand_string 2>/dev/null || grep "model name" /proc/cpuinfo 2>/dev/null | head -1 | cut -d: -f2 || echo "Unknown") - **Memory:** $(sysctl -n hw.memsize 2>/dev/null | awk '{print $1/1024/1024/1024 " GB"}' || free -h 2>/dev/null | awk '/^Mem:/{print $2}' || echo "Unknown") ## Results Summary EOF # Add metrics from env file if it exists local metrics_file="$RESULTS_DIR/metrics_${TIMESTAMP}.env" if [ -f "$metrics_file" ]; then echo "### Timing Results" >> "$REPORT_FILE" echo "" >> "$REPORT_FILE" echo "| Dataset | Load Time | Query Time | Validation |" >> "$REPORT_FILE" echo "|---------|-----------|------------|------------|" >> "$REPORT_FILE" source "$metrics_file" for ds in msmarco msmarco-v2 wikipedia cranfield; do load_var="LOAD_TIME_${ds}" query_var="QUERY_TIME_${ds}" validation_var="VALIDATION_${ds}" if [ -n "${!load_var}" ] || [ -n "${!query_var}" ] || [ -n "${!validation_var}" ]; then echo "| $ds | ${!load_var:-N/A}s | ${!query_var:-N/A}s | ${!validation_var:-N/A} |" >> "$REPORT_FILE" fi done echo "" >> "$REPORT_FILE" fi # Add log excerpts echo "## Detailed Logs" >> "$REPORT_FILE" echo "" >> "$REPORT_FILE" for log_file in "$RESULTS_DIR"/*_${TIMESTAMP}.log; do if [ -f "$log_file" ]; then local log_name=$(basename "$log_file" .log) echo "### $log_name" >> "$REPORT_FILE" echo "" >> "$REPORT_FILE" echo '```' >> "$REPORT_FILE" # Include last 100 lines of each log tail -100 "$log_file" >> "$REPORT_FILE" echo '```' >> "$REPORT_FILE" echo "" >> "$REPORT_FILE" fi done echo "" echo "Report generated: $REPORT_FILE" } # Run benchmarks if [ "$DATASET" = "all" ]; then for ds in cranfield msmarco msmarco-v2 wikipedia; do run_benchmark "$ds" done else run_benchmark "$DATASET" fi # Generate report if $DO_REPORT; then generate_report fi echo "" echo "=== Benchmark Complete ===" echo "Results in: $RESULTS_DIR" # Exit with failure if any validation failed if $VALIDATION_FAILED; then echo "" echo "ERROR: One or more validations FAILED!" echo "This indicates BM25 scoring bugs that need investigation." exit 1 fi timescale-pg_textsearch-b63fe58/benchmarks/sql/000077500000000000000000000000001525762047700216755ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/000077500000000000000000000000001525762047700236245ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/01-load.sql000066400000000000000000000050351525762047700255050ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark - Data Loading -- This benchmark loads the complete Cranfield collection and creates BM25 index -- Timing: Data loading and index creation are timed but results not validated -- Contains 1400 aerodynamics abstracts with 225 queries \set ON_ERROR_STOP on \timing on \echo 'Cranfield BM25 Benchmark - Loading Phase' \echo '======================================' -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; -- Create tables for Cranfield dataset \echo 'Creating tables...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Load the complete Cranfield dataset \echo 'Loading complete Cranfield collection (1400 documents, 225 queries)...' \i dataset.sql -- Create BM25 index for Cranfield documents \echo 'Building BM25 index (this may take time)...' CREATE INDEX cranfield_full_tapir_idx ON cranfield_full_documents USING bm25(full_text) WITH (text_config='english', k1=1.2, b=0.75); -- Verify data loading \echo 'Data loading verification:' SELECT 'Documents loaded:' as metric, COUNT(*) as count FROM cranfield_full_documents UNION ALL SELECT 'Queries loaded:' as metric, COUNT(*) as count FROM cranfield_full_queries UNION ALL SELECT 'Expected rankings:' as metric, COUNT(*) as count FROM cranfield_full_expected_rankings ORDER BY metric; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('cranfield_full_tapir_idx')) as index_size, pg_relation_size('cranfield_full_tapir_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('cranfield_full_documents')) as table_size, pg_total_relation_size('cranfield_full_documents') as table_bytes; \echo 'Load phase completed. Index ready for queries.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/02-queries.sql000066400000000000000000000146541525762047700262530ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark - Query Performance -- Tests BM25 index scan performance using the standard Cranfield IR collection -- Outputs structured timing data for regression detection \set ON_ERROR_STOP on \timing on -- Soft cap on per-statement runtime so a pathological query fails fast -- instead of hanging the benchmark. See msmarco/queries.sql for context. SET statement_timeout = '5min'; \echo '=== Cranfield BM25 Benchmark - Query Phase ===' \echo '' -- Warm up: run a few queries to ensure index is cached \echo 'Warming up index...' SELECT doc_id FROM cranfield_full_documents ORDER BY full_text <@> to_bm25query('boundary layer', 'cranfield_full_tapir_idx') LIMIT 10; SELECT doc_id FROM cranfield_full_documents ORDER BY full_text <@> to_bm25query('heat transfer', 'cranfield_full_tapir_idx') LIMIT 10; -- ============================================================ -- Benchmark 1: Query Latency (10 iterations each, median reported) -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency (10 iterations each) ===' \echo 'Running top-10 queries using the BM25 index' \echo '' -- Helper function to run a query multiple times and return median execution time CREATE OR REPLACE FUNCTION benchmark_query(query_text text, iterations int DEFAULT 10) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric) AS $$ DECLARE i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; BEGIN times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT doc_id FROM cranfield_full_documents ORDER BY full_text <@> to_bm25query($1, ''cranfield_full_tapir_idx'') LIMIT 10' USING query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Short query (2 words) \echo 'Query 1: Short query (2 words) - "boundary layer"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query('boundary layer'); \echo '' \echo 'Query 2: Medium query (4 words) - "supersonic flow heat transfer"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query('supersonic flow heat transfer'); \echo '' \echo 'Query 3: Long query (from Cranfield query 1)' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query('what similarity laws must be obeyed when constructing aeroelastic models of heated high speed aircraft'); \echo '' \echo 'Query 4: Common terms - "flow pressure"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query('flow pressure'); \echo '' \echo 'Query 5: Specific terms - "magnetohydrodynamic viscosity"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query('magnetohydrodynamic viscosity'); DROP FUNCTION benchmark_query; -- ============================================================ -- Benchmark 2: Query Throughput (all 225 Cranfield queries) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (225 Cranfield queries) ===' \echo 'Running all standard Cranfield queries sequentially' DO $$ DECLARE q RECORD; start_time timestamp; end_time timestamp; total_ms numeric := 0; query_count int := 0; BEGIN start_time := clock_timestamp(); FOR q IN SELECT query_id, query_text FROM cranfield_full_queries ORDER BY query_id LOOP PERFORM doc_id FROM cranfield_full_documents ORDER BY full_text <@> to_bm25query(q.query_text, 'cranfield_full_tapir_idx') LIMIT 10; query_count := query_count + 1; END LOOP; end_time := clock_timestamp(); total_ms := EXTRACT(EPOCH FROM (end_time - start_time)) * 1000; RAISE NOTICE 'THROUGHPUT_RESULT: % queries in % ms (avg % ms/query)', query_count, round(total_ms::numeric, 2), round((total_ms / query_count)::numeric, 2); END $$; -- ============================================================ -- Benchmark 3: Index Statistics -- ============================================================ \echo '' \echo '=== Benchmark 3: Index Statistics ===' SELECT 'cranfield_full_tapir_idx' as index_name, pg_size_pretty(pg_relation_size('cranfield_full_tapir_idx')) as index_size, pg_size_pretty(pg_relation_size('cranfield_full_documents')) as table_size, (SELECT COUNT(*) FROM cranfield_full_documents) as num_documents; -- ============================================================ -- Benchmark 4: Search Quality Validation (Precision@10) -- ============================================================ \echo '' \echo '=== Benchmark 4: Search Quality Validation ===' \echo 'Checking precision@10 against Cranfield relevance judgments' WITH query_results AS ( SELECT q.query_id, d.doc_id, ROW_NUMBER() OVER ( PARTITION BY q.query_id ORDER BY d.full_text <@> to_bm25query(q.query_text, 'cranfield_full_tapir_idx') ) as rank FROM cranfield_full_queries q CROSS JOIN LATERAL ( SELECT doc_id, full_text FROM cranfield_full_documents ORDER BY full_text <@> to_bm25query(q.query_text, 'cranfield_full_tapir_idx') LIMIT 10 ) d WHERE q.query_id <= 10 -- Sample first 10 queries for validation ), precision_calc AS ( SELECT qr.query_id, COUNT(CASE WHEN er.doc_id IS NOT NULL THEN 1 END)::float / 10.0 as precision_at_10 FROM query_results qr LEFT JOIN cranfield_full_expected_rankings er ON qr.query_id = er.query_id AND qr.doc_id = er.doc_id WHERE qr.rank <= 10 GROUP BY qr.query_id ) SELECT 'Mean Precision@10 (queries 1-10)' as metric, round(avg(precision_at_10)::numeric, 4) as value FROM precision_calc; \echo '' \echo '=== Cranfield BM25 Benchmark Complete ===' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/dataset.sql000066400000000000000000064552001525762047700260060ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark Dataset - Full Collection -- This test validates BM25 implementation using the complete Cranfield collection -- Contains 1400 aerodynamics abstracts with 225 queries -- Insert all Cranfield documents INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1, 'experimental investigation of the aerodynamics of a wing in a slipstream .', 'brenckman,m.', 'j. ae. scs. 25, 1958, 324.', 'experimental investigation of the aerodynamics of a wing in a slipstream . an experimental study of a wing in a propeller slipstream was made in order to determine the spanwise distribution of the lift increase due to slipstream at different angles of attack of the wing and at different free stream to slipstream velocity ratios . the results were intended in part as an evaluation basis for different theoretical treatments of this problem . the comparative span loading curves, together with supporting evidence, showed that a substantial part of the lift increment produced by the slipstream was due to a /destalling/ or boundary-layer-control effect . the integrated remaining lift increment, after subtracting this destalling lift, was found to agree well with a potential flow theory . an empirical evaluation of the destalling effects was made for the specific configuration of the experiment .'), (2, 'simple shear flow past a flat plate in an incompressible fluid of small viscosity .', 'ting-yili', 'department of aeronautical engineering, rensselaer polytechnic institute troy, n.y.', 'simple shear flow past a flat plate in an incompressible fluid of small viscosity . in the study of high-speed viscous flow past a two-dimensional body it is usually necessary to consider a curved shock wave emitting from the nose or leading edge of the body . consequently, there exists an inviscid rotational flow region between the shock wave and the boundary layer . such a situation arises, for instance, in the study of the hypersonic viscous flow past a flat plate . the situation is somewhat different from prandtl''s classical boundary-layer problem . in prandtl''s original problem the inviscid free stream outside the boundary layer is irrotational while in a hypersonic boundary-layer problem the inviscid free stream must be considered as rotational . the possible effects of vorticity have been recently discussed by ferri and libby . in the present paper, the simple shear flow past a flat plate in a fluid of small viscosity is investigated . it can be shown that this problem can again be treated by the boundary-layer approximation, the only novel feature being that the free stream has a constant vorticity . the discussion here is restricted to two-dimensional incompressible steady flow .'), (3, 'the boundary layer in simple shear flow past a flat plate .', 'm. b. glauert', 'department of mathematics, university of manchester, manchester, england', 'the boundary layer in simple shear flow past a flat plate . the boundary-layer equations are presented for steady incompressible flow with no pressure gradient .'), (4, 'approximate solutions of the incompressible laminar boundary layer equations for a plate in shear flow .', 'yen,k.t.', 'j. ae. scs. 22, 1955, 728.', 'approximate solutions of the incompressible laminar boundary layer equations for a plate in shear flow . the two-dimensional steady boundary-layer problem for a flat plate in a shear flow of incompressible fluid is considered . solutions for the boundary- layer thickness, skin friction, and the velocity distribution in the boundary layer are obtained by the karman-pohlhausen technique . comparison with the boundary layer of a uniform flow has also been made to show the effect of vorticity .'), (5, 'one-dimensional transient heat conduction into a double-layer slab subjected to a linear heat input for a small time internal .', 'wasserman,b.', 'j. ae. scs. 24, 1957, 924.', 'one-dimensional transient heat conduction into a double-layer slab subjected to a linear heat input for a small time internal . analytic solutions are presented for the transient heat conduction in composite slabs exposed at one surface to a triangular heat rate . this type of heating rate may occur, for example, during aerodynamic heating .'), (6, 'one-dimensional transient heat flow in a multilayer slab .', 'campbell,w.f.', 'j. ae. scs. 25, 1958, 340.', 'one-dimensional transient heat flow in a multilayer slab . in a recent contribution to the readers'' forum wassermann gave analytic solutions for the temperature in a double layer slab, with a triangular heat rate input at one face, insulated at the other, and with no thermal resistance at the interface . his solutions were for the three particular cases.. i propose here to give the general solution to this problem, to indicate briefly how it is obtained using the method of reference 2, and to point out that the solutions given by wassermann are incomplete for times longer than the duration of the heat input .'), (7, 'the effect of controlled three-dimensional roughness on boundary layer transition at supersonic speeds .', 'van driest,e.r. and mccauley,w.d.', 'j. ae. scs. 27, 1960, 261.', 'the effect of controlled three-dimensional roughness on boundary layer transition at supersonic speeds . experiments were performed in the 12-in. supersonic wind tunnel of the jet propulsion laboratory of the california institute of technology to investigate the effect of three-dimensional roughness elements (spheres) on boundary-layer transition on a tained at local mach numbers of 1.90, 2.71, and 3.67 by varying trip size, position, spacing, and reynolds number per inch . the results indicate that (1) transition from laminar to turbulent flow induced by three-dimensional roughness elements begins when the double row of spiral vortices trailing each element contaminates and breaks down the surrounding field of vorticity, (2) transition appears rather suddenly, becoming more violent with increasing roughness height relative to the boundary-layer thickness, (3) after the breakdown of the vorticity field, the strength of the spiral vortices may still persist in the sublayer of the ensuing turbulent flow, (4) lateral spacing of roughness elements has little effect upon the initial breakdown (contamination) of the laminar flow, and (5) the trip reynolds number where u and v are the velocity and kinematic viscosity at the outer edge of the boundary layer and k is roughness height, such that transition occurs at the roughness position, varies as the position reynolds number to the one-fourth power, viz., where x is trip position .'), (8, 'measurements of the effect of two-dimensional and three-dimensional roughness elements on boundary layer transition .', 'klebanoff,p.s.', 'j. ae. scs. 22, 1955, 803.', 'measurements of the effect of two-dimensional and three-dimensional roughness elements on boundary layer transition . in his study of the effect of roughness on transition, h. l. dryden found, on the basis of available data, that the effect of a two-dimensional roughness element such as a /trip wire/ could be represented reasonably well in terms of a functional relation between and, where is the reynolds number of transition based on distance from the leading edge, is the height of the roughness element, and is the boundary-layer displacement thickness at the position of the element . at his suggestion some additional data were obtained, primarily to extend the range to higher values of, during the course of an investigation of transition on a flat plate conducted at the national bureau of standards . after the results on the two- dimensional roughness elements were obtained, it appeared to be desirable to see whether a row of three-dimensional roughness elements would behave in the same way .'), (9, 'transition studies and skin friction measurements on an insulated flat plate at a mach number of 5.8 .', 'korkegi,r.h.', 'j. ae. scs. 23, 1956, 97.', 'transition studies and skin friction measurements on an insulated flat plate at a mach number of 5.8 . an investigation of transition and skin friction on an insulated flat plate, 5 by 26 in., was made in the galcit 5 by 5 in. hypersonic wind tunnel at a nominal mach number of 5.8 . the phosphorescent lacquer technique was used for transition detection and was found to be in good agreement with total-head rake measurements along the plate surface and pitot boundary- layer surveys . it was found that the boundary layer was laminar at reynolds numbers of at least 5 x 10 . transverse contamination caused by the turbulent boundary layer on the tunnel sidewall originated far downstream of the flat plate leading edge at reynolds numbers of 1.5 to 2 x 10, and spread at a uniform angle of 5 compared to 9 degree in low-speed flow . the effect of two-dimensional and local disturbances was investigated . the technique of air injection into the boundary layer as a means of hastening transition was extensively used . although the onset of transition occurred at reynolds numbers as low as 10, a fully developed turbulent boundary layer was not obtained at reynolds numbers much below 2 x 10 regardless of the amount of air injected . a qualitative discussion of these results is given with emphasis on the possibility of a greater stability of the laminar boundary layer in hypersonic flow than at lower speeds . direct skin-friction measurements were made by means of the floating element technique, over a range of reynolds numbers verified as being laminar over the complete range . with air injection, turbulent shear was obtained only for reynolds numbers greater than 2 x 10, this value being in good agreement with earlier results of this investigation . the turbulent skin-friction coefficient was found to be approximately 0.40 of that for incompressible flow for a constant value of r, and 0.46 for an effective reynolds number between 5 and 6 x 10 .'), (10, 'the theory of the impact tube at low pressure .', 'chambre,p.l. and schaaf,s.a.', 'j. ae. scs. 15, 1948, 735.', 'the theory of the impact tube at low pressure . a theoretical analysis has been made for an impact tube of the relation between free-stream mach number and the impact and free-stream pressures and densities for extremely low pressures . it is shown that the results differ appreciably from the corresponding continuum relations .'), (11, 'similar solutions in compressible laminar free mixing problems .', 'napolitano,l.', 'j. ae. scs. 23, 1956, 389.', 'similar solutions in compressible laminar free mixing problems . there are in supersonic aerodynamics many situations of practical interest wherein streams of different velocities and, in general, different stagnation pressures mix with one another . in the majority of these problems the interaction between the two streams takes place in the presence of an axial pressure gradient . its effect on the characteristics of the mixing may influence significantly the performances of the devices wherein the phenomena cited above occur . a theoretical and experimental program of research to study mixing in the presence of axial pressure gradients is being carried on at the polytechnic institute of brooklyn .'), (12, 'some structural and aerelastic considerations of high speed flight .', 'bisplinghoff,r.l.', 'j. ae. scs. 23, 1956, 289.', 'some structural and aerelastic considerations of high speed flight . the dominating factors in structural design of high-speed aircraft are thermal and aeroelastic in origin . the subject matter is concerned largely with a discussion of these factors and their interrelation with one another . a summary is presented of some of the analytical and experimental tools available to aeronautical engineers to meet the demands of high-speed flight upon aircraft structures . the state of the art with respect to heat transfer from the boundary layer into the structure, modes of failure under combined load as well as thermal inputs and acrothermoelasticity is discussed . methods of attacking and alleviating structural and aeroelastic problems of high-speed flight are summarized . finally, some avenues of fundamental research are suggested .'), (13, 'similarity laws for stressing heated wings .', 'tsien,h.s.', 'j. ae. scs. 20, 1953, 1.', 'similarity laws for stressing heated wings . it will be shown that the differential equations for a heated plate with large temperature gradient and for a similar plate at constant temperature can be made the same by a proper modification of the thickness and the loading for the isothermal plate . this fact leads to the result that the stresses in the heated plate can be calculated from measured strains on the unheated plate by a series of relations, called the /similarity laws ./ the application of this analog theory to solid wings under aerodynamic heating is discussed in detail . the loading on the unheated analog wing is, however, complicated and involves the novel concept of feedback and /body force/ loading . the problem of stressing a heated box-wing structure can be solved by the same analog method and is briefly discussed .'), (14, 'piston theory - a new aerodynamic tool for the aeroelastician .', 'ashley,h. and zartarian,g.', 'j. ae. scs. 23, 1956, 1109.', 'piston theory - a new aerodynamic tool for the aeroelastician . representative applications are described which illustrate the extent to which simplifications in the solutions of high-speed unsteady aeroelastic problems can be achieved through the use of certain aerodynamic techniques known collectively as /piston theory ./ based on a physical model originally proposed by hayes and lighthill, piston theory for airfoils and finite wings has been systematically developed by landahl, utilizing expansions in powers of the thickness ratio and the inverse of the flight mach number m . when contributions of orders and are negligible, the theory predicts a point-function relationship between the local pressure on the surface of a wing and the normal component of fluid velocity produced by the wing''s motion . the computation of generalized forces in aeroelastic equations, such as the flutter determinant, is then always reduced to elementary integrations of the assumed modes of motion . essentially closed-form solutions are given for the bending- torsion and control-surface flutter properties of typical section airfoils at high mach numbers . these agree well with results of more exact theories wherever comparisons can be fairly made . moreover, they demonstrate the increasingly important influence of thickness and profile shape as m grows larger, a discovery that would be almost impossible using other available aerodynamic tools . the complexity of more practical flutter analyses-e.g., on three-dimensional wings and panels-is shown to be substantially reduced by piston theory . an iterative procedure is outlined, by which improved flutter eigenvalues can be found through the successive introduction of higher-order terms in and . other applications to unsteady supersonic problems are reviewed, including gust response and rapid maneuvers of elastic aircraft . steady-state aeroelastic calculations are also discussed, but for them piston theory amounts only to a slight modification of ackeret''s formulas . suggestions are made regarding future research based on the new aerodynamic method, with particular emphasis on areas where computational labor can be reduced with a minimum loss of precision . it is pointed out that a mach number zone exists where thermal effects are appreciable but nonlinear viscous interactions may be neglected, and that in this zone piston theory is the logical way of estimating air loads when analyzing aerodynamic- thermoelastic interaction problems .'), (15, 'on two-dimensional panel flutter .', 'fung,y.c.', 'j. ae. scs. 25, 1958, 145.', 'on two-dimensional panel flutter . theory and experiments of the flutter of a buckled plate are discussed . it is shown that an increase in the initial deviation from flatness or a static pressure differential across the plate raises the critical value of the /reduced velocity ./ the applicability of the galerkin method to the linearized problem of flutter of an unbuckled plate has been questioned by several authors . in this paper the flutter condition was formulated in the form of an integral equation and solved numerically by the method of iteration and the method of matrix approximations, thus avoiding the constraint of assumed modes . for a plate (with finite bending rigidity) the results confirm those given by the galerkin method . an approximate analysis of the limiting form and amplitude of the flutter motion for a buckled plate is presented .'), (16, 'transformation of the compressible turbulent boundary layer .', 'mager,a.', 'j. ae. scs. 25, 1958, 305.', 'transformation of the compressible turbulent boundary layer . the transformation of the compressible turbulent boundary- layer equations to their incompressible equivalent is demonstrated analytically . the transformation is essentially the same as that for the laminar layer, first given by stewartson, except that the explicit relation between the viscosity and temperature is not required . a key point in the analysis is the modification of the stream function to include a mean of the fluctuating components and the postulate that the apparent turbulent shear, associated with an elemental mass, remains invariant in the transformation . the values of the incompressible friction coefficients and of pressure rise causing separation thus transformed show good agreement with the experimentally measured and independently reported results . an application of the transformation to the self-preserving boundary layers and to the computations of general boundary-layer flow is shown .'), (17, 'remarks on the eddy viscosity in compressible mixing flows .', 'lu ting and paul a. libby', 'polytechnic institute of brooklyn, and general applied science laboratories, inc.', 'remarks on the eddy viscosity in compressible mixing flows . in connection with a study of the wakes behind bodies in hypersonic flow carried out for the missile and space vehicle division of the general electric company, it was desired to estimate the eddy viscosity in axisymmetric, compressible wakes . because of the lack of applicable experimental data, it was found necessary to make such an estimate by rationally extending the few available data for incompressible flows to the compressible case . this suggested the application and extension of the transformations applied to turbulent boundary layers in reference infinitesimal mass are invariant with transformation, mager showed that the partial differential equations for the compressible turbulent boundary layer can be transformed to incompressible form . the validity of this assumption and of the transformations was established for several boundary-layer flows by comparison with experiment .'), (18, 'the flow field in the diffuser of a radial compressor .', 'rhyming,i.l.', 'j. ae. scs. 27, 1960, 798.', 'the flow field in the diffuser of a radial compressor . this note discusses the two-dimensional diffuser flow field in a radial compressor outside the impeller wheel . it is assumed that the diffuser has guide vanes arranged in a circular row at a radius . the impeller wheel has the radius (see fig. 1) . the flow in the diffuser starts at the circle with the radius . the velocity components, and in the r and directions of the velocity vector on this circle are prescribed together with the thermal state of the gas . the flow so prescribed on the radius will, if no disturbances are present (i.e., no boundary conditions in the flow other than zero velocity at infinity are to be fulfilled), develop in a spiral flow .'), (19, 'an investigation of the pressure distribution on conical bodies in hypersonic flows .', 'victor zakkay', 'aerodynamics laboratory, polytechnic institute of brooklyn, freeport, n.y.', 'an investigation of the pressure distribution on conical bodies in hypersonic flows . a large amount of work on conical flow fields without axial symmetry at supersonic speed is presently available . however, no apparent hypersonic approximation has yet been derived . in this note, experimental data on two elliptical cones at m = 6 are presented and a hypersonic approach obtained from physical considerations is suggested .'), (20, 'generalised-newtonian theory .', 'love,e.s.', 'j. ae. scs. 26, 1959, 314.', 'generalised-newtonian theory . author generalizes lees''s (amr 10(1957), rev. 2601) modification of newtonian theory for blunt-nose bodies to apply to pointed- nose bodies as well . the result is expressed by sin where is the local inclination of the body surface and the subscript /max/ refers to the maximum local inclination and pressure coefficient . for blunt-nose bodies and the generalized theory reverts to lees''s blunt-nose modification with given by normal shock relations . author shows, by comparison of newtonian and generalized-newtonian theory with exact solutions, the superiority of generalized-newtonian theory . he also shows that both two-dimensional and axisymmetric shapes are correlated by this generalization . results are presented in two figures that support author''s generalization and indicate the independence of the correlation from variations in both the hypersonic similarity parameter k = m(d1) and the ratio of specific heats y . reviewer believes this generalization should be of interest to those engaged in development of hypersonic hardware as well as theory .'), (21, 'on heat transfer in slip flow .', 'stephen h. maslen', 'lewis flight propulsion laboratory, naca, cleveland, ohio', 'on heat transfer in slip flow . a number of authors have considered the effect of slip on the heat transfer and skin friction in a laminar boundary layer over a flat plate . reference 1 considers this by a perturbation on the usual laminar boundary-layer analysis while some other studies.dash e.g., reference the impulsive motion of an infinite plate .'), (22, 'on slip-flow heat transfer to a flat plate .', 'oman,r.a. and scheuing,r.a.', 'j. ae. scs. 26, 1959, 126.', 'on slip-flow heat transfer to a flat plate . assuming that continuum flow energy equation in a boundary layer remains valid well into slip region and taking account of the temperature jump in a moving rarefied gas and for influence of large mean free path through appropriate boundary conditions, a solution is found for the temperature gradient in the slip region . then from maslen expression (j. aero. sci. 25, 6, 400-401, june slipping fluid to a flat plate, and behavior confirms results for small values of knudsen number .'), (23, 'skin-friction and heat transfer characteristics of a laminar boundary layer on a cylinder in axial incompressible flow .', 'seban,r.a. and bond,r.', 'j. ae. scs. 18, 1951, 671.', 'skin-friction and heat transfer characteristics of a laminar boundary layer on a cylinder in axial incompressible flow . a solution is given for the case of the laminar boundary layer of an incompressible fluid of constant properties on the exterior of a cylinder with flow parallel to the cylinder axis . this case differs from the blasius solution for flow along a flat plate by considering the effect of the curvature in a plane transverse to the flow direction . the local skin-friction and heat-transfer coefficients for a prandtl number of 0.715 are evaluated and compared to the similar magnitudes for flat plate flow, and the effect of the curvature is shown to be significant in some practical cases . recovery factors are evaluated, and this quantity is found to be insensitive to the effect of curvature of the boundary .'), (24, 'theory of stagnation point heat transfer in dissociated air .', 'fay,j.a. and riddell,f.r.', 'j. ae. scs. 25, 1958, 73.', 'theory of stagnation point heat transfer in dissociated air . the boundary-layer equations are developed in general for the case of very high speed flight where the external flow is in a dissociated state . in particular the effects of diffusion and of atom recombination in the boundary layer are included . it is shown that at the stagnation point the equations can be reduced exactly to a set of nonlinear ordinary differential equations even when the chemical reactions proceed so slowly that the boundary layer is not in thermochemical equilibrium . two methods of numerical solution of these stagnation point equations are presented, one for the equilibrium case and the other for the nonequilibrium case . numerical results are correlated in terms of the parameters entering the numerical formulation so as not to depend critically on the physical assumptions made . for the nonequilibrium boundary layer, both catalytic (to atom recombination) and noncatalytic wall surfaces are considered . a solution is represented which shows the transition from the /frozen/ boundary layer (very slow recombination rates) to the equilibrium boundary layer (fast recombination rates) . a recombination rate parameter is introduced to interpret the nonequilibrium results, and it is shown that a scale factor is involved in relating the equilibrium state of a boundary layer on bodies of different sizes . it is concluded that the heat transfer through the equilibrium stagnation point boundary layer can be computed accurately by a simple correlation formula and that the heat transfer is almost unaffected by a nonequilibrium state of the boundary layer provided the wall is catalytic and the lewis number near unity .'), (25, 'inviscid hypersonic flow over blunt-nosed slender bodies .', 'lees,l. and kubota,t.', 'j. ae. scs. 24, 1957, 195.', 'inviscid hypersonic flow over blunt-nosed slender bodies . at hypersonic speeds the drag area of a blunt nose is much larger than the drag area of a slender afterbody, and the energy contained in the flow field in a plane at right angles to the flight direction is nearly constant over a downstream distance many times greater than the characteristic nose dimension . the transverse flow field exhibits certain similarity properties directly analogous to the flow similarity behind an intense blast wave found by g. i. taylor, s. c. lin, and a. sakurai . a comparison with the experiments of hammitt, vas, and bogdonoff on a flat plate with a blunt leading edge at in helium shows that the shock-wave shape is predicted very accurately by this similarity analysis . the predicted surface pressure distribution is somewhat less satisfactory . experimental results on a hemisphere-cylinder obtained at in the galcit air tunnel indicate that not only the shock-wave shape but also the surface pressures for this body are given very closely by the similarity theory, except near the hemisphere-cylinder junction . energy considerations combined with a detailed study of the equations of motion show that flow similarity is also possible for a class of bodies of the form, provided that, where for a two-dimensional body and for a body of revolution . when the shock shape is not similar to the body shape, and the entire flow field some distance from the nose must depend to some extent on the details of the nose geometry . by again utilizing energy and drag considerations one finds that at hypersonic speeds the inviscid surface pressures generated by a blunt leading edge are larger than the pressures induced by boundary-layer growth on an insulated flat surface for an insulated blunt-nosed slender body of revolution the corresponding distance is given by . (here is free-stream reynolds number based on leading-edge thickness, or nose diameter .) in free flight these constants are replaced by 1,700 and 20, respectively, so that viscous interaction effects are important over the forward portion of a blunt- nosed slender body only for relatively low values of . however, /far downstream/ of the nose the inviscid over-pressure is small and viscous interaction phenomena will have to be taken into account .'), (26, 'inviscid leading-edge effect in hypersonic flow .', 'cheng,h.k. and pallone,a.j.', 'j. ae. scs. 23, 1956, 700.', 'inviscid leading-edge effect in hypersonic flow . current interest in the problem of inviscid-viscous interaction has led to the realization of the significant effect of the leading-edge thickness in hypersonic flow . the purpose of this note is to give an account of the downstream influence of the blunt leading edge on the basis of the hypersonic small perturbation theory .'), (27, 'newtonian flow theory for slender bodies .', 'cole,j.d.', 'j. ae. scs. 24, 1957, 448.', 'newtonian flow theory for slender bodies . as an aid to the aerodynamieist in the design of air frames for hypersonic speeds (speeds faster than about mach 5), newtonian flow theory is examined from the point of view of gas dynamics and hypersonic small-disturbance theory . the usual theory is shown to result as the first approximation of an expansion valid for small . a basic similarity parameter is introduced . a general solution of the first approximation for the flow past slender bodies (bodies which cause only a small disturbance to the stream) at zero angle of attack is given . an important condition which limits the application of the theory is noted-namely, that the pressure coefficient on the surface not fall to zero . the theory is then applied to cones and to bodies whose shape is .'), (28, 'a note on the explosion solution of sedov with application to the newtonian theory of unsteady hypersonic flow .', 'freeman,n.c.', 'j. ae. scs. 27, 1960, 77.', 'a note on the explosion solution of sedov with application to the newtonian theory of unsteady hypersonic flow . an exact analytical solution of the equations of inviscid compressible unsteady flow has been given by sedov (reference to the solution may be made through hayes and probstein) . this solution is the similarity solution for a constant-energy point explosion . in view of the recent work on problems of hypersonic flow in the limiting form of the ratio of specific heats near 1 solution in this limit and inquire what form such a solution would take . einbinder, in a recent note, has examined the solution for various but does not mention the interesting case of . it may be shown that the convergence to the limit is nonuniform over the flow field . it is also not difficult to show that the non- uniform behavior exhibited here is that which one would expect from the newtonian formulation as derived in reference 3 .'), (29, 'a simple model study of transient temperature and thermal stress distribution due to aerodynamic heating .', 'isakson,g.', 'j. ae. scs. 24, 1957, 611.', 'a simple model study of transient temperature and thermal stress distribution due to aerodynamic heating . the present work is concerned with the determination of transient temperatures and thermal stresses in simple models intended to simulate parts or the whole of an aircraft structure of the built- up variety subjected to aerodynamic heating . the first case considered is that of convective heat transfer into one side of a flat plate, representing a thick skin, and the effect of the resulting temperature distribution in inducing thermal stresses associated with bending restraint at the plate edges . numerical results are presented for the transient temperature differentials in the plate when the environment temperature first increases linearly with time and then remains constant, the period of linear increase representing the time of acceleration of the aircraft . corresponding thermal stress information is presented . the second case is that of the wide-flanged i-beam with convective heat transfer into the outer faces of the flanges . numerical results are presented for transient temperature differentials for a wide range of values of the applicable parameters and for an environment temperature variation as described above . corresponding thermal stresses in a beam of infinite length are determined . a theoretical analysis of the stress distribution in a beam of finite length is carried out and numerical results obtained for one case . an experimental investigation of temperatures and stresses in such a beam is described, and results are presented which indicate good agreement with corresponding theoretical results .'), (30, 'photo-thermoelastic investigation of transient thermal stresses in a multiweb wing structure .', 'gerard,g. and tramposch,h.', 'j. ae. scs. 26, 1959, 783.', 'photo-thermoelastic investigation of transient thermal stresses in a multiweb wing structure . photothermoelastic experiments were performed on a long multiweb wing model for which a theoretical analysis is available in the literature . the experimental procedures utilized to simulate the conditions prescribed in the theory are fully described . correlation of theory and experiment in terms of dimensionless temperature, stress, time, and biot number revealed that the theory predicted values higher than the experimentally observed maximum thermal stresses at the center of the web . detailed temperature measurements in the flange suggested that the major source of this discrepancy can be traced to the one-dimensional heat conduction analysis of the flange employed in the theory .'), (31, 'thermal buckling of supersonic wing panels .', 'hoff,n.j.', 'j. ae. scs. 23, 1956, 1019.', 'thermal buckling of supersonic wing panels . the temperature and thermal stress distributions are analyzed in multicellular supersonic wing structures . a buckling criterion is established for the panels of cover plates subjected to thermal stresses .'), (32, 'the dynamic motion of a missile descending through the atmosphere .', 'friedrich,h.r. and dore,f.j.', 'j. ae. scs. 22, 1955, 628.', 'the dynamic motion of a missile descending through the atmosphere . a method is presented for computing rapidly, yet accurately, the dynamic motion of a ballistic-type missile descending through the atmosphere . the equations of motion are separated into a set of /static/ trajectory equations (zero angle of attack) and a set of /rotational/ equations describing the oscillatory motion of the missile about its center of gravity . a transformation allows the rotational equations to be written in a manner analogous to the equation for an undamped oscillating spring mass system with the mass equal to unity and a time variable spring constant . for given initial conditions this equation can be solved to obtain the envelope of maximum angle of attack . an additional transformation allows the calculation of the complete oscillatory motion at any time during the trajectory as a function of the maximum angle of attack at that time . this solution shows that the maximum angle of attack of a missile descending through the atmosphere at relatively constant speed is reduced even when the aerodynamic damping is neglected .'), (33, 'the prospects for magneto-aerodynamics .', 'resler,e.j. and sears,w.r.', 'j. ae. scs. 25, 1958, 235.', 'the prospects for magneto-aerodynamics . the equations describing the flow of an electrically conducting fluid in the presence of electric and magnetic fields are written down with the aid of certain simplifications appropriate to aeronautical applications . in order to estimate the probable significance of magneto-aerodynamic effects, some data on conductivity of pure and /seeded/ air are first examined . dimensionless quantities representing the ratios of forces and of currents are then formed and their values studied for conditions of flight in the atmosphere . some examples of magneto-hydrodynamic and magneto- gasdynamic effects in simple flows are given . these include two cases of poiscuille flow of conducting liquids with applied magnetic fields and the case of quasi-one-dimensional gas flow with applied electrical and magnetic fields . in the last case, attractive possibilities are found for controlled acceleration or deceleration of gas at subsonic and supersonic speeds, even in constant-area channels . the behavior of the flow is characteristically different in different regimes of mach number and flow speed relative to certain /significant speeds/ that are dependent on the ratio of electrical to magnetic field strengths . these are studied, and a chart is constructed to relate the length to the speed ratio of a maximum-acceleration constant-area channel . it is concluded that the advantages that may accrue from magneto-aerodynamic methods are sufficiently attractive to justify the considerable research and engineering development that will be required . among the unsolved engineering problems are the reduction of surface resistance of electrodes in contact with a conducting gas, development of techniques for seeding, and provision of the required magnetic fields in flight .'), (34, 'constant-temperature magneto-gasdynamic channel flow .', 'kerrebrock,j.p. and marble,f.e.', 'j. ae. scs. 27, 1960, 78.', 'constant-temperature magneto-gasdynamic channel flow . in the course of investigating boundary-layer flow in continuous plasma accelerators with crossed electric and magnetic fields, it was found advantageous to have at hand simple closed-form solutions for the magneto-gasdynamic flow in the duct which could serve as free-stream conditions for the boundary layers . nontrivial solutions of this sort are not available at present, and in fact, as in the work of resler and sears, the variation of conditions along the flow axis must be obtained through numerical integration . consequently, some simple solutions of magneto-gasdynamic channel flow were sought, possessing sufficient algebraic simplicity to serve as free-stream boundary conditions for analytic investigations of the boundary layer in a physically reasonable accelerator . in particular, since the cooling of the accelerator tube is likely to be an important physical problem because of the high gas temperatures required to provide sufficient gaseous conductivity, channel flow with constant temperature appears interesting . some simple algebraic solutions for the case of a constant temperature plasma are developed in the following paragraphs .'), (35, 'stagnation point of a blunt body in hypersonic flow .', 'li,t.y. and geiger,r.e.', 'j. ae. scs. 24, 1957, 25.', 'stagnation point of a blunt body in hypersonic flow . the purpose of this paper is to present a method of calculation devised to yield all the important information on the symmetric inviscid hypersonic flow in the stagnation point region of a blunt body . the problem is the same as that considered by hayes who used a slightly different approach . it is demonstrated that hayes'' results are valid in the stagnation point region and can hence be considered a basis for constructing less restricted solutions . equations are presented giving velocity, pressure, detachment distance, and vorticity . the values of shock detachment distance and body pressure coefficient are compared with experimental data for spheres . the pressure comparison shows that the results of hayes and the theory presented herein represent a better approximation than the newtonian impact theory for hypersonic mach numbers . in conclusion, the possibility of refinements to this analysis is discussed .'), (36, 'supersonic flow around blunt bodies .', 'serbin,h.', 'j. ae. scs. 25, 1958, 58.', 'supersonic flow around blunt bodies . the newtonian theory of impact has been shown to be useful for pressure calculations on the forward facing part of bodies moving at high speed . it is now a familiar practice to use this information to calculate nonviscous velocities at the wall and then to estimate rates of heat transfer . this procedure is perhaps open to question,. heat-transfer rates depend on velocity gradients which are not given by the newtonian analysis . nor can one obtain information on boundary-layer stability or all the body stability derivatives . it seems, therefore, inevitable that, as design proceeds with these hypersonic missiles, there will be a greater need for more accurate aerodynamic theories either to predict what will happen in unfamiliar flight conditions or to effect an extrapolation from a known test result to the design condition .'), (37, 'a new technique for investigating heat transfer and surface phenomena under hypersonic flow conditions .', 'ferri,a. and libby,p.a.', 'j. ae. scs. 24, 1957, 464.', 'a new technique for investigating heat transfer and surface phenomena under hypersonic flow conditions . on the forebody of many practically interesting hypersonic vehicles, there is little interaction between the inviscid flow field and the boundary layer . therefore, inviscid flow theory can be used to determine, independent of surface phenomena, the physically interesting quantities such as shock shape, shock detachment distance, sonic line shape, and pressure distribution . furthermore, the pressure distribution so determined can then be used for the study of heat transfer, materials behavior, and other surface phenomena . thus, for these bodies, the prandtl boundary-layer concept can be utilized for the calculation of both the inviscid flow and the boundary-layer behavior . it is the purpose of this note to point out that this concept can also be applied experimentally in order to provide, in conjunction with a conventional hypersonic wind-tunnel air supply, a means for investigating hypersonic heat transfer and surface phenomena under conditions of flight reynolds numbers .'), (38, 'on the prediction of mixed subsonic/supersonic pressure distributions .', 'sinnott,c.s.', 'j. ae. scs. 27, 1960, 767.', 'on the prediction of mixed subsonic/supersonic pressure distributions . high-speed wind-tunnel results are analyzed to derive a semiempirical scheme for the prediction of transonic pressure distributions . the supersonic and subsonic parts of the flow are treated separately, and then linked by an empirical shock pressure rise relation . the significance of the empirical results is considered in relation to the physical mechanism of transonic flows . it is also shown that theoretical solutions can be improved by introducing the empirical shock relation .'), (39, 'on the flow of a sonic stream past an airfoil surface .', 'sinnott,c.s.', 'j.ae.scs. 26, 1959, 169.', 'on the flow of a sonic stream past an airfoil surface . this study of the flow about an airfoil in a near-sonic stream indicates the important factors determining the pressure distribution on the airfoil . analysis of the mach wave pattern suggests that the supersonic domain of the flow can be derived from two simple-wave flows, one arising from the mach waves reflected at the sonic line and the other from the changes in airfoil surface slope . the compressive effect of the reflected mach waves is determined quantitatively as a function of airfoil leading-edge geometry from an analysis of measured pressure distributions for uncambered airfoils,. and it is shown how this can be superimposed on the wave system from the curved surface to give an equivalent simple-wave flow over the airfoil . an application of this scheme to the calculation of the pressure distribution over an airfoil in a sonic stream gives results in good agreement with experiment .'), (40, 'experiments on boundary layer transition at supersonic speeds .', 'van driest,e.r. and boison,j.c.', 'j. ae. scs. 24, 1957, 885.', 'experiments on boundary layer transition at supersonic speeds . tests were conducted in the 12-in. continuous supersonic wind tunnel of the jet propulsion laboratory, california institute of technology, to determine the effects of surface cooling on boundary-layer transition at supersonic speeds . the effects of cooling were investigated at test section mach numbers of 1.97, smooth cone in the presence of three levels of supply-stream turbulence (0.4, 2, and 9 per cent) and several single-element roughnesses at fixed axial location . transition data were obtained optically by means of a magnified-schlieren system . the results, for the range of mach number investigated, indicate that (1) transition on a smooth cone can definitely be delayed by surface cooling, (2) transition promoted by either supply-stream turbulence or surface roughness can also be delayed by surface cooling depending upon degree of turbulence or relative roughness respectively, and (3) the adverse effects of increased turbulence and roughness decrease with increasing mach number .'), (41, 'on transition experiments at moderate supersonic speeds .', 'morkovin,m.v.', 'j. ae. scs. 24, 1957, 480.', 'on transition experiments at moderate supersonic speeds . studies of transition over a flat plate at mach number 1.76 were carried out using a hot-wire anemometer as one of the principal tools . the nature and measurements of free-stream disturbances at supersonic speeds are analyzed . the experimental results are interpreted in the light of present overall information on transition at supersonic speeds and conclusions as to further fruitful experiments are drawn .'), (42, 'the gyroscopic effect of a rigid rotating propeller on engine and wing vibration modes .', 'scanlan,r.h. and truman,j.c.', 'j. ae. scs. 17, 1950, 653.', 'the gyroscopic effect of a rigid rotating propeller on engine and wing vibration modes . in many wing vibration analyses it is found necessary to take into account the effect of flexibly mounted engines . hence, it is reasonable to ask what vibratory gyroscopic effect this flexibility may give rise to when propellers are whirling . an engine mount may be thought of as a horizontal beam cantilevered from the wing, having both horizontal and vertical flexibility . if this beam were infinitely rigid horizontally, then, when it vibrated, the gyroscopic moments induced in the propeller due to the resultant pitching motion of its axis would not produce propeller axis yaw . however, engine-mount lateral stiffness tical stiffness, so that gyroscopic effects will play a role as the propeller axis undergoes pitching vibrations at the tip of the cantilever engine mount . the purpose of this paper is to investigate this role under the assumption that the propeller itself is a rigid disc . the paper is divided into four parts . part (1) deals briefly with classical gyroscope theory . part (2) presents engine vibration mode studies-experimental photographic techniques on a model gyroscope mounted at the ends of two different cantilever beams . part (3) presents the theory of the coupled motion of an elastic wing upon which a gyroscope is mounted to simulate an engine-propeller system on an airplane . part (4) consists of an example of the theory of part (3), in which, by taking what are thought to be reasonable parameters, results are obtained showing how the whirling of a rigid propeller may materially affect wing normal mode shapes and frequencies .'), (43, 'the relation between wall temperature and the effect of roughness on boundary layer transition .', 'potter,j.l. and whitfield,j.d.', 'j. ae. scs. 28, 1961, 663.', 'the relation between wall temperature and the effect of roughness on boundary layer transition . the experimentally demonstrated rise and subsequent fall of transition reynolds number with decreasing wall- to-ambient temperature ratio has been the subject of two recent notes . in both cases it was argued that the increased effectiveness of roughness due to wall cooling was not sufficient to explain the transition-reversal phenomenon on nominally smooth bodies . in one case, the criterion for transition reversal was taken to be and in the other values of as low as eter is a reynolds number formed from velocity and kinematic viscosity based on calculated conditions at the height of roughness element k in the undisturbed, laminar boundary layer at the station of roughness location . the present note is submitted to show that another method for evaluating the effect of roughness on transition leads to an opposite conclusion .'), (44, 'tip-bluntness effects on cone pressures at m=6.85 .', 'bertram,m.h.', 'j.ae.scs. 23, 1956,898.', 'tip-bluntness effects on cone pressures at m=6.85 . there is, at present, considerable interest in the characteristies of blunted bodies from both an aerodynamic and a heat-transfer standpoint . the use of blunt shapes is contemplated to reduce the heat-transfer problem at body noses, but there are also applications for blunt noses which occur from mainly aerodynamic considerations . an actual reduction in drag may be the beneficial result of blunting the nose of a cone or a similar slender shape under certain conditions . although the sphere has received considerable treatment, the nose shapes are not necessarily tangent spheres . in the case, let us say, of a total head tube situated in the nose of a given body, the blunting may be quite flat, and nose sections blunter than spherical shape may conceivably be desirable, in some cases, from the heat- transfer standpoint . the purpose of the present investigation is to examine the aerodynamic effect of a simple type of nose blunting on a basic body . the incompressible flow of an electrically conducting fluid past a porous plate y = 0 with constant suction velocity in the presence of a transverse uniform strength has recently been investigated by gupta . in this note, the problem is generalized to take into account the effect of free convection, when a body force g per unit mass is acting in the negative x-direction parallel to the wall . the fluid is assumed to be semi- incompressible as usual . in addition to the obvious practical significance, this problem is also interesting in the sense that it provides another exact solution of the magnetohydrodynamic equations, since the only electromagnetic assumptions involved are constant properties and freedom from excessive charges .'), (45, 'an investigation of separated flows, part ii: flow in the cavity and heat transfer .', 'charwat,a.f.', 'j. ae. scs. 28, 1961, 513.', 'an investigation of separated flows, part ii: flow in the cavity and heat transfer . the first portion of this paper describes studies of the internal structure of the separated flow in a notch at a free-stream mach number of 3 . observations include.. flow visualization, spark- schlieren pictures of the fluctuations of the free shear layer, and studies of the diffusion of heat from sources placed in the separated region . the second part describes measurements of local heat transfer to the wall . the external mach number, the length-to-depth ratio of the cavity, the ratio of the oncoming boundary layer thickness to the notch depth (in the turbulent flow region), the thermal to-momentum thickness ratio of the boundary layer and, finally, the geometry of the internal boundary of the separated region are varied as systematically as possible . on the basis of these observations, a simple model of the flow in and the heat transfer across the separated region is formulated .'), (46, 'some comments on the inversion of certain large matrices .', 'bertram klein', 'convair, a division of general dynamics corp., san diego, calif.', 'some comments on the inversion of certain large matrices . the subject of matric structural analysis has been treated in two recently published papers in the journal . the authors of these papers have made a number of statements about the inversion of certain large matrices . it is the purpose of this note to bring to the attention of the reader certain facts that shed new light on this important problem . it is shown here that the situation is not as hopeless as the above- mentioned authors intimate .'), (47, 'analysis of low-aspect-ratio aircraft structures .', 'samson,s.h. and bergmann,h.w.', 'j. ae. scs. 27, 1960, 679.', 'analysis of low-aspect-ratio aircraft structures . two methods are presented for the analysis of complex low- aspect-ratio aircraft structures . both methods provide for arbitrary external loading, are general with respect to the orientation of structural members, and permit arbitrary boundary conditions . for purposes of analysis a structure is idealized as a network of flexural members with interconnected torsion boxes . in the first method, sets of linear equations are obtained by expressing boundary conditions, member deflection equations, equilibrium requirements, and slope-compatibility relationships in terms of deflections and internal forces . the solution for deflections and internal forces is then formed as the product of an inverse structural matrix and a column matrix of load functions . in the second method, the conditions at a given boundary are assembled as a column matrix and are transferred in a step by-step fashion over the entire structure to an opposite boundary . the transfer is accomplished by successive multiplications of square matrices composed independently for the different transfer ranges . the final operation is the inversion of a relatively small matrix and provides the solution for the unknown boundary conditions . comparisons of theoretical results with experimental data and electric-analog solutions are favorable .'), (48, 'supersonic flow at the surface of a circular cone at angle of attack .', 'willett,j.e.', 'j. ae. scs. 27, 1960, 907.', 'supersonic flow at the surface of a circular cone at angle of attack . formulas for the inviscid flow properties on the surface of a cone at angle of attack are derived for use in conjunction with the m.i.t. cone tables . these formulas are based upon an entropy distribution on the cone surface which is uniform and equal to that of the shocked fluid in the windward meridian plane . they predict values for the flow variables which may differ significantly from the corresponding values obtained directly from the cone tables . the differences in the magnitudes of the flow variables computed by the two methods tend to increase with increasing free-stream mach number, cone angle and angle of attack .'), (49, 'temperature and velocity profiles in the compressible laminar boundary layer with arbitrary distribution of surface temperature .', 'chapman,d. and rubesin,m.', 'j. ae. scs. 16, 1949, 547.', 'temperature and velocity profiles in the compressible laminar boundary layer with arbitrary distribution of surface temperature . an analysis is presented which enables the temperature profiles, veiocity profiles, heat transfer, and skin friction to be calculated for laminar flow over a two-dimensional or axially symmetric surface without pressure gradient but with an arbitrary analytic distribution of surface temperature . the general theory is applicable to a gas of any prandtl number, although the numerical results given herein have been computed for air . the predictions of the theory for the special case of constant surface temperature are compared with the calculations of crocco . on the basis of this comparison, it is inferred that the present theory enables heat-transfer and skin-friction calculations accurate to within about 5 per cent to be made for flight conditions up to mach numbers near 5 and to within about 1 or 2 per cent for supersonic wind-tunnel conditions up to considerably higher mach numbers . a particular effort has been made to present the results, which are simple considering their generality, in a form that can be used readily in practical applications . from the mathematical point of view, the theory is applicable to an arbitrary analytic distribution of surface temperature, but in any given practical case it is necessary that the surface-temperature distribution be approximated by a polynomial . the only unknowns in the final equations developed are the coefficients of this polynomial, so that the work involved in applying the theory in any given case depends entirely on the work involved in approximating a given surface- temperature distribution by a polynomial . an example is worked out in detail which illustrates some of the principal effects of variable surface temperature . it is shown that both positively infinite and negatively infinite heat-transfer coefficients can occur . the anomaly of infinite and negative heat-transfer coefficients is discussed and attributed to the customary definition of the heat-transfer coefficient, which is shown to be fundamentally inappropriate for flows with variable surface temperature . in the particular example considered, a conventional method for calculating the net heat transferred yields completely incorrect results . a brief qualitative discussion of the possible effects of the heat transfer on flow separation is given . in order to facilitate the use of the results, all of the principal equations developed are collected and summarized in the section entitled /practical use of results ./'), (50, 'investigation of laminar boundary layer in compressible fluids using the crocco method .', 'van driest,e.r.', 'naca tn.2597, 1952.', 'investigation of laminar boundary layer in compressible fluids using the crocco method . in the present investigation of the flow of air in a thin laminar boundary layer on a flat plate, the crocco method has been used to solve the simultaneous differential equations of momentum and energy involved in such flow . the crocco method was used because it gave accurate results for arbitrary prandtl number near unity . the prandtl number was taken at 0.75, the specific heat was held constant, and the sutherland law of viscosity-temperature variation was assumed to represent the viscosity data starting with an initial ambient temperature of -67.6 f . the main results presented here are the skin-friction and heat-transfer coefficients as functions of reynolds number, mach number, and wall-to-free-stream temperature ratio . variations of shear, velocity, temperature, and mach number across the boundary layer are included . the crocco method is discussed in detail .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (51, 'theory of aircraft structural models subjected to aerodynamic heating and external loads .', 'o''sullivan,w.j.', 'naca tn.4115, 1957.', 'theory of aircraft structural models subjected to aerodynamic heating and external loads . the problem of investigating the simultaneous effects of transient aerodynamic heating and external loads on aircraft structures for the purpose of determining the ability of the structure to withstand flight to supersonic speeds is studied . by dimensional analyses it is shown that .. constructed of the same materials as the aircraft will be thermally similar to the aircraft with respect to the flow of heat through the structure will be similar to those of the aircraft when the structural model is constructed at the same temperature as the aircraft . external loads will be similar to those of the aircraft . subjected to heating and cooling that correctly simulate the aerodynamic heating of the aircraft, except with respect to angular velocities and angular accelerations, without requiring determination of the heat flux at each point on the surface and its variation with time . acting on the aerodynamically heated structural model to those acting on the aircraft is determined for the case of zero angular velocity and zero angular acceleration, so that the structural model may be subjected to the external loads required for simultaneous simulation of stresses and deformations due to external loads .'), (52, 'procedure for calculating flutter at high supersonic speed including camber deflections, and comparison with experimental results .', 'morgan,h.g.', 'naca tn.4335, 1958.', 'procedure for calculating flutter at high supersonic speed including camber deflections, and comparison with experimental results . a method which may be used at high supersonic mach numbers is described for calculating the flutter speed of wings having camber in their deflection modes . the normal coupled vibration modes of the wing are used to derive the equations of motion . chord deflections of the vibration modes are approximated by polynomials . the wing may have a control surface and may carry external stores although no aerodynamic forces on the stores are presented . the aerodynamic forces that are assumed to be acting on the wing are obtained from piston theory and also from a quasi-steady form of a theory for two-dimensional steady flow . airfoil shape and thickness effects are taken account of in the analysis . the method is used to calculate the flutter speed of some wings which had been previously tested at mach numbers of 1.3 to 3.0 . comparison of the calculations and experiment is made for flat-plate 60 and 45 delta wings and also for an untapered 45 sweptback wing .'), (53, 'transition reynolds numbers of separated flows at supersonic speeds .', 'larson,h.k. and keating,s.j.', 'nasa tn.d349, 1960.', 'transition reynolds numbers of separated flows at supersonic speeds . experimental research has been conducted on the effects of wall cooling, mach number, and unit reynolds number on the transition reynolds number of cylindrical separated boundary layers on an ogive-cylinder model . results were obtained from pressure and temperature measurements and shadowgraph observations . the maximum scope of measurements encompassed mach numbers between 2.06 and 4.24, reynolds numbers (based on length of separation) between 60,000 and 400,000, and ratios of wall temperature to adiabatic wall temperature between 0.35 and 1.0 . within the range of the present tests, the transition reynolds number was observed to decrease with increasing wall cooling, increase with increasing mach number, and increase with increasing unit reynolds number . the wall-cooling effect was found to be four times as great when the attached boundary layer upstream of separation was cooled in conjunction with cooling of the separated boundary layer as when only the separated boundary layer was cooled . wall cooling of both the attached and separated flow regions also caused, in some cases, reattachment in the otherwise separated region . cavity resonance present in the separated region for some model configurations was accompanied by a large decrease in transition reynolds number at the lower test mach numbers .'), (54, 'method for calculation of compressible laminar boundary layer characteristics in axial pressure gradient with zero heat transfer .', 'morduchow,m. and clarke,j.h.', 'naca tn.2784, 1952.', 'method for calculation of compressible laminar boundary layer characteristics in axial pressure gradient with zero heat transfer . the karman-pohlhausen method is extended primarily to sixth-degree velocity profiles for determining the characteristics of the compressible laminar boundary layer over an adiabatic wall in the presence of an axial pressure gradient . it is assumed that the prandtl number is unity and that the coefficient of viscosity varies linearly with the temperature . a general approximate solution which permits a rapid determination of the boundary-layer characteristics for any given free-stream mach number and given velocity distribution at the outer edge of the boundary layer is obtained . numerical examples indicate that this solution will in practice lead to results of satisfactory accuracy, including the critical reynolds number for stability . for the special purpose of calculating the location of the separation point in an adverse pressure gradient, a short and simple method, based on the use of a seventh-degree velocity profile, is derived . the numerical example given here indicates that this method should in practice lead to sufficiently accurate results . for the special case of flow near a forward stagnation point it is shown that the karman-pohlhausen method with the usual fourth-degree profiles leads to results of adequate accuracy, even for the critical reynolds number .'), (55, 'separation, stability and other properties of compressible laminar boundary layer with pressure gradient and heat transfer .', 'morduchow,m. and grape,r.g.', 'naca tn.3296, 1955.', 'separation, stability and other properties of compressible laminar boundary layer with pressure gradient and heat transfer . a theoretical study is made of the effect of pressure gradient, wall temperature, and mach number on laminar boundary-layer characteristics and, in particular, on the skin-friction and heat-transfer coefficients, on the separation point in an adverse pressure gradient, on the wall temperature required for complete stabilization of the laminar boundary layer, and on the minimum critical reynolds number for laminar stability . the prandtl number is assumed to be unity and the coefficient of viscosity is assumed to be proportional to the temperature, with a factor arising from the sutherland relation . a simple and accurate method of locating the separation point in a compressible flow with heat transfer is developed . numerical examples to illustrate the results in detail are given throughout .'), (56, 'an analysis of the applicability of the hypersonic similarity law to the study of the flow about bodies of revolution at zero angle of attack .', 'ehret,d.m.', 'naca tn.2250, 1950.', 'an analysis of the applicability of the hypersonic similarity law to the study of the flow about bodies of revolution at zero angle of attack . the hypersonic similarity law as derived by tsien has been investigated by comparing the pressure distributions along bodies of revolution at zero angle of attack . in making these comparisons, particular attention was given to determining the limits of mach number and fineness ratio for which the similarity law applies . for the purpose of this investigation, pressure distributions determined by the method of characteristics for ogive cylinders for values of mach numbers and fineness ratios varying from 1.5 to 12 were compared . pressures on various cones and on cone cylinders were also compared in this study . the pressure distributions presented demonstrate that the hypersonic similarity law is applicable over a wider range of values of mach numbers and fineness ratios than might be expected from the assumptions made in the derivation . this is significant since within the range of applicability of the law a single pressure distribution exists for all similarly shaped bodies for which the ratio of free-stream mach number to fineness ratio is constant . charts are presented for rapid determination of pressure distributions over ogive cylinders for any combination of mach number and fineness ratio within defined limits .'), (57, 'applicability of the hypersonic similarity rule to pressure distributions which include the effects of rotation for bodies of revolution at zero angle of attack .', 'rossow,v.j.', 'naca tn.2399, 1951.', 'applicability of the hypersonic similarity rule to pressure distributions which include the effects of rotation for bodies of revolution at zero angle of attack . the analysis of technical note 2250, 1950, is extended to include the effects of flow rotation . it is found that the theoretical pressure distributions over ogive cylinders can be related by the hypersonic similarity rule with sufficient accuracy for most engineering purposes . the error introduced into pressure distributions and drag of ogive cylinders by ignoring the rotation term in the characteristic equations is investigated . it is found that the influence of the rotation term on pressure distribution and drag depends only upon the similarity parameter k (mach number divided by fineness ratio) . although the error in drag, due to neglect of the rotation term, is negligible at k=0.5, the error is about 30 percent at k=2.0 . charts are presented for the rapid determination of pressure distributions for rotational flow over ogive cylinders for all values of the similarity parameter between 0.5 and of mach number and fineness ratio .'), (58, 'pressure measurements on sharp and blunt 5 and 15 half-angle cones at mach number 3.86 and angles of attack to 100 .', 'amick,j.l.', 'nasa tn.d753, 1961.', 'pressure measurements on sharp and blunt 5 and 15 half-angle cones at mach number 3.86 and angles of attack to 100 . measured pressure distributions on cones are compared with modified newtonian theory . deviations as large as 14 percent of the stagnation pressure behind a normal shock are found . by combining empirical results for cylinders normal to the flow with newtonian concepts, a method of calculating pressures on cones at high angles of attack is developed . calculations by this method differ from the experimental results on sharp cones by only 2 percent of the stagnation pressure behind a normal shock . for blunted cones, additional deviations up to 8 percent are noted near the nose . schlieren pictures of the flow show an attached shock on the sharp of attack . detachment of the shock appears to be associated with the attainment of sonic speed immediately behind the shock . an orifice size effect is found which can increase the indicated pressure above the true value, if the orifice width is greater than one-tenth the local radius of curvature .'), (59, 'tables of exact laminar-boundary layer solutions when the wall is porous and fluid properties are variable .', 'brown,w.d. and donoughe,p.l.', 'naca tn.2479, 1951.', 'tables of exact laminar-boundary layer solutions when the wall is porous and fluid properties are variable . the three partial differential equations of the laminar boundary layer for two-dimensional steady-state compressible flow have been transformed into two ordinary differential equations by the method of pohlhausen, falkner, and skan . the ordinary equations include parameters for expressing the simultaneous effects of pressure gradient in the main-stream flow through a porous wall and property changes in the fluid due to large temperature differences between the wall and the free stream . a total of 58 cases have been solved numerically by the method of picard . the euler number (nondimensional pressure-gradient parameter) ranges in value from 1 (stagnation-point value) to the negative values found at the laminar separation points . three rates of flow through the porous wall were considered (including the impermeable case where the flow rate is 0) . five temperature ratios (stream temperature divided by wall temperature) were used .. the uncooled and unheated case (temperature ratio of 1), two cooled cases (temperature ratios of ture ratios of and ) . velocity, weight-flow, and temperature distributions are tabulated as are the dimensionless stream function of falkner and skan and its derivatives and the dimensionless temperature function of pohlhausen and its derivatives . for each case, displacement, momentum, and convection thicknesses, as well as nusselt number and coefficient of friction at the wall, were computed .'), (60, 'estimation forces and moments due to rolling for several slender tail configurations at supersonic speeds .', 'bobbitt,p.j. and malvestuto,f.s.', 'naca tn.2955, 1953.', 'estimation forces and moments due to rolling for several slender tail configurations at supersonic speeds . the velocity potentials, span loadings, and corresponding force and moment derivatives have been theoretically evaluated for a number of slender-tail arrangements performing a steady rolling motion at supersonic speeds . the method of analysis is based upon an application of conformal-transformation techniques . the utilization of these techniques allows the simple determination of the complex potentials for various types of two-dimensional boundary-value problems . in addition, two simple and often-used approximations to the rolling derivatives have been compared with the corresponding exact values determined by the method presented in this report . in order to show the importance of wing-tail interference, the effect of the flow field behind a rolling wing on the tail characteristics has been illustrated for a simple wing-tail arrangement .'), (61, 'on flow of electrically conducting fluids over a flat plate in the presence of a transverse magnetic field .', 'rossow,v.j.', 'naca tn.3971, 1957.', 'on flow of electrically conducting fluids over a flat plate in the presence of a transverse magnetic field . the use of a magnetic field to control the motion of electrically conducting fluids is studied . the boundary-layer solutions are found for flow over a flat plate when the magnetic field is fixed relative to the plate or to the fluid . the equations are integrated numerically for the effect of the transverse magnetic field on the velocity and temperature profiles, and hence, the skin friction and rate of heat transfer . it is concluded that the skin friction and the heat-transfer rate are reduced when the transverse magnetic field is fixed relative to the plate and increased when fixed relative to the fluid . the total drag is increased in all the cases studied .'), (62, 'similar solutions for the compressible laminar boundary layer with heat transfer and pressure gradient .', 'cohen,c.b. and reshotko,e.', 'naca tn.3325, 1955.', 'similar solutions for the compressible laminar boundary layer with heat transfer and pressure gradient . stewartson''s transformation is applied to the laminar compressible boundary-layer equations and the requirement of similarity is introduced, resulting in a set of ordinary nonlinear differential equations previously quoted by stewartson, but unsolved . the requirements of the system are .. prandtl number of 1.0, linear viscosity-temperature relation across the boundary layer, an isothermal surface, and the particular distributions of free-stream velocity consistent with similar solutions . this system admits axial pressure gradients of arbitrary magnitude, heat flux normal to the surface, and arbitrary mach numbers . the system of differential equations is transformed to an integral system, with the velocity ratio as the independent variable . for this system, solutions are found for pressure gradients varying from that causing separation to the infinitely favorable gradient and for wall temperatures from absolute zero to twice the free-stream stagnation temperature . some solutions for separated flows are also presented . for favorable pressure gradients, the solutions are unique . for adverse pressure gradients, where the solutions are not unique, two solutions of the infinite family of possible solutions are identified as essentially viscid at the outer edge of the boundary layer and the remainder essentially inviscid . for the case of favorable pressure gradients with heated walls, the velocity within a portion of the boundary layer is shown to exceed the local external velocity . the variation of a reynolds analogy parameter, which indicates the ratio of skin friction to heat transfer, is from zero to 7.4 for a surface of temperature twice the free-stream stagnation temperature, and from zero to 2.8 for a surface held at absolute zero where the value 2 applies to a flat plate .'), (63, 'hypersonic viscous flow over slender cones .', 'talbot,l.', 'naca tn.4327, 1958.', 'hypersonic viscous flow over slender cones . viscous self-induced pressures on 3 -semivertex-angle cones were measured over the range 3.7 free-stream mach number 5.8 and 0.5 viscous-interaction parameter 2.3 . the data were found to be in good agreement with results obtained by talbot on 5 cones in the range rameter 3.5 . all these data were correlated reasonably well by the viscous-interaction parameter, which is defined as where and are the mach number and reynolds number based on ideal taylor-maccoll flow conditions and c is the chapman-rubesin factor . a new method for calculating self-induced pressures is presented which takes into account the interaction between boundary-layer growth and the inviscid-flow field at the outer edge of the boundary layer . pressures calculated by this method were only 10 to 20 percent higher than the measured values .'), (64, 'unsteady oblique interaction of a shock wave with plane disturbances .', 'moore,f.k.', 'naca tn.2879, 1953.', 'unsteady oblique interaction of a shock wave with plane disturbances . analysis is made of the flow field produced by oblique impingement of weak plane disturbances of arbitrary profile on a plane normal shock . three types of disturbance are considered .. moves . the sound wave refracts either as a simple isentropic sound wave or as an attenuating isentropic pressure wave, depending on the angle between the shock and the incident sound wave . a stationary vorticity wave of constant pressure appears behind the shock . reflects as a sound wave, and a stationary vorticity wave is produced . the shock . the incident wave refracts as a stationary vorticity wave, and either a sound wave or attenuating pressure wave is also produced . computations are presented for the first two types of incident wave, over the range of incidence angles, for shock mach numbers of 1, 1.5, and .'), (65, 'convection of a pattern of vorticity through a shock wave .', 'ribner,h.s.', 'naca tn.2864, 1953.', 'convection of a pattern of vorticity through a shock wave . an arbitrary weak spatial distribution of vorticity can be represented in terms of plane sinusoidal shear waves of all orientations and wave lengths (fourier integral) . the analysis treats the passage of a single representative weak shear wave through a plane shock and shows refraction and modification of the shear wave with simultaneous generation of an acoustically intense sound wave . applications to turbulence and to noise in supersonic wind tunnels are indicated .'), (66, 'some effects of joint conductivity on the temperature and thermal stresses in aerodynamically heated skin-stiffener combinations .', 'griffith,g.e. and miltonberger,g.h.', 'naca tn.3609, 1956.', 'some effects of joint conductivity on the temperature and thermal stresses in aerodynamically heated skin-stiffener combinations . temperatures and thermal stresses in typical skin-stiffener combinations of winglike structures subjected to aerodynamic heating have been obtained with the aid of an electronic differential analyzer . variations were made in an aerodynamic heat-transfer parameter, in a joint conductivity parameter, and in the ratio of skin width to skin thickness . the results, which are presented in nondimensional form, indicate that decreasing the joint conductivity parameter lowers both the interior and the average temperature ratios, increases the peak thermal stress ratios in the skin, and may considerably increase the peak stiffener stress ratios,. increasing the aerodynamic heat-transfer parameter decreases the interior and average temperature ratios, increases the peak skin stress ratios somewhat, but greatly increases the peak stiffener stress ratios,. and increasing the ratio of skin width to skin thickness produces only moderate decreases in the peak skin stress ratios while moderately increasing the peak stiffener stress ratios .'), (67, 'dynamic stability of vehicles traversing ascending or descending paths through the atmosphere .', 'tobak and allen.', 'naca tn.4275, 1958.', 'dynamic stability of vehicles traversing ascending or descending paths through the atmosphere . an analysis is given of the oscillatory motions of vehicles which traverse ascending and descending paths through the atmosphere at high speed . the specific case of a skip path is examined in detail, and this leads to a form of solution for the oscillatory motion which should recur over any trajectory . the distinguishing feature of this form is the appearance of the bessel rather than the trigonometric function as the characteristic mode of oscillation .'), (68, 'some aspects of air-helium simulation and hypersonic approximations .', 'love,e.s.', 'nasa tn.d49, 1959.', 'some aspects of air-helium simulation and hypersonic approximations . some illustrations of the differences that may be expected between results obtained in hypersonic wind tunnels that employ air and results obtained in those that employ helium as the test medium (imperfect-gas effects are not considered) are compiled and presented herein . simple expressions are presented that demonstrate the possibility of simulating air results in helium tests and of transforming helium data to equivalent air data . nonviscous and viscous simulations are considered . in most cases, the methods and the general forms of the expressions for simulation that are derived are applicable to any two ideal gases having different ratios of specific heats .'), (69, 'predicted shock envelopes about two types of vehicles at large angles of attack .', 'kaattari,g.e.', 'nasa tn.d860, 1961.', 'predicted shock envelopes about two types of vehicles at large angles of attack . methods based on oblique- and normal-shock relationships and the continuity of mass flow through suitably chosen volume elements between the shock and body were developed to predict shock envelopes about two types of vehicles being considered for atmosphere entry . one type is a high-drag capsule shape . the other type is essentially a slender triangular wing capable of providing high lift or high drag, depending on the angle of attack . predicted and measured shock envelopes were compared for a mach number range of 3 to 15 for vehicles at high angles of attack,. good agreement was found . most of the available experimental data were in a speed and temperature range in which no important real-gas effects occurred .'), (70, 'a study of flow changes associated with airfoil section drag rise at supercritical speeds .', 'nitzburg,g.e. and crandall,s.', 'naca tn.1813, 1949.', 'a study of flow changes associated with airfoil section drag rise at supercritical speeds . a study of experimental pressure distributions and section characteristics for several moderately thick airfoil sections was made . a correlation appears to exist between the drag-divergence mach number and the free-stream mach number for which sonic velocity occurs at the airfoil crest, the chordwise station at which the airfoil surface is tangent to the free-stream direction . it was found that, since the mach number for which sonic velocity occurs at the airfoil crest can be estimated satisfactorily by means of the prandtl-glauert rule, a method is provided whereby the drag-divergence mach number of an airfoil section at a given angle of attack can be estimated from the low-speed pressure distribution and the airfoil profile . this method was used to predict with a reasonable degree of accuracy the drag-divergence mach number of a considerable number of airfoil sections having diverse shapes and a wide range of thickness-chord ratios . the pressure distributions and section force characteristics of several moderately thick airfoil sections at mach numbers above the drag-divergence mach number were analyzed . some of the characteristics of the flow over these airfoils at supercritical mach numbers are discussed .'), (71, 'laminar boundary layer behind shock advancing into stationary fluid .', 'mirels,h.', 'naca tn.3401, 1955.', 'laminar boundary layer behind shock advancing into stationary fluid . a study was made of the laminar compressible boundary layer induced by a shock wave advancing into a stationary fluid bounded by a wall . for weak shock waves, the boundary layer is identical with that which occurs when an infinite wall is impulsively set into uniform motion shocks . velocity and temperature profiles, recovery factors, and skin-friction and heat-transfer coefficients are tabulated for a wide range of shock strengths .'), (72, 'boundary layer behind shock or thin expansion wave moving into stationary fluid .', 'mirels,h.', 'naca tn.3712, 1956.', 'boundary layer behind shock or thin expansion wave moving into stationary fluid . the boundary layer behind a shock or thin expansion wave advancing into a stationary fluid has been determined . laminar and turbulent boundary layers were considered . the wall surface temperature behind the wave was also investigated . the assumption of a thin expansion wave is valid for weak expansions but becomes progressively less accurate for strong expansion waves . the laminar-boundary-layer problem was solved by numerical integration except for the weak wave case, which can be solved analytically . integral (karman-pohlhausen type) solutions were also obtained to provide a guide for determining expressions which accurately represent the numerical data . analytical expressions for various boundary-layer parameters are presented which agree with the numerical integrations within 1 percent . the turbulent-boundary-layer problem was solved using integral methods similar to those employed for the solution of turbulent compressible flow over a semi-infinite flat plate . the fluid velocity, relative to the wall, was assumed to have a seventh-power profile . the blasius equation, relating turbulent skin friction and boundary-layer thickness, was utilized in a form which accounted for compressibility . consideration of the heat transfer to the wall permitted the wall surface temperature, behind the wave, to be determined . the wall thickness was assumed to be greater than the wall thermal-boundary-layer thickness . it was found that the wall temperature was uniform (as a function of distance behind the wave) for the laminar-boundary-layer case but varied with distance for the turbulent-boundary-layer case .'), (73, 'investigation of the stability of the laminar boundary layer in a compressible fluid .', 'lees,l. and lin,c.c.', 'naca tn.1115, 1946.', 'investigation of the stability of the laminar boundary layer in a compressible fluid . in the present report the stability of two-dimensional laminar flows of a gas is investigated by the method of small perturbations . the chief emphasis is placed on the case of the laminar boundary layer . part 1 of the present report deals with the general mathematical theory . the general equations governing one normal mode of the small velocity and temperature disturbances are derived and studied in great detail . it is found that for reynolds numbers of the order of those encountered in most aerodynamic problems, the temperature disturbances have only a negligible effect on those particular velocity solutions which depend primarily on the viscosity coefficient (/viscous solutions/) . indeed, the latter are actually of the same form in the compressible fluid as in the incompressible fluid, at least to the first approximation . because of this fact, the mathematical analysis is greatly simplified . the final equation determining the characteristic values of the stability problem depends on the /inviscid solutions/ and the function of tietjens in a manner very similar to the case of the incompressible fluid . the second viscosity coefficient and the coefficient of heat conductivity do not enter the problem,. only the ordinary coefficient of viscosity near the solid surface is involved . part 2 deals with the limiting case of infinite reynolds numbers . the study of energy relations is very much emphasized . it is shown that the disturbance will gain energy from the main flow if the gradient of the product of mean density and mean vorticity near the solid surface has a sign opposite to that near the outer edge of the boundary layer . a general stability criterion has been obtained in terms of the gradient of the product of density and vorticity, analogous to the rayleigh-tollmien criterion for the case of an incompressible fluid . if this gradient vanishes for some value of the velocity ratio of the main flow exceeding 1-1/m (where m is the free stream mach number) .'), (74, 'an experimental study of the turbulen coundary layer on a shock tube wall .', 'gooderum,p.n.', 'naca tn.4243, 1958.', 'an experimental study of the turbulen coundary layer on a shock tube wall . interferometric measurements were made of the density profiles of an unsteady turbulent boundary layer on the flat wall of a shock tube . the investigation included both subsonic and supersonic flow (mach numbers of 0.50 and 1.77) with no pressure gradient and with heat transfer to a cold wall . velocity profiles and average skin-friction coefficients were calculated . effects on the velocity profile of surface roughness and flow length are examined .'), (75, 'studies of structural failure due to acoustic loading .', 'hess,n.w.', 'naca tn.4050, 1957.', 'studies of structural failure due to acoustic loading . some discussion of the acoustic fatigue problem of aircraft structures is given along with data pertaining to the acoustic inputs from some powerplants in common use . comparisons are given for results of some fatigue tests of flat panels and cantilever beams exposed to both random- and discrete-type inputs . in this regard it appears that both the stress level of the test and the type of model are significant,. hence, no generalization can be made at this time . with regard to increasing the fatigue life, it was noted that increased stiffening of a panel due to curvature and pressure differential is particularly beneficial .'), (76, 'flight measurement of wall pressure fluctuations and boundary-layer turbulence .', 'mull,h.r. and algranti,j.s.', 'nasa tn.d280, 1960.', 'flight measurement of wall pressure fluctuations and boundary-layer turbulence . the results are presented for a flight test program using a fighter type jet aircraft flying at pressure altitudes of 10,000, 20,000, and apparatus was used to measure and record the output of microphones and hot-wire anemometers mounted on the forward-fuselage section and wing of the airplane . mean-velocity profiles in the boundary layers were obtained from total-pressure measurements . the ratio of the root-mean-square fluctuating wall pressure to the free-stream dynamic pressure is presented as a function of reynolds number and mach number . the longitudinal component of the turbulent-velocity fluctuations was measured, and the turbulence-intensity profiles are presented for the wing and forward-fuselage section . in general, the results are in agreement with wind-tunnel measurements which have been reported in the literature . for example, the variation of (is the root mean square of the wall-pressure fluctuation, and q is the free-stream dynamic pressure) with reynolds number was found to be essentially constant for the forward fuselage-section boundary layer, while variations at the wing station were probably unduly affected by the microphone diameter, which was large compared with the boundary-layer thickness .'), (77, 'a comparative analysis of the performance of long range hypervelocity vehicles .', 'eggers,a.j.', 'naca tn.4046, 1957.', 'a comparative analysis of the performance of long range hypervelocity vehicles . long-range hypervelocity vehicles are studied in terms of their motion in powered flight, and their motion and aerodynamic heating in unpowered flight . powered flight is analyzed for an idealized propulsion system which rather closely approaches present-day rocket motors . unpowered flight is characterized by a return to earth along a ballistic, skip, or glide trajectory . only those trajectories are treated which yield the maximum range for a given velocity at the end of powered flight . aerodynamic heating is treated in a manner similar to that employed previously by the senior authors in studying ballistic missiles (naca tn 4047), with the exception that radiant as well as convective heat transfer is considered in connection with glide and skip vehicles . the ballistic vehicle is found to be the least efficient of the several types studied in the sense that it generally requires the highest velocity at the end of powered flight in order to attain a given range . this disadvantage may be offset, however, by reducing convective heat transfer to the re-entry body through the artifice of increasing pressure drag in relation to friction drag - that is, by using a blunt body . thus the kinetic energy required by the vehicle at the end of powered flight may be reduced by minimizing the mass of coolant material involved . the glide vehicle developing lift-drag ratios in the neighborhood of and greater than 4 is far superior to the ballistic vehicle in ability to convert velocity into range . it has the disadvantage of having far more heat convected to it,. however, it has the compensating advantage that this heat can in the main be radiated back to the atmosphere . consequently, the mass of coolant material may be kept relatively low . the skip vehicle developing lift-drag ratios from about 1 to 4 is found to be superior to comparable ballistic and glide vehicles in converting velocity into range . at lift-drag ratios below 1 it is found to be about equal to comparable ballistic vehicles while at lift-drag ratios'), (78, 'an analytical treatment of aircraft propeller precession instability .', 'reed,w.h. and bland,s.r.', 'nasa tn.d659, 1961.', 'an analytical treatment of aircraft propeller precession instability . an analytical investigation is made of a precession-type instability which can occur in a flexibly supported aircraft-engine-propeller combination . by means of an idealized mathematical model which is comprised of a rigid power-plant system flexibly mounted in pitch and yaw to a fixed backup structure, the conditions required for neutral stability are determined . the paper also examines the sensitivity of the stability boundaries to changes in such parameters as stiffness, damping, and asymmetries in the engine mount, propeller speed, airspeed, mach number, propeller thrust, and location of pitch and yaw axes . stability is found to depend strongly on the damping and stiffness in the system . with the use of nondimensional charts theoretical stability boundaries are compared with experimental results obtained in wind-tunnel tests of an aeroelastic airplane model . in general, the theoretical results, which do not account for wing response, show the same trends as observed experimentally,. however, for a given set of conditions calculated airspeeds for neutral stability are consistently lower than the measured values . evidently, this result is due to the fact that wing response tends to add damping to the system .'), (79, 'effects of extreme surface cooling on boundary layer transition .', 'jack,j.r.', 'naca tn.4094, 1957.', 'effects of extreme surface cooling on boundary layer transition . an investigation was made to determine the combined effects of surface cooling, pressure gradients, nose blunting, and surface finish on boundary-layer transition . data were obtained for various body shapes at a mach number of 3.12 and reynolds numbers per foot as high as 15x10 . previous transition studies, with moderate cooling, have shown agreement with the predictions of stability theory . for surface roughnesses ranging from 4 to 1250 microinches the location of transition was unaffected with moderate cooling . with extreme cooling, an adverse effect was observed for each of the parameters investigated . in general, the transition reynolds number decreased with decreasing surface temperature . in particular, the beneficial effects of a favorable pressure gradient obtained with moderate cooling disappear with extreme cooling, and a transition reynolds number lower than that observed on a cone is obtained . further, an increase in the nose bluntness decreased the transition reynolds number under conditions of extreme cooling .'), (80, 'effect of distributed three-dimensional roughness and surface cooling on boundary layer transition and lateral spread of turbulence at supersonic speeds .', 'braslow,a.l.', 'nasa tn.d53, 1959.', 'effect of distributed three-dimensional roughness and surface cooling on boundary layer transition and lateral spread of turbulence at supersonic speeds . an investigation was made in the langley 4 by 4-foot supersonic pressure tunnel at mach numbers of 1.61 and 2.01 to determine (1) the effect of distributed roughness on boundary-layer transition with the model surface at adiabatic wall temperature and cooled and (2) the effect of surface cooling on the lateral spread of turbulence . both distributed granular-type and single spherical roughness particles were used, and transition of the boundary layer was determined by hot-wire anemometers . the transition-triggering mechanism of the three-dimensional roughness at supersonic speeds appeared to be the same as that previously observed at subsonic speeds . in fact, the critical value of the roughness reynolds number parameter (that is, the value at which turbulent spots are initiated by the roughness) was found to be approximately the same at supersonic and subsonic speeds when complete local conditions at the top of the roughness, including density and viscosity, were considered in the formulation of the roughness reynolds number . for three-dimensional roughness at a reynolds number less than its critical value, the roughness introduced no disturbances of sufficient magnitude to influence transition . surface cooling, although providing a theoretical increase in stability to small disturbances, did not increase to any important extent the value of the critical roughness reynolds number for three-dimensional roughness particles . cooling, therefore, because of its effect on the boundary-layer thickness, density, and viscosity actually promoted transition due to existing three-dimensional surface roughness for given mach and reynolds numbers . the measured lateral spread of turbulence in the boundary layer appeared to be unaffected by the increased laminar stability derived from the surface cooling .'), (81, 'compressible laminar flow and heat transfer about a rotating isothermal disk .', 'ostrach,s. and thornton,p.', 'naca tn.4320, 1958.', 'compressible laminar flow and heat transfer about a rotating isothermal disk . the flow and heat transfer about a rotating isothermal disk are re-examined to include the effects of compressibility and property variations . if viscous dissipation is neglected, the compressible problem is correlated to the incompressible problem by assuming linear variations of viscosity and thermal conductivity with temperature . certain inaccuracies in several previous incompressible solutions are noted and corrected herein . the effect of compressibility appears as a distortion of the normal coordinate and normal velocity component and as a multiplicative factor in the heat-transfer coefficient, the nusselt number, and in the expressions for the skin-friction components and torque required to rotate the disk .'), (82, 'theoretical investigation of the ablation of a glass-type heat protection shield of varied material properties at the stagnation point of a re-entering irbm .', 'adams,e.w.', 'nasa tn.d564, 1961.', 'theoretical investigation of the ablation of a glass-type heat protection shield of varied material properties at the stagnation point of a re-entering irbm . the melting-type heat protection at the stagnation point of a re-entering irbm is treated by employing homogeneous, opaque, and nondecomposing glass shields which do not exceed a temperature of some effects due to variations of the glass properties . the ballistic re-entry vehicle has a nose diameter of 0.635 m, a ballistic factor of 3.5 x 10, a re-entry angle of 124.9 (from the vertical) at an altitude of 100 km, and a re-entry speed of 4.5 . the performance of 36 different glass shields with assumed combinations of material properties is investigated by employing a calculation method which yields practically exact, transient solutions for the problem . as a corollary, results for a certain steady flight state are also given . the discussions made it possible to derive under realistic flight conditions some thermal characteristics for the employment of thin, or light-weight, glass shields . investigation of these hypothetical glass shields leads to the conclusion that a low thermal conductivity and a high specific heat, and thus, a small thermal diffusivity are most desirable . a small thermal diffusivity yields high surface temperatures, causing a high radiative heat transfer out of the shield,. and steep temperature profiles normal to the surface, causing a small thermal penetration across the shield with little total ablation of the shield . results show that for the assumed irbm re-entry, the necessary thickness of the employed glass shields increases monotonically with thermal diffusivity which is the only material parameter affecting this thickness . a high viscosity level and a high emissivity constant of the surface of the supposedly opaque shield are also desirable,. although, these two properties exert a comparatively small influence on the overall performance when disregarding glass shields with an extremely low viscosity level .'), (83, 'discussion of solar proton events and manned space flights .', 'anderson,k. and sinchtel,c.d.', 'nasa tn.d671, 1961.', 'discussion of solar proton events and manned space flights . as a result of studies made during the international geophysical year (igy) and the international geophysical cooperation (igc), it is known that a considerable fraction of large solar flares give rise to almost pure streams of protons which reach the earth and continue to arrive for as long as 11 days . the energies of these particles lie within a very steep spectrum extending from 20 to least 500 mev . because of the frequency of large flares during times of high solar activity, and owing to the long duration of each solar proton emission, these particles were present in detectable intensity near the top of the earth''s atmosphere for about 15 percent of the time from 1957 to 1960 . the number of large flares that accelerated and released these particles during this three-year period was about 30 . the event that began on august 22, 1958 contributed greatly toward the understanding of the solar and terrestrial sequence of events, and in addition provided the first identification of the emitted particles . a flare on may of protons in the neighborhood of the earth that this phenomenon was recognized as an additional radiation hazard to manned vehicles in the high atmosphere and in most parts of the solar system . the three very intense events that occurred in july, 1959 further supported this conclusion, and the possibility of predicting such events became an important consideration . in addition to its value in the protection of human beings, effective forecasting clearly would be of great value in the detailed scientific study of this phenomenon . this paper presents a preliminary discussion of some aspects of predicting the arrival of protons at the earth following the appearance of solar activity features and, equally important, of forecasting the periods when this penetrating radiation is unlikely to occur .'), (84, 'experimental investigation of the downstream influence of stagnation point mass transfer .', 'libby,pa. and cresci,r.j.', 'j. ae. scs. 28, 1961, 51.', 'experimental investigation of the downstream influence of stagnation point mass transfer . this report presents the results of an experimental investigation of the downstream influence of localized mass transfer in the stagnation region of a blunt body under hypersonic flow conditions . the coolant is injected through a porous plug coaxial with the centerline of symmetry of the model . the tests were carried out in a wind tunnel with a mach number of 6.0, stagnation temperatures of approximately 1,600 r., and a stagnation pressure of approximately 600 psia . four different gases were injected over a range of mass flows . the heat transfer on the impermeable section was measured under isothermal wall conditions,. for the higher rates of mass flow, adiabatic surface temperatures were also determined . the theoretical analysis of the boundary-layer flow is investigated in order to establish the similarity parameters for the flow system . these parameters permit the extrapolation of the test results to other flow conditions, provided that laminar flow prevails . helium is found to be the most efficacious coolant .'), (85, 'on trails of axisymmetric hypersonic blunt bodies flying through the atmosphere .', 'feldman,s.', 'j. ae. scs. 28, 1961, 433.', 'on trails of axisymmetric hypersonic blunt bodies flying through the atmosphere . the trail left in the atmosphere by a body moving at hypersonic speeds is the subject of theoretical treatment . the times required for ionization and dissociation (and their inverse processes) to go to completion, when compared to the flow times of a gas particle, are important in determining the observable effects of hypersonic trails-i.e., emitted thermal radiation and reflection of electromagnetic waves from the trail . in order to simplify the theoretical treatment, the trail is divided into two regions .. (1) the expansion-controlled trail, which treats the behavior of the wake behind the body up to a point, along the direction of flight, where the pressure decays to the free-stream value and cooling is controlled principally by the expansion of the flow, and (2) the conduction-controlled trail, where the trail cools mainly by diffusion of heat away from the high-temperature core . the influence of the details of the body shape on the observables are discussed and a simple computational procedure for the behavior of the conduction-controlled trail is developed based on integral methods . results of calculations that assume thermodynamic equilibrium of the flow field give the values of the thermodynamic variables in the trail of a sphere, axial distributions of emitted thermal radiation, and maps of electron density distribution . it is shown that the cooling of the conduction-controlled trail is essentially due to conduction of heat and that viscous effects are not important . it is found that this portion of the trail does not widen as one proceeds downstream . flight velocities considered vary between 15,000 and 35,000 ft sec and altitudes range between 100,000 and 250,000 ft .'), (86, 'inviscid-incompressible flow theory of static peripheral jets in proximity to the ground .', 'strand,t.', 'j. ae. scs. 1961, 27.', 'inviscid-incompressible flow theory of static peripheral jets in proximity to the ground . an /exact/ flow theory of peripheral jets issuing symmetrically from a hovering aerial-ground vehicle is presented . the theory is exact insofar as no simplifying assumptions have been made in obtaining a solution of the governing inviscid, two-dimensional hydrodynamical flow equations . the results are valid for all jet thickness vehicle height ratios . the limit of applicability of existing theories (very low thickness height ratios) are defined . jet reaction, lift, and power coefficients for static conditions are introduced and computed . lift augmentation and lift power ratios are also calculated . applications to three-dimensional vehicles with rotational symmetry are indicated .'), (87, 'free-convection magnetohydrodynamic flow past a porous flat plate .', 'pau-chang lu', 'graduate assistant, department of mechanical engineering, case institute of technology, cleveland, ohio', 'free-convection magnetohydrodynamic flow past a porous flat plate . the incompressible flow of an electrically conducting fluid past a porous plate with constant suction velocity in the presence of a transverse uniform strength has recently been investigated by gupta . in this note, the problem is generalized to take into account the effect of free convection, when a body force is acting parallel to the wall . the fluid is assumed to be semi-incompressible as usual . in addition to the obvious practical significance, this problem is also interesting in the sense that it provides another exact solution of the magnetohydrodynamic equations, since the only electromagnetic assumptions involved are constant properties and freedom from excessive charges .'), (88, 'magnetohydrodynamic free-convection pipe flow .', 'cramer,k.r.', 'j. ae. scs. 28, 1961, 736.', 'magnetohydrodynamic free-convection pipe flow . it has been shown that transverse magnetic fields of practical strengths exert considerable influence on liquid-metal, free-convection, vertical, flat-plate and parallel-plate flow fields . the extent of influence was determined by the magnitude of a nondimensional parameter a which is the ratio of the hartmann number to the fourth root of the grashof number, and is a measure of the relative influence of the magnetic and buoyant forces . in this note the steady, fully developed, laminar, free-convection flow of a fluid of electrical conductivity through a fully submerged, open-ended, constant-temperature, vertical pipe located in a transverse magnetic field of strength is analyzed in terms of the same parameter . the magnitude of its influence on the velocity and temperature profiles, the surface shear and heat transfer, and the volumetric flow rate is determined .'), (89, 'an investigation of separated flows, part i: the pressure field .', 'charwat,a.f.', 'j. ae. scs. 28, 1961, 457.', 'an investigation of separated flows, part i: the pressure field . the present article describes an investigation of several types of separated regions such as blunt-base wakes and cavities formed in cutouts in the boundaries and ahead of or behind two dimensional steps in supersonic (mach numbers 2 to 4) and subsonic flow . the conditions for the existence, the geometry, and the pressure field are described in this paper . a second article (to be published) will describe investigations of the internal flow and the heat transfer across such separated regions . it is found that there is a maximum (critical) ratio of the length of the separated free-shear layer to the depth of the depression in the boundary beyond which the cavity collapses, leaving mutually independent separated regions at each protrusion . this critical length changes greatly upon laminar-turbulent transition in the oncoming boundary layer,. in either laminar or turbulent flow it is approximately independent of mach and reynolds numbers . a semiempirical correlation predicting the conditions under which the flow will span a depression of arbitrary depth is proposed . detailed pressure distributions along the boundaries of a cavity (in turbulent flow) are presented as a function of the ratio of the cavity length to the critical length, which is found to be the pertinent similarity parameter . for short notches the impact pressure due to the reversal of the inner portion of the shear layer at recompression tends to thicken the shear layer and a type of boundary layer-free stream interaction governs the pressure field . the pressure in the cavity is nearly constant and can be higher than free-stream . in long notches the shear layer bends inward at separation and curves back gradually ahead of the recompression point . the floor-pressure variation is pronounced and the recovery pressure at reattachment is small . the variation of the drag coefficient with mach number reflects the change from one to the other mechanism of recompression . detailed surveys of the mach-number distributions in a blunt-body wake and the mixing region behind its throat, as well as in the shear layer spanning a cutout in a wall, are presented and analyzed . it is found that, in general, the assumptions of the simple supersonic-wake models which rely on a principle of steady flow with mass conservation in the cavity are not adequate for cavities in which there is recompression against a boundary . results showing the influence of the thickness of the initial boundary layer (in the range of 0.3 to 3 times the notch depth) and of the geometry of the notch are also presented .'), (90, 'periodic temperature distributions in a two-layer composite slab .', 'stonecypher,t.e.', 'j. ae. scs. 27, 1960, 152.', 'periodic temperature distributions in a two-layer composite slab . an investigation to determine the feasibility of using an insulating thermal barrier to protect exposed solid propellant motors from atmospheric or environmental temperature variations has recently been completed . in one portion of this study, a solution was developed for the periodic temperature distribution in a two-layer composite slab . one exposed surface of this composite slab was adiabatic, and the other exposed surface was subjected to a sinusoidal temperature variation . the technique used in the analysis was similar to that of grober . in this note, pertinent features of the development of the solution are given .'), (91, 'periodic temperature distribution in a two-layer composite slab .', 'w. f. campbell', 'national aeronautical establishment, ottawa, ont., canada', 'periodic temperature distribution in a two-layer composite slab . in a recent contribution to the reader''s forum, under the above title, stonecypher outlined a method for finding the periodic temperature distribution in a two-layer composite slab, one exposed surface of the slab being insulated and the other subject to a sinusoidal temperature variation . perfect thermal contact between the two layers, and constant thermal properties were assumed . two years ago i drew attention in these pages to a method for determining the transient temperature in such a two-layer slab resulting from a triangular heat-input pulse . i should like to point out that this same method also is applicable to the case where one external face is given a sinusoidal temperature variation with time . the method is based on the analogy between one-dimensional heat flow and the flow of an electric current in a simple transmission line having only series resistance and parallel capacitance .'), (92, 'the analysis of redundant structures by the use of high-speed digital computers .', 'chrichlow,w.j. and haggenmacher,g.w.', 'j. ae. scs. 27, 1960, 595.', 'the analysis of redundant structures by the use of high-speed digital computers . large-scale redundant structure analyses are currently feasible by the use of modern high-speed digital computers . this capability opportunely meets the urgent need to solve complex problems which otherwise would be hopelessly beyond the capacity of the hand desk computer . however, the difficulties have now shifted from tedious hand computations to the problems of adequately representing the structure by a model and of the peculiarities of irregular geometrical configurations . a wide scope of problem types can be handled by a generalized program approach . matrix formulation is used for the organization of input data and for handling data transfer in the large complex of subroutines, including the formation of equilibrium and continuity conditions to the final loads and deflections . simultaneous treatment of thermal expansions and plasticity is included . the use of minimum-size redundant systems is emphasized, starting from the philosophy of cutting members to provide a statically determinate structure . improved numerical accuracy and problem size capacity is gained for a given computer . examples are discussed ranging from simple plane-load diffusion problems to pressurized fuselage cutouts and complex wing-fuselage-shell intersection-type problems .'), (93, 'the supersonic blunt body problem - review and extensions .', 'van dyke,m.d.', 'j. ae. scs. 25, 1958, 485.', 'the supersonic blunt body problem - review and extensions . a survey of existing analytical treatments of the supersonic or hypersonic blunt-body problem indicates that none is adequate for predicting the details of the flow field . reasons are given for the failure of various plausible approximations . a numerical method, which is simpler than others proposed, is set forth for solving the full inviscid equations using a medium-sized electronic computer . results are shown from a number of solutions for bodies that support detached shock waves described by conic sections .'), (94, 'the transverse curvature effect in compressible axially symmetric laminar boundary layer flow .', 'probstein,r.f. and elliott,d.', 'j. ae.scs. 28, 1956, 206.', 'the transverse curvature effect in compressible axially symmetric laminar boundary layer flow . the viscous transverse curvature effect in compressible axially symmetric laminar boundary-layer flow has been investigated, and it is found that the effect is characterized by the parameter which is essentially the ratio of the boundary-layer thickness to body radius . it is shown that the busemann and crocco integrals of the two-dimensional energy equation for are still valid for axially symmetric flow in which the transverse curvature effects are considered . by a generalization of mangler''s transformation it is then shown that the boundary-layer equations are reducible to an almost two-dimensional form, making the analysis simpler for two asymptotic flow regions characterized by and less than or of the order of unity . it is with the latter region that the present paper is primarily concerned, and for this case it is shown that the additional term in the momentum and energy equations, which differentiates them from the two-dimensional form, behaves like an external favorable pressure gradient . except for certain special cases it is necessary to obtain the of the order of unity by means of asymptotic expansions in ascending powers of a parameter that is small compared to unity but proportional to . it is shown how the asymptotic solutions can be found for (1) the velocity and temperature distributions for the compressible zero pressure gradient case when the body shapes are given by and and (2) the velocity distribution for incompressible flow with an external velocity of the form past a body given by . the zeroth approximation is the mangler result . for the cases of a linear external velocity distribution, similar profiles can be found for all values of . more generally it is shown that similar profiles exist if the exponents n and m satisfy the condition that . here, similar is used in the restricted meaning that the distributions are derivable from ordinary differential equations . in the case of the cone and cylinder with zero pressure gradient where the equations have been numerically integrated for, the first-order correction to the mangler formulation shows that the effect on both the skin-friction coefficient and heat-transfer rate can become appreciable in the range where is less than or of the order of unity . at a constant, the effects are increased in magnitude when either the ratio of wall to free-stream temperature, or mach number, is increased . also, all other conditions being equal, for the same value of the skin-friction coefficient and heat-transfer increase on the cylinder is greater than that on the cone . for flows with pressure gradient, the transverse curvature term behaves again like a favorable pressure gradient and tends to delay both separation and transition when compared with axially symmetric flows in which the transverse curvature effect is neglected .'), (95, 'temperature distribution and thermal stresses in a model of a supersonic wing .', 'pohle,f.v. and oliver,h.', 'j. ae. scs. 21, 1954, 8.', 'temperature distribution and thermal stresses in a model of a supersonic wing . the transient temperature distribution and the thermal stresses in an idealized wing structure considered by hoff and torda in reference 1 are determined . only the effects of aerodynamic heating and of heat conduction are included,. radiation and convection effects are neglected . the present work differs from that of reference 1 in that the conduction from the cap to the web is considered when the temperature of the cap is calculated, and the spar cap temperature is assumed to be a function of both space and time . graphs of temperature and thermal stress distributions are presented, and the results are compared with those of reference 1 .'), (96, 'review of published data on the effect of roughness on transition from laminar to turbulent flow .', 'hugh l. dryden', 'national advisory committee for aeronautics', 'review of published data on the effect of roughness on transition from laminar to turbulent flow . a review is presented of the published data on the effect of roughness, especially single roughness elements, on transition from laminar to turbulent flow, in which an attempt is made to reanalyze and correlate the available information . the reanalysis shows that the transition reynolds number of a flat plate with zero pressure gradient is a function of the ratio of the height of the roughness element to the displacement thickness of the boundary layer at the element, this functional relation being a better representation of the data than a constant critical reynolds number of the roughness element . other data show that the effects of roghness are similar in streams of different initial turbulence and that a plot of the ratio of transition reynolds number of the rough plate to that for the smooth plate against the ratio of the height of the roughness element to displacement thickness of the boundary layer at the element gives good correlation of all the data for a given shape when transition occurs downstream from the roughness element . at a certain value of the height-thickness ratio dependent on the stream speed, location of roughness element, and airstream turbulence, the transition position reaches the element and remains there as the height or the stream speed is further increased . the paper also discusses available data on the effect of distributed roughness on transition on a flat plate, as well as some of the published data on roughness effects on transition on air-foils .'), (97, 'a mixing theory for the interaction between dissipative flows and nearly isentropic streams .', 'crocco,l. and lees,l.', 'j. ae. scs. 19, 1952, 649.', 'a mixing theory for the interaction between dissipative flows and nearly isentropic streams . by means of a simplified theoretical /model,/ the present paper treats the general class of flow problems characterized by the interaction between a viscous or dissipative flow near the surface of a solid body, or in its wake, and an /outer/ nearly isentropic stream . for the present, the external flow is taken to be a plane, steady, supersonic flow, which makes a small angle with a plane surface or plane of symmetry, although the methods used can be extended to curved surfaces, to axially symmetric supersonic flows, and also to subsonic flows . the internal dissipative flow is regarded as quasi-one-dimensional and parallel to the surface on the average, with a properly defined mean velocity and mean temperature . the nonuniformity of the actual velocity distribution is taken into account only approximately by means of a relation between mean temperature and mean velocity . mixing, or the transport of momentum from outer stream to dissipative flow, is considered to be the fundamental physical process determining the pressure rise that can be supported by the flow . with the aid of this concept, a large number of flow problems is shown to be basically similar, such as boundary-layer shockwave interaction, wake flow behind blunt-based bodies (base pressure problem), flow separation in overexpanded supersonic nozzles, separation on wings and bodies, etc .'), (98, 'heat transfer by laminar flow to a rotating plate .', 'millsaps,k. and pohlhausen,k.', 'j. ae. scs. 19, 1952, 120.', 'heat transfer by laminar flow to a rotating plate . an exact solution of the heat-transfer problem for the von karman example of the laminar flow of a viscous fluid over a rotating plate is given in dimensionless form and physically discussed . the solution is explicitly given for a constant temperature on the plate with viscous dissipation included . the numerical results are given for prandtl numbers from 0.5 to 10 .'), (99, 'the fundamentals of the statistical theory of turbulence .', 'th. von karman', 'california institute of technology', 'the fundamentals of the statistical theory of turbulence . statistical theory in general considers mean values of certain quantities . in the case of the turbulent motion one is interested in mean values of velocities and of their derivatives, and in mean values of squares and products of velocities and their derivatives . it was o. reynolds who first expressed the so-called apparent or turbulent stresses by the mean values of the products of the velocity components . the different theories suggested so far have as their common objective the establishment of relations between certain mean values, e.g. between the turbulent shear stresses given by the mean products of velocity fluctuations and the derivatives of the mean velocities, i.e. the measured mean velocity gradients . in this sort of investigations the conception of the /correlation/ is of paramount importance . the late a. friedman tried to introduce the correlations as unknown variables in the hydrodynamic equations., however, he could not carry his investigations to practical results, i.e., to results which can be compared with the experimental evidence . recently, g. i. taylor had success in his analysis of /isotropic/ turbulence by means of correlation calculations, and was able to discuss, theoretically, the problem of the decay of turbulence in a windstream behind a turbulence producing device . his theory raised considerable interest because it is concerned with the important problem of wind-tunnel turbulence and its results could be compared directly with experimental work done by dryden in this country and by fage, townend and simmons in england . the present paper is concerned with two fundamental problems.. with uniform isotropic turbulence and with the turbulent friction in a parallel stream . first, the general theory of isotropic turbulence is developed . this general theory includes taylor''s consideration as a special case . however, it'), (100, 'vibration isolation of aircraft power plants .', 'taylor,e.s. and browne,k.a.', 'j. ae. scs. 6, 1938, 43.', 'vibration isolation of aircraft power plants . vibration in aircraft structure can almost always be traced to vibratory forces originating from the power plant . these forces are transmitted to the aircraft in two ways .. (1) by the action of air forces upon the surfaces of the aircraft in, or adjacent to, the slip stream of the propeller, and (2) by direct transmission of unbalanced forces from the power plant through the engine mounting . the latter has always caused the preponderance of disturbance . vibratory stresses induced in the engine mounting structure occasionally produce fatigue failures in the associated parts, and always shorten the useful life of the entire aircraft structure . more important, however, are the psychological and physiological effects of continuous vibration and its attendant noise on the passengers and crew . this may very likely be the major source of the rapid fatigue which is so intimately associated with flying . the importance and desirability of drastically reducing vibration can hardly be questioned . this paper is limited to a consideration of the directly transmitted forces and, further, considers the power plants as rigid bodies attached by flexible means to the aircraft which is also considered as a rigid body of relatively large mass . it is also limited to the case of engines and engine supporting structures having axial symmetry (radial engines), although the methods employed could easily be extended to other cases .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (101, 'laminar heat transfer over blunt-nosed bodies at hypersonic flight speeds .', 'lester lees', 'the ramo-wooldridge corporation, los angles, and california institute of technology, pasadena, california', 'laminar heat transfer over blunt-nosed bodies at hypersonic flight speeds . this paper deals with two limiting cases of laminar heat transfer over blunt-nosed bodies at hypersonic flight speeds, or high stagnation temperatures.. (a) thermodynamic equilibrium, in which the chemical reaction rates are regarded as /very fast/ compared to the rates of diffusion across streamlines., (b) diffusion as rate-governing, in which the volume recombination rates within the boundary layer are /very slow/ compared to diffusion across streamlines . in either case the gas density near the surface of a blunt-nosed body is much higher than the density just outside the boundary layer, and the velocity and stagnation enthalpy profiles are much less sensitive to pressure gradient than in the more familiar case of moderate temperature differences . in fact, in case (a), the nondimensionalized enthalpy gradient at the surface is represented very accurately by the /classical/ zero pressure gradient value, and the surface heat-transfer rate distribution is obtained directly in terms of the surface pressure distribution . in order to illustrate the method, this solution is applied to the special cases of an unyawed hemisphere and an unyawed, blunt cone capped by a spherical segment . in the opposite limiting case where diffusion is rate-controlling the diffusion equation for each species is reduced to the same form as the low-speed energy equation, except that the prandtl number is replaced by the schmidt number . the simplifications introduced in case (a) are also applicable here, and the expression for surface heat transfer rate is similar., the maximum value of the ratio between the rate of heat transfer by diffusion alone and by heat conduction alone in the case of thermodynamic equilibrium is given by.. (prandtl no./schmidt no .) when the diffusion coefficient is estimated by taking a reasonable value of atom-molecule collision cross section this ratio is 1.30 . additional theoretical and (especially) experimental studies are clearly required before these simple results are accepted .'), (102, 'advantages and limitations of models .', 'sobey,a.j.', 'j. r. ae. s. 63, 1959, 646.', 'advantages and limitations of models . summary .. the use of models for structural test investigations in the presence of kinetic heating effects is examined . the principal features of the complex process to be represented are discussed under the classifications external air flow, internal heat transfer, elastic response . of these the second is found to influence most model design, and an analysis of a typical structure is included to illustrate the various contributions to internal heat transfer .'), (103, 'theory of mixing and chemical reaction in the opposed jet diffusion flame .', 'spalding,d.b.', 'a.r.s. jnl. 31, 1961, 763.', 'theory of mixing and chemical reaction in the opposed jet diffusion flame . an idealization of the flow system used by potter and butler is analyzed . the differential equation of mixing is solved exactly, to give the location of, and burning rate in, the flame . the solutions to the chemical kinetic differential equation are discussed, relations being derived between the jet flow rate at extinction, the chemical kinetic constants and the laminar flame speed in premixed gases . it is shown that the jet flow rate at extinction is independent of the transport properties . comparison is made with the experimental data of potter, heimel and butler . it is argued that experiments must be carried out at higher reynolds numbers if the measurements are to be quantitatively analyzable .'), (104, 'similar solutions of a free convection boundary layer equation for an electrically conducting fluid .', 'reeves,b.l.', 'a.r.s. jnl. 31, 1961, 517.', 'similar solutions of a free convection boundary layer equation for an electrically conducting fluid . author investigates the existence of a class of similar solutions for free convection from a vertical flat plate, such as are known for free convection in a nonconducting fluid . the magnetic field acts transversely to the fluid motion and is assumed to remain constant in the direction perpendicular to the plate . this introduces into the momentum equation a retarding force which is a function only of x, the distance along the plate length . for similarity it is found that the magnetic inductance must vary as . if the plate temperature is constant . if n = 0, the magnetic inductance is constant while the plate temperature increases linearly with x .'), (105, 'the asymptotic boundary layer on a circular cylinder in axial incompressible flow .', 'stewartson,k.', 'q.app.math. 13, 1955, 113.', 'the asymptotic boundary layer on a circular cylinder in axial incompressible flow . in this paper the incompressible boundary layer over a circular cylinder in an axial flow is investigated far from the leading edge . if u and v are the velocity components in the x and r direction respectively and a stream function is introduced by and, then for a constant free-stream velocity has the following asymptotic form .. where the p''s are determined successively, first for s=1 and all t, then s=2 and all t, etc., from ordinary differential equations . here and log c=euler''s constant . it is shown that the effect of the curvature of the body (in planes perpendicular to the flow) is to increase the skin friction . also the case in which the free-stream velocity is proportional to (at the method breaks down), is studied . it is concluded that the effect of the curvature of the cylinder, when the boundary layer has a thickness comparable with its radius of curvature, is to delay separation .'), (106, 'the transverse potential flow past a body of revolution .', 'campbell,i.j.', 'q.j.mech.app.math. 9, 1956, 140.', 'the transverse potential flow past a body of revolution . it is shown that in the potential flow of an incompressible inviscid fluid past a body of revolution set with its axis at right angles to the stream, the velocity components at the surface along and perpendicular to the meridians vary with azimuthal angle round the body in a simple manner . this is shown by entirely elementary considerations .'), (107, 'on the mixing of two parallel streams .', 'ting,lu.', 'j.math.phys. 38, 1959, 153.', 'on the mixing of two parallel streams . using the techniques of boundary-layer theory, the proper third boundary condition for the mixing of two parallel streams is derived from the compatibility condition of the higher order approximation . it is shown that the commonly adopted third boundary condition of balancing of transverse momentum is correct only for the mixing problem of two semi-infinite incompressible streams . for the fulfillment of the proper third boundary condition, the possibility of introducing the similar solution of blasius type is examined for various cases .'), (108, 'properties of the confluent hypergeometric function .', 'a. d. macdonald', '', 'properties of the confluent hypergeometric function . the confluent hypergeometric functions have proved useful in many branches of physics . they have been used in such problems involving diffusion and sedimentation, as isotope separation and protein molecular weight determinations in the ultracentrifuge . the solution of the equation for the velocity distribution of electrons in high frequency gas discharges may frequently be expressed in terms of these functions . the high frequency breakdown electric field may then be predicted theoretically for gases by the use of such solutions together with kinetic theory . this report presents some of the properties of the confluent hypergeometric functions together with six-figure tables of the functions .'), (109, 'the production of uniform shear flow in a wind tunnel .', 'owen,p.r. and zienkiewicz,h.k.', 'j.fluid mech. 2, 1957, 521.', 'the production of uniform shear flow in a wind tunnel . a nearly uniform shear flow was obtained in the working section of a wind tunnel by inserting a grid of parallel rods with varying spacing . the function of such a grid is to impose a resistance to the flow, so graded across the working section as to produce a linear variation in the total pressure at large distances downstream without introducing an appreciable gradient in static pressure near the grid . a method of calculating a suitable arrangement of the rods is described . although this method is strictly applicable only to weakly sheared flows, an experiment made with a grid designed for a shear parameter as large as 0.45 gave results in close agreement with the theory . there was no evidence from the experiment of any large-scale secondary flow accompanying the shear--a danger inherent in an empirical attempt to grade the resistance of the grid--nor was any tendency observed for the shear to decay with increasing distance from the grid .'), (110, 'dynamics of a dissociating gas .', 'lighthill,m.j.', 'j.fluid mech. 2, 1957, 1.', 'dynamics of a dissociating gas . this is a lucid introduction to the effects of dissociation in gas dynamics . the problem in view is that of air flow past a bluff body at speeds somewhat above 2 km sec . thermodynamic equilibrium is assumed,. theories of near equilibrium for transport properties and of large departures from equilibrium being promised in parts 2 and 3 . following a survey of the equilibrium statistical thermodynamics of a pure dissociating diatomic gas, a new model is introduced . this /ideal dissociating gas/ is characterized by only three constants, the characteristic temperature, density and internal energy for dissociation . physically, it may be regarded as having its vibrational modes always just half excited (so that at low temperatures the ratio of specific heats approaches 4 3 rather then 7 5) . thermodynamic properties of the ideal gas are derived, and the oblique shock wave relations deduced in the /strong-shock/ approximation (including an elegant relation between the principal curvatures of any bow shock and the subsequent vorticity) . useful relations are given for the isentropic changes that take place along streamlines between shocks . various of these results are applied to the problem typified by a sphere flying at high mach number . the newtonian impact theory and its empirical modification are dismissed as lacking theoretical basis, in favor of the limit for large values of both mach number and density ratio across the shock . it is suggested that the zero surface pressure sometimes predicted by the latter theory corresponds to separation not of the flow but of the shock wave from the surface . an estimate is given for the subsequent shape of the shock . finally, another approximation is applied to the region near the stagnation streamline . the fluid is assumed incompressible, but rotational in accord with the shock relations,. and it is shown that a spherical shock corresponds to a concentric spherical body . the resulting surface pressure is within 1 per cent of that predicted by freeman''s second approximation based on the newtonian-plus-centrifugal solution (same j. 1 (1956),'), (111, 'the laminar boundary layer equation: a method of solution by means of an automatic computer .', 'leigh,d.c.f.', 'proc. cam. phil. s. 51, 1955, 320.', 'the laminar boundary layer equation: a method of solution by means of an automatic computer . a method, very suitable for use with an automatic computer, of solving the hartree-womersley approximation to the incompressible boundary-layer equation is developed . it is based on an iterative process and the choleski method of solving a simultaneous set of linear algebraic equations . the programming of this method for an automatic computer is discussed . tables of a solution of the boundary-layer equation in a region upstream of the separation point are given . in the upstream neighbourhood of separation this solution is compared with goldstein''s asymptotic solution and the agreement is good .'), (112, 'steady motion of conducting fluids in pipes under transverse magnetic fields .', 'shercliff,j.a.', 'proc. cam. phil.s. 49, 1953, 136.', 'steady motion of conducting fluids in pipes under transverse magnetic fields . this paper studies the steady motion of an electrically conducting, viscous fluid along channels in the presence of an imposed transverse magnetic field when the walls do not conduct currents . the equations which determine the velocity profile, induced currents and field are derived and solved exactly in the case of a rectangular channel . when the imposed field is sufficiently strong the velocity profile is found to degenerate into a core of uniform flow surrounded by boundary layers on each wall . the layers on the walls parallel to the imposed field are of a novel character . an analogous degenerate solution for channels of any symmetrical shape is developed . the predicted pressure gradients for given volumes of flow at various field strengths are finally compared with experimental results for square and circular pipes .'), (113, 'acoustical signal detection in turbulent airflow .', 'smith,m.w. and lambert,r.f.', 'j.acous.s.am. 32, 1960, 858.', 'acoustical signal detection in turbulent airflow . improvement in detected signal-to-noise ratio is obtained for a periodic signal masked by additive noise and turbulent noise backgrounds . comparisons are made between autocorrelation, crosscorrelation, and a combination of frequency filtering and crosscorrelation . although the latter method provided the greatest improvement, the crosscorrelation technique was the most successful single method . it turned out that the maximum improvement obtainable was limited by the dynamic range of the correlator computer and not by errors due to finite averaging time and scanning the delay . the improvement for signals masked by turbulent noise was found to be about 5 db less than that obtained for additive noise .'), (114, 'response of plates to a decaying and convecting randon pressure field .', 'dyer,i.', 'j.acous.s.am. 31, 1959, 922.', 'response of plates to a decaying and convecting randon pressure field . following the methods of lyon, an analysis of the vibratory response of a plate to a random pressure field is given . the pressure correlation of the random field is assumed to have a scale small compared to the plate size, to decay exponentially, and to convect with constant speed over the plate . two cases are considered, one in which the convection speed is much less than the speed of free flexural waves in the plate, the other in which the convection speed is the same order as the flexural wave speed . the mean square plate displacement is shown to be relatively independent of convection for speeds much less than the flexural wave speed, and to increase significantly for speeds in the order of the flexural wave speed . it is shown that damping is usually, but not always, an effective means of vibration reduction . in the case of convection speeds much smaller than the flexural speed, the use of hysteretic damping for reduction of the displacement response is shown to be limited by the decay of the assumed random pressure field .'), (115, 'on turbulent lubrication .', 'constantinescu,v.n.', 'proc.inst.mech.e. 173, 1959, 881.', 'on turbulent lubrication . the paper concerns the hydrodynamic turbulent motion in the lubricant layer . proceeding from the reynolds equations and introducing the approximations currently used in lubrication problems, owing to the lubricant film thickness, the general motion equations for turbulent lubrication are written . using the prandtl mixing length hypothesis, exact and approximate solutions are obtained for the velocity distribution into the lubricant layer . the results are discussed by pointing out the pressure gradient and the reynolds number influence on the velocity distributions, as well as the differences with respect to the laminar flow . in order to obtain simple formulae, the exact dependence of the rate of flow on the pressure gradient into a dimensionless form is replaced by a linear relation, the slope of which depends on the reynolds number . this approximation allows the obtainment of the pressure differential equation under a simple form . the pressure equation is integrated in case of journal bearings, by assuming a constant or a variable viscosity of the lubricant . the results are compared to the experimental data obtained by m. i. smith and d. d. fuller and the good qualitative agreement is pointed out .'), (116, 'the elliptic cylinder in a shear flow with hyperbolic velocity profile .', 'jones,e.e.', 'q.j.mech.app.math. 12, 1959, 191.', 'the elliptic cylinder in a shear flow with hyperbolic velocity profile . the stream function for the shear flow with hyperbolic velocity profile past an elliptic cylinder has been determined as an infinite series of mathieu functions . it is found that the stagnation streamline of the flow is displaced towards a region of higher velocity, this displacement increasing the main stream, (2) as the stream becomes progressively non-uniform, (3) with increase of minor axis length when the major axis length remains invariant . in each case the displacement reaches a limiting value as the cylinder moves away from the axis of symmetry of the stream . these limiting values are reached at critical distances from the axis of symmetry, which decrease as the stream becomes progressively non-uniform, but these distances are approximately independent of incidence . the pressure coefficients and the resultant force and moment coefficients associated with the cylinder have also been obtained, and investigated numerically for the flat plate type of cylinder .'), (117, 'the motion of a viscous liquid past a paraboloid .', 'mather,d.j.', 'q.j.mech.app.math. 14, 1961, 423.', 'the motion of a viscous liquid past a paraboloid . an approximate solution for the steady flow of incompressible viscous liquid past a paraboloid of revolution is described . an assumption is made for the form of the stokes stream function and substituted into the navier-stokes equations using paraboloidal coordinates . after making suitable approximations, a non-linear differential equation for a function f is deduced . the solutions of this equation depend on the reynolds number of the flow considered . examples found by numerical integration are given to illustrate the properties of the function f for reynolds numbers varying from 0.0001 to is found, and it is shown that this approximate solution tends to the perfect fluid flow away from the boundary, allowance being made for the displacement effect of what may be called the boundary layer .'), (118, 'the transonic flow of a compressible fluid through an axially symmetrical nozzle .', 'tomotika,s. and hasimoto,z.', 'j.math.phys. 29, 1950, 105.', 'the transonic flow of a compressible fluid through an axially symmetrical nozzle . by a method similar to that developed by s. tomotika and k. tamada (quart. appl. math. 7, 381-397 (1950),. these rev. 11, 275) for computing two-dimensional mixed isentropic flows in the sonic region, the flow in the vicinity of the throat of an axially symmetrical nozzle is studied . several exact solutions to von karman''s equation for axially symmetrical transonic flows are obtained and the one that gives flows through a converging and diverging nozzle is considered in detail . this solution consists of four branches of which two are rejected because of singularities . of the remaining two branches, one gives pure supersonic flow and the other gives taylor''s type of flow with a local supersonic region in the throat . by varying a parameter, the latter branch approaches two asymptotes which yield meyer''s type of asymmetrical flows .'), (119, 'conduction of fluctuating heat flow in a wall consisting of many layers .', 'vodicka,v.', 'app.sc.res. 5, 1955, 108.', 'conduction of fluctuating heat flow in a wall consisting of many layers . van gorcum has pointed to interesting and important analogies between the theory of a passive four-pole and the conduction of heat waves through stratiform bodies . this paper generalizes in certain regards van gorcum''s ideas and draws their consequences for the case of a solid, bounded by two infinite parallel planes and consisting of any number of layers made from different materials .'), (120, 'measurement of convective heat transfer by means of the reynolds analogy .', 'granville,r.a. and boxall,g.', 'brit.j.app.phys. 11, 1960, 471.', 'measurement of convective heat transfer by means of the reynolds analogy . preston''s method for measuring skin friction in pipes has been extended to include non-uniform flow, with and without pressure gradients, over flat surfaces . by means of a modified form of the reynolds analogy, the local convective heat transfer coefficient can be related to the skin friction, and it is proposed that the method be used in aerodynamic models of furnaces and in heat transfer plant of simple geometry . more investigations are required of the effects of fluid turbulence, surface roughness and surface curvature on convective heat transfer and skin friction .'), (121, 'a theory for base pressures in transonic and supersonic flow .', 'korst,h.h.', 'j.app.mech. 23, 1956, 593.', 'a theory for base pressures in transonic and supersonic flow . a physical flow model is devised based on the concepts of interaction between the dissipative shear flow and the adjacent free stream and the conservation of mass in the wake . four flow components are integrated in the model,. namely, the flow approaching the trailing edge, the expansion around the trailing edge, the mixing within the free-jet boundary, and the recompression at the end of the wake . a unique and stable solution results for the base pressure . theoretical results obtained for thin approaching boundary layer do not require empirical information and are, therefore, best suited to evaluate the merits of the theory . here emphasized is the case of isoenergetic constant-pressure mixing in the turbulent free-jet boundary and agreement is found between theory and experimental data .'), (122, 'a simplified approximate method for the calculation of the pressure around conical bodies of arbitrary shape in supersonic and hypersonic flow .', 'willi f. jacobs', 'lockheed aircraft corporation, georgia division', 'a simplified approximate method for the calculation of the pressure around conical bodies of arbitrary shape in supersonic and hypersonic flow . exact conical-flow solutions are available only for circular cones at zero angle of attack . for nonaxisymmetric cones or cones at angle of attack, only approximate methods exist . these methods are generally quite complicated and further limited to certain body shapes or certain mach-number ranges . a great need was therefore felt for a simple approximate method applicable to any arbitrarily shaped conical body at zero incidence as well as at angle of attack . such a method has been developed recently at lockheed and is presented here in abbreviated form . the method is based on the /equivalent-cone/ theory . this theory determines the pressure on a conical body utilizing information for a symmetric cone at zero angle of attack with the same normal component of the free stream with respect to the surface as the local element of the body considered . this method works relatively well at high mach numbers . however, it is quite inconsistent at lower mach numbers, especially for bodies which deviate considerably from circular cones . the equivalent-cone method does not give satisfactory results, mainly due to the fact that it considers only the local surface element on the body independent of the other body elements in the newtonian-theory manner .'), (123, 'the downstream influence of mass transfer at the nose of a slender cone .', 'cresci,r.j. and libby,p.a.', 'j.aer.scs. 29, 1962, 815.', 'the downstream influence of mass transfer at the nose of a slender cone . the influence of localized mass transfer at the nose of a slender cone under hypersonic flow conditions has been studied by experimental and theoretical means . two gaseous coolants, nitrogen and helium, are injected through a porous plug subtending a half angle of 30 . the effect of the mass transfer on the shock shape, pressure distribution, heat transfer, and transition are investigated . the experimental work involved tests in the mach-number-8.0 tunnel at pibal . the theoretical analysis involved a study of the effect of mass transfer on the shock stand-off distance and leads to an inviscid-flow parameter permitting the experimentally determined shock shape and pressure distribution to be extrapolated to other than test conditions and to other coolant gases . there is obtained the maximum value of this parameter resulting in no significant alteration of the pressure distribution on the cone and thus defining the flows in which boundary-layer-type similarity applies . significant reductions in heat transfer are obtained with injection . indeed, with small amounts of helium injection the peak heating is found to occur downstream on the cone and to be an order of magnitude less than would occur at the stagnation point without mass transfer . with nitrogen early transition is found to occur, so that local heating rates are actually increased over those prevailing at the same reynolds number without injection .'), (124, 'a summary of the supersonic pressure drag of bodies of revolution .', 'morris,d.n.', 'j.aero.scs. 28, 1961, 563.', 'a summary of the supersonic pressure drag of bodies of revolution . a number of approximate theories for supersonic and hypersonic flow over bodies of revolution at zero angle of attack are appraised by a critical comparison with characteristics and second-order results, with the use of hypersonic similarity as a basis for the comparison . most of the approximate theories are inadequate except over very limited ranges of fineness ratio and mach number . the combination of second-order supersonic theory and second-order shock-expansion theory provides consistently good results throughout the supersonic speed range . on the basis of exact (or nearly exact) supersonic solutions and a limited amount of test data and theory in the transonic region, summary design curves are developed that give the pressure drag of conical and ogive noses and conical and ogive boattails over the complete range of transonic, supersonic, and hypersonic mach numbers . other shapes can be analyzed in the same manner, provided that an equivalent amount of data is available . the analysis is made with the assumption of inviscid flow, so that the effects of boundary-layer growth, shock boundary-layer interaction, and flow separation are not included . the present correlations provide a sound basis of inviscid-flow results from which these additional viscous effects can be evaluated .'), (125, 'measurements of skin friction of the compressible turbulent boundary layer on a cone with foreign gas injection .', 'pappas,c.c. and okuno,a.f.', 'j.aero.scs. 27, 1960, 321.', 'measurements of skin friction of the compressible turbulent boundary layer on a cone with foreign gas injection . measurements of average skin friction of the turbulent boundary layer have been made on a 15 total included angle cone with foreign gas injection . measurements of total skin-friction drag were obtained at free-stream mach numbers of 0.3, 0.7, 3.5, and x 10 with injection of helium, air, and freon-12 through the porous wall . substantial reductions in skin friction are realized with gas injection within the range of mach numbers of this test . the relative reduction in skin friction is in accordance with theory--that is, the light gases are most effective when compared on a mass flow basis . there is a marked effect of mach number on the reduction of average skin friction,. this effect is not shown by the available theories . limited transition location measurements indicate that the boundary layer does not fully trip with gas injection but that the transition point approaches a forward limit with increasing injection . the variation of the skin-friction coefficient, for the lower injection rates with natural transition, is dependent on the flow reynolds number and type of injected gas,. and at the high injection rates the skin friction is in fair agreement with the turbulent boundary-layer results .'), (126, 'an investigation of two-dimensional supersonic base pressures .', 'charwat,a.f. and yakura,j.k.', 'j.aero.scs. 25, 1958, 122.', 'an investigation of two-dimensional supersonic base pressures . an investigation of the base pressure behind wedges at mach numbers 2 and 3 in the laminar and the transitional regime is reported . temperature and velocity traverses through the mixing zone are shown and exploratory investigations of the wake vortex by use of hot wires and flow-visualization techniques are described . it is found that the laminar two-dimensional base pressure agrees well with chapman''s theoretical predictions . the shear layer exhibits gross velocity distributions characteristic of the free jet mixing zone, but also shows disturbances that originate in the expansion-turning of the oncoming boundary layer . an interesting trailing vortex is observed, which is explained in terms of nonuniform mixing rate in the wake .'), (127, 'supersonic axially symmetric nozzles .', 'clippinger,r.f.', 'b. r. l. r794, 1951.', 'supersonic axially symmetric nozzles . at each of twenty-one exit mach numbers, ranging from 1.008 to 8.238, ten supersonic axially symmetric nozzle shapes with plane sonic surfaces have been computed on the eniac by the method of characteristics . the boundary of the shortest of each group of ten has a sharp edge at the sonic plane, while the others have smooth boundaries . this report describes the computational procedures and presents a sample of the results for twenty nozzles . more extensive and elaborate tables of the results of the entire computations are available at the ballistic research laboratories . nozzle contours can be obtained accurately from them by interpolation for exit mach numbers between 1.479 and 8.238 for a wide range of ratios of nozzle length to throat diameter .'), (128, 'effects of free stream vorticity on the behaviour of a viscous boundary layer .', 'li,t-y.', 'j.aero.scs. 23, 1956, 1128.', 'effects of free stream vorticity on the behaviour of a viscous boundary layer . theoretical investigation is considered of the two-dimensional steady flow field at large distance from a finite object set in a viscous incompressible fluid . study is made of coordinate-type expansions for pressure and velocity for large r, uniformly in, for fixed reynolds number, assuming exact boundary conditions at infinity and regularity of flow with zero net mass flow across a simple curve enclosing the object . mathematical nature of the distinction between parameter and coordinate-type expansions is discussed with description of inner and outer expansions and matching techniques . a feature of the expansion procedure is the introduction of an artificial parameter . inner and outer expansions are matched with the aid of known solutions of the navier-stokes equations . analysis requires simple consideration of the heat and laplace equations without resort to special methods . paper is worth studying by those interested in asymptotic expansion procedures .'), (129, 'an investigation of the noise produced by a subsonic air jet .', 'j. h. gerrard', 'university of manchester', 'an investigation of the noise produced by a subsonic air jet . to investigate the theoretical predictions of lighthill on aerodynamic sound, measurements have been made of the sound field of a 1 in. air jet issuing from a long pipe . the measurements have been made over a wide frequency band (30 to 10,000 cycles/sec.) and in one-third octave bands in this frequency range . the mean mach number at the pipe orifice was varied from 0.3 to 1.0 . the dependence of the apparent position of the noise sources on frequency and jet speed was investigated . at a given frequency a source is situated farther from the jet orifice the higher the jet speed . lower frequency sources appear farther downstream than ones of higher frequency, consistent with their association with larger eddies . the directional characteristics of the sound field at different frequencies and jet speeds are illustrated by means of scale diagrams showing lines of constant sound intensity . these sound fields are analyzed in terms of the moving quadrupole sources of lighthill''s theory and good agreement obtained . it is shown that the apparent spread of the sources at low frequencies is due to the doppler effect . at low frequency relative to the frequency of maximum power output) the radiation is predominantly that of three mutually orthogonal longitudinal quadrupoles which, except for the effect of convection upon it, has a sound field like a monopole source . at higher frequencies the sound fields of lateral and longitudinal quadrupoles predominate .'), (130, 'the behaviour of non-linear systems .', 'clauser,f.h.', 'j.aero. scs. 23, 1958, 411.', 'the behaviour of non-linear systems . many of the phenomena that occur in the world around us are governed by nonlinear relationships . in the development of the mathematical sciences, the difficulties of nonlinear analysis have hindered the formulation of nonlinear concepts that would permit us to understand such phenomena . in the present article, our progress in understanding the behavior of nonlinear systems is reviewed and an attempt is made to present the resulting concepts in such a way that they may be applied with some generality to other problems .'), (131, 'two-dimensional jet mixing of a compressible fluid .', 'pai,s.i.', 'j.aero.scs. 16, 1949, 463.', 'two-dimensional jet mixing of a compressible fluid . the mixing and divergence of a supersonic jet exhausting into a supersonic stream are investigated theoretically . in the first part of this paper, the flow is assumed to be laminar . when the velocity and temperature in the jet are different slightly from those of the surrounding stream, by the method of small perturbations and under ordinary boundary layer assumptions, the equation of motion of two-dimensional flow will be reduced to a form of the well-known equation of heat conduction, whose solution is known for any given boundary conditions . it has also been shown that the exact solution of the two dimensional jet mixing of viscous compressible fluids can be obtained by successive approximations starting with the solution of small perturbations . velocity and temperature distributions for two cases--one is the mixing of two-uniform flows and the other is the mixing of a jet of compressible fluid from a two-dimensional nozzle with full expansion exhausting into a supersonic stream--have been calculated . the properties of the jet mixing depend mainly on the momentum of the jet regardless of whether the change of momentum is due to the change of velocity or the change of temperature--i.e., the change of density . compressibility has a considerable effect on the properties of the jet . in the second part, the cases of turbulent flow are investigated . by means of reichardt''s theory of free turbulence, the turbulent shearing stress may be expressed as it has been shown in this paper that where is a constant that can be determined experimentally . the value of n lies between 0 and 1 . the exact value of n depends on the condition of mixing . when the expression of turbulent shearing stress given above is used instead of the viscous stress in the equation of motion, by suitable transformation of variables, it has been shown that the equation of two-dimensional turbulent jet mixing is identical to that of the laminar case . hence, the solution of the first part of this paper can be applied to the turbulent case, provided that the characteristic constants and n have been properly chosen .'), (132, 'viscosity effects in sound waves of finite amplitude: in survey in mechanics .', 'lighthill,m.j.', 'ed. by g.k.batchelor and r.m.davies. c.u.p. 1956.', 'viscosity effects in sound waves of finite amplitude: in survey in mechanics . this article has as its subject /the conflicting influence on sound propagation of convection on the one hand, and of diffusion and relaxation on the other/, whose importance in the determination of the structure of shock waves was first appreciated clearly by sir geoffrey taylor . as an essential introduction to the main topics, author gives an exceptionally clear and valuable account of the physical mechanisms of viscosity, thermal conductivity, and other diffusion effects, including relaxation . the classical theory of shock-wave formation is then discussed, and some extensions are made . the remainder of the article is based on the demonstration that the nonlinear equation for plane progressive sound waves, in which convection and diffusion are taken into account to a first approximation, can be transformed into burgers''s equation, the general solution of which was given by hopf and cole . this approach, in which all flows are continuous (they become discontinuous at shock waves in the limit as viscosity, etc., tend to zero), allows the author to re-derive and extend whitham''s theory of the formation and decay of weak plane shock waves, and to derive many new results, such as the velocity distributions during the union of two shock waves and during the formation of a shock wave . the application of the same idea to non-plane shock waves is also discussed, but more briefly,. in these cases, burgers''s equation is not quite such a good approximation as before . the article concludes with sections on sound waves whose reynolds numbers based on the length scale of the flow and the velocity amplitude are comparable with unity, and on the effects of relaxation on the properties of shock waves . the whole is much more than a survey, and represents a very substantial advance in the theory of sound waves . it is the finest possible tribute to sir geoffrey taylor that he should be able to inspire articles such as this and the others in this volume .'), (133, 'some effects of surface curvature on laminar boundary layer flow .', 'murphy,j.s.', 'j.aero.scs. 20, 1953, 338.', 'some effects of surface curvature on laminar boundary layer flow . the laminar flow of a viscous incompressible fluid over a two-dimensional curved surface is investigated for two cases, one in which the curvature is /large/ and the other in which it is cases are obtained as approximations from the exact equations of motion by an order-of-magnitude analysis . these equations are solved for flow over a particular surface with zero surface pressure gradient . in this analysis, the pressure gradient normal to the surface is included, and the outer boundary conditions are modified in accordance with the requirements of flow over a curved surface . the results indicate that for equal reynolds numbers, the stress on convex surfaces is less than the flat-plate value, while the stress on concave surfaces is greater than for a flat plate . the most important effect of surface curvature, for the cases considered, is the modification of the shape of the velocity profile near the /outer edge/ of the boundary layer . the requirement that a smooth transition exist between the viscous flow and the potential flow at the outer edge of the layer causes the profile to have a negative slope near the outer edge for convex surface curvature and a positive slope for concave surface curvature .'), (134, 'note on an interaction between the boundary layer and the inviscid flow .', 'antonio ferri and paul a. libby', 'department of aeronautical engineering and applied mechanics, polytechnic institute of brooklyn, brooklyn, n.y.', 'note on an interaction between the boundary layer and the inviscid flow . according to the classical boundary-layer theory the flow about bodies at reynolds numbers of aeronautical interest can be considered as composed of two regimes.. an outside inviscid flow and a thin boundary-layer region adjacent to the body . this point of view leads to the approximation that, on a slightly curved surface, throughout the layer is negligibly small . the additional assumption that the inviscid flow is irrotational leads to the requirement that is zero at the outer edge of the boundary layer . in this theory any interaction between the two regimes is accountable by a simple correction to the body shape based on the boundary-layer displacement thickness . recently, in connection with hypersonic laminar boundary layers, this classical point of view has been modified., an interaction between the two flow regimes leading to a self-induced axial pressure gradient has been considered . it is the purpose of the present note to point out another type of interaction which may be of practical importance and of fundamental interest even at mach numbers below those considered in the hypersonic boundary-layer theory and which may have to be considered in that theory .'), (135, 'the calculation of wall shearing stress from heat-transfer measurements in compressible flows .', 'nick s. diaconis', 'lewis flight propulsion laboratory, naca, cleveland, ohio', 'the calculation of wall shearing stress from heat-transfer measurements in compressible flows . it has been shown by ludwieg that the wall shearing stress of a laminar or turbulent boundary layer in an incompressible flow can be determined from a heat-transfer measurement at the surface . the instrument used in that investigation was essentially a small, locally insulated, heating element embedded in the test surface . the size of the instrument was restricted by the condition that the thermal boundary layer generated by the heating element be contained locally within the laminar sublayer . in the present analysis ludweig''s theory for such an instrument is extended to compressible flow over an insulated flat plate . with the same limitations on the design and operation of the instrument as mentioned above, it can also be assumed for compressible laminar and turbulent boundary layers that only the flow in the immediate vicinity of the wall or the laminar sublayer will be affected in the region of the heated element . this assumption then permits the use of the laminar boundary-layer equations as the governing equations for this analysis for both laminar and turbulent boundary layers .'), (136, 'recent developments in rocket nozzle configurations .', 'roa,g.v.r.', 'a.r.s.jnl. 31, 1961,1488.', 'recent developments in rocket nozzle configurations . existing configurations of supersonic portion of rocket nozzles are described and compared . survey covers bell-type conical and contoured nozzles, annular nozzles, plug nozzles, and the author''s own /e-d/ (expansion-deflection) nozzle . the latter is a bell-type nozzle in which the gases are first deflected radially outward by a small central plug, then expanded radially inward around the base of the plug, and finally deflected back to a nearly axial direction by the nozzle wall, in compressive turning .'), (137, 'the generation of sound by aerodynamic means .', 'curle,n.', 'j.roy.ae.s. 65, 1961, 724.', 'the generation of sound by aerodynamic means . a summary is given of some of the more important experimental results relating to the noise radiated from a cold subsonic turbulent jet . these are then related to the predictions of lighthill''s general theory of aerodynamic noise .'), (138, 'wakes in axial compressors .', 'pearson,h. and mckenzie,a.b.', 'j.roy.ae.s.63, 1959, 415.', 'wakes in axial compressors . the tendency in the past has been to assume that when wakes or non-uniform total head profiles are fed into an axial compressor then substantially constant static pressure prevails at the entry, the variations in total head appearing as variations in velocity . this variation in velocity causes variation in incidence on the early stage blade rows and thus can give rise to excitation of blade vibration . this assumption is implicit, for instance, in references 1 and 2, but we think has been a common assumption by most of the people working in this field . where the compressor is fed by a duct of substantially parallel walls for a reasonable length ahead, such an assumption appeared justifiable . such a duct when given an air flow test with its outlet discharging, for instance, to atmosphere instead of to the compressor, then the distribution assumed would normally be obtained and in fact many surveys of such ducts have been represented in this fashion . the object of this note is to show that, in fact, this distribution will not normally occur when the compressor is present and we may normally expect much more nearly a constant velocity into the compressor with attendant static pressure distributions to match with the total head variations ahead of the intake, with of course, the attendant curved flow to support the static pressure gradients .'), (139, 'viscous effects on pitot tubes at low speeds .', 'mcmillan,f.a.', 'j.roy.ae.s.58, 1954, 570.', 'viscous effects on pitot tubes at low speeds . measurements were made of the pressure in a blunt-nosed pitot tube, in an air stream at reynolds numbers from about 15 to 1000 . the results are expressed in terms of a pressure coefficient density of the fluid, and p and v are the static pressure and velocity in the undisturbed stream . as found in previous investigations, becomes greater than 1 at low reynolds numbers, the increase being about at a reynolds number of 50 (based on external tube radius) . in disagreement with the work of hurd, chesky, and shapiro, no decrease of below 1 was found at any reynolds number . when the values of found by various experiments are plotted against reynolds numbers based on internal tube radius, it is found that the curves are in closer agreement than when the external radius is used .'), (140, 'the determination of turbulent skin friction by means of pitot tubes .', 'preston,j.h.', 'j.roy.ae.s. 58, 1954, 109.', 'the determination of turbulent skin friction by means of pitot tubes . a simple method of determining local turbulent skin friction on a smooth surface has been developed which utilises a round pitot tube resting on the surface . assuming the existence of a region near the surface in which conditions are functions only of the skin friction, the relevant physical constants of the fluid and a suitable length, a universal non-dimensional relation is obtained for the difference between the total pressure recorded by the tube and the static pressure at the wall, in terms of the skin friction . this relation, on this assumption, is independent of the pressure gradient . the truth and form of the relation were first established, to a considerable degree of accuracy, in a pipe using four geometrically similar round pitot tubes--the diameter being taken as representative length . these four pitot tubes were then used to determine the local skin friction coefficient at three stations on a wind tunnel wall, under varying conditions of pressure gradient . at each station, within the limits of experimental accuracy, the deduced skin friction coefficient was found to be the same for each pitot tube, thus confirming the basic assumption and leaving little doubt as to the correctness of the skin friction so found . pitot traverses were then made in the pipe and in the boundary layer on the wind tunnel wall . the results were plotted in two non-dimensional forms on the basis already suggested and they fell close together in a region whose outer limit represented the breakdown of the basic assumption, but close to the wall the results spread out, due to the unknown displacement of the effective centre of a pitot tube near a wall . this again provides further evidence of the existence of a region of local dynamical similarity and of the correctness of the skin friction deduced from measurements with round pitot tubes on the wind tunnel wall . the extent of the region in which the local dynamical similarity may be expected to hold appears to vary from about to of the boundary-layer thickness for conditions remote from, and close to, separation respectively .'), (141, 'free-flight techniques for high speed aerodynamic research .', 'hamilton,j.a. and hufton,p.a.', 'j.roy.ae.s. 60, 1956, 151.', 'free-flight techniques for high speed aerodynamic research . the development rocket-borne and rocket-launched high-speed airplane model test is described . details of airborne components, telemetering units, tracking, and their calibration are also discussed . tests on controls, drag measurements, longitudinal stability evaluations, lift measurements, pressure measurements, aeroelastic estimations, and sonic bang recordings are effected . the reynolds numbers involved are much higher than are usual in the wind tunnel, and extensions of mach numbers are obtained beyond the tunnel limits, both free of the tunnel wall interference .'), (142, 'the problem of aerodynamic heating .', 'van driest,e.r.', 'aero.eng.rev. 15, 1956.', 'the problem of aerodynamic heating . paper is a good review of knowledge to date on convective heat transfer to objects moving through air at low and high speeds . theoretical and experimental information is given on recovery factors and heat-transfer coefficients for isothermal surfaces of unswept flat plates, wedges and cones with attached shock waves, and stagnation points of blunt bodies of revolution, for both laminar and turbulent boundary layers . a convenient nomograph for calculating flat plate turbulent boundary-layer heat-transfer coefficients is given . effects of surface cooling, surface roughness, and supply stream turbulence on transition are discussed and shown graphically .'), (143, 'interplanetary orbits .', 'vertregt,m.', 'j.brit.inter.s. 16, 1958, 326.', 'interplanetary orbits . the basic equations under simplified conditions for interplanetary flight are derived . for a voyage from planet to planet an unlimited number of orbits is possible . in order to give a clear survey of these possible orbits a diagram is developed from which the approximate energy-requirement, the duration, and other particulars of a voyage can be easily found .'), (144, 'heat flow in composite slabs .', 'mayer,e.', 'j.am.r.s. 22, 1952, 150.', 'heat flow in composite slabs . this paper presents the solution of the heat flow problem in composite walls under heat transfer conditions which are typical of uncooled rocket engine walls . analytic expressions in the form of fourier sums are obtained for the temperature distribution in a composite wall consisting of an inner (refractory) medium and an outer metallic) medium under newtonian heat transfer into the first medium with negligible heat transfer from the second medium to the exterior . the expressions obtained are based on a plane parallel composite slab as a representative model for relatively thin cylindrical walls, with thickness-to-radius ratio not exceeding 0.2 . the general results for the composite slab are simplified for the limiting cases of a thin refractory shield with a thick shielded medium and a thick refractory shield with a thin shielded medium .'), (145, 'skin friction in the laminar boundary layer in compressible flow .', 'young,a.d.', 'aero.quart. 1, 1949, 137.', 'skin friction in the laminar boundary layer in compressible flow . from an analysis of the work of crocco and others, semi-empirical formulae are derived for the skin friction on a flat plate at zero incidence with a laminar boundary layer . these formulae are for the general case of heat transfer, and when there is no heat transfer . the problem of heat transfer and the effect of radiation are discussed in the light of these formulae . the second formula is then utilised in the development of an approximate method for solving the momentum equation of the boundary layer on a cylinder without heat transfer . the method indicates that with increase of mach number there is a marked forward movement of separation from a flat plate in the presence of a constant adverse velocity gradient .'), (146, 'supersonic flow past slender bodies with discontinuous profile slope .', 'fraenkel and portnoy.', 'aero.quart. 6, 1955, 114.', 'supersonic flow past slender bodies with discontinuous profile slope . ward''s slender-body theory is extended to derive first approximations to the external forces on slender bodies of general cross section with discontinuous profile slope . two classes of body are considered .. bodies whose profile (typified by the local radius) is continuous between the nose and base, and certain bodies whose profile is discontinuous, such as bodies with annular or side air intakes and wing-bodies on which the wing has an unswept leading edge . (where air intakes are concerned, it is assumed that they are sharp-edged and that there is no /spillage/ of the internal flow) . the following conclusions apply to the former class of bodies . the variation of drag with mach number is found to depend only on the discontinuities in the longitudinal rate of change of the cross-sectional area, and is thus independent of cross-sectional shape . the drag itself is unchanged if the direction of the flow is reversed . the expressions for lift and moment assume the same forms as for smooth pointed bodies, the lift depending only on conditions at the base of the body . the general theory is applied to winged bodies of revolution with an unswept wing leading edge .. the results bear a marked resemblance to those obtained by ward . the results for wings alone are seen to be applicable, with one modification, to subsonic as well as to supersonic speeds .'), (147, 'supersonic flow past slender pointed wings with ?similar? cross sections at zero lift .', 'lord,w.t. and brebner,g.g.', 'aero.quart. 10, 1959, 79.', 'supersonic flow past slender pointed wings with ?similar? cross sections at zero lift . some recent theoretical work on slender pointed wings at zero lift is co-ordinated and extended . the wings considered may have any pointed plan form shape, provided that the trailing edge is straight and unswept . the root section profile and cross-section shapes are arbitrary, provided that, on any one wing, the latter are /descriptively similar/ (diamond or parabolic biconvex for instance), though not necessarily geometrically similar . the chief aim of the work is to find wings with simple geometry, low wave drag and pressure distributions which are unlikely to be seriously affected by viscous effects . wave drag and pressure distributions are calculated by slender-wing theory . general formulae, which are both simple and instructive, are given for the wave drag and the overall pressure distribution, with particular emphasis on the root pressure distribution . results for a number of wings of special interest are presented and discussed .'), (148, 'on displacement thickness .', 'lighthill,m.j.', 'j.fluid mech. 4, 1958, 383.', 'on displacement thickness . four alternative theoretical treatments of ''displacement thickness'', and, generally, of the influence of boundary layers and wakes on the flow outside them, are set out, first for two-dimensional, and then for three-dimensional, laminar or turbulent, incompressible flow . they may be called the methods of ''flow reduction'', ''equivalent sources'', ''velocity comparison'' and the principal expression obtained for the displacement thickness in three-dimensional flow may be written if, as orthogonal coordinates (x,y) specifying position on the surface, we choose x as the velocity potential of the external flow, and y as a coordinate, constant along the external-flow streamlines, such that h dy is the distance between (x,y) and z is the distance from the surface, u and v are the x and y components of velocity, and u takes the value u just outside the boundary layer .'), (149, 'expansions at small reynolds number for the flow past a sphere and a circular cylinder .', 'proudman,i. and pearson,j.r.a.', 'j.fluid mech. 2, 1957, 237.', 'expansions at small reynolds number for the flow past a sphere and a circular cylinder . this paper is concerned with the problem of obtaining higher approximations to the flow past a sphere and a circular cylinder than those represented by the well-known solutions of stokes and oseen . since the perturbation theory arising from the consideration of small non-zero reynolds numbers is a singular one, the problem is largely that of devising suitable techniques for taking this singularity into account when expanding the solution for small reynolds numbers . the technique adopted is as follows . separate, locally valid the regions close to, and far from, the obstacle . reasons are presented for believing that these ''stokes'' and ''oseen'' expansions are, respectively, of the forms where are spherical or cylindrical polar coordinates made dimensionless with the radius of the obstacle, r is the reynolds number, and and vanish with r . substitution of these expansions in the navier-stokes equation then yields a set of differential equations for the coefficients and, but only one set of physical boundary conditions is applicable to each expansion (the no-slip conditions for the stokes expansion, and the uniform-stream condition for the oseen expansion) so that unique solutions cannot be derived immediately . however, the fact that the two expansions are (in principle) both derived from the same exact solution leads to a ''matching'' procedure which yields further boundary conditions for each expansion . it is thus possible to determine alternately successive terms in each expansion . the leading terms of the expansions are shown to be closely related to the original solutions of stokes and oseen, and detailed results for some further terms are obtained .'), (150, 'integration of the boundary layer equations .', 'meksyn,d.', 'proc.roy.s.a. 237 1956, 543.', 'integration of the boundary layer equations . the equations of the boundary layer are integrated by an expression of the form where f(x) is a positive function with x=0 as the stationary point,. (x) is slowly varying,. the integral contains an unknown parameter which is found from the condition . the integral is evaluated by the method of steepest descent . the expressions obtained are usually divergent, except in few cases which include blasius''s equation,. the divergent expressions are summed by euler''s transformation . to check the procedure it is applied to falkner and skan''s equation . the results obtained are very striking,. few terms in the expansions are sufficient to obtain close agreement with hartree''s laborious numerical computations . the method is also applied to the general boundary-layer equation for the case of flow past an elliptic cylinder, measured by schubauer . the results obtained are in close agreement with schubauer''s measurements for the velocities, almost up to separation, for the position of the separation point,. and in satisfactory agreement downstream of separation .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (151, 'the generation of noise by isotropic turbulence .', 'proudman,j.', 'proc.roy.s.a 214,1952,119.', 'the generation of noise by isotropic turbulence . a finite region, with fixed boundaries, of an infinite expanse of compressible fluid is in turbulent motion . this motion generates noise and radiates it into the surrounding fluid . the acoustic properties of the system are studied in the special case in which the turbulent region consists of decaying isotropic turbulence . it is assumed that the reynolds number of the turbulence is large, and that the mach number is small . the noise appears to be generated mainly by those eddies of the turbulence whose contribution to the rate of dissipation of kinetic energy by viscosity is negligible . it is shown that the intensity of sound at large distances from the turbulence is the same as that due to a volume distribution of simple acoustic sources occupying the turbulent region . in this analogy, the whole fluid is to be regarded as a stationary and uniform acoustic medium . the local value of the acoustic power output p per mass of turbulent fluid is given approximately by the formula where a is a numerical constant, u is the mean-square velocity fluctuation, is the time, and c is the velocity of sound in the fluid . the constant a is expressed in terms of the well-known velocity correlation function f(r) by assuming the joint probability distribution of the turbulent velocities and their first two time-derivatives at two points in space to be gaussian . the numerical value is then obtained by substituting the form of f(r) corresponding to heisenberg''s theoretical spectrum of isotropic turbulence . it is found that the effects of decay make only a small contribution to the value of a, and that the order of magnitude of a is not changed when widely differing forms of the function f(r) are used .'), (152, 'on the flow of compressible fluid past an obstacle .', 'lord rayleigh, o.m., f.r.s.', '', 'on the flow of compressible fluid past an obstacle . it is well known that according to classical hydrodynamics a steady stream of frictionless incompressible fluid exercises no resultant force upon an obstacle, such as a rigid sphere, immersed in it . the development of a /resistance/ is usually attributed to viscosity, or when there is a sharp edge to the negative pressure which may accompany it (helmholtz) . in either case it would seem that resistance involves something of the nature of a wake, extending behind the obstacle to an infinite distance . when the system of disturbed velocities, although it may mathematically extend to infinity, remains as it were attached to the obstacle, there can be no resistance . the absence of resistance is asserted for an incompressible fluid., but it can hardly be supposed that a small degree of compressibility, as in water, would affect the conclusion . on the other hand, high relative velocities, exceeding that of sound in the fluid, must entirely alter the conditions . it seems worth while to examine this question more closely, especially as the first effects of compressibility are amenable to mathematical treatment .'), (153, 'on the steady motion of viscous, incompressible fluids, with particular reference to a variation principle .', 'millikan,c.b.', 'phil.mag. 7, 1929, 641.', 'on the steady motion of viscous, incompressible fluids, with particular reference to a variation principle . except in exceptional cases, it is not possible to represent the motion of a viscous incompressible liquid by means of a variation principle, but all cases of such motion that have yet been discovered belong to this class of /exceptional cases ./ the appropriate functions are given .'), (154, 'velocity and temperature distributions in the turbulent wake behind a heated body of revolution .', 'hall,a.a. and hislop,g.s.', 'proc.cam.phil.s. 34, 1938, 345.', 'velocity and temperature distributions in the turbulent wake behind a heated body of revolution . recently (see abstract 954 (1938)) goldstein made calculations based on theories of vorticity transfer, of the distributions of velocity and temperature in the turbulent wake behind a heated body of revolution, and the present authors now record an experimental determination of these distributions in a low-turbulence wind tunnel . difficulty was experienced in obtaining a truly symmetrical wake and observations have been reduced to mean values, curves of which are given .'), (155, 'on the solution of the laminar boundary layer equations .', 'howarth,l.', 'proc.roy.s.a, 164, 1938, 547.', 'on the solution of the laminar boundary layer equations . the problem of the flow along a flat plate placed edgewise to a steady stream, when a retarding pressure gradient varying linearly as the distance x from the leading edge of the plate is superposed is discussed . if y denotes distance measured perpendicular to the plate, a solution is obtained in the form of a power series in x where coefficients are functions of . differential equations are obtained for these coefficients . seven of the coefficients have been obtained with reasonable accuracy, and the eighth and ninth roughly . unfortunately it appears that about eight more terms are required to carry the solution to the point of separation,. the work involved in their determination is prohibitive . two approximate methods have been developed for determining the error when the first seven terms of the series are used as an approximation . these methods lead to the determination of the point of separation and are in agreement as to its position . if is the velocity at the edge of the boundary layer at the leading edge of the plate and is the velocity gradient, separation is found when . a method is developed for the solution of the boundary layer equations in any retarded region . it is obtained by replacing the velocity distribution at the edge of the boundary layer by a circumscribing polygon of infinitesimal sides and applying the preceding solution to each of these sides, making the momentum integral continuous at each vortex . the problem is thereby reduced to the solution of a first order differential equation .'), (156, 'the effect of shallow water on wave resistance .', 'havelock,t.h.', 'proc.roy.s.a, 100,1922,499.', 'the effect of shallow water on wave resistance . the general character of experimental results dealing with the effect of shallow water on ship resistance may be stated briefly as follows ..--at low velocities the resistance in shallow water is greater than in deep water, the speed at which the excess is first appreciable varying with the type of vessel . as the speed increases, the excess resistance increases up to a maximum at a certain critical velocity, and then diminishes . with still further increase of speed, the resistance in shallow water ultimately becomes, and remains, less than that in deep water at the same speed . the maximum effect is the more pronounced the shallower the water . for further details and references one may refer to standard treatises, but one quotation may be made in regard to the critical velocity .. /this maximum appears to be at about a speed such that a trochoidal wave travelling at this speed in water of the same depth is about times as long as the vessel . it was at one time supposed that the speed for maximum increase in resistance was that of the wave of translation . this, however, holds only for water whose depth is less than for greater depths the speed of the wave of translation rapidly becomes greater than the speed of maximum increase of resistance ./ in a recent analysis of the data, h. m. weitbrecht expresses a similar conclusion by stating that for each depth of water there is a critical velocity, but that the critical velocity does not vary as the square root of the corresponding depth .'), (157, 'the hodographic transformation in transonic flow .', 'lighthill,m.j.', 'proc. roy.s. a, 191, 1947, 323.', 'the hodographic transformation in transonic flow . the author studies the problem of finding the shape of a symmetrical nozzle with the velocity along the axis (x-axis) specified . the velocity along each streamline is assumed to increase steadily . the singularity at the sonic velocity and to the axis of the nozzle) is first studied in the physical plane by using a power series in . in the hodograph plane, the two characteristics of the hodograph differential equation passing through the sonic point and are lines of branch points . the region between these lines is a region of triple-valuedness for the stream function . outside this region is single-valued . there are also singularities at the sonic point and the point corresponding to the specified condition at the exit of the nozzle . the author then proposes to construct in the hodograph plane by at the exit velocity and (3) a finite sum regular throughout .. sin, where r is the square of the velocity and the are hypergeometric functions . the a''s are fixed by the required approximation to the specified velocity distribution along the axis . this solution is single-valued, convergent and represents except a region near the sonic point in the nozzle . for this excluded region, the author inverts the solution to obtain a power series in for 0 . this is shown to be convergent for the region of interest . the type of solution considered by the author gives a nozzle having an infinitely long supersonic part .'), (158, 'temperature charts for induction and constant temperature heating .', 'heisler,m.p.', 'a.s.m.e.trans. 14, 1947, 227.', 'temperature charts for induction and constant temperature heating . charts are presented for determining complete temperature historics in spheres, cylinders, and plates . it is shown that for values of the dimensionless time ratio x greater than 0.2 the heating equations reduce to such a simple form that for each shape two charts which give temperatures at any position within the heated or cooled bodies can be plotted . it is also shown that the usual simple heating and cooling charts can also be used for the determination of temperatures and heating times in bodies heated by a constant rate of heat generation at the surface (induction heating) . finally, a two dimensional chart is given for finding heating times in short cylinders, thereby eliminating the trial-and-error solution that is necessary when heating times are found from the present one-dimensional charts .'), (159, 'numerical methods for transient heat flow .', 'dusinberre,g.m.', 'a.s.m.e.trans. 12, 1945, 703.', 'numerical methods for transient heat flow . this paper deals with the application of numerical methods for the solution of heat-conduction problems, their generality being extended in the following ways .. may proceed most rapidly to a solution, or may proceed more slowly and with greater precision,. (b) criteria are developed for the choice of modulus to insure convergence . this is most important at a convective surface,. (c) a method is developed for handling k and c when these properties vary independently with temperature . a comprehensive appendix gives the derivations, and the use of equations and charts is demonstrated by typical examples .'), (160, 'approximate analytical solutions for hypersonic flow past slender power-law bodies .', 'mirels,h.', 'nasa r-15, 1959.', 'approximate analytical solutions for hypersonic flow past slender power-law bodies . approximate analytical solutions are presented for two-dimensional and axisymmetric hypersonic flow over blunt-nosed slender bodies whose shapes follow a power law variation . in particular, the body shape is given by where is the transverse body ordinate, is the streamwise distance from the nose, and m is a constant in the range . both zero-order solutions and first-order (small but nonvanishing values of solutions are presented, where m is the free-stream mach number and is a characteristic body or streamline slope . the zero-order shock shape is similar to the body shape for these flows . the solutions are found within the framework of hypersonic-slender-body theory . the limiting case m=1 corresponds to a wedge or cone flow . the limiting case corresponds to a constant-energy flow . the latter cases are included so that the present study may be applied to all flows wherein the zero-order shock shape is given by with m in the range . flow fields associated with shock shapes having values of m outside this range are also discussed . for all values of, except m=1, certain portions of the flow field riolate the hypersonic-slender-body approximations, while other portions are consistent with these approximations . for m=1, all portions of the flow field are consistent with the approximations . the approximate solutions are found as follows . the asymptotic form of the flow in the vicinity of the body surface is used as a guide to write approximate expressions for the dependent variables . these expressions exactly satisfy the continuity and energy equations and contain arbitrary constants which are evaluated so as to satisfy boundary conditions at the shock . the approximate solutions do not satisfy the lateral momentum equation except at the shock and (for the first-order problem) at the body surface . the results of the approximate solutions are compared with numerical integrations of the equations of motion for various values of m and (ratio of specific heats) . good agreement is noted, particularly when m and are both near one . the shock is relatively close to the body for the latter cases . sufficient results are presented to evaluate the accuracy of the approximate method for various values of m and .'), (161, 'supersonic flow past a family of blunt symmetric bodies .', 'van dyke,m.d. and gordon,h.d.', 'nasa r-1, 1959.', 'supersonic flow past a family of blunt symmetric bodies . some 100 numerical computations have been carried out for unyawed bodies of revolution with detached bow waves . the gas is assumed perfect with . free-stream mach numbers are taken as 1.2, 1.5, 2, 3, 4, 6, 10, and . the results are summarized with emphasis on the sphere and paraboloid .'), (162, 'nearly circular transfer trajectories for descending satellites .', 'low,g.m.', 'nasa r-3, 1959.', 'nearly circular transfer trajectories for descending satellites . simplified expressions describing the transfer from a satellite orbit to the point of atmospheric entry are derived . the expressions are limited to altitude changes that are small compared with the earth''s radius, and velocity changes small compared with satellite velocity . they are further restricted to motion about a spherical, nonrotating earth . the transfer orbit resulting from the application of thrust in any direction at any point in an elliptic orbit is considered . expressions for the errors in distance (miss distance) and entry angle due to an initial misalinement and magnitude error of the deflecting thrust are presented . the largest potential contributing factor towards a miss distance stems from the misalinement of the retrovelocity increment . if this velocity increment is pointed in direct opposition to the flight path, a 1 misalinement leads to a miss distance of 34.5 miles . however, it is shown that this error can be avoided by applying the velocity increment at an angle between 120 and 150 below the flight-path direction . the guidance and accuracy requirements to establish a circular orbit, in addition to the corrections applied to transform elliptic orbits into circular ones, are also discussed .'), (163, 'an analysis of the corridor and guidance requirements for supercircular entry planetary atmospheres .', 'chapman,d.r.', 'nasa r-55, 1959.', 'an analysis of the corridor and guidance requirements for supercircular entry planetary atmospheres . an analysis is presented of supercircular entry into a planet''s atmosphere giving particular attention to the corridor through which spacecraft must be guided in order to accomplish various maneuvers . a dimensionless parameter based on conditions at the conic perigee altitude is introduced for characterizing supercircular entries and conveniently prescribing corridor widths associated with elliptic, parabolic, or hyperbolic approach trajectories . the analysis applies to vehicles of arbitrary weight, shape, and size . illustrative calculations are made for venus, earth, mars, jupiter, and titan . for nonlifting vehicles having fixed aerodynamic coefficients, curves are presented of dimensionless parameters from which can be calculated the maximum deceleration, maximum rate of laminar convective heating, and total laminar heat absorbed during single-pass entry at velocities up to twice circular velocity . for lifting vehicles, curves are presented of the maximum deceleration and overshoot boundary of an entry corridor,. equations are presented for estimating laminar aerodynamic heating from the maximum deceleration . it is shown that the corridor width is independent of vehicle weight, dimensions, and drag coefficient, provided these are the same at the overshoot boundary as at undershoot . the corridors of certain planets can be broadened markedly by the application of aerodynamic lift,. for example, the 10-earth-g corridor width for single-pass, nonlifting, parabolic entry is increased from to 52, 51, and 52 miles, respectively, by employing a lift-drag ratio of 1 . the use of aerodynamic lift does not increase appreciably the corridors of mars and titan . all corridor widths decrease rapidly as the entry velocity is increased . terminal guidance requirements on accuracy of velocity and flight path angle for successfully entering various corridors are compared with analogous requirements for putting a satellite into orbit, for hitting the moon from the earth, and for achieving icbm accuracy . consideration is given to the terminal guidance problem involved in using a planet''s atmosphere--rather than rocket fuel--to effect orbital transfers from heliocentric to planeto-centric motion, thereby converting a hyperbolic approach trajectory to an elliptic orbit about the target planet . this fuel saving maneuver appears technologically feasible for certain planetary voyages, and implies the possibility of achieving a large reduction in required earth lift-off weight of chemical propulsion systems .'), (164, 'an approximate analytical method for studying entry into planetary atmospheres .', 'chapman,d.r.', 'nasa r-11, 1959.', 'an approximate analytical method for studying entry into planetary atmospheres . the pair of motion equations for entry into a planetary atmosphere is reduced to a single, ordinary, nonlinear differential equation of second order by disregarding two relatively small terms and by introduring a certain mathematical transformation . the reduced equation includes various terms, certain of which represent the gravity force, the centrifugal acceleration, and the lift force . if these particular terms are disregarded, the differential equation is linear and yields precisely the solution of allen and eggers applicable to ballistic entry at relatively steep angles of descent . if all the other terms in the basic equation are disregarded (corresponding to negligible vertical acceleration and negligible vertical component of drag force), the resulting truncated differential equation yields the solution of sanger for equilibrium flight of glide vehicles with relatively large lift-drag ratios . a number of solutions for lifting and nonlifting vehicles entering at various initial angles also have been obtained from the complete nonlinear equation . these solutions are universal in the sense that a single solution determines the motion and heating of a vehicle of arbitrary weight, dimensions, and shape entering an arbitrary planetary atmosphere . one solution is required for each lift-drag ratio . these solutions are used to study the deceleration, heating rate, and total heat absorbed for entry into venus, earth, mars, and jupiter . from the equations developed for heating rates, and from available information on human tolerance limits to acceleration stress, approximate conditions for minimizing the aerodynamic heating of a trimmed vehicle with constant lift-drag ratio are established for several types of manned entry . a brief study is included of the process of atmosphere braking for slowing a vehicle from near escape velocity to near satellite velocity .'), (165, 'skin-friction measurements in incompressible flow .', 'smith, d.w. and walker, j. h.', 'naca report r-26', 'skin-friction measurements in incompressible flow . experiments have been conducted to measure in incompressible flow the local surface-shear stress and the average skin-friction coefficient for a turbulent boundary-layer on a smooth flat plate having zero pressure gradient . the local surface-shear stress was measured by a floating-element skin-friction balance and also by a calibrated total head tube located on the surface of the test wall . the average skin-friction coefficient was obtained from boundary-layer velocity profiles . the boundary-layer profiles were also used to determine the location of the virtual origin of the turbulent boundary layer . data were obtainec for a range of reynolds numbers from 1 million to about 45 million with an attendant change in mach number from 0.11 to 0.32 . the measured local skin-friction coefficients obtained with the floating-element balance agree well with those of schultz-grunow and kempf for reynolds numbers up to 45 million . the measured average skin-friction coefficients agree with those given by the schoenherr curve in the ranges of reynolds numbers from 1 to 3 million and 30 to 45 million . in the range of reynolds numbers from 3 to 30 million the measured values are less than those predicted by the schoenherr curve . the results show that the /univeral skin-friction constants/ proposed by coles appraoch asymptotically a constant value at reynolds numbers exceeding mentioned constants and the limited reynolds number range of the present investigation, there is some doubt as to the validity of any turbulent skin-friction law written on the basis of the present results . hence, no new friction law is proposed . the frictional resistance of a flat plate was calculated by means of the momentum method and also the integrated measured local surface shear . for reynolds numbers from 14 million to 45 million both methods give about the same result,. whereas at lower values of reynolds number the momentum method based on velocity profiles uncorrected for the effects of turbulence results in a frictional resistance as much as 4 percent higher than that of the integrated shear . the measurement of local surface shear by a calibrated preston tube appears to be accurate and inexpensive . the calibration as given by preston must be modified slighlty, however, to yield the results obtained from the floating-element skin-friction balance .'), (166, 'flow of chemically reacting gas mixtures .', 'clarke,j.f.', 'coa r117, 1961.', 'flow of chemically reacting gas mixtures . suitable forms of the equations for the flow of an inviscid, non-heat-conducting gas in which chemical reactions are occurring are derived . the effects of mass diffusion and non-equilibrium amongst the internal modes of the molecules are neglected . special attention is given to the speeds of sound in such a gas mixture and a general expression for the ratio of frozen to equilibrium sound speeds is deduced . an example is given for the ideal dissociating gas . the significance of the velocity defined by the ratio of the convective derivatives of pressure and density is explained . it is the velocity which exists at the throat of a convergent-divergent duct under maximum mass flow conditions, and it is shown that this velocity depends on the nozzle geometry as well as on the ''reservoir'' conditions . as an illustration the phenomena of sound absorption and dispersion are discussed for the ideal dissociating gas . the results can be concisely expressed in terms of the frozen and equilibrium sound speeds, the frequency of the (harmonic) sound vibration and a characteristic time for the rate of progress of the reaction .'), (167, 'linearized flow of a dissociating gas .', 'clarke,j.f.', 'j.fluid mech. 7, 1960, 577.', 'linearized flow of a dissociating gas . the equations for planar two-dimensional steady flow of an ideal dissociating gas are linearized, assuming small disturbances to a free stream in chemical equilibrium . as an example of their solution, the flow past a sharp corner in a supersonic stream is evaluated and the variations of flow properties in the relaxation zone are found . numerical illustrations are provided using an ''oxygen-like'' ideal gas and comparisons made with a characteristics solution . the flow past a sharp corner can be studied in a conventional shock tube and it may be possible to verify the present theory experimentally . in particular it may prove feasible to use the results to obtain a measure of the reaction rates in the gas mixture .'), (168, 'heat conduction through a gas with one inert internal model .', 'clarke,j.f.', 'coa n102, 1960.', 'heat conduction through a gas with one inert internal model . the rate of energy transfer between parallel flat plates is evaluated when the (stagnant) gas between them is polyatomic with one inert internal mode . deviations of the thermal conductivity from the complete equilibrium of the inert mode relaxation time and the effectiveness of the walls in exciting or de-exciting this mode . the results are obtained via a linear theory consistent with small temperature differences between the plates . it is found that the eucken-value of conductivity could be exceeded if the relaxation times are non-zero and the plates very effective in exciting the inert mode . when relaxation times are very short the effect of the walls on the energy transfer rate is small, but the walls make their presence felt by distorting the temperature profiles in /boundary layers/ adjacent to the walls which are of order in thickness time) . this result is analogous to hirschfelder''s (1956) for the case of chemical reactions . for experimental measurement of conductivity in a hot wire cell type of apparatus it is shown that extrapolation of measured reciprocal conductivities to zero reciprocal pressure should load to the full eucken value . it is also shown that the slope of reciprocal apparent (measured) conductivity versus reciprocal pressure curves is a function of relaxation time as well as of the accommodation coefficients . it is quite possible that the relaxation effect here is comparable with the temperature jump effects, even for rotation in diatomic molecules .'), (169, 'on the sudden contact between a hot gas and a cold solid .', 'j. f. clarke, b.sc., ph.d., a.f.r.ae.s.', '', 'on the sudden contact between a hot gas and a cold solid . the flow induced by the sudden contact between a semi-infinite expanse of gas and a solid, initially at different temperatures, is examined on the basis of a linear continuum theory . for times large compared with the mean time between molecular collisions in the gas, the velocity and pressure disturbances are found to be concentrated around a wave front propagating out from the interface at the ambient isentropic sound speed, whilst, near to the interface, these disturbances are small and the gas temperatures are nearly equal to those predicted by the classical constant pressure heat conduction theory . the possible significance of these results in connection with reflected shock wave techniques to measure high temperature gas properties is commented upon .'), (170, 'the interaction of a reflected shock wave with the boundary layer in a shock tube .', 'mark,h.', 'naca tm.1418.', 'the interaction of a reflected shock wave with the boundary layer in a shock tube . ideally, the reflection of a shock from the closed end of a shock tube provides, for laboratory study, a quantity of stationary gas at extremely high temperature . because of the action of viscosity, however, the flow in the real case is not one-dimensional, and a boundary layer grows in the fluid following the initial shock wave . in this paper simplifying assumptions are made to allow an analysis of the interaction of the shock reflected from the closed end with the boundary layer of the initial shock afterflow . the analysis predicts that interactions of several different types will exist in different ranges of initial shock mach number . it is shown that the cooling effect of the wall on the afterflow boundary layer accounts for the change in interaction type . an experiment is carried out which verifies the existence of the several interaction regions and shows that they are satisfactorily predicted by the theory . along with these results, sufficient information is obtained from the experiments to make possible a model for the interaction in the most complicated case . this model is further verified by measurements made during the experiment . the case of interaction with a turbulent boundary layer is also considered . identifying the type of interaction with the state of turbulence of the interacting boundary layer allows for an estimate of the state of turbulence of the boundary layer based on an experimental investigation of the type of interaction .'), (171, 'a low density wind tunnel study of shock wave structure and relaxation phenomena in gases .', 'sherman,f.s.', 'naca tn.3298.', 'a low density wind tunnel study of shock wave structure and relaxation phenomena in gases . the profiles and thicknesses of normal shock waves of moderate strength have been determined experimentally in terms of the variation of the equilibrium temperature of an insulated transverse cylinder in free-molecule flow . the shock waves were produced in a steady state in the jet of a low-density wind tunnel, at initial mach numbers of 1.72 and 1.82 in helium and 1.78, the shock thickness, determined from the maximum slope of the cylinder temperature profile, varied from mean free path in the supersonic stream . a comparison between the experimental shock profiles and various theoretical predictions leads to the tentative conclusions that .. (1) the navier-stokes equations are adequate for the description of the shock transition for initial mach numbers up to 2, and (2) the effects of rotational relaxation times in air can be accounted for by the introduction of a /second/ or /bulk/ viscosity coefficient equal to about two-thirds of the ordinary shear viscosity .'), (172, 'some aerodynamic considerations of nozzle afterbody combination .', 'cortright,e.m.', 'aero. eng. rev. 15, 1956, 59.', 'some aerodynamic considerations of nozzle afterbody combination . the aerodynamic problems associated with propulsion-system installations have assumed a role of vital importance in the development of supersonic aircraft . although air-induction systems have received moderate attention in the literature, considerably less information can be found on the design and installation of turbojet exit nozzles . this condition should not be interpreted to indicate a lack of problems in jet-exit design . as flight speeds reach supersonic levels, it becomes increasingly difficult to achieve nozzle installations which are efficient over the entire speed range . the difficulties largely stem from the fact that the goals of high jet thrust and low afterbody drag are not always compatible . in many of the compromise solutions, it is generally unsatisfactory to examine isolated nozzle and afterbody performance . rather they must be treated as a unit, and the complex effects of jet interaction with the external stream must be taken into account . to accomplish this, the nozzle and air-frame designers must closely coordinate their efforts . some of the aerodynamic problems of nozzle afterbody combinations are outlined in this report . particular attention is devoted to the influence of the jet-stream interaction on both nozzle thrust and after-body drag . for this purpose, use is made of shock- boundary-layer-interaction concepts . this approach, although not precise, correctly predicts many trends and is generally enlightening .'), (173, 'the effect of a central jet on the base pressure of a cylindrical afterbody in a supersonic stream .', 'reid,j. and hastings,r.c.', 'arc r + m.3224, 1962.', 'the effect of a central jet on the base pressure of a cylindrical afterbody in a supersonic stream . this report describes an experimental investigation of the factors affecting the base flow and jet structure behind a cylindrical after-body with a central nozzle . seven interchangeable nozzles were tested . six of these were convergent-divergent, with a design mach number of 2.0, jet base diameter ratios ranging from 0.2 to 0.8 and nozzle divergence angles ranging from convergent with a jet base diameter ratio of 0.6 . in the main experimental programme the free-stream mach number was 2.0 and the boundary layer was turbulent both on the after-body and in the nozzle . measurements were made of the base pressure, the surface pressure distribution inside the nozzle, the overall thrust and the nozzle mass flow, over a range of jet pressures . this programme was supplemented by comparative tests with the jet exhausting into still air (static tests) . readings were taken of the internal nozzle pressures and the jet thrust at different jet pressures . schlieren photography was used extensively throughout . the results of the tests with external flow are presented in the form of curves showing the separate effects of jet pressure ratio, jet base diameter ratio, nozzle design mach number and nozzle divergence angle on the base pressure and overall thrust . the special case of base bleed is discussed separately . similar curves are included for the static tests . these show the effect of jet pressure ratio and nozzle geometry on the jet thrust . a general method of correlating data on annular base pressures is proposed and discussed . essentially, this method compares the pressure on an annular base with the calculated pressure on the corresponding two-dimensional base . it correlates the present results reasonably well, but is less successful when applied to more extensive data .'), (174, 'investigation at supersonic speeds of the effects of jet mach number and divergence angle of the nozzle upon the pressure of the base annulus of a body of revolution .', 'bromm,a.f. and o''donnel,r.m.', 'naca rm l54i16, 1954.', 'investigation at supersonic speeds of the effects of jet mach number and divergence angle of the nozzle upon the pressure of the base annulus of a body of revolution . an investigation has been conducted in the langley 9-inch supersonic tunnel to determine the jet effects for varying jet mach number and nozzle divergence angle upon the pressure on the base annulus of a model with a cylindrical afterbody . the tests were conducted over a wide range of jet static pressure ratios and at a reynolds number of approximately free-stream mach numbers of 1.62, 1.94, and 2.41 . all testing was conducted with an artificially induced turbulent boundary layer along the model . in the lower range of jet static pressure ratios, jet flow from a sonic or supersonic nozzle affected the pressure acting on the base annulus in essentially the same manner as shown in naca rm e53h25 which covers jet static pressure ratios up to about present results showed that the base pressure tends to level off with increasing jet static pressure ratio, and at the extreme static pressure ratios reached in tests with sonic nozzles the base pressure began to decrease . except in the lower range of jet static pressure ratios, nozzle divergence angle generally had a larger effect on the base pressures than nozzle mach number,. the increase in base pressure for a change in divergence angle from 0 to 10 was small compared to the increase when the divergence angle was changed from and other data indicates that the effects of divergence angle were reduced when the ratio of jet exit diameter to base diameter was decreased . jet mach number effects increased with increase in stream mach number .'), (175, 'experiments with static tubes in a supersonic airstream .', 'holder,d.w., north,r.j. and chinneck,a.', 'arc r + m 2782, 1953.', 'experiments with static tubes in a supersonic airstream . systematic tests have been made at a mach number of 1.6 on a family of static tubes . the variables which have been investigated are the shape of the nose, the distance of the holes downstream, and the inclination of the tube to the flow . pressure measurements have also been made in the vicinity of a shock wave and close to a wall .'), (176, 'base pressure at subsonic speeds in the presence of a supersonic jet .', 'craven,a.h.', 'coa r129, 1960.', 'base pressure at subsonic speeds in the presence of a supersonic jet . this paper presents the results of an experimental investigation into the effect of supersonic jets upon the base pressure of a bluff cylinder in a uniform subsonic flow . the ratio of jet diameter to base diameter was 0.1875 . jet stagnation pressures giving slight under-expansion of the jet cause an increase in the base pressure but for larger jet stagnation pressures the base pressure is again reduced . a simple theory, based on a momentum integral, shows the dependence of the base drag upon the jet and free stream speeds and upon the dimensions of the jet and the base .'), (177, 'the mixing of free axially-symmetrical jets of mach number 1.40 .', 'n. h. johannesen', 'department of the mechanics of fluids, university of manchester communicated by the director-general of scientific research (air), ministry of supply', 'the mixing of free axially-symmetrical jets of mach number 1.40 . axially-symmetrical, supersonic, fully-expanded jets of diameter about 0.75 in. and of mach number 1.40 issuing into an atmosphere at rest were investigated by schlieren and shadow photography and by pressure traversing . the development of the jets was found to depend critically on the strength of the shock waves in the core of the jet at the nozzle exit . with strong shock waves present the jet spread very rapidly and was very unsteady . the jet did in some cases break up into large eddies of the same size as the diameter of the jet . when no disturbances were present in the core of the jet the spreading was far more gradual and the jet showed only slight unsteadiness . the turbulent mixing region of the first part of the jet with strong shock waves was investigated in detail by pitot tubes . the first inch was found to correspond to a two-dimensional half-jet . the velocity profiles were similar and well represented by the error integral . the rate of spreading was only half the value for low-speed flow . by integrations across the mixing region the entrainment and the loss of kinetic energy were determined . these quantities were found to agree well with the values estimated by assuming an error-integral velocity profile .'), (178, 'on full dispersed shock waves in carbon dioxide .', 'griffith,w.c. and kenny,a.', 'j.fluid mech. 3, 1957, 286.', 'on full dispersed shock waves in carbon dioxide . it is pointed out that, for shock mach numbers between 1 and that the adjustments in the energy in all the degrees of freedom proceed slowly and in parallel and occur over a distance large compared with the mean free path . theoretical velocity profiles for such shock waves are given and found to be in excellent agreement with interferometric shock-tube observations .'), (179, 'an analysis of base pressure at supersonic speeds and comparison with experiment .', 'chapman,d.', 'naca tn.2137, 1950.', 'an analysis of base pressure at supersonic speeds and comparison with experiment . in the first part of the investigation an analysis is made of base pressure in an inviscid fluid, both for two-dimensional and axially-symmetric flow . it is shown that for two-dimensional flow, and also for the flow over a body of revolution with a cylindrical sting attached to the base, there are an infinite number of possible solutions satisfying all necessary boundary conditions at any given free-stream mach number . for the particular case of a body having no sting attached only one solution is possible in an inviscid flow, but it corresponds to zero base drag . accordingly, it is concluded that a strictly inviscid-fluid theory cannot be satisfactory for practical applications . since the exact inviscid-fluid theory does not adequately describe the conditions of a real fluid flow, an approximate semi-empirical theory for base pressure in a viscous fluid is developed in a second part of the investigation . the semi-empirical theory is based partly on inviscid-flow calculations, and is restricted to airfoils and bodies without boat-tailing . in this theory an attempt is made to allow for the effects of mach number, reynolds number, profile shape, and type of boundary-layer flow . the results of some recent experimental measurements of base pressure in two-dimensional and axially-symmetric flow are presented for purposes of comparison . some experimental results also are presented concerning the support interference effect of a cylindrical sting, and the interference effect of a reflected bow wave on measurements of base pressure in a supersonic wind tunnel .'), (180, 'boundary layer over a flat plate in presence of shear flow .', 'ting,l.', 'phys. fluids, 13, 1960, 78.', 'boundary layer over a flat plate in presence of shear flow . the governing equations of an incompressible boundary layer over a flat plate in the presence of a shear flow with finite vorticity are derived . for large vorticity, a similarity solution is obtained . for moderate vorticity, one of the governing equations is replaced by an approximate one for which similarity solutions exist .'), (181, 'some problems on heat conduction in stratiform bodies .', 'vodicka,v.', 'j. phys. soc. japan, 14, 1959, 216.', 'some problems on heat conduction in stratiform bodies . problems on heat conduction in multilayer bodies lead usually to complicated calculations . the present paper gives an idea of specific difficulties arising in the case of infinite composite solides . general deductions are applied to a special class of questions .'), (182, 'effect of roughness on transition in supersonic flow .', 'van driest,e.r. and blumer,c.b.', 'agard r255, 1960.', 'effect of roughness on transition in supersonic flow . further experiments carried out in the 12-inch supersonic wind tunnel of the jet propulsion laboratory of the california institute of technology to investigate the effect of three-dimensional roughness elements (spheres) on boundary-layer transition on a 10 transfer are reported herein . the local mach number for these tests was minimum (effective) size of trip required to bring transition to its lowest reynolds number varies as the one-fourth power of the distance from the apex of the cone to the trip . use of available data at other mach numbers indicates that the mach number influence for effective tripping is taken into account by the simple expression .'), (183, 'properties of impact pressure probes in free molecule flow .', 'harris,e.l. and patterson,g.n.', 'utia r52, 1958.', 'properties of impact pressure probes in free molecule flow . an expression has been derived for the mass flow through a circular tube in free molecule flow when the tube and gas are in relative motion . the gas entering the tube is assumed to have a maxwellian distribution function and the molecular reflection process at the wall is assumed to be diffuse . the theory has been used to determine the pressure read by an impact probe in free molecule flow . although the general expressions derived apply to any value of gas velocity and tube size, the detailed calculations for the pressure probe are difficult except for the case of low speeds and long tubes . an experimental check of the theory has been carried out using impact probes in a whirling arm apparatus and in the utia low density wind tunnel . agreement between theory and experiment is quite satisfactory .'), (184, 'scale models for thermo-aeroelastic research .', 'molyneux,w.g.', 'rae tn.struct.294, 1961.', 'scale models for thermo-aeroelastic research . an investigation is made of the parameters to be satisfied for thermo-aeroelastic similarity . it is concluded that complete similarity obtains only when aircraft and model are identical in all respects, including size . by limiting consideration to conduction effects, by assuming the major load carrying parts of the structure are in regions where the flow is either entirely laminar, or entirely turbulent, and by assuming a specific relationship between reynolds number and nusselt number, an approach to similarity can be achieved for small scale models . experimental and analytical work is required to check on the validity of these assumptions . it appears that existing hot wind tunnels will not be completely adequate for thermo-aeroelastic work, and accordingly a possible layout for the type of tunnel required is described . automatic programmed control of the tunnel would appear to be necessary .'), (185, 'some possibilities of using gas mixtures other than in aerodynamic research .', 'dean r. chapman', '', 'some possibilities of using gas mixtures other than in aerodynamic research . a study is made of the advantages that can be realized in compressible-flow research by employing a substitute heavy gas in place of air . most heavy gases considered in previous investigations are either toxic, chemically active, or (as in the case of the freons) have a ratio of specific heats greatly different from air . the present report is based on the idea that by properly mixing a heavy monatomic gas with a suitable heavy polyatomic gas, it is possible to obtain a heavy gas mixture which has the correct ratio of specific heats and which is nontoxic, nonflammable, thermally stable, chemically inert, and comprised of commercially available components . calculations were made of wind-tunnel characteristics for 63 gas pairs comprising 21 different polyatomic gases properly mixed with each of three monatomic gases (argon, krypton, and xenon) . for a given mach number, reynolds number, and tunnel pressure, a gas-mixture wind tunnel having the same specific-heat ratio as air would be appreciably smaller and would require much less power than a corresponding air wind tunnel . analogous though different advantages can be realized in compressor research and in firing-range research . the most significant applications, perhaps, arise through selecting and proportioning a gas mixture so as to have at ordinary wind-tunnel temperatures certain dimensionless characteristics which air at flight temperatures possesses but which air at ordinary wind-tunnel temperatures does not possess . characteristics which involve the relaxation time (or bulk viscosity), the variation of viscosity with temperature, and the variation of specific heat with temperature fall within this category . other applications arise in heat-transfer research since certain gas mixtures can be concocted to have any prandtl number in the range at least between 0.2 and 0.8 .'), (186, 'base pressure in supersonic flow .', 'gadd,g.e., holder,d.w. and regan,j.d.', 'arc cp271, 1956.', 'base pressure in supersonic flow . the problem of accurately predicting the pressure and wake configuration at the base of bodies in supersonic flow is an extremely important one inasmuch as a sizeable portion of the total drag of a given body may be attributable to the low pressure in this region . although a great deal of theoretical and experimental work has been done in this field, there does not yet exist a satisfactory method for accurate predictions . this paper represents an excellent effort to experimentally confirm analytically deduced concepts . a large amount of experimental data on body shapes such as wedges, cones, and cone-cylinders has been obtained over a range of mach numbers up to 4 . the data are thoroughly discussed with respect to analytical deductions . on the basis of the evidence accumulated it is concluded that the boundary-layer thickness has only a small effect on the base pressure for axisymmetric bodies and for two-dimensional bodies when the base height-to-chord ratios are of the order . reviewer believes this report is a significant contribution in the field of base pressure and wake flow phenomena .'), (187, 'investigation of separated flows in supersonic and subsonic streams with emphasis on the effect of transition .', 'chapman, d.r., kuehn, d. m. and larson, h. k.', 'naca report 1356', 'investigation of separated flows in supersonic and subsonic streams with emphasis on the effect of transition . experimental and theoretical research has been conducted on flow separation associated with steps, bases, compression corners, curved surfaces, shock-wave boundary-layer reflections, and configurations producing leading-edge separation . results were obtained from pressure-distribution measurements, shadow-graph observations, high-speed motion pictures, and oil-film optics . the maximum scope of measurement encompassed mach numbers between 0.4 and 3.6, and length reynolds numbers between 4000 and 5000000 . the principal variable controlling pressure distribution in the separated flows was found to be the location of transition relative to the reattachment and separation positions . classification is made of each separated flow into one of three regimes .. and /turbulent/ with transition upstream of separation . by this means of classificaiton it is possible to state rather literal results regarding the steadiness of flow and the influence of reynolds number within each regime . for certain pure laminar separations a theory for calculating dead-air pressure is advanced which agrees well with subsonic and supersonic experiments . this theory involves no empirical information and provides an explanation of why transition location relative to reattachment is important . a simple analysis of the equations for interaction of boundary-layer and external flow near either laminar or turbulent separation indicates the pressure rise to vary as the square root of the wall shear stress at the beginning of interaction . various experiments substantiate tnis variation for most test conditions . an incidental observation is that the stability of a separated laminar mixing layer increases markedly with an increase in mach number . the possible significance of this observation is discussed .'), (188, 'an analysis of base pressure at supersonic velocities and comparison with experiment .', 'chapman, dean r .', 'naca report 1051', 'an analysis of base pressure at supersonic velocities and comparison with experiment . in the first part of the investigation an analysis is made of base pressure in an inviscid fluid, both for two-dimensional and axially symmetric flow . it is shown that for two-dimensional flow, and also for the flow over a body of revolution with a cylindrical sting attached to the base, there are an infinite number of possible solutions satisfying all necessary boundary conditions at anh given free-stream mach numger . for the particular case of a body having no sting attached only one solution is possible in an inviscid flow, but it corresponds to zero base drag . accordingly, it is concluded that a strictly inviscid-flow theory cannot be satisfactory for practical applications . an approximate semi-empirical analysis for base pressure in a viscous fluid is developed in a second part of the investigation . the semi-empirical analysis is based partly on inviscid-flow calculations . in this theory an attempt is made to allow for the effects of mach number, reynolds number, profile shape, and type of boundary-layer flow . some measurements of base pressure in two-dimensional and axially symmetric flow are presented for purposes of comparison . experimental results then are presented concerning the support interference effect of a cylindrical sting, and the interference effect of a reflected air wave on measurements of base pressure in a supersonic wind tunnel .'), (189, 'experimental investigation of base pressure on blunt-trailing-edge wings of supersonic velocities .', 'chapman,d.r., wimbrow,w.r. and kester,r.h.', 'naca r1109.', 'experimental investigation of base pressure on blunt-trailing-edge wings of supersonic velocities . measurements of base pressure are presented for 29 blunt-trailing-edge wings having an aspect ratio of 3.0 and various airfoil profiles . the different profiles comprised thickness ratios between 0.05 and 0.10, boattail angles between --2.9 and 20, and ratios of trailing-edge thickness to airfoil thickness between 0.2 and 1.0 . the tests were conducted at mach numbers of 1.25, 1.5, 2.0, and 3.1 . for each mach number, the reynolds number and angle of attack were varied . the lowest reynolds number investigated was 0.2 x 10 and the highest was 3.5 x 10 . measurements on each wing were obtained separately with turbulent flow and laminar flow in the boundary layer . span-wise surveys of the base pressure were conducted on several wings . the results with turbulent boundary-layer flow showed only small effects on base pressure of variations in reynolds number, airfoil profile shape, boattail angle, and angle of attack . the principal variable affecting the base pressure for turbulent flow was the mach number . at the highest mach number investigated (3.1), the ratio of boundary-layer thickness to trailing-edge thickness also affected the base pressure significantly . the results obtained with laminar boundary-layer flow to the trailing edge showed that the effect of reynolds number on base pressure was large . in all but a few exceptional cases the effects on base pressure of variations in angle of attack and in profile shape upstream of the base were appreciable though not large . the principal variable affecting the base pressure for laminar flow was the ratio of boundary-layer thickness to trailing-edge thickness . for a few exceptional cases involving laminar flow to the trailing edge, the effects on base pressure of variations in profile shape, boattail angle, and angle of attack were found to be unusually large . in such cases the variation of base pressure with angle of attack was discontinuous and exhibited a hysteresis . stroboscopic schlieren observations at a mach number of 1.5 indicated that these apparently special phenomena were associated with a vortex trail of relatively high frequency .'), (190, 'on magnetohydrodynamic shock waves .', 'kanwal,r.', 'j.math.mech. 9, 1960, 681.', 'on magnetohydrodynamic shock waves . in the earlier attempts at finding the jump conditions across a hydromagnetic shock wave (1, 2, 3) various simplifying assumptions regarding the shape of the shock and the dimensions and the character of the motion are made . from that analysis it is possible to write down the jump conditions in a higher degree of generality (4) . the shock conditions for magnetohydrodynamic flows can, however, be derived in their full generality with the help of the transport equation as used by thomas (5) in the derivation of shock conditions in conventional gas dynamics . the purposes of this paper are .. cover the present more general case . that every flow and field quantity downstream from the shock wave is expressible separately in terms of the known values of these quantities upstream from the shock wave . in this rearranged form of the equations, various effects of the shock wave can be easily read off . the shock conditions along the same lines as in conventional gas dynamics .'), (191, 'a theory for the core of a leading edge vortex .', 'hall,m.g.', 'rae r.aero.2644, 1960.', 'a theory for the core of a leading edge vortex . in the flow past a slender delta wing at incidence can be observed a roughly axially symmetric core of spiralling fluid, formed by the rolling up of the shear layer that separates from a leading edge . the aim in this report is to predict the flow field within this vortex core, given appropriate conditions at its outside edge . the basic assumptions are core . in addition it is assumed that the flow is axially symmetric and incompressible . together, these admit outer and inner solutions for the core from the equations of motion . for the outer solution the sub-core is ignored, and the flow is taken to be inviscid (but rotational) and conical . the resulting solution consists of simple expressions for the velocity components and pressure . for the inner solution, which applies to the diffusive sub-core, the flow is taken to be laminar, and approximations, some based on the boundary conditions and some analogous to those of boundary layer theory, are made . the solution obtained in this case is a first approximation, and is presented in tabular form . a sample calculation yields results which are in good qualitative and fair quantitative agreement with experimental measurements .'), (192, 'on the hypersonic viscous flow past slender bodies of revolution .', 'yashura,m.', 'j.phys.soc. japan, 11, 1956, 878.', 'on the hypersonic viscous flow past slender bodies of revolution . a similar solution of the hypersonic viscous flow past slender bodies of revolution is deduced for a special case when the radial coordinate of the body surface at section x is proportional to x, where the radial coordinate have the comparable order value with the thickness of the boundary layer . here, /similar/ is used in the direct meaning that distributions in the boundary layer keep the similar form lengthwise . calculations are accomplished for the region of strong interaction between the boundary layer and the shock wave . from several calculations it may be expected that if the thickness of the body becomes small, the thickness of the layer in which the longitudinal velocity component u is rapidly decreased also becomes small, and in the major part of the boundary layer, only the normal component v is increased . further if the thickness of the body is increased, then, the height of the shock wave, the pressure on the wall, and the shear stress at the wall are also increased while the boundary layer thickness is decreased . the nose region is excluded by the reason that the ordinary boundary layer theory will be invalid there .'), (193, 'a study of inviscid flow about air foils at high supersonic speeds .', 'eggers, a.j., syvertson, c.a., and krqus, s.', 'naca report 1123', 'a study of inviscid flow about air foils at high supersonic speeds . steady flow about curved airfoils at high supersonic speeds is investigated analyticially . with the assumption that air behaves as a diatomic gas, it is found the the shock-expansion method may be used to predict the flow about curved airfoils up to extremely high mach numbers, provided the flow deflection angles are not too close to those corresponding to shock detachment . this result applies not only to the determination of the surface pressure distribution, but also to the determination of the whole flow field about an airfoil . verification of this observation is obtained with the aid of the method of characteristics by extensive calculations of the pressure gradient and shock-wave curvature at the leading edge, and by calculations of the pressure distribution on a 10-percent-thick biconvex airfoil at 0 angle of attack . an approximation to the shock-expansion method for thin airfoils at high mach numbers is also investigated and is found to yield pressures in error by less than 10 percent at mach numbers above three and flow deflection angles up to 25 . this slender-airfoil method is relatively simple in form and thus may prove useful for some engineering purposes . effects of caloric imperfections of air manifest in disturbed flow fields at high mach numbers are investigated, particular attention being given to the reduction of the ratio of specific heats . so long as this ratio does not decrease appreciably below to include the effects of these imperfections, should be substantially as accurate as for ideal-gas flows . this observation is verfied with the aid of a generalized shock-expansion method and a generalized method of characteristics employed in forms applicable for local air temperatures up to about 5000 rankine . the slender-airfoil method is modified to employ an average value of the ratio of specific heats for a particular flow field . this simplified method has essentially the same accuracy for imperfect-gas flows as its counterpart has for ideal-gas flows . an approximate flow analysis is made at extremely high mach numbers where it is indicated that the ratio of specific heats may approach close to 1 . in this case, it is found that the shock-expansion method may be in considerable error,. however, the busemann method for the limit of infinite free-stream mach number and specific-heat ratio of 1 appears to apply with reasonable accuracy .'), (194, 'general theory of airfoil sections having arbitrary shape or pressure distribution .', 'allen,h.j.', 'naca r833, 1945.', 'general theory of airfoil sections having arbitrary shape or pressure distribution . in this report a theory of thin airfoils of small camber is developed which permits either the velocity distribution corresponding to a given airfoil shape, or the airfoil shape corresponding to a given velocity distribution to be calculated . the procedures to be employed in these calculations are outlined and illustrated with suitable examples .'), (195, 'correlation of theoretical and photo-thermoelastic results on thermal stresses in idealized wing structure .', 'tramposch,h. and gerard,g.', 'j.app.mech. 27, 1960.', 'correlation of theoretical and photo-thermoelastic results on thermal stresses in idealized wing structure . after a rather complete exploratory program described in previous papers, the photo-thermoelastic method was applied to the experimental evaluation of the thermal-stress theories . the new technique was correlated with several theories which analyzed the transient thermal stresses in idealized wing structures of high-speed aircraft . various theories were investigated which represented the same idealized wing models and differed from each other only in the simplifying assumptions regarding the temperature distributions in skin and webs . the theories were evaluated by duplicating the boundary and initial conditions on plastic models and then by correlating the theories with the observed fringe orders in nondimensional form . a significant general conclusion was reached after correlating the available theories and experimental results . owing to simplifying assumptions concerning the thermal behavior in the flanges, thermal stresses predicted by the available theories are all higher than the experimental observation . in some cases the discrepancy is as great as 30 per cent .'), (196, 'pressure distributions . axially symmetric bodies in oblique flow .', 'campbell,i.j. and lewis,r.g.', 'arc cp213, 1955.', 'pressure distributions . axially symmetric bodies in oblique flow . a simple picture, known from the work of i. lotz, of the flow over the forward part of a body of revolution in oblique flow is derived here from entirely elementary considerations . the pressure at any point of the (forward part of the) body at any angle of incidence depends on three parameters whose values vary along the body . the variation of these parameters along the body can be determined from a relatively small number of wind tunnel or water tunnel measurements . the necessary water tunnel measurements have been made for four axially symmetric head shapes . additional measurements have been made to illustrate the theoretical conclusions . the data for each head shape are adequate for a determination of the pressure coefficient at any point on the head shapes at any angle of incidence (up to 6, say) . in particular they can be used to determine the peak suction at any angle of incidence and so the conditions for the onset of cavitation on the head .'), (197, 'pressure distributions on three bodies of revolution to determine the effect of reynolds number up to and including the transonic speed range .', 'swihart,j.m. and whitcomb,c.f.', 'naca rm l53h04, 1953.', 'pressure distributions on three bodies of revolution to determine the effect of reynolds number up to and including the transonic speed range . this paper presents the results of an investigation conducted in the langley 16-foot transonic tunnel to determine the effects of varying reynolds number on the pressure distribution on a transonic body of revolution at angles of attack through the transonic speed range . the effect of a change in sting cone angle on the pressure distributions and a comparison of experimental incremental pressures with theory is also included . the models were tested through a mach number range from 0.60 to 1.09 . the reynolds number range based on body length was from 9 x 10 to 39 x diameter was 1.3 x 10 to 4.53 x 10 for the model at 8 angle of attack . an increase in reynolds number from 9 x 10 to 39 x 10 affected the longitudinal pressure distributions very slightly . these effects were of such a nature as to cause an increase of 0.05 in the normal-force coefficient of the body when tested in the subcritical cross-flow reynolds number range . this increase is in agreement with theoretical approximations . a comparison between experimental and theoretical values of the incremental pressure coefficient due to angle of attack indicated good agreement except at angles where separated flow areas existed over the body . the effect of a change in sting-cone angle from 5 to 9 on the pressure distribution of the 120-inch model was negligible up to a mach number of 1.05 . at this mach number the effect was to cause a small increase in the velocity over the rear of the body .'), (198, 'investigation of a systematic group of naca 1 - series cowlings with and without spinners .', 'nichols,m.r. and keith,a.l.', 'naca r950, 1949.', 'investigation of a systematic group of naca 1 - series cowlings with and without spinners . an investigation has been conducted in the langley propeller-research tunnel to study cowling-spinner combinations based on the naca 1-series nose inlets and to obtain systematic design data for one family of approximately ellipsoidal spinners . in the main part of the investigation, 11 of the related spinners were tested in various combinations with 9 naca open-nose cowlings, which were also tested without spinners . the effects of location and shape of the spinner, shape of the inner surface of the cowling lip, and operation of a propeller having approximately oval shanks were investigated briefly . in addition, a study was conducted to determine the correct procedure for extrapolating design conditions determined from the low-speed test data to the design conditions at the actual flight mach number . the design conditions for the naca 1-series cowlings and cowling-spinner combinations are presented in the form of charts from which, for wide ranges of spinner proportions and rates of internal flow, cowlings with near-maximum pressure recovery can be selected for critical mach numbers ranging from spinners and the effects of the spinners and the propeller on the cowling design conditions are presented separately to provide initial quantitative data for use in a general design procedure through which naca 1-series cowlings can be selected for use with spinners of other shapes . by use of this general design procedure, correlation curves established from the test data, and derived compressible-flow equations relating the inlet-velocity ratio to the surface pressures on the cowling and spinner, naca 1-series cowlings and cowling-spinner combinations can be designed for critical mach numbers as high as 0.90 .'), (199, 'measurement of two dimensional derivatives on a wing-aileron-tab system .', 'wight,k.c.', 'part i, arc r + m 2934, 1955. part ii arc r + m 3029, 1958.', 'measurement of two dimensional derivatives on a wing-aileron-tab system . measurements have been made of the direct two-dimensional damping and stiffness derivatives for a in incompressible flow . corrections arising from the apparatus are discussed and reference is made to an attempt to measure the direct tab derivatives . the effects are shown of frequency parameter, amplitude of oscillation, reynolds number, aileron angle and position of transition on the wing . variation with frequency parameter is substantially the same as for vortex-sheet theory and variation of amplitude produces little change in both derivatives . at the lowest reynolds number there is little change in both derivatives with variation of aileron angle for the condition of natural transition, but at higher reynolds numbers the stiffness derivatives increase at . a forward movement of transition reduces the stiffness derivatives at the smaller aileron angles, but at, at the lowest reynolds number, an increase results . similar trends are observed for the damping derivatives above . comparison with vortex-sheet theory shows that the measured values of the stiffness and damping derivatives are approximately 0.6 of the theoretical values . measurements have been made of the direct tab derivatives and cross aileron-tab derivatives for a per cent aileron and 4 per cent (approx.) tab . in addition some measurements of the direct aileron derivatives have been made for comparison with earlier results together with a number of static derivatives for the wing and controls . the influence is shown of frequency parameter, reynolds number, position of transition, mean tab angle and sealing of the control hinge gaps . some tests have been made with the ailcron set at minus 8 deg and the tab at plus 12 deg for which condition the hinge moment on the aileron was zero . reasonable agreement with the values given by the /equivalent profile/ theory is shown for both direct damping derivatives and for the direct tab stiffness derivative . the direct aileron stiffness derivative shows some departure from the theoretical value when . at and the natural transition, comparison with the values given by flat-plate theory gives the following approximate factors, where suffix denotes the theoretical values ..'), (200, 'calculation of derivatives for a cropped delta wing with subsonic leading edges oscillating in a supersonic airstream .', 'watson,j.', 'arc r + m 3060, 1958.', 'calculation of derivatives for a cropped delta wing with subsonic leading edges oscillating in a supersonic airstream . the lift, pitching moment and full-span constant-chord control hinge-moment are derived for a cropped delta wing describing harmonic plunging and pitching oscillations of small amplitude and low-frequency parameter in a supersonic air stream . it is assumed that (a) the wing has subsonic leading edges, (b) the wing is sufficiently thin and the mach number sufficiently supersonic to permit the use of linearised theory . expressions for the various derivative coefficients are obtained for a particular delta wing of aspect ratio 1.8 and taper ratio these are avaluated and tabulated for mach numbers 1.1, 1.15, 1.2, 1.3, 1.4, 1.5, 1.6 and 1.944 .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (201, 'supersonic flow past oscillating airfoils including nonlinear thickness effects .', 'van dyke,m.d.', 'naca r1183, 1954.', 'supersonic flow past oscillating airfoils including nonlinear thickness effects . a solution to second order in thickness is derived for harmonically oscillating two-dimensional airfoils in supersonic flow . for slow oscillations of an arbitrary profile, the result is found as a series including the third power of frequency . for arbitrary frequencies, the method of solution for any specific profile is indicated, and the explicit solution derived for a single wedge . nonlinear thickness effects are found generally to reduce the torsional damping, and so to enlarge the range of mach numbers within which torsional instability is possible . this destabilizing effect varies only slightly with frequency in the range involved in dynamic stability analysis, but may reverse to a stabilizing effect at high flutter frequencies . comparison with a previous solution exact in thickness suggests that nonlinear effects of higher than second order are practically negligible . the analysis utilizes a smoothing technique that replaces the actural problem by one involving no kinked streamlines . this stratagem eliminates all consideration of shock waves from the analysis, yet yields the correct solution for problems that actually contain shock waves .'), (202, 'aircraft flutter .', 'williams,j.', 'arc r + m 2492, 1951.', 'aircraft flutter . the term flutter is used here to denote maintained or violent oscillations of a structure due to aerodynamic forces acting in conjunction with both elastic and inertial forces . attention is restricted to this particular branch of the more general field of aeroelasticity, which embraces buffeting, divergence, and reversal of control, as well as flutter,. airscrew flutter is not specifically considered . the monograph is divided into three main parts, each of which has been made self-contained for the convenience of readers . in the first part, general methods for the investigation of aircraft flutter, by theoretical analysis and by experiments on flutter models, are set out and discussed . a detailed account of the aerodynamic theory of wings in non-uniform motion is not included, since this has already been provided elsewhere, but methods for the evaluation of the aerodynamic forces required in a theoretical flutter analysis are logically developed, and a bibliography of researches on the aerodynamic theory is given in the appendix . investigations on specific types of aircraft flutter--namely wing flutter, control surface flutter, and tab flutter--are discussed in part these various types of flutter are considered, but the practical details of flutter-prevention devices are omitted . finally, in part 3, methods for the experimental determination of airloads on oscillating aerofoil systems are described, and available airload measurements are analysed and compared with theoretical results . an attempt has been made to refer in the text to all relevant british work reported by the early part of 1947 . foreign work has been mentioned in parts 1 and 2 only where necessary for the sake of completeness, but in part 3 and the appendix all relevant foreign references known to the author have been included . matrix notation has been used for the theoretical treatment in part 1, but otherwise its use has been avoided .'), (203, 'calculated velocity distributions and force derivatives for a series of high-speed aerofoils .', 'sinott,c.s.', 'arc r + m 3045.', 'calculated velocity distributions and force derivatives for a series of high-speed aerofoils . the polygon method of woods is used to calculate the velocity distribution over a number of two-dimensional aerofoils at low incidence, subcritical flows only being considered . lift slopes and aerodynamic centres at zero lift are also calculated . some comparisons with experimental results are made, and these show good agreement at zero incidence .'), (204, 'a study of the application of airfoil section data to the estimation of the high subsonic speed characteristics of swept wings .', 'hunton,l.w.', 'naca rm a55c23, 1955.', 'a study of the application of airfoil section data to the estimation of the high subsonic speed characteristics of swept wings . estimates of the variation with mach number of the aerodynamic characteristics of swept wings are made on the basis of airfoil section data combined with span-loading theory . the analysis deals with examinations of some 26 wings and wing-body combinations ranging in sweep angle from 30 to 60 and for mach numbers between 0.6 and 1.0 . results of the study indicate that the two-dimensional section data afford good qualitative information for such high-speed aerodynamic characteristics as the variation with mach number of drag, zero-lift pitching-moment coefficient, and lift coefficient for flow separation . quantitative estimates of the force and moment divergence mach numbers could not be made with any degree of certainty from the airfoil data alone . somewhat improved quantitative estimates for a given configuration were obtainable by basing the estimates on the measured characteristics for a wing of similar plan form but different section, and adjusting for the effects of differences in section on the basis of section data .'), (205, 'a correlation of airfoil section data with the aerodynamic loads measured on a 45 sweptback wing at subsonic mach numbers .', 'walker,h.j. and maillard,w.c.', 'naca rm a55c08, 1955.', 'a correlation of airfoil section data with the aerodynamic loads measured on a 45 sweptback wing at subsonic mach numbers . an investigation has been made of the possibility of correlating airfoil section data with measured pressure distributions over a 45 sweptback wing in the mach number range from 0.50 to 0.95 at a free-stream reynolds number of approximately 2 million . the wing had an aspect ratio of 5.5, a taper ratio of 0.53, naca 64a010 sections normal to the quarterchord line, and was mounted on a slender body of revolution . at mach numbers of 0.85 and below, and for wing normal-force coefficients below the maximum normal-force coefficient for an infinite-aspect-ratio wing yawed 45 to the flow (derived from airfoil section data by simple sweep relations), good correlation was obtained over most of the wing between wing-section and two-dimensional-airfoil pressure distributions . for greater normal-force coefficients lateral boundary-layer flow permitted the inboard wing sections to rise to high maximum section normal-force coefficients . the effectiveness of this lateral boundary-layer flow disappeared towards the tip . for all mach numbers, the influence of plan-form effects on the pressure distributions limited the quality of the correlation at the 20- and 95-percent-semispan stations . above a mach number of about 0.85 the shock waves originating at the juncture of the body and the wing trailing edge spread over the span, preventing further application of two-dimensional data . the spanwise load distributions at moderate normal-force coefficients could be predicted from span-loading theory for the entire mach number range of the tests .'), (206, 'the applications of the polygon method to the calculation of the compressible subsonic flow round two-dimensional profiles .', 'woods,l.c.', 'arc cp115, 1953.', 'the applications of the polygon method to the calculation of the compressible subsonic flow round two-dimensional profiles . this paper sets out the method now used by the author of applying the polygon method to the calculation of the compressible subsonic flow round two-dimensional aerofoils . tables have been constructed which can be used for all aerofoil shapes, putting the polygon method on the same footing numerically as goldstein''s method has the advantage over approximation 3 that it can be applied in the following cases which are beyond the scope of goldstein''s method .. conventional aerofoils, (b) the low-speed flow about very thick aerofoils, e.g., in reference 3 it is applied to circular cylinders, (c) the flow about symmetric aerofoils between either straight or constant pressure walls, (d) flow in asymmetric channels, and (e) more difficult problems of the flow about aerofoils in the presence of one or two constraining walls (to be published) . a method of calculating lift and moment coefficients, and their rates of change with incidence (a) is also given in the paper . as an example the velocity distribution and the rates of change of the lift and moment coefficients with a are calculated for the aerofoil r.a.e.104 at values of m (mach number at infinity) of 0, and 0.7, for various values of the incidence, a . the velocity distributions for zero incidence are found to be in fair agreement with the corresponding experimental results . the results at incidence are in satisfactory agreement with the experimental results, not for the same incidence, but for the same lift coefficient . it is found, for example, that at m = 0.7 the theory for a = 0.8 agrees best with experiment for a = 1.0, when the lift coefficients are approximately the same .'), (207, 'laminar boundary layer oscillations and transition on a flat plate .', 'schubauer,g.b. and skramstad,h.k.', 'naca r909, 1948.', 'laminar boundary layer oscillations and transition on a flat plate . this is an account of an investigation in which oscillations were discovered in the laminar boundary layer along a flat plate . these oscillations were found during the course of an experiment in which transition from laminar to turbulent flow was being studied on the plate as the turbulence in the wind stream was being reduced to unusually low values by means of damping screens . the first part of the paper deals with experimental methods and apparatus, measurements of turbulence and sound, and studies of transition . a description is then given of the manner in which oscillations were discovered and how they were found to be related to transition, and then how controlled oscillations were produced and studied in detail . the oscillations are shown to be the velocity variations accompanying a wave motion in the boundary layer, this wave motion having all the characteristics predicted by a stability theory based on the exponential growth of small disturbances . a review of this theory is given . the work is thus experimental confirmation of a mathematical theory of stability which had been in the process of development for a period of approximately 40 years, mainly by german investigators .'), (208, 'the hall effect in the viscous flow of ionized gas between parallel plates under transverse magnetic field .', 'sato,h.', 'j.phys.soc. japan, 16, 1961, 1427.', 'the hall effect in the viscous flow of ionized gas between parallel plates under transverse magnetic field . the electrical conductivity of an ionized gas is anisotropic in the presence of magnetic field (hall effect) . the conductivity is expressed by a tensor in the same form for both fully and partially ionized gases . by the use of modified ohm''s law and conventional magnetohydrodynamical equations the incompressible viscous flow between parallel plates under the transverse magnetic field is analyzed and an exact solution is obtained when the magnetic reynolds number is small . the numerical results reveal a remarkable effect of anisotropy of conductivity . the acceleration and deceleration of viscous ionized gas under combined electric and magnetic fields are also calculated .'), (209, 'boundary layer induced noise in the interior of aircraft .', 'ribner,h.s.', 'utia r37, 1956.', 'boundary layer induced noise in the interior of aircraft . at high speeds the turbulent boundary layer washing the airplane fuselage excites appreciable skin vibration, promoting strong noise in the interior . the fluctuating exciting pressure distribution can be represented as a pattern of moving waves (fourier integral) . a running ripple in the skin follows underneath each wave, and the noise is ultimately due to these ripples . the acoustic effects of the running ripples are calculated for an infinite sheet,. this is considered the main result of the paper . supersonically moving ripples radiate strong sound in the form of mach waves,. subsonically moving ripples radiate no sound . formulas for the mean square surface pressure and the energy flux are obtained for an assumed idealized turbulent pressure spectrum . the results are adapted to provide a tentative estimate of the noise generated at subsonic speeds in a practical fuselage . the running ripples are almost noise-free, but multiple reflections at the frames and stringers promote standing waves . an assumption is used to link the two kinds of waves, and this leads to provisional calculations of noise level .. on this basis the noise level is predicted to vary as for thin boundary layers, changing progressively to for thick layers ( = external air density, = speed, = layer thickness, = panel thickness) . some comparisons are made with experiment . finally, an idea for minimizing the noise is presented .'), (210, 'propeller in yaw .', 'ribner,h.s.', 'naca r820, 1945.', 'propeller in yaw . it was realized as early as 1909 that a propeller in yaw develops a side force like that of a fin . in 1917, r. g. harris expressed this force in terms of the torque coefficient for the unyawed propeller . of several attempts to express the side force directly in terms of the shape of the blades, however, none has been completely satisfactory . an analysis that incorporates induction effects not adequately covered in previous work and that gives good agreement with experiment over a wide range of operating conditions is presented herein . the present analysis shows that the fin analogy may be extended to the form of the side-force expression and that the effective fin area may be taken as the projected side area of the propeller . the effective aspect ratio is of the order of 8 and the appropriate dynamic pressure is roughly that at the propeller disk as augmented by the inflow . the variation of the inflow velocity, for a fixed-pitch propeller, accounts for most of the variation of side force with advance-diameter ratio v nd . the propeller forces due to an angular velocity of pitch are also analyzed and are shown to be very small for the pitching velocities that may actually be realized in maneuvers, with the exception of the spin . further conclusions are .. a dual-rotating propeller in yaw develops up to one-third more side force than a single-rotating propeller . a yawed single-rotating propeller experiences a pitching moment in addition to the side force . the pitching moment is of the order of the moment produced by a force equal to the side force, acting at the end of a lever arm equal to the propeller radius . this cross-coupling between pitch and yaw is small but possibly not negligible . the formulas for propellers in yaw derived herein (with the exception of the compressibility correction) and a series of charts of the side-force derivative calculated therefrom have been presented without derivation in an earlier report .'), (211, 'effect of slight blunting of leading edge of an immersed body on the flow around it at hypersonic speed .', 'chernyi,g.g.', 'nasa tt f-35, 1960.', 'effect of slight blunting of leading edge of an immersed body on the flow around it at hypersonic speed . manufacturing and maintainance of ideally sharp leading edges and noses is practically impossible, hence a discrepancy arises between the theory established for sharp edges and actual flow around slightly blunted edges, where a detached shock is formed with a subsonic adjacent region . semi-empirical method is worked out showing that the pressure distribution in the vicinity of the leading edge is the same for different thin profiles having the same shape of bluntness on their edges or noses . the data for a flat plate can be used for all of them . for moderate supersonic speed the pressure on the remaining body is practically unaffected by the nose bluntness, and can be computed from a sharp-edge theory . for high supersonic speed a slight blunting of the edge can considerably alter the pattern of flow over a large region . the method consists in replacing blunted edge by action of concentrated forces on the flow,. it is applied to blunted wedge where it shows doubling of the drag computed by classic theory, and to cones, where the drag of a blunted cone may become smaller than that of a sharp one .'), (212, 'theory and tunnel tests of rotor blade for supersonic turbines .', 'stratford,b.s. and sansome,g.e.', 'arc r + m 3275, 1960.', 'theory and tunnel tests of rotor blade for supersonic turbines . in special circumstances where a large work output is required from a turbine in a single stage it is necessary to use high pressure ratios across the nozzle blades, thus producing supersonic velocities at inlet to the rotor . as part of an investigation into such turbines, several designs for the inter-blade passages of the rotor have been tested in a two-dimensional tunnel, a design theory being developed concurrently . the first design, featuring constant passage width and curvature as in steam-turbine practice, but having thin leading and trailing edges, was found to suffer from focusing of the compression waves from the concave surface, with consequent flow separation from the opposite convex surface . it gave a velocity coefficient of measured at an inlet mach number of 1.90 and turning angle of 140 deg . the measured value compares favourably with values from previous steam tests, where the results have been in the range from 0.65 to 0.92 . from theoretical reasoning, and from additional test observations, a subsequent passage was designed having an inlet transition length of small curvature, leading to a free-vortex passage of double the transition curvature,. a small amount of contraction was incorporated . schlieren photographs showed the flow in this passage to be almost shock free . a thin region of low-energy air existed close to the convex surface, but liquid-injection tests located only one small bubble of reversed flow . pressure traverses at exit indicated a velocity coefficient of 0.952, based on the area-mean total pressure . when allowance is made for turning angle and reynolds number this result appears to compare quite favourably with previous work . it would seem that the optimum blade pitching in a turbine would be about 20 to 30 per cent closer than in a two-dimensional cascade . however, the resultant pitching tends to become very close, except at very large turning angles, with the result that in some applications difficulties could arise in the practical design and manufacture . several uncertainties remain and the present design must be regarded as still experimental .'), (213, 'the performance of supersonic turbine nozzles .', 'stratford,b.s. and sansome,g.e.', 'arc r + m 3273, 1959.', 'the performance of supersonic turbine nozzles . an investigation has been conducted at the national gas turbine establishment into the performance of turbines having high pressure ratios per stage . the present report discusses the mode of operation of supersonic nozzles for such turbines, and describes a cascade experiment . both theory and experiment demonstrate that the conditions imposed upon the supersonic flow immediately downstream of the nozzles (e.g., by a following row of rotor blades) exert an overriding influence upon the nozzle outlet flow angle, and hence upon the maximum pressure ratio obtainable across the nozzle--providing that the axial component of velocity is subsonic . this is an important difference from the more familiar flow of subsonic turbine nozzles, where, for example, the downstream gas angle is controlled predominantly by the nozzle blade shape and spacing . a suitable test technique using a closed-jet tunnel is demonstrated . the particular nozzles tested, of convergent-divergent form, had a straight-sided divergent portion of to axial direction) and a design mach number of 2 . the flow was found to be well behaved as regards shock pattern, losses, and starting over the range of pressure ratios tested--between 9 1 and 19 1 . in particular the efficiency at the design pressure ratio of 16.6 1 was high, the velocity coefficient calculated from traverses of pitot and static tubes being 0.98 . for the conversion of pitot to total pressure at a mach number of 2.5 a high accuracy is important in the measurement of the static pressure,. nevertheless readings from a conventional four-hole instrument appear to be reliable .'), (214, 'on the testing of supersonic compressor cascades .', 'staniforth,r.', 'ngte r212, 1957.', 'on the testing of supersonic compressor cascades . to facilitate the development of high speed axial-flow compressors, an investigation was made into the possibility of measuring blade performance in a stationary cascade at supersonic speeds . a suitable technique was developed and the losses in a variety of cascades were measured, but these losses were too high for the blading to have any possible application . it was concluded that if a useful compressor is to result, it is essential to test the cascades at mach numbers close to the existing technique was suitable only for zero incidence tests, and thus a new approach is necessary . some of the fundamentals of this cascade testing at low supersonic speeds are discussed in the light of the current understanding of the mode of operation of supersonic compressors at transonic speeds .'), (215, 'the test performance of highly loaded turbine stages designed for high pressure ratio .', 'johnston,i.h. and dransfield,d.c.', 'ngte r235, 1959.', 'the test performance of highly loaded turbine stages designed for high pressure ratio . a blade design for a highly loaded two-stage turbine is described and the test performance of the turbine is presented . some of the factors affecting the performance and matching of turbine blade rows operating at supersonic gas velocity are discussed and investigated by means of tests on a three-dimensional nozzle cascade tunnel and on a variety of single-stage turbine builds .'), (216, 'the supersonic axial flow compressor .', 'kantrowitz,a.', 'naca r974, 1950.', 'the supersonic axial flow compressor . an investigation has been made to explore the possibilities of axial-flow compressors operating with supersonic velocities into the blade rows . preliminary calculations showed that very high pressure ratios across a stage, together with somewhat increased mass flows, were apparently possible with compressors which decelerated air through the speed of sound in their blading . the first phase of this investigation, which has been reported in naca acr l5d20, was the development of efficient supersonic diffusers to decelerate air through the speed of sound . the present report is largely a general discussion of some of the essential aerodynamics of single-stage supersonic axial-flow compressors . in the supersonic flow about isolated bodies, large energy losses usually occur due to wave systems which extend far from the bodies . supersonic flow entering a cascade is considered and, in this case, the possibility of entirely eliminating this extended wave system is demonstrated,. thus, no reason for supersonic compressors to be necessarily inefficient is apparent . the conditions that occur as the flow through the compressor is being started are discussed and a hypothesis as to the type of transonic flow which will be encountered is proposed . as an approach to the study of supersonic compressors, three possible velocity diagrams are discussed briefly . because of the encouraging results of this study, an experimental single-stage supersonic compressor has been constructed and tested in freon-12 . in this compressor, air decelerates through the speed of sound in the rotor blading and enters the stators at subsonic speeds . a pressure ratio of about 1.8 at an efficiency of about 80 percent has been obtained .'), (217, 'flow pattern in a converging-diverging nozzle .', 'oswatitsch,k. and rothstein,w.', 'naca tm.1215.', 'flow pattern in a converging-diverging nozzle . the present report describes a new method for the prediction of the flow pattern of a gas in the two-dimensional and axially symmetrical case . it is assumed that the expansion of the gas is adiabatic and the flow stationary . the several assumptions necessary on the nozzle shape effect, in general, no essential limitation on the conventional nozzles . the method is applicable throughout the entire speed range,. the velocity of sound itself plays no singular part . the principal weight is placed on the treatment of the flow near the throat of a converging-diverging nozzle . for slender nozzles formulas are derived for the calculation of the velocity components as function of the location .'), (218, 'intensity, scale and spectra of turbulence in mixing region of free subsonic jet .', 'laurence,j.', 'naca r1292, 1956.', 'intensity, scale and spectra of turbulence in mixing region of free subsonic jet . the intensity of turbulence, the longitudinal and lateral correlation coefficients, and the spectra of turbulence in a 3.5 inch-diameter free jet were measured with hot-wire anemometers at exit mach numbers from 0.2 to 0.7 and reynolds numbers from the results of these measurements show the following .. (1) near the nozzle (distances less than 4 or 5 jet diam downstream of the nozzle) the intensity of turbulence, expressed as percent of core velocity, is a maximum at a distance of approximately increasing mach and or reynolds number . at distances greater than 8 jet diameters downstream of the nozzle, however, the maximum intensity moves out and decreases in magnitude until the turbulence-intensity profiles are quite flat and approaching similarity . (2) the lateral and longitudinal scales of turbulence are nearly independent of mach and or reynolds number and in the mixing zone near the jet vary proportionally with distance from the jet nozzle . (3) farther downstream of the jet the longitudinal scale reaches a maximum and then decreases approximately linearly with distance . (4) near the nozzle the lateral scale is much smaller than the longitudinal and does not vary with distance from the centerline, while the longitudinal scale is a maximum at a distance from the centerline of about mum moves out from the centerline . (6) a statistical analysis of the correlograms and spectra yields a /scale/ which, although different in magnitude from the conventional, varies similarly to the ordinary scale and is easier to evaluate .'), (219, 'on the strength distribution of noise sources along a jet .', 'ribner,h.s.', 'utia r51, 1958.', 'on the strength distribution of noise sources along a jet . the spatial distribution of noise sources along a jet is investigated by application of lighthill''s theory to regions of ''similar'' profiles . the analysis refers to the noise power emitted by a ''slice'' of jet (section between two adjacent planes normal to the axis) as a function of distance x of the slice from the nozzle . it is found that this power is essentially constant with x in the initial mixing region (x law), then further downstream (say 8 or 10 diameters from the nozzle) falls off extremely fast (x law or faster) in the fully developed jet . because of this striking attenuation of strength with distance, it is concluded that the mixing region produces the bulk of the noise and must dominate in muffler behavior,. conversely, the ''fat'' part of the jet must contribute much less to the total noise power than is commonly supposed . powell''s experiments on the effects of nozzle velocity profile on total noise power are interpreted qualitatively . the behavior of multiple-nozzle or corrugated mufflers, both as to overall quieting and frequency-shifting, is also interpreted in the light of the results . the possibility emerges that such mufflers may be improved without serious thrust loss by the addition of a sound-attenuating shroud .'), (220, 'a general purpose analogue correlator for the analysis of random noise signals .', 'g. a. allcock, a.m.i.e.e., a.m.brit.i.r.e. p. l. tanner, m.sc. (eng), grad. i.e.e. k. r. mclachlan, a.m.brit. i.r.e.', '', 'a general purpose analogue correlator for the analysis of random noise signals . a large proportion of the current research programme of the department of aeronautics and astronautics is concerned with the study of jet noise and boundary layer pressure fluctuations and their effect on aircraft structures . early in the work it was decided that for a complete description of the random processes involved it would be necessary in the experimental programme to make correlation measurements in addition to the more standard spectrum and amplitude distribution measurements . it was also felt that it would be desirable from the university point of view to construct a general purpose correlator which could later be used on other types of work . to this end it was decided to give the correlator a wider bandwidth than might strictly have been necessary for the problems on hand . subsequent development work has amply justified this decision .'), (221, 'a theoretical study of annular supersonic nozzles .', 'lord,w.t.', 'arc r + m 3227, 1961.', 'a theoretical study of annular supersonic nozzles . this paper is concerned with the design of annular supersonic nozzles to produce uniform flow in supersonic wind tunnels which are axi-symmetrical and which have an internal coaxial circular cylinder throughout . symmetrical two-dimensional and conventional axi-symmetrical nozzles are special cases of annular nozzles . proposals are made for design criteria sufficient to ensure that the flow inside a nozzle is free from limit lines and shock waves,. the criteria for (symmetrical) two-dimensional and (conventional) axi-symmetrical nozzles are new . the two outstanding procedures for designing two-dimensional and axi-symmetrical nozzles are generalised to apply to annular nozzles . one of the design procedures is mainly analytical and the other is mainly numerical,. the analytical expressions in both procedures are made much more complicated by the presence of the internal cylinder but the numerical process criteria and the mainly numerical design procedure are successfully applied to the design of a particular annular nozzle .'), (222, 'the flow over delta wings at low speeds with leading edge separation .', 'marsden,d.j., simpson,r.w. and rainbird,w.j.', 'coa r114, 1957.', 'the flow over delta wings at low speeds with leading edge separation . a low speed investigation of the flow over a 40 apex angle delta wing with sharp leading edges has been made in order to ascertain details of the flow in the viscous region near the leading edge of the suction surface of the wing . a physical picture of the flow was obtained from the surface flow and a smoke technique of flow visualization, combined with detailed measurements of total head, dynamic pressure, flow directions and vortex core positions in the flow above the wing . surface pressure distributions were also measured and integrated to give normal force coefficients . the results of this investigation were compared with those of other experimental investigations and also with various theoretical results . in particular, the normal force coefficients, vortex core positions and attachment line positions were compared with the theoretical results of mangler and smith, reference 19 . it was found that .. exist on the upper surface of the wing outboard of and below the main vortices . these secondary vortices are formed as a result of separation of the boundary layers developing outboard of the top surface attachment lines .'), (223, 'a note on the theory of the stanton tube .', 'gadd,g.e.', 'arc r + m 3147, 1958.', 'a note on the theory of the stanton tube . existing theories for the stanton tube are critically reviewed, and the paper then outlines a simple method which predicts the calibration function at high reynolds numbers to the right order of magnitude .'), (224, 'quasi-cylindrical surfaces with prescribed loadings in the linearised theory of supersonic flow .', 'jones,j.g.', 'rae tn.aero.2769.', 'quasi-cylindrical surfaces with prescribed loadings in the linearised theory of supersonic flow . a formula for the velocity field in terms of a given surface distribution of vorticity is applied to points lying on the surface . an equation giving the shape of a quasi circular-cylindrical surface in terms of a prescribed loading is derived . as an example a half ring wing with prescribed loading is discussed .'), (225, 'elliptic cones alone and with wings at supersonic speeds .', 'jorgensen,l.h.', 'naca tn4045,1957', 'elliptic cones alone and with wings at supersonic speeds . to help fill the gap in the knowledge of aerodynamics of shapes intermediate between bodies of revolution and flat triangular wings, force and moment characteristics for elliptic cones have been experimentally determined for mach numbers of 1.97 and sectional axis ratios from 1 through 6 and with lengths and base areas equal to circular cones of fineness ratios 3.67 and 5 have been studied for angles of bank of 0 and 90 . elliptic and circular cones in combination with triangular wings of aspect ratios 1 and 1.5 also have been considered . the angle-of-attack range was from 0 to about 16, and the reynolds number was 8x10, based on model length . in addition to the forces and moments at angle of attack, pressure distributions for elliptic cones at zero angle of attack have been determined . the results of this investigation indicate that there are distinct aerodynamic advantages to the use of elliptic cones . with their major cross-sectional axes horizontal, they develop greater lift and have higher lift-drag ratios than circular cones of the same fineness ratio and volume . in combination with triangular wings of low aspect ratio, they also develop higher lift-drag ratios than circular cones with the same wings . for winged elliptic cones, this increase in lift-drag ratio results both from lower zero-lift drag and drag due to lift . visual-flow studies indicate that, because of better streamlining in the crossflow plane, vortex flow is inhibited more for an elliptic cone with major axis in the plane of the wing than for a circular cone with the same wing . as a result, vortex drag resulting from lift is reduced . shifts in center of pressure with changes in angle of attack and mach number are small and about the same as for circular cones . comparisons of theoretical and experimental force and moment characteristics for elliptic cones indicate that simple linearized (flat plate) wing theory is generally adequate even for relatively thick cones . zero-lift pressure distributions and drag can be computed using van dyke''s second-order slender-body theory . for winged circular cones, a modification of the slender-body theory of naca rep. 962 results in good agreement of theory with experiment .'), (226, 'aerofoil theory of a flat delta wing at supersonic speeds .', 'robinson,a.', 'rae r.aero.2151, 1946.', 'aerofoil theory of a flat delta wing at supersonic speeds . lift, drag, and pressure distribution of a triangular flat plate moving at a small incidence at supersonic speeds are given for arbitrary mach number and aspect ratio . the values obtained for lift and drag are compared with the corresponding values obtained by strip theory . the possibility of further applications of the analysis leading up to the above results is indicated .'), (227, 'a technique for improving the predictions of linearised theory on the drag of straight edge wings .', 'randall,d.g.', 'arc cp394, 1957.', 'a technique for improving the predictions of linearised theory on the drag of straight edge wings . the curve of drag against mach number for straight-edged wings, calculated by using the linearised theory of supersonic flow, displays discontinuities in slope at the various mach numbers for which the edges are sonic . these features, which are not observed in practice, are due to the fact that linearised theory predicts an infinite pressure along a subsonic or sonic edge . it is shown that if the linearised equation of supersonic flow is used to determine the flow over straight-edged wings, but the linearised boundary condition is replaced by the full placed by plausible values . on this basis a simple method is derived for improving the linearised predictions of the drag of straight-edged wings which exhibits satisfactory agreement with experimental results . while the technique is not directly applicable to ridge lines, an artifice renders them amenable to similar treatment .'), (228, 'navier-stokes solutions at large distances from a finite body .', 'i-dee chang', '', 'navier-stokes solutions at large distances from a finite body . this paper is concerned with a theoretical investigation of the flow field at large distances from an object moving through a viscous fluid . the discussion will be restricted to the case of two-dimensional stationary incompressible flow . the object will be assumed to be of finite size . the domain of the fluid is infinite and it is assumed that there are no other boundaries for the fluid except that of the given object . the reynolds number will be assumed to have a fixed value., thus we shall not consider the limiting cases of the reynolds number tending to zero or to infinity .'), (229, 'interference between the wings and tail surfaces of a combination of slender body, cruciform wings and cruciform tail set at both incidence and yaw .', 'owen,p.r. and anderson,r.g.', 'rae r.aero.2471, 1952.', 'interference between the wings and tail surfaces of a combination of slender body, cruciform wings and cruciform tail set at both incidence and yaw . the interference between the wings and the tail surfaces of a combination of circular body, low aspect ratio cruciform wings and cruciform tail in an inviscid flow is analysed using the slender body theory . the system may be subjected to both incidence and yaw and, in general, the tail fins may be staggered angularly with respect to the main wings . the method is a development of that used by owen and maskell in r.a.e. report no. aero.2441 to analyse similar effects on a system set at zero yaw . simple expressions to determine the strengths and positions of the trailing vortices (supposed to be rolled-up) downstream of the main wings are given, and from them the forces on the tail are deduced . when the tail surfaces are triangular and of low aspect ratio an exact solution is obtained from slender body theory .. but for rectangular tail surfaces of moderate or high aspect ratio, it is suggested that the changes in lift and sideforce on the tail caused by the wing vortex field can be estimated approximately from the mean upwash and sidewash angles evaluated over the respective tail spans . formulae for these means angles are presented .'), (230, 'interference between the wings and tail plane of a slender wing-body tailplane combination .', 'owen,p.r. and maskell,e.c.', 'rae r.aero.2441, 1951.', 'interference between the wings and tail plane of a slender wing-body tailplane combination . an approximate method of predicting the interference between the wings and the tailplane of a slender wing-body-tailplane combination in an inviscid flow is developed, in order to explain the change in centre of pressure position with incidence which has been found to occur in wind tunnel and flight tests on guided weapons . incidence changes in one plane only, normal to the plane containing the wings and the tail surfaces, have been considered . the method is based on slender body theory and the assumption that the wing trailing vortices roll-up completely before they reach the tailplane,. it is, therefore, applicable to weapons equipped with low aspect ratio wings far separated from the tail surfaces . when the tail surfaces are triangular and of low aspect ratio, an analytical solution is given for the effect of the wing downwash field on the tail lift . for high aspect ratio, rectangular tail surfaces it is suggested by comparison with experimental data, that the tail lift may be estimated approximately from the value of the mean downwash angle across the tail span . a summary of the method is given in para.5 which, in conjunction with the introduction, may be read independently of the rest of the report .'), (231, 'practical calculation of second-order supersonic flow past non-lifting bodies of revolution .', 'van dyke,m.d.', 'naca tn.2744.', 'practical calculation of second-order supersonic flow past non-lifting bodies of revolution . calculation of second-order supersonic flow past bodies of revolution at zero angle of attack is described in detail, and reduced to routine computation . use of an approximate tangency condition is shown to increase the accuracy for bodies with corners . tables of basic functions and standard computing forms are presented . the procedure is summarized so that one can apply it without necessarily understanding the details of the theory . a sample calculation is given, and several examples are compared with solutions calculated by the method of characteristics .'), (232, 'accuracy of approximate methods for predicting pressure on pointed non-lifting bodies of revolution in supersonic flow .', 'ehret,d.m.', 'naca tn.2764.', 'accuracy of approximate methods for predicting pressure on pointed non-lifting bodies of revolution in supersonic flow . the accuracy and range of applicability of the linearized theory, second-order theory, tangent-cone method, conical-shock-expansion theory and newtonian theory for predicting pressure distributions on pointed bodies of revolution at zero angle of attack are investigated . pressure distributions and integrated pressure drag obtained by these methods are compared with standard values obtained by the method of characteristics and the theory of taylor and maccoll . three shapes, cone, ogive, and a modified optimum body, are investigated over a wide range of fineness ratios and mach numbers . it is found that the linearized theory is accurate only at low values of the hypersonic similarity parameter number to body fineness ratio) and that second-order theory appreciably extends the range of accurate application . the second-order theory gives good results on ogives when the ratio of the tangent of maximum surface angle to the tangent of the mach angle is less than 0.9 . tangent-cone method cannot be widely applied with good accuracy . in general, the conical-shock-expansion theory predicts pressure and drag within engineering accuracy when the hypersonic similarity parameter is greater than 1.2 . although newtonian theory gives good accuracy, except for cones, at the highest values of the hypersonic similarity parameter investigated, it is less accurate than the conical-shock-expansion theory .'), (233, 'the theoretical wave drag of some bodies of revolution .', 'fraenkel,l.e.', 'rae r.aero.2420.', 'the theoretical wave drag of some bodies of revolution . this report investigates the wave drag of bodies of revolution with pointed or open-nose forebodies and pointed or truncated afterbodies . the ''quasi-cylinder'' and ''slender-body'' theories are reviewed, a reversibility theorem is established, and the concept of the interference effect of a forebody on an afterbody is introduced . the theories are applied to bodies whose profiles are either straight or parabolic arcs, formulae and curves being given for forebody and afterbody drag, and for the interference drag . the results of the two theories are compared and are seen to agree well in the region of geometries where both theories are applicable .'), (234, 'a second order shock-expansion method applicable to bodies of revolution near zero lift .', 'syvertson,c.a. and denis,d.h.', 'naca tn.3527.', 'a second order shock-expansion method applicable to bodies of revolution near zero lift . a second-order shock-expansion method applicable to bodies of revolution near zero lift is developed . expressions defining the pressures on noninclined bodies are derived by the use of characteristics theory in combination with properties of the flow predicted by the generalized shock-expansion method . this result is extended to inclined bodies to obtain expressions for the normal-force and pitching-moment derivatives at zero angle of attack . the method is intended for application under conditions between the ranges of applicability of the second-order potential theory and the generalized shock-expansion mehtod - namely, when the ratio of free-stream mach number to nose fineness ratio is in the neighborhood of 1 . for noninclined bodies, the pressure distributions predicted by the second-order shock-expansion method are compared with existing experimental results and with predictions of other theories . for inclined bodies, the normal-force derivatives and locations of the center of pressure at zero angle of attack predicted by the method are compared with experimental results for mach numbers from 3.00 to 6.28 . fineness ratio 7, 5, and 3 cones and tangent ogives were tested alone and with cylindrical afterbodies up to 10 diameters long . in general, the predictions of the present method are found to be in good agreement with the experimental results . for non-inclined bodies, pressure distributions predicted with the method are in good agreement with existing experimental results and with distributions obtained with the method of characteristics . for inclined bodies, the normal-force derivatives per radian (for normal-force coefficients referenced to body base area) are predicted within 0.2 and the locations of the center of pressure are predicted within 0.2 body diameters . on the basis of these results, the second-order shock-expansion method appears applicable for values of the ratio of free-stream mach number to nose fineness ratio from 0.4 to 2 .'), (235, 'on the minimisation and numerical evaluation of wave drag .', 'eminton,e.', 'rae r.aero.2564.', 'on the minimisation and numerical evaluation of wave drag . a fourier analysis of the linearised theory expression for the zero-lift wave drag of a smooth, slender body in terms of its cross-sectional area distribution is used to derive the area distribution which minimises the expression for given length, volume, nose area, base area and n intermediate areas . another minimal deduced from this by relaxing the restriction on volume is used to evolve a method for the numerical evaluation of the original expression . two practical applications of these results are discussed . the first is in the design of wing-body combinations to have small drag rise at transonic speeds . the second is in the calculation of the wave drag of wing-body combinations at zero lift,. an example is constructed to illustrate the method and to give an indication of its accuracy .'), (236, 'criteria for thermodynamic equilibrium in gas flow .', 'rudin,m.', 'phys.fluids, 1, 1958.', 'criteria for thermodynamic equilibrium in gas flow . when gases flow at high velocity, the rates of internal processes may not be fast enough to maintain thermodynamic equilibrium . by defining quasi-equilibrium in flow as the condition in which the temperature, pressure, density, and velocity deviate by less than a fixed, small percentage from what they would be if the flowing gas could actually be in thermodynamic equilibrium, criteria are derived for determining whether quasi-equilibrium is a stable condition in the flow . by use of excitation of molecular vibration as an example, the general properties of criteria curves are discussed and interpreted . a discussion is given of how to use these results to determine definitely whether a flow is or is not in thermodynamic equilibrium . applications to dissociating gases, to mixtures, and to the phenomenon of /choking/ in a laval nozzle are given special consideration . for cases when application of the criteria predict nonequilibrium, equations are provided in a form useful for numerical forward integration along streamlines .'), (237, 'a compressor routine test code .', 'n. a. dimmock', 'communicated by the deputy controller aircraft (research and development), ministry of aviation', 'a compressor routine test code . the routine testing of aircraft-type compressors.dash in the main, axial-flow, multi-stage compressors.dash requires a compromise between research accuracy and the practical considerations . this test code is the outcome of a survey of compressor testing techniques and instrumentation, initiated and subsequently discussed and endorsed by the aerodynamics sub-committee of the gas turbine collaboration committee . the code aims at defining methods of measurement and weighting whereby compressor performance can be obtained sufficiently accurately for a realistic and direct comparison to be made between one compressor and another . the measurement of a quantity at a point in the fluid flow, and the averaging and weighting of such measurements have been treated separately as far as is possible . the recommendations are given in the main text, whilst additional discussion on these is put into the appendices .'), (238, 'on a determination of the pitot-static tube factor at low reynolds numbers, with special reference to the measurement of low air speeds .', 'ower,e. and johansen,f.c.', 'arc r + m 1437, 1931.', 'on a determination of the pitot-static tube factor at low reynolds numbers, with special reference to the measurement of low air speeds . reasons for enquiry--to provide a standard instrument for the calibration of low speed anemometers .'), (239, 'design and calibration at low speeds of a static tube and a pitot-static tube with semi-ellipsoidal nose shapes .', 'kettle,d.j.', 'rae tn.aero.2247, 1953.', 'design and calibration at low speeds of a static tube and a pitot-static tube with semi-ellipsoidal nose shapes . a new static tube and a new pitot-static tube have been designed and calibrated in the no.1 and the no.2 11 ft x 8 ft wind tunnels of the r.a.e., using a long static tube, the error of which is believed to be very small, as a standard for comparison . the results show that the static pressure measured by these tubes is in error due to the supporting strut and to the nose shape of the tube by an amount which may be calculated for positions of the static slot, or holes, greater than 10 tube diameters ahead of the strut . the readings show no measurable scale effect in the speed range 100-230 ft sec . the static tube is insensitive to yaw in the range 1 with a square-edged slot and is even less sensitive to yaw when the slot edges are rounded . the turbulence of the tunnel has an effect on the static pressure reading .'), (240, 'a theoretical analysis of heat transfer in regions of separated flow .', 'application to turbulent separations is made for a prandtl number of', 'unity in low-speed flow without injection . all calculations are for the case of zero boundary-layer thickness at the position of separation . for alminar separations the differential equations for viscous flow at arbitrary mach number are solved for the enthalpy and velocity profiles within the thin layer where mixing with dead air takes place . results are presented in tabular form for prandtl numbers between 0.1 and 10 . the rate of heat transfer to a separated laminar region in air laminar boundary layer having the same constant pressure . injection of gas into the separated region is calculated to have a powerful effect in reducing the rate of heat transfn to the wall . it is calculated that a moderate quantity of gas injection reduces to zero the heat transfer in a laminar separated flow .', 'a theoretical analysis of heat transfer in regions of separated flow . the flow field analyzed consists of a thin, constant pressure viscous mixing layer separated from a solid surface by an enclosed region of low-velocity air (/dead air/) . the law of conservation of energy is employed to relate calculated conditions within the separated mixing layer to the rate of heat transfer at the solid surface . this physical speed is app ied to alminar separations in compressible flow for various prandtl numbers, including consideration of the case where air is injected into the separated region .'), (241, 'laminar mixing of a non-uniform stream with a fluid at rest .', 'nash,j.f.', 'arc 22245, 1960.', 'laminar mixing of a non-uniform stream with a fluid at rest . a theoretical analysis is made of the constant pressure laminar mixing process between a stream having an initial boundary layer velocity profile, and a fluid at rest . the present theory follows the methods of w. tollmien and s. i. pai with certain modifications . the results apply to incompressible flow, but can be extended to the compressible case without difficulty .'), (242, 'an approximate theory of base pressure in two dimensional flow at supersonic speeds .', 'kirk,f.n.', 'rae tn.aero.2377, 1954.', 'an approximate theory of base pressure in two dimensional flow at supersonic speeds . an approximate theory of the base pressure in two-dimensional flow at supersonic speeds is presented using asimplified representation of the flow and some of the findings of tollmien''s work on turbulent mixing in incompressible flow . good qualitative predictions of the effects of a boundary layer, of bleed air and of boat-tailing are obtained .'), (243, 'investigation with an interferometer of the turbulent mixing of a free supersonic jet .', 'gooderum,p.b., wood,g.p. and brevoort,m.', 'naca r963, 1950.', 'investigation with an interferometer of the turbulent mixing of a free supersonic jet . the free turbulent mixing of a supersonic jet of mach number of which a description is given, was used for the investigation . density and velocity distributions through the mixing zone have been obtained . it was found that there was similarity in distribution at the cross sections investigated and that, in the subsonic portion of the mixing zone, the velocity distribution fitted the theoretical distribution for incompressible flow . it was found that the rates of spread of the mixing zone both into the jet and into the ambient air were less than those of subsonic jets .'), (244, 'an improved smoke generator for use in the visualisation of airflow, particularly boundary layer flow at high reynolds numbers .', 'preston,j.h. and sweeting,n.e.', 'arc r + m 2023, 1943.', 'an improved smoke generator for use in the visualisation of airflow, particularly boundary layer flow at high reynolds numbers . and rapid method by which boundary layer flow was rendered visible has been previously described in the journal of the royal aeronautical society . it gave promise of being useful at the highest tunnel speeds provided a denser smoke could be obtained, which at the same time was free from the troublesome deposits associated with the wood smoke . of the aerodynamics division attempts were made by the fuel research station to improve the density of the wood smoke and to reduce the deposits . these they showed were conflicting requirements, and whilst some improvement was effected, it was not sufficient for observation in the new tunnels at high speeds . the staff of the director-general of scientific research and development, ministry of supply, was then approached and it was decided to develop an oil smoke generator from a simple generator of this type which was demonstrated to us . this has been done successfully . the final apparatus in contrast to the wood smoke generator is light and compact . it takes only a few minutes to start and can be run as long as desired . improvement on the wood smoke both as regards density and freedom from deposits, which cause premature transition . the density and quality of the smoke are now under control . smokes ranging from a light smoke of bluish white colour to a heavy smoke dense white in appearance can be obtained . the oil smoke retains the advantages of the wood smoke in that it is non-corrosive and non-irritant, and the smell can be tolerated even when it is present in a considerable concentration . a certain amount of condensation is inevitable with oil smokes, but with suitable precautions troubles arising from this can be avoided . a dry solid smoke made by melting a hard wax was successfully generated with the same apparatus . unfortunately because of its flocculent nature this smoke gave rise to solid deposits when passed through bore tubing, leading eventually to complete blockage . this seems to be a feature of solid smokes . the apparatus has been used to determine transition and laminar separation points on model wings in a number of the national physical laboratory tunnels . smoke filaments have been maintained in the laminar state up to wind speeds of 180 ft. sec. in the new tunnels . there is much to be said for making a standard practice of visualising boundary layer flow on models, particularly as the technique is simple and rapid . it would greatly assist the interpretation of force measurements and the more detailed explorations of the boundary layer by total head tubes and hot wires . the use of oil smoke is not limited to boundary layer flow visualisation . the apparatus described in this report would seem to be particularly suited for educational work in small demonstration tunnels .'), (245, 'the ground effect on the jet flap in two dimensions .', 'huggett,d.j.', 'arc 19,713, 1957.', 'the ground effect on the jet flap in two dimensions . this paper presents the results of the first part of an experimental investigation of the ground effect on simple jet flap aerofoils . in this part of the work an aerofoil having a 58.1 deg jet flap was tested under two-dimensional conditions . the pressure lift on the aerofoil was measured, with the ground at fixed positions, for varying jet momentum coefficients . it was found that the effect of the ground on the pressure lift was very small up to a certain critical jet coefficient . on increasing the jet coefficient beyond the critical value a marked loss of pressure lift was observed . this critical value referred to is approximately the same as the jet coefficient at which the jet first hits the ground . some significant, though highly tentative comments, are made regarding the practical application of this work to the take-off characteristics of a jet flapped aircraft .'), (246, 'the design of minimum drag tip fins . with an appendix - on the conformal transformation of a wing with a fin .', 'falkner,v.m.', 'arc r + m 2279, 1945.', 'the design of minimum drag tip fins . with an appendix - on the conformal transformation of a wing with a fin . the report describes an investigation into the design of minimum drag tip fins by lifting line theory . the work is based on an exact solution of the conformal transformation which is applicable to this problem following the method of trefitz . three types of solution are treated, corresponding to symmetrical upper and lower fins, single upper or lower fins, and unequal upper and lower fins . a representative range of solutions for circulation distribution along wing and fins has been calculated for each of the three cases by the use of elliptic and theta functions . a detailed account is given, with examples, of the procedure for calculating the plan of wing and fins, the lift and induced drag, and the setting of the fins .'), (247, 'the calculation of the pressure distribution on thick wings of small aspect ratio at zero lift in subsonic flow .', 'weber,j.', 'arc r + m 2993.', 'the calculation of the pressure distribution on thick wings of small aspect ratio at zero lift in subsonic flow . the method of expressing the velocity increment over aerofoils directly in terms of the section ordinates wings of finite aspect ratio . the wings considered are untapered in plan-form but may be tapered in thickness . the section can be of any given shape so that in this sense the analysis is more general than that of refs. 3 to 6 which deal with wings of biconvex section . the coefficients required in the calculation are tabulated for the centre-section of straight and swept-back wings of aspect ratios 0.5,. 1,. 2,. and 4, the wing of infinite aspect ratio having been treated in ref. 1 . the remaining calculations can be made very quickly . since wings of very small aspect ratio can be treated also by the method of slender-body theory, the relations between linear theory, slender-body theory, and linearised slender-body theory are discussed . for the special case of ellipsoids, the results obtained from the various methods are compared with the exact solution .'), (248, 'the application of lighthill formula for numerical calculation of pressure distributions on bodies of revolution at supersonic speed and zero angle of attack .', 'ohman,l.', 'saab tn.45, 1960.', 'the application of lighthill formula for numerical calculation of pressure distributions on bodies of revolution at supersonic speed and zero angle of attack . an integral expression, given by lighthill and based on linearized theory, for the external supersonic flow over the surface of slender pointed or ducted bodies of revolution at zero angle of attack is shown to give a good approximation of the exact flow for a much wider mach number and thickness range than could be expected from linearized theory . a numerical method, based on this expression, is developed and applied for digital computing . some results from applying the digital computing procedure for determining the pressure distribution and wave drag for various bodies of revolution are given .'), (249, 'formulae for the computation of the functions employed for calculating the velocity distribution about a given aerofoil .', 'watson,e.j.', 'arc r + m 2176.', 'formulae for the computation of the functions employed for calculating the velocity distribution about a given aerofoil . in order to determine the velocity distribution about an arbitrary aerofoil, it is necessary to evaluate the functions and (in the notation of aerofoil theory) when is given numerically . if the values of are specified at 2n points equally spaced about the circle into which the aerofoil is transformed, the formulae obtained here may be used to calculate these functions at the same points . formulae are also given for calculating the integrals of or, since these have application to the design of aerofoils by thwaites''s numerical method . the simplicity of the formulae for and enables the effect on the velocity distribution of a local change of shape readily to be determined by making n large . this is discussed in 3 . the formulae are collected in the appendix, and a table of the coefficients for the case n = 20 is given .'), (250, 'pressure distributions at zero lift for delta wings with rhombic cross sections .', 'eminton,e.', 'arc cp.525, 1960.', 'pressure distributions at zero lift for delta wings with rhombic cross sections . the linearised theory of thin wings is used to calculate pressure distributions over delta wings with rhombic cross sections . a deuce programme has been written for the calculation and some of the results are compared with those of slender thin wing theory .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (251, 'a collection of longitudinal stability derivatives of wings at supersonic speeds .', 'naysmith,a.', 'rae tn.aero.2423, 1956.', 'a collection of longitudinal stability derivatives of wings at supersonic speeds . a collection has been made of theoretical data, for wings alone, on those stability derivatives that govern the short-period oscillation of aircraft travelling at supersonic speeds . all the derivatives available have been obtained by means of the linear theory, and so the information given is subject to the usual limitations . the information has been presented in what is hoped is the most convenient form to show its extent, and to expose the parts of the field where experimental investigation is most needed .'), (252, 'an investigation of interference effects on similar models of different size in various transonic tunnels in the u.k. .', 'f. o/hara and l. c. squire, r.a.e.', 'and a. b. haines, a.r.a.', 'an investigation of interference effects on similar models of different size in various transonic tunnels in the u.k. . details are given of a programme of tests being made on similar swept-wing models in transonic tunnels of different types . force measurement results at subsonic speeds in the r.a.e. 3 ft. by 3 ft. slotted tunnel show only small interference effects for models of moderate blockage at low incidence., at higher incidences, the interference effect on lift becomes appreciably greater than estimated by theory, and significant pitching moment differences occur, apparently due to wall interference on the wing flow field . comparable but smaller effects are evident in the results from the a.r.a. 9 ft. by 8 ft. perforated tunnel . at speeds just above m = 1, the force fluctuates as speed is increased, because of wave reflection interference . the magnitude of the fluctuations diminishes as speed is further increased and this reduction is more marked in the perforated tunnel . pressure measurements along the top of the body at zero incidence show delay in shock movements at high subsonic speeds indicating a blockage effect on speed., the effect is larger in the perforated tunnel though smaller than predicted by theory . above m = 1, both expansion and shock waves are strongly reflected in the slotted tunnel but considerable alleviation, particularly of shock waves, is achieved in the perforated tunnel, for which an analysis of the effects is given, showing for example, the effect of the open-area distribution of the walls .'), (253, 'on the ground level disturbance from large aircraft flying at supersonic speeds .', 'lilley,g.m. and spillman,j.j.', 'coa n103.', 'on the ground level disturbance from large aircraft flying at supersonic speeds . the whitham-walkden theory for the estimation of the strength of shock waves at ground level from aircraft flying at supersonic speeds is applied to the case of a typical projected supersonic civil transport aeroplane . if a figure of 2 lb sq.ft. (including a factor of 2 for ground reflection) is taken as an upper limit for the acceptable strength of the bow wave from such an aircraft it is shown that restrictions on the climb and flight plan will be involved . the advantage of the employment of larger engines with or without afterburning is discussed, with reference also to the penalties involved owing to the increase in weight of the aircraft and its direct operating costs . finally it is suggested that an aircraft of given volume could be designed, by suitable choice of thickness and lift distribution, to minimise the strength of the shock waves in the far field .'), (254, 'boundary layers with suction and injection . a review of published work on skin friction .', 'craven,a.h.', 'coa r136.', 'boundary layers with suction and injection . a review of published work on skin friction . available data on the effects of suction and injection on skin friction are summarised and compared . it is shown that injection into a turbulent boundary layer can produce a skin friction coefficient lower than the laminar value at the same reynolds number on an impermeable plate .'), (255, 'an approximate solution of the turbulent boundary layer equations in incompressible and compressible .', 'lilley,g.m.', 'coa r134, 1960.', 'an approximate solution of the turbulent boundary layer equations in incompressible and compressible . if over the ''outer region'' of the boundary layer, where the mean velocity varies but little from its value outside the shear layer, a virtual eddy viscosity is defined, which is constant over the outer region but varies in the direction of the mainstream, a solution of the turbulent boundary layer equations can be found which satisfies the appropriate boundary conditions . the solution leads to a compatibility condition for the virtual eddy viscosity in terms of the wall shear stress, the boundary layer momentum thickness and the mainstream velocity, at least for the case of a constant external velocity . this compatibility condition, which can be expressed as for moderate to high reynolds numbers, where is the shear velocity, is the boundary layer thickness and is the virtual eddy (kinematic) viscosity, is just the condition townsend (1956) found for the equilibrium of the large eddies . the numerical value of the constant derived by townsend agrees with ours for reynolds numbers (based on x) of about . with this relation for an equation, analoguous to the momentum integral equation solution, can be found for as a function of local freestream velocity, with one disposable parameter .'), (256, 'an experimental study of the glancing interaction between a shock wave and a turbulent boundary layer .', 'stanbrook,a.', 'arc cp.555, 1960.', 'an experimental study of the glancing interaction between a shock wave and a turbulent boundary layer . an experimental study has been made at mach numbers from 1.6 to 2.0 of the interaction between the turbulent boundary layer on a side wall of a wind tunnel and the shock wave produced by a plate mounted on the wall . under these conditions the shock wave boundary layer interaction was three dimensional at least over the region investigated (up to 10 boundary layer thicknesses from the plate) . it was found that the boundary layer was separated by a shock wave of strength type occur on the sides of fuselages at the wing fuselage junction and may therefore be important with regard to the design of waisted shapes .'), (257, 'on turbulen flow between parallel plates .', 'pai,s.i.', 'j.app.mech. 20, 1953, 109.', 'on turbulen flow between parallel plates . the reynolds equations of motion of turbulent flow of incompressible fluid have been studied for turbulent flow between parallel plates . the number of these equations is finally reduced to two . one of these consists of mean velocity and correlation between transverse and longitudinal turbulent-velocity fluctuations only . the other consists of the mean pressure and transverse turbulent-velocity intensity . some conclusions about the mean pressure distribution and turbulent fluctuations are drawn . these equations are applied to two special cases .. one is poiseuille flow in which both plates are at rest and the other is couette flow in which one plate is at rest and the other is moving with constant velocity . the mean velocity distribution and the correlation can be expressed in a form of polynomial of the co-ordinate in the direction perpendicular to the plates, with the ratio of shearing stress on the plate to that of the corresponding laminar flow of the same maximum velocity as a parameter . these expressions hold true all the way across the plates, i.e., both the turbulent region and viscous layer including the laminar sublayer . these expressions for poiseuille flow have been checked with experimental data of laufer fairly well . it also shows that the logarithmic mean velocity distribution is not a rigorous solution of reynolds equations .'), (258, 'the effect of turbulence on slider bearing lubrication .', 'chou,y.t. and saibei,e.', 'j.app.mech. 25, 1959, 122.', 'the effect of turbulence on slider bearing lubrication . based on prandtl''s mixing-length mechanism, the pressure equation for turbulent flow in slider-bearing lubrication is derived . an analytical solution is given and compared with the one for laminar flow . it is found that the turbulent effect increases the pressure and consequently, the load-carrying capacity . however, the power loss also increases .'), (259, 'second order theory for unsteady supersonic flow past slender pointed bodies of revolution .', 'revell, j. d.', 'j. ae. scs.1960, 730.', 'second order theory for unsteady supersonic flow past slender pointed bodies of revolution . the thermodynamic properties (z = pv rt, e rt, h rt, s r, and pressure) are given for equilibrium mixtures of dissociated and ionized molecules and atoms of the elements nitrogen and oxygen having the low temperature composition of .78847 n and .21153 o . the tabulated properties of this mixture (a close approximation to the properties of air) are given at close intervals from 2000 to and 10 times the normal density . the results are based on chemical equilibria between the species o, o, n, n, no, no, no, no, o, o, o, o, n, n, n and electrons . the method of presentation permits later corrections for the effect of argon and co and the contribution of intermolecular forces . the calculations are based on 9.758 e.v. as the dissociation energy of molecular nitrogen and 1.45 e.v. as the electron affinity of atomic oxygen .'), (260, 'a critical review of skin friction and heat transfer solutions of the laminar boundary layer of a flat plate .', 'rubesin,m.w. and johnson,h.a.', 'asme trans. 1949.', 'a critical review of skin friction and heat transfer solutions of the laminar boundary layer of a flat plate . a review is made of existing literature concerned with the analytical investigation of the velocity and temperature distributions in the boundary layers of a heated (or cooled) flat plate . the plate is postulated infinitely thin and is parallel to a uniform fluid stream . the more recent solutions include the combined effects of frictional dissipation and variable fluid properties . only the results pertaining to the transfer phenomena occurring at the plate surface are included, i.e., skin drag and over-all heat transfer,. the individual temperature and velocity distributions leading to these results are omitted .'), (261, 'experiments on axi-symmetric boundary layers along a long cylinder in incompressible flow .', 'yashura,m.', 'trans. japan soc.ae.sc. 2, 1959.', 'experiments on axi-symmetric boundary layers along a long cylinder in incompressible flow . experiments on axi-symmetric boundary layers along a long cylinder were made especially to investigate the effect of transverse curvature on the velocity profile . laminar velocity profiles were measured and compared with theoretical ones with good accuracy . a representative profile was plotted to see the effect of transverse curvature, which showed small, but obvious effect accompanied by increasing skin friction . the transition of the flow from laminar to turbulent was observed, and its reynolds number was estimated to occur at 1.2 1.8x10 in the present experiment . the turbulent profile was also measured and plotted by using the coordinates to express the wall law deduced by richmond, from which it was estimated that, as the ratio of the momentum thickness to body radius increases, the profile near the outer layer tends to bend down relative to the line of logarithmic wall law .'), (262, 'the formation of a blast wave by a very intense explosion .', 'taylor,g.i.', 'proc. roy. soc. a, 201, 1950, 159.', 'the formation of a blast wave by a very intense explosion . this paper was written early in 1941 and circulated to the civil defence research committee of the ministry of home security in june of that year . the present writer had been told that it might be possible to produce a bomb in which a very large amount of energy would be released by nuclear fission--the name atomic bomb had not then been used--and the work here described represents his first attempt to form an idea of what mechanical effects might be expected if such an explosion could occur . in the then common explosive bomb mechanical effects were produced by the sudden generation of a large amount of gas at a high temperature in a confined space . the practical question which required an answer was .. would similar effects be produced if energy could be released in a highly concentrated form unaccompanied by the generation of gas .qm this paper has now been declassified, and though it has been superseded by more complete calculations, it seems appropriate to publish it as it was first written, without alteration, except for the omission of a few lines, the addition of this summary, and a comparison with some more recent experimental work, so that the writings of later workers in this field may be appreciated . an ideal problem is here discussed . a finite amount of energy is suddenly released in an infinitely concentrated form . the motion and pressure of the surrounding air is calculated . it is found that a spherical shock wave is propagated outwards whose radius r is related to the time t since the explosion started by the equation where is the atmospheric density, e is the energy released and s a calculated function of, the ratio of the specific heats of air . the effect of the explosion is to force most of the air within the shock front into a thin shell just inside that front . as the front expands, the maximum pressure decreases till, at about 10 atm., the analysis ceases to be accurate . at 20 atm. 45 of the energy has been degraded into heat which is not available for doing work and used up in expanding against atmospheric pressure . this leads to the prediction that an atomic bomb would be only half as efficient, as a blast-producer, as a high explosive releasing the same amount of energy . in the ideal problem the maximum pressure is proportional to r, and comparison with the measured pressures near high explosives, in the range of radii where the two might be expected to be comparable, shows that these conclusions are borne out by experiment .'), (263, 'cylindrical shock waves produced by instantaneous energy release .', 'lin,s.c.', 'j.app.phys. 25, 1954, 54.', 'cylindrical shock waves produced by instantaneous energy release . taylor''s analysis of the intense spherical explosion has been extended to the cylindrical case . it is found that the radius r of a strong cylindrical shock wave produced by a sudden release of energy e per unit length grows with time t according to the equation where is the atmospheric density and is a calculated function of the specific heat ratio . for is found to be approximately unity . for this case, the pressure behind the shock wave decays with radius r according to the relation . applying the results of this analysis to the case of hypersonic flight, it can be shown that the shock envelope behind a meteor or a high-speed missile is approximately a paraboloid given by where d and v denote the total drag and the velocity of the missile, respectively, and x is the distance behind the missile .'), (264, 'asymptotic solution of the two dimensional oscillating aerofoil problem for high subsonic mach numbers .', 'eckhaus,w.', 'proc. 9th int. con. app.mech. 1956.', 'asymptotic solution of the two dimensional oscillating aerofoil problem for high subsonic mach numbers . a new method has been given, for obtaining asymptotic solutions of a boundary value problem for the wave equation . the method is simpler than the method previously given by burger, and leads to a result identical with burger''s result .'), (265, 'some instabilities arising from the interaction between shock waves and boundary layer .', 'lambourne,n.c.', 'npl aero.348, 1958.', 'some instabilities arising from the interaction between shock waves and boundary layer . a brief review is made of the available information concerning the flow fluctuations and instabilities arising from shock-induced separation in the flow over aerofoils and wings . the influence this phenomenon has on the oscillatory behaviour of aerofoils and control surfaces is also briefly discussed . a more detailed consideration is devoted to a recent investigation at the n.p.l. into the part played by shock-induced separation in the instability of a control surface .'), (266, 'exact solution of the neumann problem . calculation for non- circulatory plane and axially symmetric flows about or within arbitrary boundaries .', 'smith,a.n.c. and pierce,j.', '3rd nat. con. app. mech. 1958.', 'exact solution of the neumann problem . calculation for non- circulatory plane and axially symmetric flows about or within arbitrary boundaries . an exact general method of solving the neumann or second boundary-value problem has been developed and has been applied to the calculation of low-speed flows about or within bodies of almost any shape, provided the flow is either plane or has axial symmetry . solid-body, inlet, and purely internal flow problems can be solved . the method is capable of dealing with several bodies at once in the presence of one another, and consequently interference problems can be treated with ease . boundaries need not be solid, that is, flows involving area suction can be calculated . velocities can be computed not only for points on the surface of the body but for the entire flow field . a surface source distribution is used as a basis for solution . this leads to a fredholm integral equation of the second kind, which is solved as a set of linear algebraic equations, usually by a modified seidel method . at the present time the solution is programed on the ibm 704 edpm to solve the flow about any body that has the previously mentioned characteristics and whose profile can be defined satisfactorily by no more than 300 coordinate points . a number of solutions are presented, to show both the scope of the method and its accuracy . computations require from three minutes to two hours, depending upon the shape of the body and the number of points used to define it .'), (267, 'steady and transient free convection of an electrically conducting fluid from a vertical plate in the presence of a magnetic field .', 'gupta,s.', 'app. sc. res. 9, 1959.', 'steady and transient free convection of an electrically conducting fluid from a vertical plate in the presence of a magnetic field . an analysis is made for the laminar free convection and heat transfer of a viscous electrically conducting fluid from a hot vertical plate in the case when the induced field is negligible compared to the imposed magnetic field . it is found that similar solutions for velocity and temperature exist when the imposed magnetic field (acting perpendicular to the plate) varies inversely as the fourth root of the distance from the lowest end of the plate . explicit expressions for velocity, temperature, boundary layer thickness and nusselt number are obtained and the effect of a magnetic field on them is studied . it is found that the effect of the magnetic field is to decrease the rate of heat transfer from the wall . in the second part, the method of characteristics is employed to obtain solutions of the time-dependent hydromagnetic free convection equations (hyperbolic) of momentum and energy put into integral form . the results yield the time required for the steady flow to be established, and the effect of the magnetic field on this time is studied .'), (268, 'several magnetohydrodynamic free-convection solutions .', 'cramer,k.r.', '5th nat. heat transfer con. 1962.', 'several magnetohydrodynamic free-convection solutions . the influence of transverse magnetic fields on the laminar free-convection flow of liquid metals over a vertical flat plate and between vertical parallel plates is examined for specific wall temperature variations and prandtl numbers . the extent of influence on the flow and temperature fields is determined by the magnitude of a nondimensional influence parameter which is the ratio of the magnetic force to the buoyant force . in general, increasing the magnetic field strength decreases the magnitude of the velocity, wall shear, and surfaces heat transfer and increases the temperature throughout the fluid . analytical results demonstrate that magnetic fields of practical strengths exert considerable influence on liquid metal free-convection flow fields .'), (269, 'on a laminar free-convection flow and heat transfer of electrically conducting fluid on a vertical flat plate in the presence of a transverse magnetic field .', 'mori,y.', 'trans. japan soc. ae. sc. 2, 1959.', 'on a laminar free-convection flow and heat transfer of electrically conducting fluid on a vertical flat plate in the presence of a transverse magnetic field . the free-convection flow and heat transfer of an electrically conducting fluid on a vertical plate in the presence of a transverse magnetic field is analysed for a magnetic field fixed to the electrically non-conducting wall . the boundary layer equations for self-preserving flows are integrated numerically for the prandtl number of unity, and the effect of the transverse magnetic field on the velocity profile, temperature profile and rate of heat transfer is discussed . it is concluded that the heat transfer rate is reduced as the magnetic field intensity is increased .'), (270, 'on combined free and forced convection laminar magnetohydrodynamic flow and heat transfer in channels with transverse magnetic field .', 'mori,y.', 'int. devel. in heat transfer,1961.', 'on combined free and forced convection laminar magnetohydrodynamic flow and heat transfer in channels with transverse magnetic field . combined free and forced convective heat transfer in vertical channels has been studied by many researchers . due to the need for engineering design information there have been many papers concerning cases of fully developed flow with varying wall temperature . forced flows in a channel of electrically conducting fluid with a transverse magnetic field have been studied and the large effects of a magnetic field on the flow pattern have been established . flows of combined free and forced convection in electrically conducting fluids in vertical channels with a transverse magnetic field are expected to attract attention in future engineering applications, for example, in a magneto-hydrodynamic generator or in plasma studies . however, except for a report by gershuni and zhukhovitskii (1) concerning a particular case, no general study has been published . this paper is a general treatment of fully developed, free and forced convective, laminar, magneto-hydrodynamic flow in a vertical channel with a transverse magnetic field . it includes combined free and forced convective flows in channels without a magnetic field reported by ostrach (2), tao (3), etc. as special cases . hartmann flow (4) is included in the other limit .'), (271, 'an experimental test of compressibility transformation for turbulent boundary layer .', 'squire,w.', 'j. ae. sc. 29, 1962.', 'an experimental test of compressibility transformation for turbulent boundary layer . discussion of various turbulent-boundary-layer theories, in the light of experimental measurements by matting and co-workers . the application of (1) the mager insulated-wall transformation, and and illustrated graphically .'), (272, 'oscillatory aerodynamic coefficients for a unified supersonic hypersonic strip theory .', 'rodden, w. +. and revell, j.d.', 'j. ae. scs. 1960, 451.', 'oscillatory aerodynamic coefficients for a unified supersonic hypersonic strip theory . the shock tube is shown to be a feasible research tool for conducting boundary-layer transition experiments . the use of the shock tube permits the study of transition with highly cooled boundary layers, as may be encountered on hypersonic vehicles . boundary-layer transition investigations have been made on optically polished pyrex hemisphere-cylinder and ellipse-cylinder models with stagnation-to-wall enthalpy ratios between 4.5 and roughness estimated to be less than 1 microinch (rms) . transition was detected by measurements of the heat-transfer rates on the model surface . the shock tube experiments indicated that a characteristic feature of transition of a highly cooled boundary layer on a hemisphere was the simultaneous occurrence of transition over the entire supersonic portion of the hemisphere . this implies that transition first occurred in the sonic region . the transition reynolds number (based on local fluid properties at the outer edge of the boundary layer and the momentum thickness) in the sonic region increased from about 225 to 325 as the stagnation- to-wall enthalpy ratio increased from about 9.5 to 29.5 . transition occurred along the cylindrical portion of the hemisphere-cylinder model at a nearly constant momentum thickness reynolds number, increasing from about 400 to 625 as the stagnation- to-wall enthalpy ratio increased from about 9.5 to 29.5 . the highly cooled boundary layers obtained on the cylindrical portion of the shock tube hemisphere-cylinder model provided an extension of nasa transition results obtained on a cooled hemisphere-cone-cylinder model in a wind tunnel . the transition reynolds numbers obtained from these shock tube data were of the same order of magnitude as the minimum transition reynolds numbers obtained in the wind-tunnel experiments . the results indicate that, for practical purposes, boundary-layer cooling is not a critical transition parameter for blunt bodies with a highly cooled boundary layer resulting from a stagnation- to-wall enthalpy ratio of about 3 to 30 . that is, the transition reynolds number did not vary significantly with boundary- layer cooling in this cooling range, but transition always occurred at a low reynolds number (between about 350,000 and 750,000 based on local external properties and a distance along the body surface from the stagnation point) . the boundary-layer history (body shape history) appeared to be an important parameter affecting the magnitude of the reynolds number for transition and the amount of increase in the transition reynolds number with increased boundary-layer cooling . that is, transition occurred at a lower reynolds number on the ellipse-cylinder configuration than on the hemisphere-cylinder . also, the increase in transition reynolds number with an increase in boundary-layer cooling was even less significant for the ellipse-cylinder than the hemisphere-cylinder .'), (273, 'flow past slender blunt bodies - a review and extension .', 'capiaux,r. and karchmar,l.', 'paper 61-201-1904, nat. ias-ars joint meeting, 1961.', 'flow past slender blunt bodies - a review and extension . a numerical solution of the inviscid flow field about slender blunt bodies of revolution has been developed through a combination of two methods .. the van dyke solution in the subsonic flow region at the nose, and the method of characteristics in the supersonic region . the results are compared with second-order blast wave theory and with experimental data,. and the respective merits and deficiencies of the two theoretical methods are pointed out . the results of the numerical solution are further used in a discussion of the entropy layer, to propose a possible criterion of entropy layer thickness .'), (274, 'analysis of quartz and teflon shields for a particular re-entry mission .', 'adams,e.w.', 'heat transfer and fluid mech. inst. 1961, 222.', 'analysis of quartz and teflon shields for a particular re-entry mission . the transient performance of ablation type heat protection shields is treated herein for the surface of a vehicle returning from outer space to the earth . the vehicle weighs 8640 kg, has a ballistic factor of 500 lb ft, re-enters with a speed of 11 km sec at ratio of 0.5, and is subjected to a maximum deceleration of 7.7 times the gravity constant . by use of well known equations for the heat transfer and the mass transfer at a heated surface, a numerical calculation method is derived which, for the investigated ablation processes, yields exact transient solutions of the fundamental system of partial differential equations . the method is applied to various quartz shields and to one teflon shield, which all evaporate so readily under the conditions of the problem at hand that practically no flow of molten shield material exists . the solutions also show comparatively small temperature changes parallel to the surface . the results show that the nose of the vehicle is cooled predominantly by the evaporation of the quartz or the teflon,. the rest of the vehicle''s surface is cooled by radiation of the quartz or evaporation of the teflon . the large mass transfer effects on the nose of the vehicle are detrimental since the resulting low surface temperatures prevent the radiative heat transfer out of the shield, which does not involve any mass loss, from being the desirable governing cooling factor .'), (275, 'the effect of lift on entry corridor depth and guidance requirements for the return lunar flight .', 'wong,t. and slye,r.', 'nasa r-80, 1960.', 'the effect of lift on entry corridor depth and guidance requirements for the return lunar flight . corridors for manned vehicles are defined consistent with requirements for avoiding radiation exposure and for limiting values of peak deceleration . use of lift increases the depth of the entry corridor . mid-course guidance requirements appear to be critical only for the flight-path angle . increasing the energy of the transfer orbit increases the required guidance accuracy for the flight-path angle . corrective thrust applied essentially parallel to the local horizontal produces the maximum change in perigee altitude for a given increment of velocity . energy required to effect a given change in perigee altitude varies inversely with range measured from the center of the earth .'), (276, 'reaction tests of turbine nozzles for supersonic velocities .', 'keenan,j.h.', 'asme trans. 1949, 773.', 'reaction tests of turbine nozzles for supersonic velocities . a machine for testing turbine nozzles by the reaction method, which was described in a previous paper, was used to test a series of convergent-divergent turbine nozzles . the results of these tests, along with the test of a convergent turbine nozzle, are compared with each other and with analytical values . two kinds of analytical values are employed, namely, the usual values obtained from an assumed isentropic expansion from inlet state to exhaust pressure, and the values obtained from the assumption that the processes in the nozzle are isentropic except for a normal shock which takes up a position in the nozzle such as to cause the stream to fill the exit area at the exhaust pressure whenever possible . this latter kind of analytical value involves no shock when the exit area can be filled at the exhaust pressure by means of isentropic processes only, or when the exhaust pressure is lowered so far that the shock has passed out of the passage . the agreement of the test results with the calculated results of this latter kind is good, and the disagreement which exists can be attributed largely to separation at the shock and to transmission of exhaust-pressure effects upstream through the boundary layer .'), (277, 'study of flow conditions and deflection angle at exit of two-dimensional cascade of turbine rotor blades at critical and supercritical pressure ratios .', 'hauser,c.h., plohr,h.w. and sonder,g.', 'naca rm e9k25, 1950.', 'study of flow conditions and deflection angle at exit of two-dimensional cascade of turbine rotor blades at critical and supercritical pressure ratios . an analysis was made of the flow conditions downstream of a cascade of turbine rotor blades at critical and supercritical pressure ratios . the results of five theoretical methods for determining the deflection angle are compared with those of an experimental method using the conservation-of-momentum principle and static-pressure surveys, and also are compared with an analysis of schlieren photographs of the flow downstream of the blades . a two- dimensional cascade of six blades with an axial width of 1.80 inches was used for the static-pressure surveys and for some of the schlieren photographs . in order to determine the flow conditions several blade chords downstream of the cascade, schlieren photographs were taken of the flow through a cascade of 18 blades having an axial width of 0.60 inch . for the blade design studied, even at static-to-total pressure ratios considerably lower than that required to give critical velocity at the throat section, the flow was deflected in the tangential direction as predicted for the incompressible case . as the pressure ratio was lowered further, the aerodynamic loading of the rear portion of the blade reached a maximum value and remained constant . after this condition was attained, the expansion downstream of the cascade took place with a constant tangential velocity so that no further increase in the amount of turning across the blade row and no further increase in the loading of the blade was available .'), (278, 'on source and vortex distributions in the linearised theory of steady supersonic flow .', 'robinson,a.', 'q.j.mech.app.math. 1948.', 'on source and vortex distributions in the linearised theory of steady supersonic flow . the hyperbolic character of the differential equation satisfied by the velocity potential in linearized supersonic flow entails the presence of fractional infinities in the fundamental solutions of the equation . difficulties arising from this fact can be overcome by the introduction of hadamard''s finite part of an infinite integral . together with the definition of certain counterparts of the familiar vector operators this leads to a natural development of the analogy between incompressible flow and linearized supersonic flow . in particular, formulae are derived for the field of flow due to an arbitrary distribution of supersonic sources and vortices . applications to aerofoil theory, including the calculation of the downwash in the wake of an aerofoil, are given in a separate report (ref. 9) .'), (279, 'supersonic drag calculations for a cylindrical shell wing of semicircular cross section combined with a central body of revolution .', 'beane,b.j. and ryna,b.m.', 'douglas sm22627, 1956.', 'supersonic drag calculations for a cylindrical shell wing of semicircular cross section combined with a central body of revolution . a semi-circular ring wing with a body of revolution on the axis is studied to find the wave and the vortex drag for various chordwise lift distributions and for three values of a parameter describing the wing geometry . using the wave drag obtained from the chordwise loading that gives the least drag, together with the vortex and skin friction drags, the maximum lift to drag ratio for each wing geometry is computed . compared to the estimates made by lomax and heaslet, somewhat lower drags are found .'), (280, 'the surface oil flow technique as used in high speed wind tunnels in the united kingdom .', 'stanbrook,a.', 'rae tn.aero.2712, 1960.', 'the surface oil flow technique as used in high speed wind tunnels in the united kingdom . an examination has been made of the various versions of the surface oil flow technique used in different high speed wind tunnels . to provide background information for this investigation some systematic tests were made on a simple model in a small supersonic tunnel . the experience gained made it possible to explain many of the variations in terms of the different operating conditions of the tunnels . the time taken to form a pattern on a typical model is, to a first approximation, directly proportional to the value of the parameter, the factor being 36,000 12,000 . the time taken appears to be independent of the initial thickness of the oil sheet . a general procedure for the development of oil mixtures for any purpose is suggested .'), (281, 'higher order approximations for relaxation oscillations .', '', '', 'higher order approximations for relaxation oscillations . the problem of solving asymptotic developments for all quantities involved in relaxation oscillations has been solved by haag . this paper indicates how one can carry out such developments in a case which is simple enough to be treated explicitly .'), (282, 'jet effects on base pressure of conical afterbodies at mach 1. 91 and 3. 12 .', 'baughman,l.e. and kochendorfer,f.d.', 'naca rm e57e06.', 'jet effects on base pressure of conical afterbodies at mach 1. 91 and 3. 12 . data are presented which show the effect of a jet on base pressure for a series of conical afterbody-jet-nozzle combinations having boat-tail angles that varied from 0 to 11 and base-to-jet diameter ratios that varied from 1.11 to 2.67 . the jet nozzles had exit angles from 0 to 20 and were designed for exit mach numbers from 1.0 to 3.2 . pressure ratios up to 30 were tested for both a cold (air) and a hot numbers of 1.91 and 3.12 . in general, base pressure increased for increasing values of boat-tail angle, nozzle angle, jet temperature, and jet total pressure and for decreasing values of base-to-jet diameter ratio, jet mach number, and free-stream mach number . the addition of tail surfaces produced only small changes in base pressure . for all variables, base pressure is governed by the maximum pressure rise that can be supported by the wake fluid in the region of the trailing shock . the wake pressure ratio is in turn governed by the jet and free-stream mach numbers adjacent to the wake region and by the state of the boundary layer on the boattail and on the nozzle . values of wake pressure ratio computed using the theory of korst, page, and childs were in good agreement with experimental values for convergent nozzles .'), (283, 'laminar heat transfer around blunt bodies in dissociated air .', 'kemp,n.h., rose,p.h. and detra,r.w.', 'j.ae.sc. 26, 1959.', 'laminar heat transfer around blunt bodies in dissociated air . a method of predicting laminar heat-transfer rates to blunt, highly cooled bodies with constant wall temperature in dissociated air flow is developed . attention is restricted to the case of axisymmetric bodies at zero incidence, although two-dimensional bodies could be treated the same way . the method is based on the use of the /local similarity/ concept and an extension of the ideas used by fay and riddell . a simple formula is given for predicting the ratio of local heat-transfer rate to stagnation-point rate . it depends on wall conditions and pressure distribution, but not on the thermodynamic or transport properties of the hot external flow, except at the stagnation point . experimental heat-transfer rates obtained with correct stagnation-point simulation and high wall cooling in shock tubes are also presented and compared with the theoretical predictions . on the whole, the agreement is good, although in regions of rapidly varying pressure there is evidence that the local similarity assumption breaks down, and the theory underestimates the actual heat-transfer rate by up to 25 per cent .'), (284, 'the divergence of supersonic wings including chordwise bending .', 'biot,m.a.', 'j. ae. scs. 23, 1956.', 'the divergence of supersonic wings including chordwise bending . the static aeroelastic stability or divergence problem is investigated for thin supersonic wings when not only the spanwise bending and twist are taken into account but also the chordwise bending . the problem is treated in successive phases of increasing complexity from the two-dimensional curling-up of the leading edge to the three-dimensional stability of the cantilever wing . several methods of approach are developed including the nonlinear aspects of the structure and the aerodynamics . results indicate a strong dependence of stability on poisson''s ratio and the magnitude of the deformation .'), (285, 'on the flutter of panels at high mach numbers .', 'hedgepeth,j.m.', 'j.ae.scs. 23, 1956.', 'on the flutter of panels at high mach numbers . there have recently arisen some questions as to the possibility of panel flutter at high dynamic pressures and mach numbers . in addition, some doubts have been raised about the convergence of the galerkin method when applied to such problems . this note is intended to shed light on these matters .'), (286, 'effect of roll on dynamic instability of symmetric missiles .', 'murphy,c.h.', 'j. ae. scs. 21, 1954.', 'effect of roll on dynamic instability of symmetric missiles . this note attempts to extend the discussion by stating a slightly neater form of generalized stability conditions and describing certain experimental results on dynamic instability .'), (287, 'some theoretical low-speed loading characteristics of swept wings in roll and sideslip .', 'bird,j.d.', 'naca r969, 1950.', 'some theoretical low-speed loading characteristics of swept wings in roll and sideslip . the weissinger method for determining additional span loading for incompressible flow is used to find the damping in roll, the lateral center of pressure of the rolling load, and the span loading coefficients caused by rolling for wing plan forms of various aspect ratios, taper ratios, and sweep angles . in addition, the applicability of the method to the determination of certain other aerodynamic derivatives is investigated, and corrections for the first-order effects of compressibility are indicated . the agreement obtained between experimentally and theoretically determined values for the aerodynamic coefficients indicates that the method of weissinger is well suited to the calculation of the additional span loading caused by rolling and for the calculation of such resulting aerodynamic derivatives of wings as do not involve considerations of tip suction .'), (288, 'the rolling up of the trailing vortex sheet and its effect on the downwash behind wings .', 'spreiter,j.r. and sacks,a.h.', 'j. ae. scs. 18, 1951.', 'the rolling up of the trailing vortex sheet and its effect on the downwash behind wings . the motion of the trailing vortices associated with a lifting wing is investigated by theoretical and visual-flow methods for the purpose of determining the proper vortex distribution to be used for downwash calculations . both subsonic and supersonic speeds are considered in the analysis . it is found that the degree to which the vortices are rolled up depends upon the distance behind the wing and upon the lift coefficient, span loading, and aspect ratio of the wing . while the rolling up of the trailing vortices associated with high aspect-ratio wings is of little practical importance, it is shown that, with low-aspect-ratio wings, the trailing vortex sheet may become essentially rolled up into two trailing vortex cores within a chord length of the trailing edge . the downwash fields associated with the two limiting cases of the flat vortex sheet and the fully rolled-up vortices are investigated in detail for both subsonic and supersonic speeds . the intermediate case in which the rolling-up process is only partially completed at the tail position is also discussed .'), (289, 'a theoretical study of the aerodynamics of slender cruciform-wing arrangements and their wakes .', 'spreiter,j.r. and sacks,a.h.', 'naca r1296, 1957.', 'a theoretical study of the aerodynamics of slender cruciform-wing arrangements and their wakes . a theoretical study is made of some cruciform-wing arrangements and their wakes by means of slender-body theory . the basic ideas of this theory are reviewed and equations are developed for the pressures, loadings, and forces on slender cruciform wings and wing-body combinations . the rolling-up of the vortex sheet behind a slender cruciform wing is considered at length and a numerical analysis is carried out using 40 vortices to calculate the wake shape at various distances behind an equal-span cruciform wing at 45 bank . analytical expressions are developed for the corresponding positions of the rolled-up vortex sheets using a 4-vortex approximation to the wake, and these positions are compared with the positions of the centroids of vorticity resulting from the numerical analysis . the agreement is found to be remarkably good at all distances behind the wing . photographs of the wake as observed in a water tank are presented for various distances behind a cruciform wing at 0 and 45 bank . for 45 bank, the distance behind the wing at which the upper two vortices pass between the lower two is measured experimentally and is found to agree well with the the calculation of loads on cruciform tails is considered in some detail by the method of reverse flow, and equations are developed for the tail loads in terms of the vortex positions calculated in the earlier analyses .'), (290, 'dynamic stability of a missile in rolling flight .', 'bolz,r.e.', 'j. ae. scs. 19, 1952.', 'dynamic stability of a missile in rolling flight . the paper sets down the equations of motion for a symmetric rolling missile with respect to axes attached to the missile . the missile may be jet (or rocket) propelled or coasting under accelerating or decelerating conditions, respectively, wherein the variable rolling velocity is derived from intentionally or unintentionally /canted/ fins and or wings . the equations contain a force and moment system that includes, in addition to the usual forces and moments, those due to magnus effects, misaligned surfaces, canted surfaces, jet misalignment, and the linear accelerations in the plane normal to the missile axis . the results present general stability criteria for a rolling missile which are summarized in the /discussion of stability ./'), (291, 'sweepback effects in the turbulent boundary-layer shock-wave interaction .', 'stalker,r.j.', 'j. ae. scs. 27, 1960.', 'sweepback effects in the turbulent boundary-layer shock-wave interaction . experiments are reported on the interaction of turbulent boundary layers and shock waves with sweptback configurations . they show that the peak pressure rise at separation, the upstream influence ahead of separation, and the pressure rise at reattachment for moderate sweep angles can all be understood by simple extensions of available two-dimensional theories .'), (292, 'rapid laminar boundary layer calculations by piece-wise application of similar solutions .', 'smith,a.m.o.', 'j. ae. scs. 23, 1956.', 'rapid laminar boundary layer calculations by piece-wise application of similar solutions . a method is presented for the rapid calculation of the incompressible laminar boundary layer in an arbitrary flow around either a two-dimensional or a rotationally-symmetrical body . the solution is obtained without recourse to von karman''s momentum equation by means of a coarse step-by-step procedure in which each segment of the velocity distribution is approximated by one of the falkner-skan family of similar flows . solutions have at least as much accuracy as those of any other one-parameter approximate method, and in certain cases the solutions become exact . in regions of accelerating velocity, the accuracy appears to be very high . in decelerating flows, separation is predicted somewhat early compared with exact solutions that is, the method is conservative in contrast to the von karman-pohlhausen procedure which sometimes fails to predict separation that actually exists . the method is the most rapid hand procedure known to the author, provided the full history of the boundary layer is required . if only a thickness such as is needed at one point on a surface, then it is about equal in speed to the quadrature method . but, if several values of or other properties along a surface are required, it is appreciably faster than the quadrature method . characteristically, only four steps are needed between the forward stagnation point and the pressure peak . once the velocity-distribution data are available, each step in a two-dimensional calculation requires about 5 minutes, using a slide rule .'), (293, 'recent studies on the effect of cooling on boundary layer transition at mach 4.', 'wisniewski,r.j. and jack,j.r.', 'j. ae. scs. 28, 1961.', 'recent studies on the effect of cooling on boundary layer transition at mach 4. the advent of high-speed flight has necessitated the study of boundary-layer transition on highly cooled bodies . investigations such as those of references 1-4 have concentrated on this problem and have indicated, contrary to the trends predicted by small-disturbance theory, that premature transition can be found with cooling . this phenomenon, commonly called detail in references 2-5 . the purpose of this note is to report some recent transition data obtained on a cooled cone in a mach 4 wind tunnel . the model, a sharp-tip cone (included angle 13.5), was cooled by liquid nitrogen to a temperature of -340 f . the cooling method and the data analysis are similar to that described in reference 3 .'), (294, 'an investigation of laminar transitional and turbulent heat transfer on blunt-nosed bodies in hypersonic flow .', 'cresci,r.j. and mackenzie,d.a.', 'j. ae. scs. 27, 1960.', 'an investigation of laminar transitional and turbulent heat transfer on blunt-nosed bodies in hypersonic flow . laminar, transitional, and turbulent heating rates have been measured by means of the shrouded model technique . the reynolds number was varied over a ninefold range,. the enthalpy ratio (stagnation to wall) varied from 2.3 to approximately 1.5 . two different pressure distributions were imposed on the model which consisted of a spherically capped cone . the experimental data are compared to the laminar hypersonic boundary-layer theory and shown to be in good agreement on the conical portion of the model . on the spherical portion the data are approximately 20 per cent higher than the theoretical prediction . some of this discrepancy can be attributed to radiation to the nose of the model . the fully developed turbulent heat-transfer data are compared to two theories .. (1) a relatively simple turbulent theory which is based on recent theoretical work and which takes into account the upstream history of the boundary layer, and (2) the flat-plate reference-enthalpy theory, which depends on only /local/ conditions . although both theories are in reasonable agreement with the data, the latter method is simpler and somewhat more accurate . for transitional flow the theory mentioned first can be readily modified in order to permit reasonable estimates of transitional heat transfer to be obtained . on this basis it is possible to estimate laminar, transitional, and fully developed turbulent heat transfer under hypersonic blunt-body conditions . the behavior of transition reynolds number based on momentum thickness is also discussed and shown to be in quantitative agreement with recent shock-tube measurements .'), (295, 'a note on transitional heat transfer under hypersonic conditions .', 'constantino economos and paul a. libby', 'research assistant and professor of aeronautical engineering, respectively polytechnic institute of brooklyn, brooklyn, n.y.', 'a note on transitional heat transfer under hypersonic conditions . in references 1 and 2 there were presented experimental data on transitional heat transfer on a blunt body under hypersonic-flow conditions obtained by the shroud technique . the data were compared with a theoretical prediction of transitional heat transfer based on a suggestion of persh . the agreement between theory and experiment in the transitional region was found to be ''qualitatively good and quantitatively fair'' . it is the purpose of this note to present some additional transitional data obtained in conventional wind-tunnel tests and to indicate a means for improving somewhat the agreement between transitional theory and experiment .'), (296, 'notes on waves through gases at pressures small compared with the magnetic pressure, with applications to upper atmosphere aerodynamics .', 'lighthill,m.j.', 'j. fluid mech. 9, 1960, 465.', 'notes on waves through gases at pressures small compared with the magnetic pressure, with applications to upper atmosphere aerodynamics . most treatments of magnetohydrodynamic waves have confined physical interpretation to cases when the alfven velocity a is small compared with the sound velocity a . here we consider the ''low-beta situation'', in which a is much larger than a . then, except for two modes with wave velocity a the only possible waves are longitudinal ones, propagated unidirectionally along lines of magnetic force with velocity a . these can be interpreted as sound waves, confined to effectively rigid magnetic tubes of force . hall-current effects do not alter these conclusions (in contrast to the high-beta situation), and finite conductivity introduces only small dissipation . an application is made to the flow pattern around a body moving through the f layer of the ionosphere, where, although neutral particles have a very large mean free path, charged particles interact electrostatically and, it is argued, may be regarded as forming a continuous fluid whose movement is independent of that of the neutral particles . a body moving at satellite speed or below would then excite the above-mentioned unidirectional sound waves, but no waves at much faster alfven velocity . these considerations suggest that its movement would be accompanied by a v-shaped pattern of electron density (figure 2), which might be in part responsible for some anomalous radar echoes that have been reported .'), (297, 'compressibility effects in magneto-aerodynamic flows past thin bodies .', 'mccune,j.e. and resler,e.l.', 'j. ae. scs. 27, 1960.', 'compressibility effects in magneto-aerodynamic flows past thin bodies . the effects of compressibility on the steady motion of a highly conducting fluid past thin cylindrical bodies in the presence of a magnetic field are studied . procedures are developed for the solution of this class of magnetoaerodynamic problems over the entire mach number range and for all ratios of magnetic to fluid-dynamic pressure . the results obtained are analogous either to the ackeret theory or the prandtl-glauert rule of conventional aerodynamics, depending on the relative values of the flow speed and the appropriate speed of propagation of magnetoacoustic disturbances . the methods used and the physical interpretation of the solutions obtained vary according to the orientation of the magnetic field with respect to the flow direction . the results of the theory are explained in terms of the anisotropic propagation of magnetoacoustic pulses studied previously by several authors .'), (298, 'incompressible wedge flows of an electrically conducting viscous fluid in the presence of a magnetic field .', 'yen,k.t.', 'j. ae. scs. 27, 1960.', 'incompressible wedge flows of an electrically conducting viscous fluid in the presence of a magnetic field . the purpose of this note is to discuss the two-dimensional flow of an electrically conducting viscous fluid past a wedge in the presence of a magnetic field . the governing differential equations and boundary conditions are given and analyzed .'), (299, 'magnetohydrodynamic flow past a semi-infinite plate .', 'meksyn,d.', 'j. ae.scs. 29, 1962.', 'magnetohydrodynamic flow past a semi-infinite plate . the flow of viscous electrically conducting fluid past a semi-infinite plate is considered . the applied constant magnetic field and the constant on-coming velocity of the fluid are in the direction parallel to the plate . in addition to reynolds number the flow in the boundary layer depends on two parameters and . the two simultaneous ordinary nonlinear differential equations are solved by the asymptotic method for the cases when and respectively . the main results obtained are as follows . the equations can be solved exactly for and . the perturbation effect from infinity when k is large depends on, whereas the perturbation effect from zero when k is small depends on . for large k, including there is no solution for . it is assumed that the fluid is incompressible with constant physical properties .'), (300, 'on a particular class of similar solutions of the equations of motion and energy of a viscous fluid .', 'reeves,b.l. and kippenhan,c.j.', 'j. ae. scs. 29, 1962.', 'on a particular class of similar solutions of the equations of motion and energy of a viscous fluid . by introducing the similarity concept to the two-dimensional, incompressible navier-stokes equations and energy equation, a particular class of solutions is found . two general types of flows are considered .. (1) laminar free convection--i.e., flows which take place due to a body force--and (2) laminar forced convection . for free convection on vertical plates, similar solutions are obtained for two different power-law surface temperature variations, and it is shown that one of these solutions constitutes a new type of boundary problem . results of numerical integrations of the equations are compared with solutions of the similar boundary-layer equations for free convection, and it is demonstrated that a range of surface temperature variations exists for which the boundary layer equations are no longer valid . for forced convection, it is shown that the use of similarity transformations provides an alternate method of deriving the ordinary differential equations for some well-known solutions, such as couette and stagnation point flows . solutions are obtained for radial converging or diverging flows between plane surfaces when the temperatures of the surfaces vary as arbitrary powers of the distance from the orgin . results of numerical integrations of the ordinary differential equations are presented for prandtl numbers of 0.01 and 1.0 and for linear surface temperature variations . some rather surprising results are obtained for diverging flows when separation occurs and some revealing comparisons with results from boundary-layer theory are made .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (301, 'approximate design of sharp-cornered supersonic nozzles .', 'm. a. rahman', 'nack and sunderland consulting mech. and elec. engineers', 'approximate design of sharp-cornered supersonic nozzles . a modified parabolic curve appears to be in close proximity to that obtained by either the method of characteristics or the wave method . thus an attempt has been made to use analytic geometry to determine approximately the contour of a two-dimensional, sharp-cornered supersonic nozzle in a very short time .'), (302, 'approximations for the thermodynamic and transport properties of high temperature air .', 'hansen, c. f.', 'nasa tr r 50, 1959 .', 'approximations for the thermodynamic and transport properties of high temperature air . the thermodynamic and transport properties of high-temperature air are found in closed form starting from approximate partition functions for the major components in air and neglecting all minor components . the compressibility, enthalpy, entropy, the specific heats, the speed of sound, the coefficients of viscosity and of thermal conductivity, and the prandtl numbers for air are tabulated from 500degree to 15,000degree k over a range of pressure from 0.0001 to 100 atmospheres . the energy of air and the mol fractions of the major components of air can be found from the tabulated values for compressibility and enthalpy . it is predicted that the prandtl number for fully ionized air, which is in complete equilibrium, will become small compared to unity, the order of transparent to heat flux .'), (303, 'effect of variable heat recombination on stagnation point heat transfer .', 'fenster,s.j. and neyman,r.j.', 'j. ae. scs. 29, 1962.', 'effect of variable heat recombination on stagnation point heat transfer . earlier studies assume an average heat of formation of atoms based upon external flow conditions . it is shown that equilibrium heat transfer decreases by 35 for a typical mach number 24 case when allowance is made for the proportions of air components . the variable recombination energy also results in atom mass fractions which are realistically less for equilibrium than frozen situations throughout the cold-wall boundary layer .'), (304, 'first-order approach to a strong interaction problem in hypersonic flow over an insulated flat plate .', 'oguchi,h.', 'univ. tokyo aero.res r330, 1958.', 'first-order approach to a strong interaction problem in hypersonic flow over an insulated flat plate . the present paper concerns with the strong interaction phenomenon over an insulated semi-infinite flat plate with a sharp leading edge . in particular the main interest is in the consistent treatment in which the boundary-layer solution may be joined continuously with the inviscid solution regarding flow variables including pressure, normal velocity, temperature (or streamwise velocity) and density . it is shown that the behavior of the inviscid solution may be consistent with that of the boundary-layer solution to at least first-order approximation that is correct to the order of, where m is the mach number of undisturbed flow, r the reynolds number based on the distance from leading edge and the ratio of specific heats . then the first-order boundary-layer problem is formulated under such an external circumstance and an attempt is made for arriving at the solution . actual calculations are carried out for both cases of air and helium . from the solution it is found that the region in which the viscous effect plays a significant role is ranged over from 0 to a certain finite value of n, say n, in terms of the similarity coordinate n in the corresponding incompressible boundary layer . the numerical results moreover indicate that the induced pressure is considerably smaller than the estimate of lees (7) obtained by his approximate method in which the effect of the first-order induced pressure on the boundary layer is ignored and no survey of the first-order boundary-layer equation is made . the present results are also found to be in excellent agreement with experimental data recently obtained in helium flow by erickson (15) .'), (305, 'hypersonic strong viscous interaction on a flat plate with surface mass transfer .', 'li,t.y. and gross,j.f.', 'heat transfer and fluid mech. inst. 1961, 146.', 'hypersonic strong viscous interaction on a flat plate with surface mass transfer . the present report gives an account of the development of an approximate theory to the problem of hypersonic strong viscous interaction on a flat plate with mass-transfer at the plate surface . the disturbance flow region is divided into inviscid and viscous flow regions . the hypersonic small perturbation theory is applied to the solution of the inviscid flow region . the method of similar solutions of compressible laminar boundary layer equations is applied to the treatment of the viscous flow region . the law of surface mass-transfer for similar solutions is derived . the pressure and the normal velocity are matched between the inviscid and viscous flow solutions . formulas for induced surface pressure, boundary layer thickness, skin friction coefficient, and heat transfer coefficient are obtained . numerical results and their significance are discussed . future improvements are indicated .'), (306, 'second approximation to laminar compressible boundary layer on flat plate in slip flow .', 'maslen,s.h.', 'naca tn.2818, 1952.', 'second approximation to laminar compressible boundary layer on flat plate in slip flow . the first-order solution for the laminar compressible boundary-layer flow over a flat plate at constant wall temperature is given . the effect of slip at the wall as well as the interaction between the boundary-layer flow and the outer stream flow are taken into consideration . the solution is obtained explicitly in terms of the known zero order, or continuum, solution . no assumptions regarding the prandtl number or viscosity-temperature law need be made . it is found that the first-order solution gives a decrease in heat transfer and, for supersonic flow, an increase in skin friction . for subsonic flow there is no first-order shear effect . the change in heat transfer is due to slip and the change in friction is due to the interaction of the zero- and first-order velocities at the outer edge of the boundary layer .'), (307, 'an approximate solution of hypersonic laminar boundary layer equations and its application .', 'nagakura,t. and naruse,h.', 'j. phys. soc. japan, 12, 1957, 1298.', 'an approximate solution of hypersonic laminar boundary layer equations and its application . approximate formulae of the displacement thickness and the skin friction of the hypersonic laminar boundary layer are derived by use of von karman''s integral method, assuming the heat-insulated wall, the prandtl number of unity and chapman and rubesin''s formula for the variation of viscosity with temperature . the results obtained are compared with some exact solutions . because of the good agreement, it seems that these formulae are very useful . these formulae, together with the tangent-wedge-approximation, are applied to the viscous flow over slender bodies with a sufficiently sharp leading edge . as an example, the pressure distribution over a flat plate is calculated numerically over the entire region of the surface . comparison with other author''s theoretical results as well as experimental values is made .'), (308, 'on the hypersonic viscous flow past a flat plate with suction or injection .', 'yasuhara,m.', 'j. phys. soc. japan, 12, 1957, 177.', 'on the hypersonic viscous flow past a flat plate with suction or injection . the hypersonic viscous flow past a flat plate with suction or injection is dealt with by karman-pohlhausen''s method in special cases when suction or injection velocity proportional to, especially for the region of strong interaction between the shock wave and the boundary layer, were p is the pressure on the plate and x is the distance measured along the plate from its leading edge . several numerical examples are given, which shows similar effects of injection to those in the case of incompressible flow that the injection makes all the height of the shock wave, the thickness of the boundary layer and the pressure on the plate larger than those in the case of no injection . on the contrary, in the case of suction no remarkable change both in the height of the shock wave and the pressure on the plate can be seen and only the velocity profile in the boundary layer is affected by the suction .'), (309, 'on the motion of a flat plate at high speed in a viscous compressible fluid, ii, steady motion .', 'stewartson,k.', 'j. ae. scs. 22, 1955, 303.', 'on the motion of a flat plate at high speed in a viscous compressible fluid, ii, steady motion . the theory of the steady flow of a viscous compressible fluid past a flat plate at high mach number due to lees and probstein is extended by a more complete discussion of the flow in the inviscid layer between the shock wave and the boundary layer . it is shown that similar solutions exist in this layer, analogously to those found by li and nagamatsu in the boundary layer, and that the two may be joined to give, allowing one minor assumption, a full account of the flow . it is shown that the boundary-layer equations may be reduced to those for an incompressible fluid and that the von karman-pohlhausen method describes the flow in it with good accuracy . the tangent wedge approximation for the pressure on the plate, used by lees and his collaborators, is found to be in deficit by 10 per cent for air . finally, it is shown that the theory for weak interaction cannot be extended further without a complete knowledge of the flow .'), (310, 'hypersonic viscous flow over a flat plate .', 'lees, l. and probstein, r. f.', 'princeton univ. aero eng. r195, 1952 (abstract by e.m.keen)', 'hypersonic viscous flow over a flat plate . in dealing with the steady laminar viscous flow over a semi-infinite flat plate some of the following topics are discussed . the streamline in the boundary layer over a leading edge of given thickness . the rate of growth of the boundary layer in the main stream, and causes of pressure variations . asymptotic solutions for thn downstream flow region, including the joining interaction of shock waves at the leading edge . pressure variations in the interanl viscous flow layer and in external inviscid flow considered as prandtl meyer flow . in cases of streamline deflection, the free stream mach number, zero pressure gradient, and surface pressure distribution . asymptotic solutions for cases of fluid injection of a cool gas . prandtl heat transfer . the joining interaction between the external inviscid flow and the internal viscous flow layer . steady laminar hpyersonic viscous flow over a flat wedge and a cone .'), (311, 'a method for predicting the onset of buffeting and other separation effects from wind tunnel tests on rigid models .', 'pearcey, h. h.', 'n.p.l. aero. 358, fm 2763. dec. 1958 .', 'a method for predicting the onset of buffeting and other separation effects from wind tunnel tests on rigid models . the method is based on the observation of the divergence that occurs in the variation of mean static pressure at the trailing edge of an aircraft wing at the critical stage in the development of boundary-layer separation when its influence first spreads to the trailing edge and thereby to the overall flow . the significance of the trailing-edge pressure variations and their connection with the effects that separation has on the mean and unsteady loads is discussed for various types of separation . good prediction can be obtained from wind-tunnel tests, or warning provided in flight, for low-speed separations and for shock-induced ones up to the stage at which the shock wave reaches the trailing edge . related divergences in wake width, lift coefficient, or shock position can also be used . pressure measurements at other isolated points often indicate the type of separation . certain special considerations apply for swept wings . the various flow changes that are considered are illustrated by schlieren photographs and described in an appendix .'), (312, 'chordwise pressure distributions over several naca 16 series airfoils at transonic mach numbers up to 1.25 .', 'ladson,c.l.', 'nasa memo 6-1-59l, 1959.', 'chordwise pressure distributions over several naca 16 series airfoils at transonic mach numbers up to 1.25 . a two-dimensional wind-tunnel investigation of the pressure distributions over several naca 16-series airfoils with thicknesses of and design lift coefficients of the langley airfoil test apparatus at transonic mach numbers from 0.7 to number from 2.4 x 10 to 2.8 x 10 and in angle of attack from -10 to and schlieren flow photographs are presented without analysis .'), (313, 'on alternative forms for the basic equations of transonic flow theory .', 'spreiter,j.r.', 'j. ae. scs. 21, 1954.', 'on alternative forms for the basic equations of transonic flow theory . attention has been called by numerous authors to the possibility of certain alternative forms for the equations for transonic flow about thin wings . it is the purpose of this note to contribute to this discussion and to indicate some reasons for the selection of one form of these in preference to another more widely used form .'), (314, 'simplified method for determination of the critical height of distributed roughness particles for boundary layer transition at mach numbers from 0 to 5.', 'braslow,a.l. and knox,e.c.', 'naca tn.4363, 1958.', 'simplified method for determination of the critical height of distributed roughness particles for boundary layer transition at mach numbers from 0 to 5. a simplified method has been devised for determination of the critical height of three-dimensional roughness particles required to promote premature transition of a laminar boundary layer on models of airplanes or airplane components in a wind tunnel with zero heat transfer . a single equation is derived which relates the roughness height to a reynolds number based on the roughness height and on local flow conditions at the height of the roughness, and charts are presented from which the critical roughness height can be easily obtained for mach numbers from 0 to 5 . a discussion of the use of these charts is presented with consideration of various model configurations . the method has been applied to various types of configurations in several wind-tunnel investigations conducted by the national advisory committee for aeronautics at mach numbers up to 4, and in all cases the calculated roughness height caused premature boundary-layer transition for the range of test conditions .'), (315, 'scale effects at high subsonic and transonic speeds and methods for fixing transition in model experiments .', 'haines,a.b., holder,d.w. and pearcey,h.h.', 'arc r + m3012, 1954.', 'scale effects at high subsonic and transonic speeds and methods for fixing transition in model experiments . the major scale effects at high subsonic and transonic speeds arise from differences between the conditions under which laminar and turbulent boundary layers separate, and in how they behave after separation . for turbulent boundary layers, these conditions and behaviour do not vary greatly as the reynolds number is changed and in many examples, it has been shown that they are similar for the turbulent layers that occur naturally at high reynolds number and for boundary layers in which transition to turbulent flow is fixed artificially . the scale effects arising in wind-tunnel tests made at low reynolds number may, therefore, often be minimised by fixing transition to turbulent flow by introducing an artificial disturbance such as that produced by excrescences attached to the surface . the fact that the effects of separation are often less severe for laminar layers than for the turbulent layers that are likely to be encountered at full scale, makes it all the more important to do this whenever possible . several methods which can be used to fix transition are described, and the results obtained by using them are compared . in general, in experiments in two-dimensional flow, good agreement is found, and explanations can be advanced for cases in which discrepancies occur . several uncertainties and difficulties that arise in fixing transition are discussed and illustrated by examples . in particular, special care is needed in interpreting the results obtained with transition fixed at very low reynolds numbers (say, less than about r = 1 x 10 based on local chord for wings of about 0.1 thickness chord ratio and possibly higher reynolds numbers for thinner wings) . the difficulties of fixing transition satisfactorily are increased for three-dimensional wings, particularly if they are swept-back or highly tapered (i.e., small chord and reynolds number near the tip) and if the tests cover a large range of incidence including high incidences for which the flow may separate from very close to the leading edge . under these circumstances, it is frequently necessary to place the excrescences at different chordwise positions for low and high angles of incidence, and this is inconvenient in practice . more research is needed before sound recommendations can be made as to how and where transition should be fixed on such models, particularly since in routine testing, it is often not possible to check the effects of transition-fixing fully . in the sections dealing with three-dimensional tests, examples are given of the spurious results that have been avoided successfully by fixing transition, of the conditions where even at low reynolds numbers artificial fixing of transition may not be necessary to give a turbulent boundary layer ahead of the shock, and of the conditions under which there are some doubts whether the methods used for fixing transition have been satisfactory .'), (316, 'the occurrence and development of boundary layer separations at high incidences and high speeds .', 'pearcey, h. h.', 'r + m 3109, arc 17901, september 1955 .', 'the occurrence and development of boundary layer separations at high incidences and high speeds . this note describes the manner in which the onset of the effects of boundary-layer separation varies with mach number for two-dimensional aerofoils, and discusses the influence of section shape as far as it is known . a brief qualitative description is given of the mechanism underlying the development of the separated flow and its effects, followed by a discussion of some of the ways in which this is likely to differ for swept-back wings at high speeds . finally, the need is emphasized for continued work in a broadening field .'), (317, 'non-equilibrium flow of an ideal dissociating gas .', 'freeman, n. c.', 'j. fluid mech. v. 4, 1958 .', 'non-equilibrium flow of an ideal dissociating gas . the theory of an''ideal dissociating''gas developed by lighthill/1957/for conditions of thermodynamic equilibrium is extended to non-equilibrium conditions by postulating a simple rate equation for the dissociation process/including the effects of recombination/ . this equation contains the''equilibrium''parameters of the lighthill theory plus a further dissociation phenomena . the behaviour of this gas is investigated in flow through a strong normal shock wave and past a bluff body . the assumption is made that the gas receives complete excitation of its rotational and vibrational degrees of freedom in an infinitesimally thin region according to the familiar rankine-hugoniot shock wave relations before dissociation begins . the variation of the relevant thermodynamic variables down-stream of this region is then computed in a few particular cases . the method used in the latter case is an extension of the''newtonian''theory of hypersonic inviscid flow . in particular, the case of a sphere is treated in some detail . the variation of the shock shape and the sphere diameter to the length scale of the dissociation process, is exhibited for conditions extending from completely undissociated flow to dissociated flow in thermal equilibrium . results would indicate that significant and observable changes from the undissociated values occur, although values for the non-equilibrium parameter are not, at present, available .'), (318, 'inviscid hypersonic flow past blunt bodies .', 'maslen,s.h. and moeckel,w.e.', 'j. ae. scs. 24, 1957.', 'inviscid hypersonic flow past blunt bodies . two methods are shown for the calculation of the flow field between a blunt body and the shock associated with it for the case of hypersonic flow . real gas effects are included . the solutions consider only symmetric flows--that is, symmetric bodies at zero incidence . one method consists in tracing successive stream tubes around the body and leads to iterations on the initially assumed position of the shock . the second is an integral method closely analogous to the karman-pohlhausen procedure for boundary layers . a distinction is made between round-nosed and flat-nosed bodies, and both cases are discussed . a specific example corresponding to a re-entry missile situation is calculated,. the two methods agree within a few per cent . comparison is also made with other known solutions in the stagnation region .'), (319, 'propagation of weak disturbances in a gas subject to relaxation effects .', 'moore,f.k. and gibson,w.e.', 'j. ae. scs. 27, 1960.', 'propagation of weak disturbances in a gas subject to relaxation effects . a generalized wave equation is derived for sound disturbances in a gas when relaxation effects connected with, for example, molecular vibration or dissociation are important . solutions involving discontinuous wave fronts are presented, and it is shown that, under certain assumptions, the complete wave equation reduces to a variant of the telegraph equation . detailed solutions are presented for disturbance fields produced by a wavy wall in subsonic and supersonic flow and a simple wedge in supersonic flow . this study is viewed as a step in the development of a theory of small disturbances of a high-temperature gas, as is found behind the shock in hypersonic flight .'), (320, 'comment on improved numerical solution of the blasius problem with three-point boundary conditions .', 'leigh, d. c.', 'j. aero. sc. v. 29, may 1962 .', 'comment on improved numerical solution of the blasius problem with three-point boundary conditions . attention is drawn to a previous accurate solution to the problem .'), (321, 'improved numerical solution of the blasius problem with three-point boundary conditions .', 'christian, w. j.', 'j. aero. sc. v. 28, november, 1961 .', 'improved numerical solution of the blasius problem with three-point boundary conditions . the blasius equation describes the velocity distribution resulting from laminar, constant-pressure mixing of a stationary fluid layer and a moving stream . in connection with a numerical procedure for the univac based on analytic continuation of the function f'' . high-speed computers now make it feasible to use analytic continuation for numerical integration of single-point boundary-value problems such that, within the limits of taylor''s expansion, truncation error may be made arbitrarily small . a brief description of the application of the routine is given .'), (322, 'on the numerical solution of the blasius problem with three-point boundary conditions .', 'toba, k.', 'j. aero. sc. v. 29, p480-1, april, 1962 .', 'on the numerical solution of the blasius problem with three-point boundary conditions . relates to a technique for approximate determination of the initial parameters . the technique is an application of the asymptotic integration method introduced by meksyn and has been applied to the computation of the skin friction for shock-generated boundary-layer flow .'), (323, 'vorticity interaction at an axisymmetric stagnation point in a viscous incompressible fluid .', 'kemp,n.h.', 'j. ae. scs. 26, 1959.', 'vorticity interaction at an axisymmetric stagnation point in a viscous incompressible fluid . the purpose of the present note is to give an exact solution of the incompressible navier-stokes equations at an axisymmetric stagnation point with vorticity in the oncoming flow which varies linearly with distance from the axis . this solution has application to the hypersonic axisymmetric blunt body problem, for which lighthill has shown the vorticity in the inviscid shock layer is very nearly of this form .'), (324, 'vorticity effect on the stagnation point flow of a viscous incompressible fluid .', 'rott, n. and lenard, m.', 'j. aero. sc. v. 26, august, 1959 .', 'vorticity effect on the stagnation point flow of a viscous incompressible fluid . the effect of vorticity on axisymmetric stagnation point boundary layer calculations is investigated by calculating a perturbation to the stagnation point flow . the shear caused by the vorticity effect is found to be surprisingly large,.the slope of the shear curve /at zero vorticity/ as calculated by kemp agrees perfectly with the value deduced in this note .'), (325, 'heat transfer to constant property laminar boundary layer flows with power function free stream velocity and wall temperature variation .', 'levy,s.', 'j.ae.scs. 19, 1952.', 'heat transfer to constant property laminar boundary layer flows with power function free stream velocity and wall temperature variation . numerical computations have been performed for the boundary-layer form of the energy equation for incompressible flows with power-function variation of free-stream velocity (u = cx) and of wall temperature (t = ax), the pertinent solutions of the momentum equation in this case being those of hartree . the numerical computations given herein are to some extent a repetition of those given by schuh and by chapman and rubesin, the object of the present computations being the resolution of discrepancies appearing in the previous solutions and an extension of their range . ibm machine calculations were employed in the finite difference calculation presently utilized, the results thereof covering a range of wall-temperature function exponents from values of m(4, 1, 0, -0.0904) . the accuracy of the numerical computations is examined in detail, and the accuracy of the computed functions at the wall, which determine the heat-transfer rate, is estimated to be within 2 per cent . examination of the results reveals that the results of schuh for the flat plate are in error . for the range of the calculations, it was found that the local heat-transfer coefficient can, with the exception of large negative values, be expressed within 5 per cent as where the exponent of the prandtl number varies from 0.254 to 0.367 for -0.0904 and where the function can be approximated by the equation'), (326, 'forst-order slip effects on the compressible laminar boundary layer over a slender body of revolution in axial flow .', 'shen,s.f. and solomon,j.m.', 'j.ae.scs. 28, 1961.', 'forst-order slip effects on the compressible laminar boundary layer over a slender body of revolution in axial flow . analysis of the compressible boundary layer with transverse curvature in first order slip flow . no boundary-layer interaction effects are considered and only the zero pressure-gradient case is examined .'), (327, 'on local flat plate similarity in the hypersonic boundary layer .', 'moore,f.k.', 'j. ae. scs. 28, 1961.', 'on local flat plate similarity in the hypersonic boundary layer . a study is made of lees'' /local flat-plate similarity/ rule for the hypersonic laminar boundary layer . it is shown that this rule is exact under assumptions commonly invoked in the inviscid theory of hypersonic flow . beginning from this theoretical basis, a modified local flat-plate similarity scheme is derived, involving separate rules for velocity and enthalpy profiles, and is compared with exact similarity solutions and with the existing theory of hypersonic leading-edge interaction .'), (328, 'the boundary layer near the stagnation point in hypersonic flow past a sphere .', 'herring,t.r.', 'j. fluid mech. 7, 1960, 257.', 'the boundary layer near the stagnation point in hypersonic flow past a sphere . flow properties behind shock waves caused by bluff bodies traveling at supersonic speeds are of major importance in missile and high-speed aircraft design . paper presents a mathematical solution for the laminar boundary layer near the stagnation point of a sphere . surface temperature is free-stream static and shock is strong . air is assumed calorically and thermally perfect with a prandtl number of 0.72 and a dynamic viscosity directly proportional to temperature . based on work of homann (zamm 16, p. 153, 1936) and lighthill simultaneous differential equations for the velocity and temperature profiles . these are solved by numerical integration along a normal to the surface using a digital computer . results are presented as functions of free-stream mach number, reynolds number, and specific heat ratio . as increases, boundary-layer thickness is shown to decrease while shock stand-off distance increases . stand-off distance also decreases with increasing and decreasing specific heat . for constant and specific heat ratio, the product of skin-friction coefficient and the square root of decreases with increasing only approaching a constant value at greater than 10,000 . reviewer''s comment is concerned with the perfect gas assumption for air . author suggests that the effects of dissociation on flow properties are accounted for by a proper choice of specific heat ratio . a consideration of the kinetics of chemical reaction in the cooled boundary layer emphasizes the oversimplification of this approach . the effect on transport properties could have been approximated in present analysis by changing the prandtl number to one more representative of the existing pressures and temperatures .'), (329, 'various aerodynamic characteristics in hypersonic rarefied gas flow .', 'probstein,r.f. and kemp,n.h.', 'j. ae. scs. 27, 1960.', 'various aerodynamic characteristics in hypersonic rarefied gas flow . this paper considers the problem of calculating viscous aerodynamic characteristics of blunt bodies at hypersonic speeds and at sufficiently high altitudes where the appropriate mean free path becomes too large for the use of familiar boundary-layer theory but not so large that free molecule concepts apply . results of an order-of-magnitude analysis are presented to define the regimes of rarefied gas flow and the limits of continuum theory . based on theoretical and experimental evidence, the complete navier-stokes equations are used as a model, except /very close/ to the free molecule condition . this model may not necessarily give the shock wave structure in detail but satisfies overall conservation laws and should give a reasonably accurate picture of all mean aerodynamic quantities . in this /intermediate/ regime there are two fundamental classes of problems .. a /viscous layer/ class and a /merged layer/ class, the latter corresponding to a larger degree of rarefaction . for the viscous layer class there is a thin shock wave, but the shock layer region between the shock and the body is fully viscous, although the viscous stresses and conductive heat transfer are small at the shock wave boundary . here, the use of the navier-stokes equations with outer boundary conditions given by the hugoniot relations is justified . for the merged layer class, the shock wave is no longer thin, and the navier-stokes equations can be used to give a solution which includes the shock structure and has free-stream conditions as outer boundary conditions . a simpler procedure is presented for /incipient merged/ conditions where the shock may no longer be considered an infinitesimally thin discontinuity but where it has not thickened sufficiently to entail the /fully merged layer/ analysis . in this case we approximate the shock by a discontinuity obeying conservation laws which include curvature effects, viscous stresses, and heat conduction . for a sphere and cylinder it is shown that the navier-stokes equations can be reduced to ordinary differential equations for both the viscous and merged layer class of problems . solutions of these equations, when used in connection with hypersonic flow problems, are in general only valid in the stagnation region . to illustrate the viscous layer solutions, numerical calculations have been performed for a sphere and cylinder with the assumption of constant density in the shock layer, which is a useful approximation at hypersonic speeds . to illustrate the merged layer solution, calculations have been carried out for a sphere using the incipient merged layer approximation . results are presented for detachment distance, surface shear, and heat-transfer rate in the stagnation region of a highly cooled sphere flying at hypersonic speed . with decreasing reynolds number, the shear and heat transfer are shown to increase above the extrapolated boundary-layer values in the viscous layer regime and then to begin falling in the incipient merged regime . as the reynolds number decreases in the incipient merged regime, the density in the shock layer increases, and the static and stagnation enthalpy behind the shock decrease . calculations performed for an insulated sphere show that, with decreasing reynolds number in the incipient merged regime, the density in the shock layer decreases,. the total enthalpy behind the shock and at the stagnation point increase so that they are higher than the free-stream total enthalpy,. and the stagnation-point pressure behaves like the total enthalpy . for the highly cooled cylinder in the viscous layer regime, the same quantities are presented as for the sphere . the increase found in shear and heat transfer above extrapolated boundary-layer theory is small, in agreement with vorticity interaction theory . a discussion is given of the behavior of available experimental data for viscous flow quantities in the intermediate regime and the behavior predicted by the results of the present calculations . qualitative agreement is indicated .'), (330, 'taylor instability of finite surface waves .', 'emmons,h.w., chang,c.r. and watson,b.c.', 'j. fluid mech. 7, 1960.', 'taylor instability of finite surface waves . the instability of the accelerated interface between a liquid (methanol or carbon tetrachloride) and air has been investigated experimentally for approximate sinusoidal disturbances of wave-number range from well below to well above the cut-off . the growth rates are measured and compared with theoretical results . a third-order theory shows the phenomena of overstability which is found in the experimental results . some measurements of later stages of growth agree moderately well with the available theory and disclose some additional phenomena of bubble competition, helmholtz instability with transition to turbulence, and jet instability with production of drops .'), (331, 'effects of surface tension and viscosity on taylor instability .', 'bellman,r. and pennington,r.h.', 'q. app. math. 12, 1954.', 'effects of surface tension and viscosity on taylor instability . the model used is that of two fluids of infinite depth, with the interface initially in the form of a sine wave with amplitude small compared to wave length . the fluids are considered incompressible, and only the linear terms in the equations of hydrodynamics are used . the first four sections discuss the effects of surface tension and viscosity . the fifth gives a few numerical results to illustrate the main points of the preceding sections .'), (332, 'similitude of hypersonic real-gas flows over slender bodies with blunted noses .', 'cheng,h.k.', 'j. ae. scs. 26, 1959, 575.', 'similitude of hypersonic real-gas flows over slender bodies with blunted noses . on the basis of the hypersonic small-perturbation theory, the laws of similitude for hypersonic inviscid flow fields over thin or slender bodies are examined, and the restrictions to ideal gases with constant specific heats and to bodies with pointed noses are removed . only steady plane or axisymmetric flows are considered . inspection of the governing system of equations shows that a similitude law exists for flow fields, under local thermal equilibrium, having the same free-stream atmosphere . for flows of ideal gas with constant specific heats, the requirement of the same free-stream atmosphere--i.e., the same composition, pressure, and density--can be replaced by the requirement of the same ratio of specific heats . for flows over blunted wedges or cones, special laws of similitude can be obtained . application of the similarity rules is examined for the case of hypersonic flows of an ideal gas with over flat plates with blunt leading edges, and for the case of equilibrium air flows over wedges . the possibility of simulating nonequilibrium flows over slender or thin bodies is also pointed out .'), (333, 'boundary-layer interaction on a yawed infinite wing in hypersonic flow .', 'robert j. whalen', 'principal scientist, flight sciences laboratory, inc., buffalo, n.y.', 'boundary-layer interaction on a yawed infinite wing in hypersonic flow . the equations are given for the laminar boundary-layer equations on a yawed infinite wing for constant wall temperature, under the combined howorth and mangler transformation . diagrams show the relatively small influence of yaw, the increase of boundary-layer secondary flow, and the variation of the local heat transfer rate with yaw .'), (334, 'influence of the leading-edge shock wave on the laminar boundary layer at hypersonic speeds .', 'lester lees', 'california institute of technology', 'influence of the leading-edge shock wave on the laminar boundary layer at hypersonic speeds . in order to bring out the importance of the leading-edge region at hypersonic speeds, the influence of the leading-edge shock wave on the laminar boundary layer is investigated in two simple cases of steady flow over a semi-infinite, insulated flat plate.. (1) sharp leading edge., (2) blunt leading edge, as approximated by a normal shock wave . the streamlines that enter the boundary layer over a large region of the plate surface has previously crossed the shock wave very near the leading-edge, where the shock is strong and highly curved . consequently, the temperature at the outer edge of the boundary layer is appreciably higher than free-stream temperature, and the vorticity there is not zero . the effects of this shock-wave larger than the usual /errors/ made in the boundary-layer theory, and an estimate of these effects can therefore be obtained within the framework of that theory . the numerical magnitude of the shock-wave influence is found to be appreciable . for the case of the blunt leading edge the slope of the curve of induced pressures plotted against the hypersonic interaction parameter closely approaches the experimental data of hammitt and bogdonoff obtained in helium at large values of this parameter . these approximate results show that the influence of the leading-edge region at hypersonic speeds requires careful theoretical and experimental study .'), (335, 'the interaction between boundary layer and shock waves in transonic flow .', 'liepmann, h. w.', 'j. aero. sc. v. 13, december, 1946 .', 'the interaction between boundary layer and shock waves in transonic flow . experiments of transonic flow past a circular arc profile show that the shock-wave pattern and the pressure distribution are strongly dependent upon the state of the boundary layer . a change from laminar to turbulent boundary layer at a given mach number changes the flow pattern considerably . shock waves can interact with the boundary layer in a manner similar to a reflection from a free jet boundary . these shock waves are not distinctly discernible from pressure distribution measurements .'), (336, 'simplified laminar boundary layer calculations for bodies of revolution and for yawed wings .', 'rott,n. and crabtree,l.f.', 'j. ae. scs. 19, 1962.', 'simplified laminar boundary layer calculations for bodies of revolution and for yawed wings . since the introduction of momentum methods in boundary-layer calculations by von karman and pohlhausen, many improvements have been proposed . an especially simple solution reduces the problem to a quadrature . here, it is proposed to extend these methods to elementary three-dimensional cases and to compressible laminar boundary-layer calculations . for comparison, the corresponding problems for the turbulent boundary layer are also discussed briefly .'), (337, 'boundary layer transition with gas injection .', 'scott,c.j. and anderson,g.e.', 'j. ae. scs. 25, 1958.', 'boundary layer transition with gas injection . the mass-injection process has been proposed as a method of cooling aerodynamic surfaces, and, since the amount of coolant required to maintain practical wall temperatures is considerably larger for turbulent than for laminar boundary layers, knowledge of the effect of the cooling method on the transition process is certainly important . exploratory studies reported here were conducted at mach number 3.7 to ascertain the effects of gas injection on the stability of the laminar boundary layer on a conical surface .'), (338, 'mass transfer cooling at mach number 4. 8.', 'leadon,b.m., scott,c.j. and anderson,g.e.', 'j. ae. scs. 25, 1958.', 'mass transfer cooling at mach number 4. 8. mass-transfer experiments on a 5 mil wire porous cone of 20 total angle have been conducted at using air and helium injection . details of the experimental technique are described in references 1 and 2 . in the laminar boundary layer the recovery factors and heat-transfer coefficients measured with zero injection agreed within per cent with theory . transition reynolds numbers observed on the porous cone with zero injection were half as large as observed on a smooth, impermeable model of identical geometry in the same channel, but injection of large amounts of air or helium did not cause transition to move forward from its zero-injection position on the porous cone . distributed roughness of this type apparently does not disturb impermeable wall theory, but it masks whatever effective roughness may be caused by discrete pore injection .'), (339, 'experimental evaluation of heat transfer with transpiration cooling in a turbulent boundary layer at m=3 .2.', 'bartle,e.r. and leadon,b.h.', 'j. ae. scs. 27, 1960.', 'experimental evaluation of heat transfer with transpiration cooling in a turbulent boundary layer at m=3 .2. it is found that for prescribed velocity field, electrical field and conductivity, the current can be calculated by integration . work is related to analytic investigation of the boundary layer in a physically reasonable accelerator .'), (340, 'analysis of effects of diffusion of a foreign gas into the laminar boundary layer of a supersonic flow of air in a tube .', 'radbill,j.r. and kaye,j.', 'j.ae.scs. 26, 1959.', 'analysis of effects of diffusion of a foreign gas into the laminar boundary layer of a supersonic flow of air in a tube . adiabatic wall temperatures and recovery factors are calculated for pipe flows with an entrance mach number of 5 and with uniform injection of helium . predicted values of the recovery factor increase slowly with increasing injection rate and with increasing distance from the tube entrance .'), (341, 'the analytical design of an axially symmetric laval nozzle for a parallel and uniform jet .', 'foelsch,k.', 'j. ae. scs. 16, 1949.', 'the analytical design of an axially symmetric laval nozzle for a parallel and uniform jet . the equations for the nozzle''s contours are derived by integration of the characteristic equations of the axially symmetric flow . since it is not possible to integrate these equations mathematically in an exact form, it was necessary to find a way to approximate the calculations . the approximation offers itself by considering and comparing the conditions of the flow in a cone with those in a nozzle, as a linearization of the characteristic equations . the first part of the report deals with equations for the transition curve by which the conical source flow is converted into a parallel stream of uniform velocity . the equations are derived by integration along a mach line of the flow in the region where the conversion takes place . a factor f is introduced expressing a relation between the direction and the velocity of the flow along a certain mach line . f remains undetermined and is not involved in the final equations . in the second part of the report, the spherical sonic flow section is converted into a plane circular section of the throat . the nozzle''s contour adjacent to the throat is formed by the arc of a circle connected with the transition curve by a straight line . the gas dynamic properties of the boundary mach line are calculated in table 1, the use of which shortens the calculations considerably .'), (342, 'effect of diffusion fields on the laminar boundary layer .', 'smith,j.w.', 'j.ze.scs. 21, 1954.', 'effect of diffusion fields on the laminar boundary layer . a theory is developed which describes the effect of a general diffusion field on the dynamic and thermal characteristics of a laminar boundary layer on a flat plate in steady compressible flow . fluid properties are considered as functions of temperature and local concentration of the foreign gas . the diffusion field is described by a differential equation that relates convective and diffusion transfer and which considers diffusion currents arising from gradients of concentration and temperature . by means of the usual transformations the system is reduced to a set of ordinary differential equations, which in turn are transformed into a set of integral equations . the latter is amenable to solution by the method of successive approximations . the theory and results have bearing on the problem of control and reduction of aerodynamic heating at hypersonic speeds . the special feature of this approach lies in the utilization of diffusion fields for the purpose of reducing the detrimental effects of viscous dissipation . although the theory is adapted to a fuller investigation of this problem, the numerical examples considered involve mainly diffusion fields of helium, with which good results have been achieved at mach numbers 8 and 12 . whereas at the higher mach number the influx of heat was practically eliminated, a reversal in the direction of heat flow has been effected at the lower mach number .'), (343, 'transpiration cooling experiments in a turbulent boundary layer at m=3 .', 'leadon,b.m. and scott,c.j.', 'j. ae. scs. 23, 1956.', 'transpiration cooling experiments in a turbulent boundary layer at m=3 . turbulent recovery factor and heat-transfer measurements have been made on a porous flat wall section at a nominal mach number of 3.0 and a reynolds number of approximately 4 x 10 using both air and helium as the transpired gas . measured heat-transfer coefficients correlate well with the compressible theory of rubesin for air and qualitatively with simple film theory for either coolant, indicating that the heat transfer from a turbulent boundary layer can be reduced by transpiration cooling to well below that of the uncooled boundary layer at the same reynolds number .'), (344, 'some experimental techniques in mass transfer cooling .', 'leadon,b.m.', 'aero/space eng. 18, 1959.', 'some experimental techniques in mass transfer cooling . author introduces his survey by a brief review of the history of investigations dealing with boundary layers on impermeable solid surfaces, and notes that no true theory exists for turbulent boundary layers, the success of studies in this area having been due to the introduction of artificial, if ingenious, assumptions which permitted empirical correlations fd data . the terminology introduced by the author for distinguishing the different situations involving mass transfer from the wall to the stream may give rise to some objections . for instance, /film cooling/ need not refer only to the injection of a liquid, since applications involving gas film cooling exist . also, his restriction of the term /transpiration cooling/ to refer to the injection through a porous surface of a gas only of the same composition as the exterior stream does not enjoy universal usage . the influence of mass transfer on heat transfer through laminar boundary layers and on the transition from laminar to turbulent flow is described, with consideration given to the question of the net effect of the stabilizing influence of surface cooling and the destabilizing influence of injection . reviewer suggests that author''s inaccurate statement to the effect that /thus far the higher energy conditions do not threaten to involve turbulent injection, so turbulent boundary-layer research enjoys a fairly academic serenity broken only by its own frustrations/ be excused on grounds of poetic license, although it ignores the efforts being devoted to the pressing practical problems of erosive burning of solid propellants (possibly the most common example of a complete /aerothermochemical/ problem involving distributed surface heat and mass transfer with chemical reaction in a flow system) and of effusion cooling of rocket nozzles, both of which involve turbulent boundary-layer conditions . author emphasizes the tedious experimental problems involved in research on boundary layers with blowing, and notes the desirability of velocity distribution measurements, especially in turbulent injection layers . the observation that no good data on concentration profiles in the case of the diffusion boundary layer have been published may be an overstatement, since author''s bibliography overlooks the work of j. berger (/contribution a l''etude de l''injection parietale,/ doctor''s thesis, university of paris, memorial des poudres 38 (annex), p. 1,. paris, imprimerie nationale, 1956) .'), (345, 'the interaction of shock waves with boundary layer on a flat surface .', 'barry,f.w., shapiro,a.h. and neumann,e.p.', 'j. ae. scs. 18, 1951, 229.', 'the interaction of shock waves with boundary layer on a flat surface . the development of supersonic compressors, supersonic diffusers, and high-speed aircraft points to the increasing importance of the interaction between shock waves and boundary layers . the experimental work reported here is intended to (1) provide a better understanding of the nature of the shock boundary-layer interaction, (2) serve as a guide and stimulus to theoretical work, and (3) develop an empirical method for predicting the effects of the interaction . experiments were performed on the reflection of an oblique shock from a boundary layer on a flat surface at a mach number of 2.05 . the effects of shock strength and boundary-layer regime were explored . the results are in the form of schlieren photographs, constant-density contours found from interferometer photographs, and static pressure distributions at the plate surface .'), (346, 'measurements of turbulent friction on a smooth flat plate in supersonic .', '', '', 'measurements of turbulent friction on a smooth flat plate in supersonic . direct measurements of supersonic local skin friction, using the floating-element technique, are presented for mach numbers from bulent flow and transition are emphasized, although some measurements in the laminar regime are included . the observed effect of compressibility is to reduce the magnitude of turbulent skin friction by a factor of two at a mach number of 4.5 and a reynolds number of about 10 . the boundary-layer momentum-integral equation for constant pressure is verified within a few per cent by two experimental methods . typical static pressure measurements are presented to show that transition can be detected by observing disturbances in pressure associated with changes in displacement thickness of the boundary layer . it is found that the turbulent boundary layer cannot be defined experimentally for values of less than about 2,000, where is the momentum thickness . for larger values of there is a unique relationship between local friction coefficient and momentum-thickness reynolds number at a fixed mach number . the appendix compares the present measurements at m = 2.5 with experimental data from other sources .'), (347, 'boundary layer measurements in hypersonic flow .', 'hill,f.k.', 'j. ae. scs. 21, 1954, 433.', 'boundary layer measurements in hypersonic flow . experimental data are presented on boundary-layer formation, heat transfer, and skin-friction coefficient at mach numbers of the wall of a conical nozzle in the presence of a favorable pressure gradient and several rates of heat transfer . the reynolds number based on momentum thickness varied from 1,500 to 3,500 . comparison is made with data at lower mach numbers and with the semiempirical theory of von karman . the existing data up to mach numbers of nine indicate agreement to within 5 per cent when compared with a form of the wilson theory, but it is clear that the effects of heat transfer and pressure gradients present problems which require extensive study and experiment in the future .'), (348, 'turbulent boundary layer in compressible fluids .', 'van driest,e.r.', 'j.ae.scs. 18, 1951, 145.', 'turbulent boundary layer in compressible fluids . the continuity, momentum, and energy differential equations for turbulent flow of a compressible fluid are derived, and the apparent turbulent stresses and dissipation function are identified . a general formula for skin friction, including heat transfer to a flat plate, is developed for a thin turbulent boundary layer in compressible fluids with zero pressure gradient . curves are presented giving skin-friction coefficients and heat-transfer coefficients for air for various wall-to-free-stream temperature ratios and free-stream mach numbers . in the special case when the boundary layer is insulated, this general formula yields skin-friction coefficients higher than those given by the von karman wall-property compressible-fluid formula but lower than those given by the von karman incompressible-fluid formula . heat transfer from the boundary layer to the plate generally increases the friction and heat-transfer coefficients .'), (349, 'numerical solution of the boundary layer equations without similarity assumptions .', 'kramer,r.f. and lieberstein,h.', 'j. ae. scs. 26, 1959, 508.', 'numerical solution of the boundary layer equations without similarity assumptions . the crocco transformation combined with a mangler transformation is used to carry the boundary-layer problem for axially symmetric blunt bodies into a form suitable for direct numerical computation without introduction of similarity assumptions . conditions which in the original problem appear at infinity now are brought to a finite straight line, and the body is transformed to a parallel line . data can be generated on the stagnation line the equations are a parabolic system of two second-order equations, the boundary-value problem is analogous to the slab problem for the heat equation . an implicit difference equation is used to reduce stability difficulties . special techniques in forming the difference equation result in a linear system of algebraic equations to be solved on any given line of integration, and these solutions are computed from recursion relations generated by back substitution . for bluntnosed bodies with approach flow mach numbers greater than 8 (approximately), large temperature gradients occur across a thin boundary layer of dissociated gas, and it is necessary to use real-gas effects, approximated here by certain fits to the gas tables . a case is computed, however, for a lower mach number approach flow using perfect-gas theory to provide a standard against which similarity solutions may be tested .'), (350, 'laminar jet mixing of two compressible fluids with heat release .', 's. i. pai', 'university of maryland', 'laminar jet mixing of two compressible fluids with heat release . the laminar jet mixing problems with heat release have been formulated . a general discussion of the solution of these problems is also given . the important parameters of these problems are brought out . some specific cases of the jet mixing problem, such as jet mixing of one compressible fluid, isothermal jet mixing of two compressible fluids, and isovel jet mixing of two compressible fluids with heat release, are discussed in detail.'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (351, 'thermal distributions in jeffrey-hamel flows between nonparallel plane walls .', 'millsaps, k. and pohlhausen, k.', 'j. aero. sc. v. 20. march 1953, pp 187-196 .', 'thermal distributions in jeffrey-hamel flows between nonparallel plane walls . the authors give the exact solution for the thermal distributions for the steady laminar flow of a viscous incompressible fluid between non-parallel plane walls held at a constant temperature . the velocity profiles are determined with the aid of jacobian elliptic functions by using the jeffery-hamel solution of the hydrodynamic problem . it is shown that in this special case the energy equation giving the temperature profiles can be reduced to an ordinary linear differential equation with variable coefficients . after the introduction of dimensionless parameters, numerical solutions are given for diverging and converging channels with total openings of 10degree for the possible combinations of three reynolds numbers and five prandtl numbers .'), (352, 'on heat transfer over a sweat-cooled surface in laminar compressible flow with a pressure gradient .', 'murduchow,m.', 'j. ae. scs. 19, 1952, 705.', 'on heat transfer over a sweat-cooled surface in laminar compressible flow with a pressure gradient . a simple expression is derived for the normal injection velocity distribution theoretically required to maintain a given uniform temperature along a porous surface in the laminar boundary-layer region of a compressible flow with a given velocity distribution outside of the boundary layer . this expression is valid for any given free-stream mach number but is based on a prandtl number of unity and on the assumption that the viscosity coefficient varies linearly with the temperature . by using the dorodnitsyn type of transformation, the variation of fluid properties even in the case of zero mach number is taken into account . this study is of particular practical interest in connection with the sweat-cooling of turbine blades and of airfoil surfaces in high speed flow . the method of analysis consists of applying the karman-pohlhausen method to both the momentum and energy boundary-layer equations and of using an additional heat balance equation, involving the coolant temperature . a closed-form approximate solution of the equations is then derived . numerical examples for flow in the immediate vicinity of a stagnation point and for a typical type of flow over a turbine blade are given .'), (353, 'the effect of helium injection at an axially symmetric stagnation point .', 'h. hoshizaki and h. j. smith', 'missile and space division, lockheed aircraft corporation, sunnyvale, calif.', 'the effect of helium injection at an axially symmetric stagnation point . an effective means of protecting the surface of a hypersonic re-entry vehicle is to inject small quantities of a lightweight gas into the boundary layer through a porous wall . this process, which is known as mass-transfer cooling, protects the surface in two ways . first of all, as the injected gas or coolant passes from the reservoir through the wall to the surface, a considerable quantity of heat is absorbed as its temperature is raised from the reservoir temperature to the wall surface temperature . characteristically, lightweight gases have relatively high specific heats . secondly, the transfer of mass and enthalpy by convection and diffusion normal to the surface alters the characteristics of the boundary layer in such a manner as to reduce the temperature gradient at the wall, and, hence, the conductive heat transfer at the wall . this is sometimes referred to as the blowing effect .'), (354, 'laminar heat-transfer and pressure measurements over blunt-nosed cones at large angle of attack .', 'victor zakkay', 'research associate, polytechnic institute of brooklyn, freeport, n.y.', 'laminar heat-transfer and pressure measurements over blunt-nosed cones at large angle of attack . tests have been conducted at a mach number of 6, in the pibal hypersonic facility, in order to determine the heat-transfer and pressure distributions over a slender blunted cone at angles of attack of erature ratio, stagnation to wall, was approximately 2.3 . the model tested has a sperical nose diameter of 1.0 in., a base diameter of 3.75 in., and a cone half-angle of 20 degrees . the measurements were made at 5 peripheral stations on the model . in this note the experimental results at a 15 degree angle of attack are presented . a more detailed analysis of the results for all angles of attack is presented in reference 1 .'), (355, 'the injection of air into the dissociated hypersonic laminar boundary layer .', 'sinclaire m. scala', 'research engineer missile and ordnance systems department, general electric company, philadelphia, pa.', 'the injection of air into the dissociated hypersonic laminar boundary layer . in first approximation, dissociated air may be treated as a binary mixture of air atoms and air molecules . in order to include the effects of mass transfer into the boundary layer, it becomes necessary to introduce a third chemical species and hence a second diffusion equation . we have avoided this complexity by considering the injection of air molecules into the boundary layer, and hence the theoretical treatment is accomplished within the framework of a binary mixture gas .'), (356, 'on optimum nose curves for missiles in the super-aerodynamic regime .', 'tan,h.s.', 'j. ae. scs. 25, 1958, 56.', 'on optimum nose curves for missiles in the super-aerodynamic regime . author shows that the differential equations defining the minimum drag body shapes for free molecule flow that were developed and numerically integrated by w. j. carter (amr 11 (1958), rev. realized, however, that numerical or analytical integration of the second-order differential equation is unnecessary since, for the flow conditions considered, the first integral to the euler equation can be written prior to the substitution of the expression defining the pressure coefficient .'), (357, 'optimum nose shapes for missiles in the super-aerodynamic region .', 'carter,w.j.', 'j. ae. scs. 24, 1957, 527.', 'optimum nose shapes for missiles in the super-aerodynamic region . the mechanics of the kinetic theory of gases is employed to describe the drag force on the nose of a missile moving in the super-aerodynamic region of the atmosphere . three separate cases are considered--ideal specular reflection, specular-type reflection from a slightly rough surface, and surface absorption followed by random emission of the striking molecules . the calculus of variations is employed to obtain the differential equation of the nose shape which minimizes the drag force for each of the three cases . the resulting differential equations are then solved by a numerical procedure . the drag coefficients for the optimum nose shapes are likewise determined and these are compared with the drag coefficients given by other nose shapes . it is further shown that the drag coefficients arising when specular-type reflections occur are significantly dependent on the nose shape . when surface absorption followed by random emission occurs, the drag coefficient is not strongly dependent on either the missile nose shape or the fineness ratio of the nose .'), (358, 'on the model of the free shock separation, turbulent boundary layer .', 'mager,a.', 'j. ae. scs. 1956, 181.', 'on the model of the free shock separation, turbulent boundary layer . by free shock-separated boundary layers, one means that type of separation where the flow downstream of the separation region is free to adjust to any direction that may result from the shock-boundary-layer interaction process . a detailed model of the free shock-separated turbulent boundary layer is postulated, and the pressure rise following from this model is estimated and compared with experiments . the results are applied to the prediction of separation in an overexpanded nozzle .'), (359, 'note on the hypersonic similarity law for an unyawed cone .', 'lees,l.', 'j. ae. scs. 18, 1951.', 'note on the hypersonic similarity law for an unyawed cone . it is now known that the hypersonic similarity law derived for slender cones and ogival bodies under the assumption, is applicable for mach numbers as low as 3 . this note makes use of a series development to infer the hypersonic similarity law for unyawed cones from the taylor-maccoll differential equations and associated boundary conditions . a simple approximate formula for the function of the similarity law is obtained, and the drag function computed with this formula is compared with kopal''s numerical results and, for very slender cones, with von karman''s linearized formula .'), (360, 'lift on inclined bodies of revolution in hypersonic flow .', 'grimminger, g., williams, e. p., and young, g.', 'j. aero. sc. v. 17, november, 1950 .', 'lift on inclined bodies of revolution in hypersonic flow . the importance of body lift lies in the fact that at moderate angles of attack and high mach number it can constitute an appreciable part of the total lift of a winged missile . in this paper an attempt has been made to analyze body lift in hypersonic flow by an approximate method and, together with a correlation of existing experimental data, to indicate the probable variation of body lift over a wide range of mach numbers extending from low supersonic to hypersonic . the method of analysis of hypersonic flow over inclined bodies of revolution employed herein has been denoted as the hypersonic approximation . it is an improvement on the newtonian corpuscular theory of aerodynamics, since it considers the centrifugal forces resulting from the curved paths of the air particles in addition to the impact /newtonian/ forces .'), (361, 'the flow of a viscous liquid past a flat plate at small reynolds number .', 'tomotika,s. and yosinobu,h.', 'j. math.phys. 35, 1956.', 'the flow of a viscous liquid past a flat plate at small reynolds number . the authors repeat the earlier calculations of piercy and winny (proc. roy. soc. london. ser. a. 140 (1933), earlier works were known to be different from each other . the careful analysis of the present authors shows that the skin-friction coefficient up to the second approximation agrees perfectly with that of piercy and winny .'), (362, 'three-dimensional effect of flutter in a real fluid .', 'wen-hwa chu', 'senior research engineer, department of mechanical sciences, southwest research institute, 8500 culebra road, san antonio 6, texas', 'three-dimensional effect of flutter in a real fluid . in ref. 1, an alternative semi-empirical formulation for flutter in a real fluid is given . for more accurate determination of the empirical coefficients, the three-dimensional effect of finite span should be taken into account . following reissner''s approximation for large-aspect-ratio rectangular wings, the boundary-value problem governing the downwash w and the vorticity distribution .'), (363, 'an alternative formulation of the problem of flutter in real fluids .', 'chu,w.h. and abramson,h.n.', 'j. ae. scs. 26, 1959.', 'an alternative formulation of the problem of flutter in real fluids . it is well known, in steady flow, that the actual lift curve slope is somewhat less than that predicted by inviscid flow theory, even at small angles of attack . as the stall angle is approached, the lift curve slope continually decreases and thus deviates even more from the theoretical value . pinkerton employed the measured circulation to determine the pressure distribution and found that the resulting prediction of the moment is considerably improved over that given by the classical theory . this amounts to replacing the conventional kutta-joukowski condition with the condition that the total lift should agree with the measured value, and this, in turn, completely determines the flow pattern . practically, this is accomplished by giving a fictitious camber to the profile . since potential flow theory is valid outside of the boundary layer, once the boundary-layer thickness is known, the potential flow may be corrected for the displacement thickness and the viscous wake by appropriate source distributions . the boundary layer cannot be evaluated, of course, until the potential flow is known and the circulation is applied . a criterion to determine the circulation, by generalizing the kutta-joukowski condition, was proposed by preston and spence by assuming that the pressure at the trailing edge shall have the same value when determined from the potential-flow values above and below the airfoil . this procedure gives qualitative information concerning viscous effects in steady flow .'), (364, 'a method for analysing the insulating properties of the laminar compressible boundary layer .', 'libby,pa. and pallone,a.', 'j. ae. scs. 21, 1954.', 'a method for analysing the insulating properties of the laminar compressible boundary layer . in some cooling problems associated with high energy flows it may be convenient to localize strongly the cooling, as for example by injecting a coolant through an upstream porous strip, and to depend on the insulating properties of the boundary layer to reduce, or to eliminate completely the need for further cooling on the surface downstream of the highly cooled section . this upstream cooling technique may be of interest in connection with optical windows in hypersonic wind tunnels, and on radomes, wings, and bodies of high-speed aircraft and missiles . in this paper a method for investigating the insulating properties of a laminar compressible boundary layer on a two- dimensional surface with zero heat transfer is presented . the physical situation considered thus corresponds to the case in which the heat transfer downstream of the strongly cooled section is completely eliminated . of practical concern is how the temperature of the uncooled surface varies in the downstream direction from its low initial value and thus how the low energy layer established by the upstream cooling insulates the downstream surface . the karman integral method extended to both the momentum and energy partial differential equations of the boundary layer has been used . the station, at which cooling and or injection ceases, corresponds to a discontinuity in boundary conditions and thus in solutions . at this point the flux of mass, momentum, and energy within the boundary layer has been made continuous by the introduction of three additional parameters in the velocity and stagnation enthalpy profiles . thus the velocity and stagnation enthalpy profiles have both been taken as sixth degree polynomials . the resulting two integral-differential equations are then solved for two unknown functions of the distance along the wall . these two functions are related to the boundary-layer thickness and to the wall temperature . initial conditions corresponding to a given initial wall temperature and an initial boundary-layer thickness are prescribed . exact closed-form solutions for the case of zero axial pressure gradient are obtained . for flows with significant pressure gradients, numerical solutions are required in general . several numerical examples of practical interest are presented .'), (365, 'the homogeneous boundary layer at an axisymmetric stagnation point with large rates of injection .', 'libby,p.a.', 'j. ae. scs. 29, 1962.', 'the homogeneous boundary layer at an axisymmetric stagnation point with large rates of injection . this report presents a theoretical analysis of the boundary layer at an axisymmetric stagnation point with large rates of air injection . the results of a previous investigation indicated that for localized mass transfer in the stagnation region, the rates of injection are considerably greater than those usually treated . the exact stagnation-point boundary-layer equations are integrated numerically for an approximate representation of the gas properties . the two-point boundary conditions are treated in a new manner which is useful for various boundary-layer and mixing problems . the exact solutions indicate that for large rates of injection the boundary layer is closely represented by an inner isothermal shear flow and by and exterior, relatively thin region, in which the flow variables change to their free-stream values . an integral method based on profiles suggested by the exact solutions is developed and shown to lead to accurate predictions of the integral thicknesses which are of interest for a study of the downstream influence of the stagnation-point mass transfer .'), (366, 'helium injection into the boundary layer at an axisymmetric stagnation point .', 'fox,h. and libby,p.a.', 'j. ae. scs. 1962, 921.', 'helium injection into the boundary layer at an axisymmetric stagnation point . this report presents a theoretical analysis of the boundary layer at an axisymmetric stagnation point with large rates of helium injection . the exact stagnation-point boundary-layer equations are integrated numerically with approximate representations of the gas properties . the treatment of the two-point boundary-value problem employed herein is shown to be useful for various boundary-layer and mixing problems . the exact solutions indicate that for large rates of injection the boundary layer can be represented by a thick, inner layer of constant shear, temperature, and composition and by a relatively thin outer region in which the flow variables adjust to their free-stream values . an inviscid-flow model is shown to lead to accurate predictions of this shear layer and will thus provide sufficiently accurate profiles for use in the study of the downstream influence of stagnation-point mass transfer . the heat transfer to the stagnation point is also considered . tabulations of the eigenvalues for a variety of wall conditions and injection rates are given .'), (367, 'control system and analysis and design via the second method of lyapunov .', 'kalman, r. e. and bertram, j. e.', 'trans. asme. series d, v. 82, june 1960 .', 'control system and analysis and design via the second method of lyapunov . the/second method/of lyapunov is the most general approach currently in the theory of stability of dynamic systems . after a rigorous exposition of the fundamental concepts of this theory, applications are made to/a/ stability of linear stationary, linear nonslationary, and nonlinear systems,./b/estimation of transient behavior,./c/control-system optimization,./d/design of relay servos . the discussion is essentially self-contained, with emphasis on the thorough development of the principal ideas and mathematical tools . only systems governed by differential equations are treated here . systems governed by difference equations are the subject of a companion paper .'), (368, 'some problems of polar missile control .', 'best,d.', 'j.roy.ae.soc., 64, 1960.', 'some problems of polar missile control . a polar-controlled missile is one in which manoeuvre is carried out by rotations about roll and pitch axes, that is, in the manner of a conventional aeroplane . this paper discusses some problems in the application of this form of control to homing missiles . in comparison with the alternative cartesian configuration, this method presents some special design problems . in the former case, it is often possible to resolve the motion into two planes and consider the pitch and yaw control systems as independent two-dimensional problems . this simplification is not possible in the case of polar control and it is usually necessary to consider the whole three-dimensional system . the equations of motion which result are, in general, not susceptible to analysis . because of this, the design of control systems requires extensive use of simulators .'), (369, 'an approximate solution of the supersonic blunt body problem for prescribed arbitrary axisymmetric shapes .', 'traugott,s.', 'j. ae. scs. 27, 1960, 361.', 'an approximate solution of the supersonic blunt body problem for prescribed arbitrary axisymmetric shapes . the integral method of belotserkovskii has been carried out to the first approximation for arbitrary blunt axisymmetric bodies in supersonic or hypersonic flight . this method is direct, in that it gives the surface-pressure distribution and shock shape for a prescribed body . results obtained by numerical integration for several body shapes at several mach numbers are compared to experimental results with good agreement . it is also shown that the method can be successfully applied to pointed bodies with attached shock . in the stagnation region, simple relationships are found from the equations of the first approximation which connect the surface-velocity gradient, shock curvature, shock-detachment distance, and body curvature . these relations are also correlated with experiment for a variety of shapes as a function of mach number . the correlations permit a rapid estimate of the stagnation-point velocity gradient, important for heat-transfer calculations, for any blunt body from the shock stand-off distance . a method for a higher approximation is described, for which, in contrast to the higher approximations of belotserkovskii, a large number of simultaneous total differential equations with unknown parameters does not occur . one form of this method has been studied numerically . results are given which, though only partially successful, indicate the amount of improvement to be expected from a higher approximation .'), (370, 'theoretical pressure distribution on a hemisphere-cylinder combination .', 'anthony casaccio', 'research assistant, aerodynamics laboratory, polytechnic institute of brooklyn, freeport, n.y.', 'theoretical pressure distribution on a hemisphere-cylinder combination . in recent years great use has been made of approximate methods for the determination of the pressure distribution on blunt-nosed bodies and afterbodies at high mach numbers . for quasi-spherical bodies it has been suggested that modified newtonian theory in combination with a prandtl-meyer expansion be used on the nose portion, the two laws being matched at the point where the pressure gradients are equal . no simple approximation, however, has been found for flat-nosed bodies . as for the pressure distribution on the afterbody, the blast-wave analogy has been suggested for general nose shapes but particular afterbody profiles . the purpose of the present note is to compare these approximate estimates with a more accurate determination of the flow field about a hemisphere-cylinder in an ideal gas flow . it was felt that since experimental investigations in air at this mach number are scarce and very difficult to obtain, the comparison would be of interest . the basis of comparison is the flow field as it results from a numerical integration of the exact equations governing the motion of the ideal fluid .'), (371, 'note on tip-bluntness effects in the supersonic and hypersonic regimes .', 'bennett,f.d.', 'j. ae. scs. 24, 1957, 314.', 'note on tip-bluntness effects in the supersonic and hypersonic regimes . in a recent letter, m. h. bertram presents some data on flows at m = 6.85 around 10 half-angle cones with blunted tips . since the demarcation between the supersonic and hypersonic regimes is not sharp and since one expects hypersonic flows to be generally similar to those at lower mach numbers-- especially where viscous effects do not predominate throughout the entire field of interest--it is of some value to compare bertram''s results with those obtained by giese and bergdolt for 15 half-angle cones at m = 2.45 . following the observation by charters and stein that drag coefficient measurements on blunted cones imply a reynolds number effect, giese and bergdolt study the convergence to conical flow of the perturbed flow about a cone with truncated tip . they employ the mach-zehnder interferometer and the conical flow criterion as analytical tools .'), (372, 'an experimental investigation of flow about simple blunt bodies at a nominal mach number of 5. 8.', 'oliver,r.e.', 'j. ae. scs.23, 1956, 177.', 'an experimental investigation of flow about simple blunt bodies at a nominal mach number of 5. 8. an experimental investigation was conducted in the galcit hypersonic wind tunnel to determine flow characteristics for a series of blunt bodies at a nominal mach number of 5.8 and free-stream reynolds numbers per in. of measured values for the pressure coefficient distributions are compared with a modified newtonian expression . the agreement is very good for the three-dimensional bodies and is fair for the circular cylinder transverse to the free-stream flow direction . a complete report of the investigation is given in a galcit hypersonic wind tunnel memorandum .'), (373, 'the generalized expansion method and its application to bodies travelling at high supersonic airspeeds .', 'eggers,a.j., savin,r.c. and syvertson,c.a.', 'j.ae.scs. 22, 1955, 231.', 'the generalized expansion method and its application to bodies travelling at high supersonic airspeeds . it is demonstrated that the shock-expansion method can be generalized to treat a large class of hypersonic flows, only one of which is flow about airfoils . this generalized method predicts the whole flow field, including shock-wave curvatures and resulting vorticity, providing that (1) disturbances originating on the surface of an object are largely absorbed in shock waves with which they interact and (2) disturbances associated with the divergence of stream lines in tangent planes to the surface are of secondary importance compared to those associated with the curvature of stream lines in planes normal to the surface . it is shown that these conditions may be met in three-dimensional as well as two-dimensional hypersonic flows . when they are met, surface streamlines may be taken as geodesics, which, in turn, may be related to the geometry of the surface . the validity of the generalized shock-expansion method for three-dimensional hypersonic flows is checked by comparing predictions of theory with experiment for the surface pressures and bow shock waves of bodies of revolution . the bodies treated are two ogives having fineness ratios of 3 and 5 . tests were conducted at mach numbers from 2.7 to 6.3 and angles of attack up to 15 degrees in the 10- by 14-in. supersonic wind tunnel of the ames aeronautical laboratory . at the lower angles of attack, theory and experiment approach agreement when the ratio of mach number to fineness ratio--that is, the hypersonic similarity parameter--exceeds 1 . at the larger angles of attack, theory tends to break down, as would be expected, on the leeward sides of the bodies . as a final point, it is inquired if the two-dimensionality of inviscid hypersonic flows has any counterpart in hypersonic boundary-layer flows . the question is answered in the affirmative, and results of experiment are employed to provide a partial check of this conclusion .'), (374, 'an investigation of optimum zoom climb techniques .', 'kelly,h.j.', 'j. ae.scs. 26, 1959, 794.', 'an investigation of optimum zoom climb techniques . the problem of optimal zoom climb maneuvering of a turbojet aircraft has been investigated using the mayer formulation of the calculus of variations . the euler-lagrange equations governing optimum symmetric flight have been integrated numerically by digital computation . discontinuities in thrust arising from turbojet afterburner blowout have been treated, and conditions which must be satisfied across the interface generated by the discontinuity have been derived . arbitrary control techniques have been compared with the optimum, and it has been found that performance is relatively insensitive to piloting technique unless a time limitation is imposed which requires high maneuvering load factors .'), (375, 'steady flow in the laminar boundary layer of a gas .', 'illingworth,c.r.', 'proc. roy. soc. a, 199, 1949.', 'steady flow in the laminar boundary layer of a gas . if the boundary-layer equations for a gas are transformed by mises''s transformation, as was done by karman tsion for the flow along a flat plate of a gas with unit prandtl number, the computation of solutions is simplified, and use may be made of previously computed solutions for an incompressible fluid . for any value of the prandtl number, and any variation of the viscosity with the temperature t, after the method has been applied to flow along a flat plate (a problem otherwise treated by crocco), the flow near the forward stagnation point of a cylinder is calculated with dissipation neglected, both with the effect of gravity on the flow neglected and with this effect retained for vertical flow past a horizontal cylinder . the approximations involved by the neglect of gravity are considered generally, and the cross-drift is calculated when a horizontal stream flows past a vertical surface . when, and the boundary is heat-insulated, it is shown that the boundary-layer equations for a gas may be made identical, whatever be the main stream, with the boundary-layer equations for an incompressible fluid with a certain, determinable, main stream . the method is also applied to free convection at a flat plate variation with altitude of the state of the surrounding fluid neglected) and to laminar flow in plane wakes, but for plane jets the conditions, previously imposed by howarth, are also imposed here in order to obtain simple solutions .'), (376, 'transformation between compressible and incompressible boundary layer equations .', 'van le,n.', 'j. ae. scs. 20, 1953.', 'transformation between compressible and incompressible boundary layer equations . it is proposed to show that the boundary-layer equation of compressible flow can be reduced to that of incompressible flow . such work was initiated by stewartson and by rott and crabtree . in the following some of the restrictions imposed by references 1 and 2 will be removed, and it will be shown that the transformation from compressible boundary layer to incompressible boundary layer can be applied to the laminar, as well as turbulent, case . a direct method will be used for this purpose .'), (377, 'a turbulent analog of the stewartson-illingworth transformation .', 'culick,f.e. and hill,j.', 'j. ae. scs. 25, 1958.', 'a turbulent analog of the stewartson-illingworth transformation . the stewartson-illingworth transformation is applied to the integral momentum equation for compressible boundary-layer flow, leaving the x-coordinate transformation unspecified, however . it is shown that the transformed equation is the integral momentum equation for incompressible flow if (a) the effect of compressibility on the boundary-layer shape parameter h can be represented by and (b) the x-coordinate transformation is chosen to be suitably related to the ratio of skin-friction coefficients in compressible and incompressible flows . experimental evidence is presented which shows that condition (a) is satisfied for turbulent boundary layers up to m = 5 . an x-transformation is chosen according to (b) and an equation is presented which gives the turbulent boundary-layer growth in compressible flow in terms of a simple quadrature . the predictions of this equation are then compared with some measurements on wind-tunnel nozzles .'), (378, 'engineering relations for friction and heat transfer to surfaces in high velocity flow .', 'eckert, e.', 'j. aero. sc. v.22, august 1955, pp 585-587', 'engineering relations for friction and heat transfer to surfaces in high velocity flow . in calculations of thermodynamic heating for high speed missiles parameters have been used based on relationships which hold for constant-property fluids . the validity of this procedure has been verified recently in a survey of heat transfer in which a relationship for the reference temperature was developed . a calculation procedure for laminar and turbulent boundary layers, based on this relationship, is given .'), (379, 'reverse flow and variational theorems for lifting surfaces in nonstationary compressible flow .', 'flax,a.h.', 'j. ae. scs. 20, 1953.', 'reverse flow and variational theorems for lifting surfaces in nonstationary compressible flow . a reverse-flow theorem for compressible nonsteady flow, valid within the limits of linearized theory, is derived . this theorem gives a general class of relations between linearized solutions for lifting surfaces in direct and reverse flow . based on the same considerations used to establish the theorem, an adjoint variational principle, which may be useful in approximate solutions of non-steady lifting surface problems, is obtained . to illustrate the uses of the reverse-flow theorem, it is applied to the determination of relations between aerodynamic coefficients in direct and reverse flow and to the obtaining of influence functions for total lift, pitching moment, and rolling moment for a wing oscillating with arbitrary motion and surface deformation, in terms of the pressure distributions for simpler cases in reverse flow .'), (380, 'effect of quasi-steady air forces on incompressible bending-torsion flutter .', 'dugundi,j.', 'j. ae. scs. 1958.', 'effect of quasi-steady air forces on incompressible bending-torsion flutter . explicit solutions are obtained for the bending-torsion flutter of a two-dimensional airfoil in incompressible flow under the assumptions that the theodorsen function, c(k) is set equal to a real constant, and the diagonal virtual mass terms are negligible . for the case of small bending to torsion frequency ratio, a comparison is made of these quasi-steady solutions with an earlier empirical expression suggested by theodorsen and garrick for the nonsteady case, and the effect of the c(k) function is indicated . the importance of the c.g. location for these small cases is re-emphasized, and the possibility of flutter at zero air speed is indicated .'), (381, 'the axisymmetric boundary layer on a long thin cylinder .', 'glauert,m.b. and lighthill,m.j.', 'proc. roy. soc. a, 230, 1955, 188.', 'the axisymmetric boundary layer on a long thin cylinder . the laminar boundary layer in axial flow about a long thin cylinder is investigated by two methods . one (2) is a pohlhausen method, based on a velocity profile chosen to represent conditions near the surface as accurately as possible . the other (3) is an asymptotic series solution, valid far enough downstream from the nose for the boundary-layer thickness to have become large compared with the cylinder radius . another series solution (due to seban, bond and kelly) is known, valid near enough to the nose for the boundary layer to be thin compared with the cylinder radius . the pohlhausen solution shows good agreement with both series, near and far from the nose, and enables an interpolation to be made (4) between them in the extensive range of distances from the nose for which neither is applicable . the final recommended curves, for the variation along the cylinder of skin friction, boundary-layer displacement area and momentum defect area, are displayed in graphical and tabular form (figure 1 and table 1) and are expected to be correct to within about 2 . the velocity near the wall is closely proportional to the logarithm of the distance from the axis,. this is the profile used in the pohlhausen method . the analogy with the distribution of mean velocity in turbulent flow over a flat plate is discussed at the end of 2 .'), (382, 'a note on the laminar boundary layer on a circular cylinder in axial incompressible flow .', 'howard r. kelly', 'u.s. naval ordnance test station, inyokern, china lake, calif.', 'a note on the laminar boundary layer on a circular cylinder in axial incompressible flow . a correction is made for the equation to compute the ratio of the displacement thickness on a cylinder to the corresponding thickness on a flat plate .'), (383, 'integration of the boundary layer equations for a plane in compressible flow with heat transfer .', 'meksyn, d.', 'proc. roy. soc. series a. v. 231, 1955. pp 274-180 .', 'integration of the boundary layer equations for a plane in compressible flow with heat transfer . the equations of motion of compressible viscous flow with vanishing pressure gradient past a plane are integrated in semi-convergent expressions, for the case when the physical constants depend on temperature and the prandtl number is close to unity . simple expressions are obtained for the temperature and velocity distributions in the boundary layer, the drag coefficient, and their dependence on the physical constants,.they contain the well-known results and several new ones . for the case when the temperature of the boundary is either above, or not much below, the temperature of the main flow, the results obtained closely agree with crocco''s numerical computations .'), (384, 'application of second-order shock-expansion theory to several types of bodies of revolution .', 'lavender,r.e. and deep,r.a.', 'j. ae. scs. 23, 1956, 1052.', 'application of second-order shock-expansion theory to several types of bodies of revolution . second-order shock-expansion theory is utilized to obtain equations for the initial normal force curve slope, initial pitching moment curve slope, and zero-lift wave drag for several type bodies of revolution . bodies considered are the cone-cylinder, cone-cylinder-frustum, cone-cylinder-frustum-booster, cone-frustum, and cone-frustum booster .'), (385, 'on a generalised porous-wall ?couette type? flow .', 'lilley,g.m.', 'j. ae. scs. 26,1959, 685.', 'on a generalised porous-wall ?couette type? flow . in a recent paper, the problem of a /couette-type/ flow in which the fixed wall is porous has been considered . the results quoted in the above reference can be obtained rigorously by the method stated below in which a different interpretation to one of the parameters is made .'), (386, 'a generalised porous-wall ?couette type? flow .', 'cramer,k.r.', 'j. ae. scs. 26, 1959, 121.', 'a generalised porous-wall ?couette type? flow . recently, it was observed that the two existing boundary-layer texts (references 1 and 2) did not contain a solution for the case of couette flow with a constant, uniformly distributed suction or blowing . thus, the following analysis considers a /couette-type/ flow between a stationary flat surface and a slightly inclined flat plate moving at a constant velocity . in addition, the flow is subjected to a constant, uniformly distributed suction or blowing at the fixed surface .'), (387, 'heat transfer for laminar flow in an annulus with porous wall .', 'inman,r.m.', 'j. ae. scs. 26, 1959, 532.', 'heat transfer for laminar flow in an annulus with porous wall . temperature profiles and heat-transfer rates of established incompressible flow through an annulus channel with porous walls of constant temperatures are determined at different injection rates . axial conduction and viscous dissipation are, as usual, neglected . injecting fluid is tacitly assumed to have the same temperature as the porous wall .'), (388, 'the pressure gradient induced by shear flow past a flat plate .', 'glauert,m.b.', 'j. ae. scs. 1962, 540.', 'the pressure gradient induced by shear flow past a flat plate . article is a continuation of an earlier note on papers by li on a semiinfinite plate in a uniform shear flow . li had deduced from the form of his equations that stream vorticity caused an induced pressure gradient in the flow . later papers by li and murray (amr 15(1962), rev. 7157) support the induced pressure gradient theory . the author notes, however, that the mathematics used by li and murray are not acceptable and the problem thus not resolved . the present note sets up simple models of complete flows examinable by elementary means . author holds that analyses demonstrate conclusively that no pressure gradient is induced in the boundary layer on a flat plate in a limited region of shear flow . he notes that the original question in the case of unbounded shear remains obscure--and anyway an unlimited shear layer is not of great practical importance .'), (389, 'simple shear flow past a flat plate in a compressible viscous fluid .', 'li,t.y.', 'j. ae. scs. 22, 1955, 724.', 'simple shear flow past a flat plate in a compressible viscous fluid . by transformation of variables, the problem of a simple shear flow of a compressible fluid over a flat plate is reduced to the corresponding problem for an incompressible fluid . the prandtl number of the compressible fluid is assumed to be unity and its viscosity to be a linear function of temperature .'), (390, 'some panel-flutter studies using piston theory .', 'johns,d.j.', 'j. ae. scs. 25, 1958.', 'some panel-flutter studies using piston theory . the use of piston theory was recently advocated for supersonic aeroelastic analyses, including the problem of panel flutter, and this has stimulated the investigation reported here . linear piston theory is mainly considered, but some effects of introducing higher order terms are discussed . flutter of rectangular simply supported panels and of elliptically shaped clamped-edge panels is considered, and some justification is provided for the use of /static/ aerodynamic forces and the neglect of aerodynamic damping . hence, it is concluded that ackeret loading gives more exact results than piston theory . solution of the flutter equations is made by applying galerkin''s method to a rayleigh-type analysis using assumed modes of deformation .'), (391, 'flutter of rectangular simply supported panels at high supersonic speeds .', 'hedgepeth,j.m.', 'j. ae. scs. 24, 1957.', 'flutter of rectangular simply supported panels at high supersonic speeds . the problem of panel flutter of rectangular simply supported plates subjected to supersonic flow over one surface is treated theoretically . the assumption is made, and subsequently verified, that the /static/ approximation to the aerodynamic flutter forces yields flutter boundaries with satisfactory accuracy for mach numbers greater than about 2 . two panel flutter analyses are performed using this static approximation in conjunction with thin-plate theory--one employs aerodynamic strip theory, the other aerodynamic surface theory . the influence of mach number, dynamic pressure, panel aspect ratio, and midplane stress on the panel thickness required to prevent flutter is determined for extensive ranges of these parameters .'), (392, 'natural frequencies of rectangular plates with edges elastically restrained against rotation .', 'jogarao,v. and lakshmikantham,c.', 'j. ae. scs. 1957.', 'natural frequencies of rectangular plates with edges elastically restrained against rotation . plates with attachments to heavier members along the edges can be described as having edges elastically restrained against rotation, in many cases uniformly along each edge . at the edges, setting slope, when is the edge bending moment with always positive, the elastic restraint can be analytically defined with describing respectively, clamped and simply supported edges . in this note natural frequencies of such plates are calculated mainly following the nomenclature of dana young .'), (393, 'the shear flow along a flat plate with uniform suction .', 'sakurai,t.', 'j.ae.scs. 24, 1957.', 'the shear flow along a flat plate with uniform suction . recently, several authors have investigated the boundary layer in a shear flow . in this note, an exact solution of the navier-stokes equations will be presented, which represents the boundary layer along an infinite flat plate with uniform suction situated in a shear flow .'), (394, 'the viscous flow near a stagnation point when the external flow has uniform vorticity .', 'j. t. stuart', 'national physical laboratory, teddington, middlesex, england', 'the viscous flow near a stagnation point when the external flow has uniform vorticity . in view of the recent controversy between li and glauert on the nature of the solution of the boundary-layer equations when the external flow is rotational, it seems worthwhile to draw attention to a certain exact solution of the navier-stokes equations which lends support to glauert''s point of view .'), (395, 'new methods in heat flow analysis with application to flight structures .', 'biot,m.a.', 'j. ae. scs. 24, 1957.', 'new methods in heat flow analysis with application to flight structures . new methods are presented for the analysis of transient heat flow in complex structures, leading to drastic simplifications in the calculation and the possibility of including nonlinear and surface effects . these methods are in part a direct application of some general variational principles developed earlier for linear thermodynamics . they are further developed in the particular case of purely thermal problems to include surface and boundary-layer heat transfer, nonlinear systems with temperature-dependent parameters, and radiation . the concepts of thermal potential, dissipation function, and generalized thermal force are introduced, leading to ordinary differential equations of the lagrangian type for the thermal flow field . because of the particular nature of heat flow phenomena, compared with dynamics, suitable procedures must be developed in order to formulate each problem in the simplest way . this is done by treating a number of examples . the concepts of penetration depth and transit time are introduced and discussed in connection with one-dimensional flow . application of the general method to the heating of a slab, with temperature-dependent heat capacity, shows a substantial difference between the heating and cooling processes . an example of heat flow analysis of a supersonic wing structure by the present method is also given and requires only extremely simple calculations . the results are found to be in good agreement with those obtained by the classical and much more elaborate procedures .'), (396, 'variational and lagrangian thermodynamics of thermal convection-fundamental shortcomings of the heat transfer coefficient .', 'biot,m.a.', 'j. ae. scs. 29, 1962.', 'variational and lagrangian thermodynamics of thermal convection-fundamental shortcomings of the heat transfer coefficient . extension of previous analyses, indicating the possibility of extending the thermodynamics of irreversible processes to systems which are not in the vicinity of an equilibrium state and for which onsager''s relations are not verified . this involves generalizations beyond the narrow field of heat transfer and to principles of wider range than those of current nonequilibrium thermodynamics .'), (397, 'a sublayer for fluid injection into the incompressible turbulent boundary layer .', 'turcotte,d.l.', 'j. ae. scs. 27, 1960.', 'a sublayer for fluid injection into the incompressible turbulent boundary layer . a sublayer region is introduced in which the intensity of turbulence grows at a prescribed rate . the decrease in wall shear stress due to fluid injection into the boundary layer is found under the hypothesis that the effect of injection is restricted to the sublayer region . experimental measurements of the velocity profiles with fluid injection substantiate this hypothesis . the theoretical decrease in wall shear stress is in good agreement with experiment,. the solution is particularly simple and for small values of the injection parameter it contains no arbitrary parameters . the theory provides a similarity parameter which differs from the one in general use .'), (398, 'heat transfer in turbulent shear flow .', 'rannie,w.d.', 'j. ae. scs. 23, 1956.', 'heat transfer in turbulent shear flow . the problems of heat transfer in turbulent shear flow along a smooth wall are discussed from the point of view of von karman''s well-known 1939 paper on the analogy between fluid friction and heat transfer . methods for extending the analysis to higher prandtl numbers are suggested .'), (399, 'conduction of heat in composite slabs .', 'jaeger, j. c.', 'quart. appl. math. v. 8 july, 1950 . pp 187-198', 'conduction of heat in composite slabs . a method of calculating the total quantity of heat that passes through a unit area from zero time to time t is developed . allowance is made for surface resistance by regarding each contact resistance as an additional layer of the appropriate thermal resistance and zero heat capacity'), (400, 'buckling stress of clamped rectangular plates in shear .', 'budiansky,b. and connor,r.w.', 'naca tn.1559, 1948.', 'buckling stress of clamped rectangular plates in shear . by consideration of antisymmetrical, as well as symmetrical, buckling configurations, the theoretical shear buckling stresses of clamped rectangular flat plates are evaluated more correctly than in previous work . the results given, which represent the average of upper and lower-limit solutions obtained by the lagrangian multiplier method, are within percent of the true buckling stresses .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (401, 'inviscid hypersonic airflows with coupled non-equilibrium processes .', 'hall,j.g., eschenroeder,a.w. and marrone,p.v.', 'ias paper 62-67, 1962.', 'inviscid hypersonic airflows with coupled non-equilibrium processes . analyses have been made of the effects of coupled chemical rate processes in external inviscid hypersonic airflows at high enthalpy levels . exact (numerical) solutions have been obtained by the inverse method for inviscid airflow over a near-spherical nose under flight conditions where substantial nonequilibrium prevails through the nose region . typical conditions considered include nose radii of the order of 1 ft at an altitude of 250,000 ft and velocities of 15,000 and 23,000 ft per sec . the results illustrate the general importance of the coupling among the reactions considered . these included dissociation-recombination, bimolecular-exchange, and ionization reactions . the exact solutions show the bimolecular, no exchange reactions to be important in blunt-nose flow for the kinetics of no and n, as they are in the case of a plane shock wave . an important difference between blunt-nose flow and plane shock flow, however, is the gasdynamic expansion in the curved shock layer of the former . this expansion reduces post-shock reaction rates . as a consequence, in the regime studied the oxygen and nitrogen-atom concentrations tend to freeze in the nose region at levels below those for infinite-rate equilibrium . the reduction below the equilibrium dissociation level can be large, particularly for nitrogen dissociation at higher velocities . in the regime considered, the chemical kinetics are dominated by two-body collision processes . the inviscid nose flow, including coupled nonequilibrium phenomena, is thus amenable to binary scaling for a given velocity . the binary scaling is demonstrated for a range of altitude and scale by correlation of the exact solutions for given velocity and a constant product of ambient density and nose radius . this similitude, which can also scale viscous nonequilibrium and radiation phenomena in the shock layer, provides a useful flexibility for hypersonic testing where it is applicable . the afterbody inviscid-flow problem is briefly discussed in the light of the results for the nose flow .'), (402, 'magnetohydrodynamics shocks .', 'de hoffman,f. and teller,e.', 'phys. rev. 80, 1950, 693.', 'magnetohydrodynamics shocks . a mathematical treatment of the coupled motion of hydrodynamic flow and electromagnetic fields is given . two simplifying assumptions are introduced .. first, the conductivity of the medium is infinite, and second, the motion is described by a plane shock wave . various orientations of the plane of the shock and the magnetic field are discussed separately, and the extreme relativistic and unrelativistic behavior is examined . special consideration is given to the behavior of weak shocks, that is, of sound waves . it is interesting to note that the waves degenerate into common sound waves and into common electromagnetic waves in the extreme cases of very weak and very strong magnetic fields .'), (403, 'magnetohydrodynamic shock waves .', 'helger,l.', 'astrophys. j. 117, 1953, 177.', 'magnetohydrodynamic shock waves . an interpretation of the de hoffman-teller shock-wave equations for an infinitely conducting medium is given analogous to the classical interpretation of the ordinary hydrodynamic shock-wave equations of rankine and hugoniot . two cases of interest are considered as a consequence of this theory . it is shown that weak magnetic fields in interstellar clouds will be amplified, and, if external mechanisms are available to reduce the compressional effects of shock waves, the field will reach a value, where p is the pressure . also, some aspects of the internal motions of prominences are considered ,. it is shown that gauss will yield results in accord with the observational material .'), (404, 'two dimensional transonic flow past airfoils .', 'kuo,y.a.', 'naca tn.2356.', 'two dimensional transonic flow past airfoils . this report concerns the problem of constructing solutions for transonic flows over symmetric airfoils . the aspect of the problem emphasized is, of necessity, not how to form a solution for compressible flow but how to simplify the initial phase of the problem, namely, the mapping of the incompressible flow . in the case of the symmetric joukowski airfoil without circulation, the mapping is relatively simple, but the coefficients in the power series are difficult to evaluate . as a result, the problem requires simplification . instead of the exact incompressible flow past the airfoil, an approximate flow is used, which is derived from a combination of source and sink . this flow differs only slightly from the exact one when the thickness is small . by the same method, the flow with circulation is also considered . after the incompressible-flow functions are approximated in this fashion, the numerical calculation of the corresponding compressible flow, by the hodograph theory, does not present any essential difficulty .'), (405, 'tables of thermal properties of gases .', 'joseph hilsenrath, chalres beckett, william bendict, liila fano, harold hoge, joseph masi, ralph nuttall, yeram touloukian, harold woolley', 'nbs circular 564 (1955)', 'tables of thermal properties of gases . tables of thermodynamic and transport properties of air, argon, carbon dioxide, carbon monoxide, hydrogen, nitrogen, oxygen, and steam .'), (406, 'on the behaviour of boundary layers at supersonic speeds .', '', '', 'on the behaviour of boundary layers at supersonic speeds . this paper considers the implications of recent advances in knowledge of the behaviour of boundary layers in supersonic flow . only the simplest case is considered-dashthat of the two-dimensional boundary layer on a flat plate, with nominal zero longitudinal pressure and temperature gradients . it is shown that the empirical/intermediate enthalpy/used with success in approximations for skin friction, etc., of laminar boundary layers is closely the same as the mean enthalpy with respect to velocity . furthermore, the mean enthalpics of laminar and turbulent boundary layers may be the same . a nonrigorous approach is made to the problems of self-induced pressure gradients, and the indications are that their effects on laminar skin friction, etc., may become noticeable at mach numbers greater than 5 and they increase as the surface temperature builds up towards zero heat-transfer conditions . the effects with turbulent boundary layers may not be so severe . finally, the results are applied to give an idea of the magnitude of the drag and aerodynamic heating problems up to m 10, and one result is that, if there is any conflict at the higher mach numbers between surface conditions required for high radiative emissivity and those which may be thought necessary for preserving a laminar boundary layer, then it may be better to choose the former .'), (407, 'stationary convection flow of an electrically conducting liquid between parallel plates in a magnetic field .', 'gershuni,g.z. and shukhovitskii,e.m.', 'soviet physics,34, 1958.', 'stationary convection flow of an electrically conducting liquid between parallel plates in a magnetic field . a study is made of the stationary convection of an electrically conducting liquid in the space between two parallel plates, heated to different temperatures, in the presence of a magnetic field . the distribution of velocity, temperature, and induced fields are found, and the convective heat flow is calculated .'), (408, 'on convective motion of a conducting fluid between parallel vertical plates in a magnetic field .', 'regirer,s.a.', 'soviet physics, 37, 1960.', 'on convective motion of a conducting fluid between parallel vertical plates in a magnetic field . stationary convective motion of a conducting fluid between vertical parallel plates in a magnetic field is considered . an exact solution of the magnetohydrodynamic equations is obtained for the case of a constant vertical temperature gradient . the critical value of grasshof''s number is determined for the case when the temperature of both plates is the same .'), (409, 'on the base pressure resulting from the interaction of a supersonic external stream with a sonic or subsonic jet .', 'chow, w. l.', 'j. aero. sc. march, 1959 . p. 176-180 .', 'on the base pressure resulting from the interaction of a supersonic external stream with a sonic or subsonic jet . it is shown that the two-dimensional base pressure problems relating to base bleed into the wake of blunt-trailing-edge airfoils, or the interaction between an external supersonic or sonic slipstream with a sonic or subsonic jet stream of a jet engine, can be calculated by theoretical considerations . constant-pressure, isoenergetic, turbulent mixing between the streams and the stagnant fluid in the wake is assumed . the theoretical calculations are in good agreement with the experimental results .'), (410, 'the supersonic flow about a blunt body of revolution for gases at chemical equilibrium .', 'gravalos,f.g., edelflet,i.h. and emmons,h.', '9th int. astro. fed. 1958.', 'the supersonic flow about a blunt body of revolution for gases at chemical equilibrium . the supersonic flow about a blunt body of revolution for gases at chemical equilibrium . a method to determine the shock wave, and its location, about a body of revolution moving at supersonic speeds is given . the method provides also the means to compute the flow characteristics in the shock layer . the fluid in which the motion takes place is assumed to be in chemical equilibrium within the shock layer ,. its thermochemical properties must be known . the essential new features of the method are .. a) it solves the direct problem, i. e., the initial data are the conditions upstream and the body shape ,. b) the integration of the fundamental equations is done in the physical plane and the difficulties inherent to other, less direct, mathematical formulations of the problem are avoided . a physical interpretation of the method is made which is in accord with the analytical definition of the problem .'), (411, 'data on shape and location of detached shock waves in cones and sphere .', 'herberle,j.w., wood,g.p. and gooderum,p.b.', 'naca tn.2000, 1950.', 'data on shape and location of detached shock waves in cones and sphere . accurate experimental data are given on the shape and the location of detached shock waves on cones and spheres at mach numbers from 1.17 to 1.81 . the data are correlated to obtain equations that describe the shock waves . this knowledge of the shock waves should be useful in calculations of the pressure distribution and the pressure drag of the fore part of cones and spheres . the experimental data on shock waves are compared with theory .'), (412, 'critical combinations of shear and transverse direct stress for an infinitely long flat plate with edges elastically restrained against rotation .', 'batdorf,s.b. and houbolt,j.c.', '', 'critical combinations of shear and transverse direct stress for an infinitely long flat plate with edges elastically restrained against rotation . an exact solution and a closely concurring approximate energy solution are given for the buckling of an infinitely long flat plate under combined shear and transverse direct stress with edges elastically restrained against rotation . it was found that an appreciable fraction of the critical stress in pure shear may be applied to the plate without any reduction in the transverse compressive stress necessary to produce buckling . an interaction formula in general use was shown to be decidedly conservative for the range in which it is supposed to apply .'), (413, 'turbulent skin friction at high mach numbers and reynolds numbers in air and helium . nasa r82, 1960 .', 'matting,f.w., chapman,d.r., nyholm,j.r. and thomas,a.g.', 'naca r847, 1946.', 'turbulent skin friction at high mach numbers and reynolds numbers in air and helium . nasa r82, 1960 . results are given of local skin-friction measurements in turbulent boundary layers over an equivalent air mach number range from 0.2 to 9.9 and an over-all reynolds number variation of 2x10 to 100x10 . direct force measurements were made by means of a floating element . flows were two-dimensional over a smooth flat surface with essentially zero pressure gradient and with adiabatic conditions at the wall . air and helium were used as working fluids . an equivalence parameter for comparing boundary layers in different working fluids is derived and the experimental verification of the parameter is demonstrated . experimental results are compared with the results obtained by several methods of calculating skin friction in the turbulent boundary layer .'), (414, 'the problem of resistance in compressible fluids .', 'von karman,t.', '5th volta cong. 1955,226.', 'the problem of resistance in compressible fluids . this report is restricted to the resistance of bodies of revolution and of cylindrical bodies of infinite length moving with uniform velocity in a compressible fluid . in the case of bodies of revolution it will be assumed that the direction of the movement is parallel to the axis of symmetry . it will be assumed that the fluid satisfies the equation of state of perfect gases, i. e. const., where p denotes the pressure, the density and t the absolute temperature . in addition to obeying this equation the fluid is characterized by the statement that the intrinsic energy of the unit mass amounts to where for simplicity''s sake the specific heat will be expressed in work rather than heat units . the ratio between the specific heat at constant pressure and the specific heat at constant volume will be denoted by . it is known that the value of x depends upon the number of degrees of freedom of the molecules,. if this number is denoted by n . for air the value x = 1.4 will be used . the limiting case x = 1 will be referred to as that of a assumed that in the range considered and are independent of the temperature .'), (415, 'the aerodynamic design of section shapes for swept wings .', 'pearcey, h.h.', '2nd inter. congr. of int. council of aero. sc. september, 1960 .', 'the aerodynamic design of section shapes for swept wings . an extension of work of lock and rogers and the result of cooperation by n.p.l., r.a.e. and members of the british aircraft industry to achieve a satisfactory design for an aircraft cruising at low supersonic speeds . knowledge of shock-wave prediction, onset of wave drag and shock-induced separation allows the basic design to be generalized for a wide range of parameters . unpublished work by bagley on the relation of aerodynamic coefficients and geometry is used . the role of upper surface velocity distribution is noted and methods for predicting pressure distributions with shock waves are reviewed for both subsonic and transonic flows .'), (416, 'methods of boundary-layer control for postponing and alleviating buffeting and other effects of shock-induced separation .', 'pearcey, h.h. and stuart, c.m.', 'smf fund paper, no. f.f. -dash 22, 1959 .', 'methods of boundary-layer control for postponing and alleviating buffeting and other effects of shock-induced separation . the use of boundary-layer control to increase the separation-free margins of mach number and lift coefficient beyond the cruise point of high-speed aircraft may often be preferred to design changes that impair the cruising performance or the landing and take-off characteristics . the factors that influence the choice of method and details of its application are discussed, emphasising particularly the need to maintain effectiveness over most of the chord to cover the wide range of separation positions encountered as the shock moves over the wing with changing flight conditions . research at the national physical laboratory that has embraced high-velocity blowing, vane and air-jet vortex generators, and, in a preliminary way, distributed suction, is briefly described . the relative merits of the various methods are discussed, and some results achieved in their application are given . for vortex generators, the importance is stressed of the vortex paths determined by the interactions of neighbouring vortices and their images . thus, systems of counter-rotating vortices always leave the surface in pairs and lose their effectiveness . co-rotating systems are therefore preferred for many applications . blowing, which in wind-tunnel tests gives results as good as or better than vortex generators and does not have the disadvantage of a drag penalty at cruise, has not yet been assessed in flight . air-jet vortex generators, which would also avoid the drag penalty, show promise of producing significant effects with relatively small blowing pressures and quantities .'), (417, 'on the stability of two dimensional parallel flows .', 'lin,c.c.', 'pt.iii - stability in a viscous fluid. q. app. math. 3, 1945, 273.', 'on the stability of two dimensional parallel flows . this is the last part of the author''s theory of the stability of plane laminar motion . (for parts 1 and 2, cf. the same quart. 3, 117-142, 218-234 (1945),. these rev. 7,225,226 .) the stability character of a viscous fluid is considered in detail . the author proceeds first to give a proof of a criterion of stability due to heisenberg .. if a velocity profile has an number and phase velocity, the disturbance with the same wave number is unstable in the real fluid when the reynolds number is sufficiently large . this destabilizing effect of viscosity is one of the most interesting phenomena in the general stability theory,. its physical and mathematical significance is carefully discussed . the author then discusses the behavior of the so-called neutral curve for the two characteristic types of velocity distribution, the boundary layer type profile and the symmetrical profile . the asymptotic behavior of the neutral curve is discussed first . the main difference between profiles with and without a point of inflection is that the two branches of the neutral curve approach and for profiles with a flex, but both converge to for the profile without a flex . the most important results are as follows . for sufficiently large reynolds number r . (2) there always exists a minimum r below which the motion is stable . a similar result was obtained by synge from energy considerations . synge found a limiting curve below which the motion is necessarily stable . the author''s discussion of the asymptotic behavior of the curves shows further that there always exists a maximum value of a beyond which the motion is stable for all reynolds numbers . hence the qualitative shape of the curve is determined . the author proceeds to show that simple approximate expressions for the stability limit can be obtained from his general analysis for a given velocity profile . these approximate stability limits for plane poiseuille flow and blasius flow are found to be r=5906 and r=502 . the reynolds numbers are based on the width of the channel and the displacement thickness, respectively . finally, the method for computing the complete instability curve is presented and the plane poiseuille case and the blasius problem worked out in detail . the stability limit for blasius flow had been given before by tollmien and schlichting . the present more exact computations agree well with tollmien''s result as far as the minimum critical reynolds number is concerned . the value found here is r=420 . the neutral curve for poiseuille motion had not been obtained before . the minimum critical number here is found to be r=5314 . the agreement with the estimate from the simple criterion mentioned above is thus very good . a discussion of the physical significance of the viscous effects and of future developments concludes the paper .'), (418, 'transition form laminar to turbulent shear flow .', 'morkovin,m.v.', 'asme trans, 80, 1958,1121.', 'transition form laminar to turbulent shear flow . recent experimental studies of transition from laminar to turbulent shear flows are reviewed . certain common features are emphasized and related to the stability theories of viscous shear layers . the three-dimensional character, the unsteadiness, and the nonlinear and random behavior of the latter stages of the transition process are also examined .'), (419, 'the design of intermediate vertical stiffeners on web plates subjected to shear .', 'rockey,k.c.', 'aero. quart. 1956, 275.', 'the design of intermediate vertical stiffeners on web plates subjected to shear . the correct design of intermediate vertical stiffeners on web plates subjected to shear becomes very important when the web plates are designed to operate at loads close to their buckling loads . this paper presents details of an extensive series of tests conducted on stiffened web plates subjected to shear . from the analysis of the results obtained from these tests, new empirical relationships between the flexural rigidity and spacing of the intermediate stiffeners and the buckling stress of the stiffened web plate have been obtained . one interesting and important feature of these new relationships is that they define more clearly than hitherto the difference in the behaviour of single-and double-sided stiffeners .'), (420, 'an experimental study of the flow field about swept and delta wings with sharp leading edges .', 'jaszlics, i. and trilling, l.', 'j. aero. sc. august 1959 . p 487 - 494, 544 .', 'an experimental study of the flow field about swept and delta wings with sharp leading edges . a series of experiments was performed to define the flow field on the upper surface of high aspect ratio swept wings and narrow delta wings at high angles of attack . it was found that near the root section of either type of wing the flow is conical . the edge of the vortex sheet which originates at the leading edge is a straight line whose position relative to the leading edge depends only on incidence . on swept wings, the vortex edge turns down-stream as soon as the vortex sheet covers the front half of the wing chord, and the flow under the vortex sheet outboard of that turning point is uniform and parallel to the leading edge of the wing . on narrow delta wings, the conical symmetry persists almost to the trailing edge .'), (421, 'analytic study of induced pressure on long bodies of revolution with varying nose bluntness at hypersonic speeds .', 'van hise,v.', 'nasa r78, 1960.', 'analytic study of induced pressure on long bodies of revolution with varying nose bluntness at hypersonic speeds . a systematic study of induced pressures on a series of bodies of revolution with varying nose bluntness has been made by using the method of characteristics for a perfect gas . the fluid mediums investigated were air and helium and the mach number range was from 5 to 40 . a study of representative shock shapes was also made . flow parameters obtained from the blast-wave analogy gave good correlations of induced pressures and shock shapes . the induced-pressure correlations yielded empirical equations for air and helium which cover the complete range of nose bluntness considered . (nose fineness ratios varied from 0.4 to 4.) available experimental results were in good agreement with the characteristics solutions . properties connected with the concept of hypersonic similitude enabled correlations of the calculations to be made with respect to nose shape, mach number, and ratio of specific heats .'), (422, 'bending of a square plate with two adjacent edges free and the others clamped or simply supported .', 'leissa,a.w. and niedenfuhr,f.w.', 'a.i.a.a. jnl. 1, 1963, 116.', 'bending of a square plate with two adjacent edges free and the others clamped or simply supported . the title problems were solved for the two cases .. (1) uniform transverse loading, (2) a concentrated force at the free corner . a function is chosen to exactly satisfy the biharmonic equation while the boundary conditions are enforced at a number of points plied at discrete points around the boundary for each of the four problems and the resulting 35 simultaneous equations were solved on an ibm 704 . tables listing the values of deflection and bending moments are presented . this paper provides useful information on the solution of these problems which are intractable by analytical methods .'), (423, 'an experimental investigation of the flow over blunt-nosed cones at a mach number of 5. 8.', 'machell,r.m. and o''bryant,w.t.', 'guggenheim aero. lab. memo 32, 1956.', 'an experimental investigation of the flow over blunt-nosed cones at a mach number of 5. 8. shock shapes were observed and static pressures were measured on spherically-blunted cones at a nominal mach number of 5.8 over a range of reynolds numbers per inch from 97,000 to 238,000, for angles of yaw from 0 to 8 . six combinations of the bluntness ratios 0.4, 0.8, and 1.064 with the cone half angles 10, 20, and 40 were used in determining the significant parameters governing pressure distribution . the pressure distribution on the spherical nose for both yawed and unyawed bodies is predicted quite accurately by the modified newtonian theory given by, where is the angle between the normal to a surface element and the flow direction ahead of the bow shock . cone half angle was found to be the significant parameter in determining the pressure distribution near the nose-cone junction and over the conical afterbody . on the 40 spherical nosed cone models the flow overexpanded with respect to the taylor-maccoll pressure in the region of the spherical-conical juncture, after which the pressure returned rapidly to the taylor-maccoll value . for models with smaller cone angles the region of minimum pressure occurred farther back on the conical portion of the model, and the taylor-maccoll pressure was approached more gradually . the shape of the pressure distributions as described in nondimensional coordinates was independent of the radius of the spherical nose and of the reynolds number over the range of reynolds number per inch between .97 x 10 and 2.38 x 10 . integrated results for the pressure foredrag of the models at zero yaw compared very closely with the predictions of the modified newtonian approximation, except for models with large cone angles and small nose radii, where the drag approaches the value given by the taylor-maccoll theory for sharp cones .'), (424, 'cantilever plate with concentrated edge load .', 'holl,d.l.', 'j.app.mech. 1937, 8.', 'cantilever plate with concentrated edge load . the author gives, by the method of finite differences, an approximate solution of the problem of a finite length of a cantilever plate which bears a concentrated load at the longitudinal free edge . all the boundary conditions are taken into account, and the plate action is determined approximately at all points of the plate . the author points out that a secondary maximum transverse stress occurs at the clamped edge nearest the loading point, and that the longitudinal stress is greatest directly under the loading point .'), (425, 'the solution of elastic plate problems by electrical analogies .', 'r. h. macneal,', 'pasadena, calif.', 'the solution of elastic plate problems by electrical analogies . a dynamic-analogy method for the solution of elastic plate problems is described in this paper . the electrical circuits developed here can be set-up and studied on an electric-analog computer . problems involving deflections under constant load, transient vibrations, or normal modes can be solved in this way . the method of applying boundary conditions to plates with irregular edges is given, together with a detailed description of the representation of the boundary conditions for a rectangular variable-thickness plate . solutions that have been obtained on the cal tech electric-analog computer are presented for the static deflections and normal modes of a rectangular cantilever plate .'), (426, 'preliminary analysis of axial flow compressors having supersonic velocity at the entrance of the stator .', 'ferri,a.', 'naca rm l9g06, 1949.', 'preliminary analysis of axial flow compressors having supersonic velocity at the entrance of the stator . a supersonic compressor design having supersonic velocity at the entrance of the stator is analyzed on the assumption of two-dimensional flow . the rotor and stator losses assumed in the analysis are based on the results of preliminary supersonic cascade tests . the results of the analysis show that compression ratios per stage of 6 to 10 can be obtained with adiabatic efficiency between 70 and 80 percent . consideration is also given in the analysis to the starting, stability, and range of efficient performance of this type of compressor . the desirability of employing variable-geometry stators and adjustable inlet guide vanes is indicated . although either supersonic or subsonic axial component of velocity at the stator entrance can be used, the cascade test results suggest that higher pressure recovery can be obtained if the axial component is supersonic .'), (427, 'flow of gas through turbine lattices .', 'deich,m.e.', 'naca tm.1393, 1956.', 'flow of gas through turbine lattices . paper is a translation of chap. 7 of the book /technical gas-dynamics/ (see amr 9, rev 1869) . the topics treated are best shown by the list of paragraph headings . they are .. 7-1 . geometrical and gasdynamical parameters of the lattices,. fundamentals of flow through lattices,. 7-2 . theoretical methods of investigation or plane potential flow of incompressible fluid through a lattice,. 7-3 . electro-hydrodynamic analogy,. 7-4 . forces acting on an airfoil in a lattice,. theorem of joukowsky for lattices,. 7-5 . fundamental characteristics of lattices,. 7-6 . friction losses in plane lattice at subsonic velocities,. 7-7 . edge losses in plane lattice at subsonic velocities,. 7-8 . several results of experimental investigations of plane lattices at small subsonic velocities,. 7-9 . flow of gas through lattice at large subsonic velocities,. critical mach number for lattice,. 7-10 . profile losses in lattices at large subsonic velocities,. 7-11 . flow of a gas through reaction lattices at supersonic pressure drops,. 7-12 . impulse lattices in supersonic flow,. 7-13 . losses in lattices at near sonic and supersonic velocities,. 7-14 . computation of angle of deflection of flow in overhang section of a reaction lattice at supersonic pressure drops,. 7-15 . characteristic features of three-dimensional flow in lattices .'), (428, 'the quasi-cylinder of specified thickness and shell loading in supersonic flow .', 'portnoy,j.', 'aero. quart. 11, 1960, 387.', 'the quasi-cylinder of specified thickness and shell loading in supersonic flow . the methods of the operational calculus are used to obtain a linear approximation to the shape of the mean camber surface of a quasi-cylinder in a supersonic flow in terms of its shell thickness and loading distributions . the analysis deals with a generalised quasi-cylinder ,. that is one which, although lying close to a mean cylinder, need not possess axial symmetry . the quasi-cylinder is also permitted to be within the small disturbance field of other separate components, e.g. a centre-body . because the linearised theory is inadmissable for internal duct flows close to and beyond the first reflected characteristic cone, the present solution is likewise invalid close to and beyond the position where this characteristic meets the mean cylinder . the work given here enables the camber shapes of /ring-wings/, which have been used theoretically to reduce or even nullify the wave-drag of a central slender-body, to be found . an example illustrates the general method .'), (429, 'a description of the r. a. e. high speed supersonic tunnel .', 'poole,j.a.', 'rae tn.aero.2678,1960', 'a description of the r. a. e. high speed supersonic tunnel . an account is given of the high supersonic speed tunnel now nearing completion . the design philosophy is reviewed, the principal features are described and some of the more interesting development problems are noted .'), (430, 'calibration of the flow in the mach 4 working section of the 4ft . x 3ft . high supersonic speed wind tunnel at rae bedford .', 'andrews,d.r. and brown,c.s.', 'rae tn.aero.2820, 1962.', 'calibration of the flow in the mach 4 working section of the 4ft . x 3ft . high supersonic speed wind tunnel at rae bedford . mach number and flow angle distributions in the working section of the mach 4 nozzle of the 4 ft x 3 ft high-supersonic-speed wind tunnel are presented for a range of total pressure and humidity .'), (431, 'free-flight measurements of the zero-lift drag and base pressure on a wind tunnel interference model (m=0 . 8 - 1. 5) .', 'greenwood,g.h.', 'rae tn.aero.2725, 1960.', 'free-flight measurements of the zero-lift drag and base pressure on a wind tunnel interference model (m=0 . 8 - 1. 5) . five free-flight models were flown to measure the zero-lift drag and body base pressure on a standard wind tunnel interference model over a mach number range of 0.84 to 1.48 . roughness bands on the wings and body of the model are shown to produce a small but definite increase in the zero-lift drag at all mach numbers . the measured drag is in fair agreement with corresponding measurements made in various transonic tunnels with differences that could plausibly be explained as the effects of tunnel interference . the effect of a simulated wind tunnel support sting is shown to increase the base pressure . the discrepancy between models with and without a sting is greatest at subsonic speeds and progressively decreases with increasing mach number until at m = 1.4 the sting has no effect on base pressure .'), (432, 'theoretical damping in roll and rolling moment due to differential wing incidence for slender cruciform wings and wing-body combinations .', 'adams,g.j. and dugan,d.w.', 'naca r1088, 1952.', 'theoretical damping in roll and rolling moment due to differential wing incidence for slender cruciform wings and wing-body combinations . a method of analysis based on slender-wing theory is developed to investigate the characteristics in roll of slender cruciform wings and wing-body combinations . the method makes use of the conformal mapping processes of classical hydrodynamics which transform the region outside a circle and the region outside an arbitrary arrangement of line segments intersecting at the origin . the method of analysis may be utilized to solve other slender cruciform wing-body problems involving arbitrarily assigned boundary conditions . in the present report, the application of the method has shown .. differential incidence of both pairs of opposite surfaces of the cruciform wing-body combinations are practically independent of the body-diameter-maximum-span ratio up to a value of this ratio of 0.3 . arrangement is only 62 percent greater than that for a corresponding planar wing-body combination . dence of both pairs of the opposing surfaces of the cruciform wing-body arrangement, is only 52 percent greater than that for a corresponding planar wing-body combination . unit surface deflection) of the cruciform wing-body arrangement having four equally deflected panels is therefore 94 percent of the corresponding planar wing-body combination .'), (433, 'application of two dimensional vortex theory to the prediction of flow fields behind wings of wing-body combinations at subsonic and supersonic speeds .', 'rogers,a.w.', 'naca tn.3227, 1954.', 'application of two dimensional vortex theory to the prediction of flow fields behind wings of wing-body combinations at subsonic and supersonic speeds . a theoretical investigation has been made of a general method for predicting the flow field behind the wings of plane and cruciform wing and body combinations at transonic or supersonic speeds and slender configurations at subsonic speeds . the wing trailing-vortex wake is represented initially by line vortices distributed to approximate the spanwise distribution of circulation along the trailing edge of the exposed wing panels . the afterbody is represented by corresponding image vortices within the body . two-dimensional line-vortex theory is then used to compute the induced velocities at each vortex and the resulting displacement of each vortex is determined by means of a numerical stepwise integration procedure . the method was applied to the calculation of the position of the vortex wake and the estimation of downwash at chosen tail locations behind triangular-wing and cylindrical-body combinations at supersonic speeds . the effects of such geometric parameters as aspect ratio, angle of attack and incidence, ratio of body radius to wing semi-span, and angle of bank on the vortex wake behind wings of wing-body combinations were studied . the relative importance of wing vortices, the corresponding image vortices within the body, and body crossflow indetermining the the total downwash was assessed at a possible tail location . it was found that the line-vortex method of this report permitted the calculation of vortex paths behind wings of wing-body combinations with reasonable facility and accuracy . a calculated sample wake shape agreed qualitatively with one observed experimentally, and sample results of the line-vortex method compared well with an available exact crossflow-plane solution . an empirical formula was derived to estimate the number of vortices required per wing panel for a satisfactory computation of downwash at tail locations . it was found that the shape of the vortex wake and the ultimate number of rolled-up vortices behind a wing depend on the circulation distribution along the wing trailing edge . for the low-aspect-ratio plane wing and body combinations considered, it appeared that downwash at horizontal tail locations is largely determined except near the tail-body juncture by the wing vortices alone for small ratios of body radius to wing semispan, and by the body upwash alone for large values of that ratio .'), (434, 'contributions of the wing panels to the forces and moments of supersonic wing-body combinations at combined angles .', 'spahr,j.r.', 'naca tn.4146, 1958.', 'contributions of the wing panels to the forces and moments of supersonic wing-body combinations at combined angles . a wind-tunnel investigation was conducted at a mach number of 1.96 and at reynolds numbers (based on the mean aerodynamic chord of the exposed wing) of 0.36 and 1.03 million to determine the normal forces, pitching moments, and rolling moments contributed by each wing panel of a cruciform-wing and body combination over a wide range of combined angles of pitch and roll . the wings were triangular of aspect ratio 2, and the body was an ogive-cylinder combination . the effects of forebody length and roughness and of the presence of the adjacent panels on these panel contributions were determined . the results of the investigation show that large changes in the panel forces and moments can occur as the result of combined angles . a general theoretical method based on slender-body and strip theories was found to yield results in good agreement with the wind-tunnel measurements . these comparisons indicate that the changes in the panel characteristics due to combined angles are caused primarily by a cross coupling between the side-wash velocities due to angle of attack and sideslip and by the presence of forebody vortices due to crossflow separation . it was found that an increase in forebody length increases the effect of the forebody vortices because of the dependence of the strength of these vortices on the forebody length .'), (435, 'application of similar solutions to calculations of laminar heat transfer on bodies with yaw and large pressure gradients in high speed flow .', 'beckwith,i.e. and cohen,n.b.', 'nada tn.d625, 1961.', 'application of similar solutions to calculations of laminar heat transfer on bodies with yaw and large pressure gradients in high speed flow . an integral method for the rapid calculation of heat-transfer distributions on yawed cylinders of arbitrary cross-sectional shape and on bodies of revolution in high-speed flows is developed for laminar boundary layers . the method involves the quadrature of a function of the pressure distribution (assumed given) and satisfies the integral energy equation with the assumption of local similarity, wherein the actual boundary-layer profiles at every station are replaced by corresponding profiles from a family of similar solutions . the method is compared with other local similarity methods and with experimental heat-transfer data on a circular cylinder and on a body of revolution designed for large axial pressure gradients . good agreement between theory and data is obtained and it is shown that the present integral method, in both its complete and simplified form, gives generally better agreement with the data than certain other local similarity methods . numerical examples are presented showing that the effect of sweep and gas properties on heat-transfer distribution is small .'), (436, 'heat transfer in planetary atmospheres at super-satellite speeds .', 'hoshizaki, h.', 'ars prep. 2173-61, august 1961 .', 'heat transfer in planetary atmospheres at super-satellite speeds . the main purpose of this investigation is to examine the dependence of heat transfer in planetary atmospheres on the total enthalpy up to flight velocities of 50,000 ft/sec where a large proportion of the atoms are ionized . the /total thermodynamic and transport property/ concept discussed by hirshfelder /j.chem.phys.,26/2/,feb.,1957/ is used .'), (437, 'hypervelocity stagnation point heat transfer .', 'scala, s. m. and warren, w. r.', 'arsj. jan. 1962 .', 'hypervelocity stagnation point heat transfer . this analysis includes the specific contributions of atoms, molecules, tions are .. /i/ partially ionized air can be approximated as a four-component gas including n2, n, n and e,. /ii/ the gas is in local thermochemical equilibrium,. /iii/ there is no charge separation,. /iv/ thermal diffusion is neglected,. /v/ no electrical or magnetic fields,. low re effects are neglected .'), (438, 'stagnation point heat transfer measurements at super satellite speeds .', 'offenhartz,e., wisblatt,h. and flagg, r.f.', 'j. roy. aero. soc., 66, 1962.', 'stagnation point heat transfer measurements at super satellite speeds . brief description of experiments performed by using shock tube techniques for measurement of the stagnation point heating of a blunt body over a stagnation enthalpy range of 650 to 900, corresponding to velocities between 32,000 ft. per sec. and 39,000 ft per sec., respectively . data thus provided are used for comparison with theory .'), (439, 'a factor affecting transonic leading edge flow separation .', 'wood,g.p. and gooderum,p.b.', 'naca tn.3804, 1956.', 'a factor affecting transonic leading edge flow separation . a change in flow pattern that was observed as the free-stream mach number was increased in the vicinity of 0.8 was described in naca technical note 1211 by lindsey, daley, and humphreys . the flow on the upper surface behind the leading edge of an airfoil at an angle of attack changed abruptly from detached flow with an extensive region of separation to attached supersonic flow terminated by a shock wave . in the present paper, the consequences of shock-wave--boundary-layer interaction are proposed as a factor that may be important in determining the conditions under which the change in flow pattern occurs . when the mach number is high enough, the attached-flow pattern exists because then the shock wave is far enough behind the leading edge to keep the influence of the high pressure behind the shock wave from extending through the boundary layer to the immediate vicinity of the leading edge and affecting the flow there . some experimental evidence in support of the importance of shock-wave--boundary-layer interaction is presented .'), (440, 'compilation of information on the transonic attachment of flows at the leading edge of airfoils .', 'lindsey,w.f. and landrum,e.', 'naca tn.4204, 1958.', 'compilation of information on the transonic attachment of flows at the leading edge of airfoils . schlieren photographs have been compiled of the two-dimensional flow at transonic speeds past 37 airfoils having variously shaped profiles, some of which are related and vary in thickness and camber . the data for these airfoils were analyzed to provide basic information on the flow changes involved and to determine factors affecting transonic-flow attachment, which is a transition from separated to unseparated flow at the leading edges of two-dimensional airfoils at fixed angles of attack as the subsonic mach number is increased .'), (441, 'evaluation of high angle-of-attack aerodynamic derivative data and stall-flutter prediction techniques .', 'halfman,r.l., johnson,h.c. and haley,s.m.', 'naca tn.2533, 1951.', 'evaluation of high angle-of-attack aerodynamic derivative data and stall-flutter prediction techniques . the problem of stall flutter is approached in two ways . first, using the m.i.t.-naca airfoil oscillator, the aerodynamic reactions on wings oscillating harmonically in pitch and translation in the stall range have been measured, evaluated, and correlated where possible with available published data, with the purpose of providing empirical information where no aerodynamic theory exists . the major effects of reynolds number, airfoil shape, and reduced frequency on the aerodynamic reactions have been reaffirmed . no instances of negative damping were observed in pure translatory motion and the ranges of negative damping occurring in pure pitch had the same general trends noted by other experimenters . data on the time-average values in the stall range of both lift and moment are presented for the first time . second, the results of numerous experimental observations of stall flutter have been reviewed and the various known attempts at its prediction have been examined, compared, and extended . the sharp drop in critical speed and change to a predominantly torsional oscillation usually associated with the transition from classical to stall flutter is apparently primarily but not entirely caused by the marked changes in moment due to pitch . fairly good stall-flutter predictions have been reported only when adequate empirical data for this aerodynamic reaction happened to be available for the desired airfoil shape, reynolds number range, and reduced-frequency range . a semiempirical method of predicting the variations of moment in pitch with airfoil shape, reduced frequency, initial angle of attack, and amplitude of oscillation has been presented .'), (442, 'some effects of variations in several parameters including fluid density on the flutter speed of light uniform cantilever wings .', 'woolston,d.c. and castile,g.e.', 'naca tn.2558, 1951.', 'some effects of variations in several parameters including fluid density on the flutter speed of light uniform cantilever wings . an experimental investigation has been made of some effects of variations in several parameters, including fluid density, on the flutter characteristics of light uniform cantilever wings . the assortment of wings tested covered a variety of positions of the elastic axis and center of gravity and values of the aspect ratio of 8, 6, and 4 . the relative-density parameter (where k is representative of the ratio of fluid density to wing mass) was varied over a range of values from 1.2 to nearly 14 . special emphasis has been placed on the lower values . the experimental investigation has been supplemented by an analytical investigation based on the two-dimensional aerodynamic theory for incompressible flow . in a few instances corrections for the effects of finite span have been made . in general, the theoretical results followed the trends indicated by experiment except at very low values of the relative-density parameter . for these low values the analytical considerations employed indicated a freedom from flutter not found experimentally . at higher values of the flutter-speed coefficient is shown to decrease with decreasing values of and to be nearly proportional to the inverse of the square root of the air density .'), (443, 'calculated and measured pressure distributions over the midspan section of the naca 4412 airfoil .', 'pinkerton, r. m.', 'naca r. 563, 1936 .', 'calculated and measured pressure distributions over the midspan section of the naca 4412 airfoil . pressures were simultaneously measured in the variable-density tunnel at 54 orifices distributed over the midspan section of a 5 by 30 inch rectangular model of the n.a.c.a. 4412 airfoil at 17 angles of attack ranging from -dash 20degree to 30degree at a reynolds number of approximately 3,000,000 . accurate data were thus obtained for studying the deviations of the results of potential-flow theory from measured results technique are presented . it is shown that theoretical calculations made either at the effective angle of attack or at a given actual lift do not accurately describe the observed pressure distribution over an airfoil section . there is therefore developed a modified theoretical calculation that agrees reasonably well with the measured results of the tests of the n.a.c.a. 4412 section and that consists of making the calculations and evaluating the circulation by means of the experimentally obtained lift at the effective angle of attack,. i.e., the angle that the chord of the model makes with the direction of the flow in the region of the section under consideration . in the course of the computations the shape parameter is modified, thus leading to a modified or an effective profile shape that differs slightly from the specified shape .'), (444, 'an approach to the flutter problem in real fluids .', 'rott,n. and george,m.b.t.', 'inst. aero. scs. perp.509, 1955.', 'an approach to the flutter problem in real fluids . an approximate theory of airfoils in unsteady motion in a viscous fluid is proposed, in which viscous effects are accounted for by relaxing the kutta condition and replacing it by a relation derived from experiments in steady flow . applications here, are limited to moderate viscous effects below the stall . the possibility of one-degree- of-freedom flutter is discussed under this assumption . the discussion is partly extrapolated to the domain of stall flutter . some possibilities of further development of this theory for the stalled case are indicated .'), (445, 'on the application of mathieu functions in the theory of subsonic compressible flow past oscillating airfoils .', 'reissner,e.', 'naca tn.1961.', 'on the application of mathieu functions in the theory of subsonic compressible flow past oscillating airfoils . an account is given of explicit solutions in terms of mathieu function functions of the problem of two-dimensional subsonic compressible flow past oscillating airfoils . the results are applied to the calculation of three-dimensional corrections for the two-dimensional theory and the effect of the incorporation of the three-dimensional effects on the mathieu function solution of the two-dimensional problem is shown . the developments are formal and must be supplemented by an appreciable amount of numerical calculations before the theory can be applied to specific problems .'), (446, 'wake of a satellite traversing the ionosphere .', 'rand,s.', 'phys. fluids, 3, 1960.', 'wake of a satellite traversing the ionosphere . the particle treatment is applied to a study of the structure of the wake behind a charged body moving supersonically through a low-density plasma . for the case of a body whose dimensions are considerably smaller than a debye length, a solution is obtained which is very similar in structure to the solution obtained by using the linearized fluid dynamics equation . for the case of a disk whose radial dimensions are much larger than a debye length, two conical regions are found in the wake . at the surface of each of these cones, over thicknesses of the order of a debye length, the ion and electron densities are increased over their ambient values . formulae for the electrohydrodynamic drag on a wire, and on a large disk are obtained .'), (447, 'motion of thin bodies in a highly rarefied plasma .', 'yoshira,h.', 'phys.fluids,4, 1961.', 'motion of thin bodies in a highly rarefied plasma . magnetic effects are considered negligible, and the velocity of the body is in a range between the electron and positive ion thermal speeds . the self-consistent field approach is used in which the electron distribution is assumed to be maxwellian, while the positive ion distribution function is given by the /collision-free/ boltzmann equation . it is assumed that the ion reflection at the body surface is specular, and the body is sufficiently thin so that the ion distribution function is a small perturbation of a maxwellian distribution . the solution for the simple case of a dielectric body with a given surface charge, as well as some general properties to be expected for a conducting body are given .'), (448, 'induction drag on a large negatively charged satellite moving in a magnetic-field-free ionosphere .', 'wyatt,p.j.', 'j. geophys. rev. 65, 1960.', 'induction drag on a large negatively charged satellite moving in a magnetic-field-free ionosphere . an induction drag, experienced by a charged satellite during its traversal of the ionosphere, has been theoretically postulated by several authors . previous ''exact'' treatments of the problem are inapplicable to large systems, and the semiempirical approach of jastrow and pearse may yield somewhat questionable results . the present description initially considers the satellite as a completely permeable spherical shell of charge, thus avoiding the difficult boundary conditions introduced by the ''exact'' linearized treatment . the effects of permeability are then shown to be approximately removable by means of an iterative process . a final result, apparently valid to within an order of magnitude, is obtained for the drag force arising solely from electrical effects . its magnitude is considerably less than that obtained by jastrow and pearse .'), (449, 'interaction of a charged satellite with the ionosphere .', 'davis,a.h. and harris,i.', 'nasa tn. d704, 1961.', 'interaction of a charged satellite with the ionosphere . the problem of the ion density distribution around a charged satellite has been treated by a numerical method which does not require linearization of the equations or restriction to infinitesimal objects . however, magnetic field effects were not considered, and a number of other simplifying assumptions were required . some sample calculations for spherical satellites are presented, illustrating the general character of the satellite wake . calculations of the so-called /charge drag/ were also made, yielding results qualitatively similar to those previously obtained by jastrow and pearse .'), (450, 'some physical interpretations of magnetohydrodynamic duct flows .', 'fujihiko sakao', 'university of tokyo, tokyo, japan', 'some physical interpretations of magnetohydrodynamic duct flows . this note presents some physical interpretations of magnetohydrodynamic duct flows with various boundary conditions viewed in the light of the effects of conducting walls on the pattern of electric current, taking examples from published results on rectangular ducts . the current patterns are illustrated in fig. 1 for rectangular ducts having various combinations of conducting and nonconducting walls, a uniform magnetic field being applied in the horizontal direction .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (451, 'liapunov''s methods in automatic control theory .', 'parks, p. c.', 'control, november and december 1962 .', 'liapunov''s methods in automatic control theory . the work of a. m. liapunov and his theory of stability is discussed . the second method of liapunov is shown to have applications for linear equations with real constant coefficients, for a proof of the routh-hurwitz criterion, and linear equations with periodic coefficients . practical examples include non-linear stability problems of control, and the functions have uses in other areas of control systems .'), (452, 'symmetric joukowsky airfoils in shear flow .', 'tsien,h.s.', 'q. app. math. 1, 1943, 130.', 'symmetric joukowsky airfoils in shear flow . the velocity components of the fluid far from the airfoil are given by where c is the chord of the airfoil, and k are constants, u and v are velocity components in the directions of the coordinates x and y . the solution is sought in the form of the stream function and satisfies laplace''s equation . a general expression for for vanishing disturbance velocities at points far from the origin is written, and the flows due to a source, a vortex and a solid circular cylinder in shear flow are considered as examples . typical streamline patterns are shown for these cases . from the eulerian equations of motion the author obtains the expression for in terms of the parameter and derivatives of . the general form of is introduced and the appropriate solution for the pressure p is obtained . by integration around a contour enclosing the body, expressions are obtained, analogous to the blasius formulae, for the force and couple on any cylinder in this type of flow . these formulae are applied to the case of a symmetrical joukowsky airfoil . the method of conformal transformation is employed in the determination of . the boundary condition of tangential flow at the airfoil surface must be satisfied by the total flow in the airfoil plane, but this condition leads to a boundary condition for in the transformed plane . the kutta-joukowsky condition of finite velocity at the trailing edge also leads to a condition on in this plane . from these conditions and the general expression for the circulation and the strengths of the doublets and quadruplets required for the force and moment are determined . hence, the formulae for lift and moment coefficient are obtained . these involve, in addition to the usual (potential-flow) terms, terms proportional to . the ten functions that appear in the expressions for the lift and moment coefficients are tabulated for values of the thickness ratio between 0 and 1 . the aerodynamic-center position and the coefficient of the moment about the aerodynamic center are also calculated and are presented graphically as functions of .'), (453, 'the influence of two-dimensional stream shear on airfoil maximum lift .', '', '', 'the influence of two-dimensional stream shear on airfoil maximum lift . the cornell aeronautical laboratory is conducting a program of theoretical and experimental research on low-speed aerodynamics as applied to stol and vtol aircraft . the objective of this program is to re-examine certain aspects of classical aerodynamic information, in the light of low-speed flight requirements, with the aim of seeking aerodynamic processes which might be exploited to enhance law-speed performance . one aspect of propeller-driven aircraft which has recently received increasing attention is the existence of strong gradients of longitudinal velocity, or shear, in the propeller slipstream . this slipstream shear interacts with a wing surface and can alter the wing characteristics . in theoretical treatments of a wing interacting with a propeller slipstream, the first important simplification is the replacement of the slipstream with an ideal uniform jet, free of all velocity gradients . the application of these theories requires that one equate the actual slipstream to an effective uniform jet . one method employed is to assume the uniform jet has a momentum flux equal to the average in the propeller slipstream . these and similar procedures are well founded on momentum considerations., however, the implicit assumption is that the flow nonuniformity, the shear, does not influence the wing characteristics .'), (454, 'several approximate analyses of the bending of a rectangular cantilever plate by uniform normal pressure .', 'nash,w.a.', 'j.app.mech. 1952, 33.', 'several approximate analyses of the bending of a rectangular cantilever plate by uniform normal pressure . three methods of approximating the deflections and moments occurring in a rectangular cantilever plate subjected to uniform normal pressure over its entire surface are presented in this paper . the first is the application of the well-known finite-difference procedure . the second and third are collocation methods, one based upon polynomial solutions of the lagrange equation, the other employing /mixed/ hyperbolic-trigonometric terms satisfying this equation . in the last two methods the boundary conditions are satisfied exactly along the clamped edge and at a finite number of points along the free edges of the plate . the results obtained for the particular case of a cantilever plate with uniform normal load indicate that the use of a relatively small number of points in the collocation method yields values of deflections and moments that are in substantial agreement with those given by the finite-difference procedure . it cannot be concluded from these results that the collocation method using the assumed functions will give satisfactory results with fewer points than the finite-difference method for cantilever plates with loading different from the one investigated .'), (455, 'modified cross-lees mixing theory for supersonic separated and reattaching flows .', 'glick,h.s.', 'galcit hyp. res. proj. memo 53, 1960.', 'modified cross-lees mixing theory for supersonic separated and reattaching flows . re-examination of the crocco-lees method has shown that the previous quantitative disagreement between theory and experiment in the region of flow up to separation was caused primarily by the improper c(k) relation assumed . a new c(k) correlation, based on low-speed theoretical and experimental data and on supersonic experimental results has been developed and found to be satisfactory for accurate calculation of two-dimensional, laminar, supersonic flows up to separation . a physical model which incorporates the concept of the /dividing/ streamline and the results of experiment . according to this physical model, viscous momentum transport is the essential mechanism in the zone between separation and the beginning of reattachment, while the reattachment process is, on the contrary, an essentially inviscid process . this physical model has been translated into crocco-lees languages using a semiempirical approach, and approximate c(k) and f(k) relations have been determined for the separated and reattaching regions . the results of this analysis have been applied to the problem of shockwave, laminar-boundary-layer interaction, and satisfactory a study of separated and reattaching regions of flow has led to quantitative agreement with experiment has been achieved .'), (456, 'a study of flow fields about some typical blunt-nosed slender bodies .', 'vaglio-laurin,r. and trella,m.', 'pibal r.623, 1960.', 'a study of flow fields about some typical blunt-nosed slender bodies . complete inviscid flow fields about three model axisymmetric configurations have been determined numerically . configurations decreasing bluntness) and flight conditions have been selected so as to indicate separately effects of nose shape, drag coefficient, flight mach number, and thermodynamic behavior of the gas (either ideal calorically perfect gas or air in equilibrium dissociation) . results are presented for thirteen cases . particular attention is devoted to interpretation and, when possible, correlation of pressure distributions on, and shock shapes about, the cylindrical afterbodies . it is found that .. (a) the correlation of pressure distributions on bodies having nonspherical noses involves interpretive modifications of the law suggested by blast wave analogy . also shocks about these bodies are not described by parabolae,. (b) for all configurations there is substantial influence of gas behavior on shock shape,. this, however, can be correlated in terms of the gas conditions along a generally defined streamline,. (c) the shock layer can generally be divided into two regions (the first bound by the body and the aforementioned streamline, the second delimited by this streamline and the shock) wherein flow properties can either be approximated by simple laws or correlated .. (d) for each configuration knowledge of the complete flow field in one flight condition (even pertaining to ideal gas flow) can be used to estimate features of flows under general flight conditions including those where equilibrium dissociation is encountered .'), (457, 'on laminar boundary-layer flow near a position of separation .', 'goldstein,s.', 'q. j. mech.app. mech. 1, 1948, 43.', 'on laminar boundary-layer flow near a position of separation . singularities are considered in the solution of the laminar boundary-layer equations at a position of separation . a singularity of the type here considered occurred in a careful numerical computation by hartree for a linearly decreasing velocity distribution outside the boundary layer ,. it may occur generally . whenever it does occur, the boundary-layer equations cease to be valid at and near separation on the upstream side, and also downstream of separation . the work suggests that singularities may arise in the solution of non-linear parabolic equations due to their non-linearity . the formulae found may help computers of laminar boundary layers, who desire more than a rough solution, to have an end-point at which to aim .'), (458, 'a new series for calculation of steady laminar boundary layer flows .', 'gortler,h.', 'j. math. mech. 6, 1957, 1.', 'a new series for calculation of steady laminar boundary layer flows . a new and general method for solving problems of plane and steady laminar boundary layer flows in incompressible fluids with arbitrary outer pressure distribution is developed . this method is based on the introduction of the dimensionless quantities as new independent spatial variables . ordinates, u(x) the given outer velocity distribution, v the kinematic viscosity .) the solution of the boundary layer problem is then given as a power series in e with coefficient functions depending on n . this series is a formally exact solution of the boundary layer problem . the new series solution has the following qualities .. have the significance only of cartesian coordinates, the influence of wall curvature being neglected in boundary layer theory, the new coordinates are adjusted to the data of the special problem in any case of application . the new variables represent a logical development of former efforts in the field of boundary-layer flow calculation . with other series solutions known for some special cases is that the leading term of the new series satisfies exactly the outer boundary condition at all cross-sections along the wall . therefore, the succeeding terms give corrections only in the inner part of the boundary layer . accordingly, taking also no. 1 into account, the zero order term by itself gives a good approximation for the boundary layer flow .'), (459, 'on the solution of the laminar boundary layer equations .', 'tani,i.', 'j.phys.soc.japan, 4, 1949, 149.', 'on the solution of the laminar boundary layer equations . the theory of the laminar boundary layer offers a means of determining the skin friction under the assumption of a given velocity distribution outside the boundary layer . owing to the mathematical difficulties, however, exact solutions are possible only when the velocity distribution is expressed as a simple function of the distance along the surface . more complicated velocity distributions necessitate recourse to the method of expansion in series or that of step-by-step calculations, but the labor involved is too great for the methods to be of practical use . approximate method due to pohlhausen (1921), which had long been recommended for general use, gives a reasonably accurate solution in a region of accelerated flow, but recently its adequacy in a region of retarded flow has been questioned . separation of flow may actually occur where the solution of pohlhausen fails to give it . more recently howarth solution, which gives fairly reasonable results in a region of retarded flow . howarth''s solution essentially consists in solving the boundary layer equations for the particular case in which the velocity u outside the boundary layer decreases linearly with the distance x measured along the surface, and utilizing the solution by replacing the actual distribution of u by a circumscribing polygon of infinitesimal sides . therefore, it is assumed that the velocity distribution at any section depends on the velocity gradient du/dx at that section only, being affected by the conditions upstream only in so far as this affects the momentum thickness 0 . in other words, the velocity distribution across the boundary layer is determined by a parameter .'), (460, 'correlated incompressible and compressible boundary layers .', 'stewartson,k.', 'proc. roy. soc. a, 200, 1949, 84.', 'correlated incompressible and compressible boundary layers . the boundary-layer equations for a compressible fluid are transformed into those for an incompressible fluid, assuming that the boundary is thermally insulating, that the viscosity is proportional to the absolute temperature, and that the prandtl number is unity . various results in the theory of incompressible boundary layers are then taken over into the compressible theory . in particular, the existence of method for retarded flows is applied to determine the point of separation for a uniformly retarded main stream velocity . a comparison with an exact solution is used to show that this method gives a closer approximation than does pohlhausen''s .'), (461, 'approximate methods fore predicting separation properties of laminar boundary layers .', 'curle,n. and skan,s.w.', 'aero. quart. 8, 1957, 257.', 'approximate methods fore predicting separation properties of laminar boundary layers . some new solutions for steady incompressible laminar boundary layer flow, obtained by gortler, have been used to test the accuracy of two methods which are commonly used to predict separation . a modification of stratford''s criterion for separation is given in this paper and is probably the most accurate and the simplest of all methods at present in use . modified numerical functions are also given for thwaites''s method of predicting the main characteristics of the boundary layer over the whole surface, which improve the accuracy of the method .'), (462, 'photo-thermoelasticity .', 'gerard,g and gilbert,a.c.', 'j.app.mech. 24, 1957.', 'photo-thermoelasticity . this paper summarizes the optical and physical properties of the photoelastic model material paraplex p-43 over the temperature range from room temperature to -40 f . descriptions are presented of techniques and equipment developed to obtain the modulus of elasticity, the material fringe value, and the thermal-expansion coefficient as a function of temperature . experimental investigations were conducted into the plane-stress problems of a disk contracting upon an elastic inclusion and the transient thermal-stress field produced by a temperature differential suddenly applied to the upper edge of a long beam . the data are correlated with theory using the material properties obtained in the calibration phase . also included are photographic results of an exploratory investigation of the thermal-shock phenomenon produced by the sudden application of a temperature differential upon plastic beams of various length-depth ratios .'), (463, 'physical properties of plastics for photo-thermoelastic investigation .', 'tramposch,h. and gerard,g.', 'j.app. mech. 25, 1958.', 'physical properties of plastics for photo-thermoelastic investigation . the optical and physical properties of paraplex p43, castolite, and epoxy resin hysol 6000-op, which are potentially of interest in photothermoelastic investigations, were investigated over a temperature range from +100 to -60 f . results on the thermal-expansion coefficient, the material fringe value, and the modulus of elasticity as functions of temperature are presented . also evaluated were thermal properties of importance in heat conduction . photothermoelastic figures of merit, which rate the optical sensitivity of materials in photothermoelastic applications, as well as a new method to determine this figure in a relative manner are presented .'), (464, 'flow studies on flat plate delta wings at supersonic speeds .', 'michael,w.h.', 'naca tn.3472, 1955.', 'flow studies on flat plate delta wings at supersonic speeds . an experimental study has been made to investigate some aspects of the nature of the flow around delta wings . vapor-screen, pressure-distribution, and ink-flow studies were made at a mach number of 1.9 on a series of semispan delta-wing models with slender wedge airfoil sections and very sharp leading edges . the models had semiapex angles ranging from 5 to 31.75 . separated regions of vorticity existed along the chords of all the wings in the series tested . concentrated vortex cores were found only on wings of very small semiapex angles . for wings with medium and large semiapex angles, the separated vorticity was concentrated in a region extending over the outboard part of the span and lying close to the wing upper surface . the results show that theoretical aerodynamic calculations, such as those in naca tn 3430, utilizing a single, separated vortex pair above the wing upper surface to represent the separated vorticity can be applied at supersonic speeds for very slender wings .'), (465, 'slender delta wings with sharp edges at zero lift .', 'weber,j.', 'rae tn.aero.2508.', 'slender delta wings with sharp edges at zero lift . several slender wings of delta planform with sharp edges have been investigated theoretically at zero lift at subsonic and at supersonic speeds . most of the wings have diamond-shaped cross sections and are intended to lead to a type of flow with leading-edge separation in the lifting condition . the pressure distributions and overall normal-pressure drags resulting from various theoretical methods are compared with one another and some discussion is included concerning the possibility of achieving the results, calculated for an inviscid stream, in a real flow in the presence of a viscous layer around the body .'), (466, 'development of the vapour screen method of flow visualization in the 3ft tunnel at rae bedford.', 'mcgregor,i.', '', 'development of the vapour screen method of flow visualization in the 3ft tunnel at rae bedford. the vapour screen method of flow visualisation in supersonic wind tunnels is outlined, and the development of a suitable technique for use in the 3 ft tunnel described, together with the associated optical and photographic equipment . the results of tests to determine the humidity required to produce an optimum density of fog in the working section over the mach number range temperature discussed . numerous vapour screen photographs of the flow over and behind delta wings are included and some comparisons made with the corresponding surface oil-flow patterns . the process of condensation, the physical and optical properties of the resulting fog, and the formation of the vapour screen picture are all considered in some detail . the effects of humidity on the mach number and static pressure in the working section were investigated and the results are compared with theoretical estimates at a nominal mach number of 2.0 . it is shown that the adverse effects of condensation on the flow at high mach numbers may be alleviated by the use of liquids with a lower latent heat of evaporation than water, and some results obtained at a mach number of the possibility of extending the vapour screen technique to transonic and subsonic speeds is also considered, and some results obtained at a mach number of 0.85 are included .'), (467, 'thin airfoil theory based on approximate solution of the transonic flow equation .', 'spreiter, j. r. and alksne, a. y.', 'naca tn 3970, may, 1957 .', 'thin airfoil theory based on approximate solution of the transonic flow equation . the present paper describes a method for the approximate solution of the nonlinear equations of transonic small disturbance theory . although the solutions are nonlinear, the analysis is sufficiently simple that results are obtained in closed analytic form for a large and significant class of nonlifting airfoils . application to two-dimensional flows with free-stream mach number near 1 leads, for instance, to general expressions for the determination of the pressure distribution on an airfoil of specified geometry and for the shape of an airfoil having a prescribed pressure distribution and gives, furthermore, the correct variation of pressure with mach number at mach number 1 . for flows that are subsonic everywhere, the method yields a pressure-correction formula that is more accurate than the prandtl-glauert rule and compares favorably with existing higher approximations . for flows that are supersonic everywhere, the method yields the equivalent, in transonic approximation, of simple wave theory . results obtained by application of these general expressions are shown to correspond closely to existing solutions and to experimental data for a wide variety of airfoils .'), (468, 'a refinement of the linearised transonic flow theory .', 'hosowaka,a.', 'j.phys.soc. japan, 15, 1960.', 'a refinement of the linearised transonic flow theory . a new method is proposed to calculate the velocity and pressure distributions around a thin symmetrical aerofoil or a slender body of revolution flying at transonic speed . it is essentially a refinement of the linearized transonic flow theory due to oswatitsch and maeder, such that a correction term is introduced to take account of the nonlinear character of the transonic flow . as examples of application, a symmetrical circular-arc aerofoil and a circular-arc body of revolution in the sonic flow are dealt with, and the results are found to be in good agreement with experiments, except for the rear portion in the latter case .'), (469, 'linearised transonic flow about slender bodies at zero angle of attack .', 'maeder,p.f. and thommen,h.u.', 'asme trans. j.app.mech. 28, 1961.', 'linearised transonic flow about slender bodies at zero angle of attack . the simple linearized transonic flow theory as originally proposed by oswatitsch and keune(1) and by the present authors (2) is improved by considering and partially correcting its error . in this manner a theory which is easy to apply and which should be valid for a great number of smooth bodies is obtained . this improved theory predicts shock waves in the lower transonic regions . it is applied to a number of significant body and airfoil shapes and its predictions are compared with experiments and results of other theoretical investigations .'), (470, 'some notes for the small disturbance linear theory of the method of local linearisation of the flow over an airfoil at mach number of unity .', 'miyai,y.', 'proc. 10th japan nat. cong. app. mech. iii-4, 1960, 207.', 'some notes for the small disturbance linear theory of the method of local linearisation of the flow over an airfoil at mach number of unity . in this paper, the pressure distribution at the surface of a symmetrical non-lifting aerofoil with free stream mach number of unity has been investigated by means of the small-disturbance linear theory or the method of local linearization . and by comparing with the calculated results based on an hodograph method, the accuracy of these approximate methods has been evaluated . moreover, when these approximate methods are used for the calculation of the pressure coefficient, some notes necessary to obtain more correct results have been discussed .'), (471, '', '', '', ''), (472, 'waves in supersonic flow .', '', '', 'waves in supersonic flow . in this chapter we shall mainly consider problems of steady, two- dimensional (plane) supersonic flow . using the fact that in this case there is a steady wave system, we shall find solutions by an indirect approach . that is, we shall first study the conditions under which simple stationary waves may exist in the flow, and then find the flow boundaries to which they correspond or which may be fitted to them . in this procedure the limited upstream influence in a supersonic field is very helpful, for it allows flows to be analyzed or constructed step by step, which is a method that is not possible in the subsonic case .'), (473, 'freeman method .', '', '', 'freeman method . the freeman method (ref. 26) is similar to chester''s method in that the newtonian-plus-centrifugal solution (eq. with the von mises transformation . a method of successive approximations is applied to both plane and axially symmetric blunt-nosed bodies for small and infinite free-stream mach number . formulas for the streamlines, shock shape, and pressure distribution are determined to this approximation . a number of special shapes are treated in ref. 26, and in certain cases the theory has a singular point where the first approximation to the pressure vanishes., that is, for a sphere (see eq. 7-113) . as in chester''s method, the theory is not applicable where the pressure becomes too small .'), (474, 'laminar mixing of a compressible fluid .', 'dean r. chapman', '', 'laminar mixing of a compressible fluid . a theoretical investigation of the velocity profiles for laminar mixing of a high-velocity stream with a region of fluid at rest has been made assuming that the prandtl number is unity . a method which involves only quadratures is presented for calculating the velocity profile in the mixing layer for an arbitrary value of the free-stream mach number . detailed velocity profiles have been calculated for free-stream mach numbers of 0, 1, 2, 3, and 5 . for each mach number, velocity profiles are presented for both a linear and a 0.76-power variation of viscosity with absolute temperature . the calculations for a linear variation are much simpler than those for a 0.76-power variation . it is shown that by selecting the constant of proportionality in the linear approximation such that it gives the correct value for the viscosity in the high-temperature part of the mixing layer, the resulting velocity profiles are in excellent agreement with those calculated by a 0.76-power variation .'), (475, 'the velocity distribution in the laminar boundary layer between parallel streams .', 'lock,r.c.', 'q. j. mech. app. math. 4, 1951, 42.', 'the velocity distribution in the laminar boundary layer between parallel streams . a method is given for obtaining the solution of the laminar boundary layer equations for the steady flow of a stream of viscous incompressible fluid over a parallel stream of different density and viscosity . an approximate solution is also obtained by means of the momentum equation . it is shown that the solutions depend only on the ratio of the velocities of the two streams and on the product of the corresponding density and viscosity ratios . numerical results are given, in the case where the lower fluid is at rest, for four values of and also when for one non-zero value of the velocity ratio .'), (476, 'the blasius equation with three-point boundary conditions .', 'napolitano, i. g.', 'quart. appl. math. v. 16, no. 4, pp 397-408, 1958 .', 'the blasius equation with three-point boundary conditions . the blasius equation subject to three-point boundary conditions, describing the interaction between two parallel streams, is solved by way of a series in terms of ascending powers of the ratio equals /u1 -dash u2//u1, where the u1''s are the outer streams'' velocities . the first three terms of the series are analytically expressed in terms of the repeated integrals of the complementary error function /im erfc / and of the repeated integrals of the square of the successive integrals of the complementary error function /jmin erfc n/ . these functions often appear in problems leading to extended heat-conduction type of equations . a recurrence formula for jmin erfc n is established and formulae relating the functions in erfc /-dashn/ and jmjn erfc to available tabulated values of the functions in erfc /n/ are derived . the first three approximations to the blasius function and to its first two derivatives are also presented in tabulated form with four significant figures . test on the convergence of the series has been made by comparison with some exact solutions obtained by high speed computing machine . the comparison, extended to the physically essential quantities, shows that .. second and first derivatives . yield extremely accurate results . the errors in the first two derivatives of the blasius functions are always contained within less than one per cent .'), (477, 'laminar boundary layers at the interface of co-current parallel streams .', 'potter,o.e.', 'q. j. mech. app. math. 10, 1957, 302.', 'laminar boundary layers at the interface of co-current parallel streams . the approximate solution of keulegan(1) for the steady flow of a stream of viscous incompressible fluid over another at rest is extended to the case where both fluids are moving co-current but at different velocities . this solution utilizes a sextic polynomial for the velocity distribution in the boundary layers . the solutions depend only on the ratio of the velocities of the two streams and on the product of the corresponding viscosity and density ratios . numerical results are given for seven values of at one value of . lock(2) has published an exact solution with a numerical result for and the sextic polynomial solution is evaluated f40umerical result for and the sextic indicates that in general the sextic polynomial is more accurate than the quartic polynomial but that the advantage is not great .'), (478, 'tabulation of the blasius function with blowing and suction .', 'emmons,h.w. and leigh,d.c.', 'arc cp.1913, 1953.', 'tabulation of the blasius function with blowing and suction . authors tabulate solutions of f'''''' + ff'''' = 0 for the velocity distribution in a boundary layer . for each solution f''(0) = 0, the third boundary condition is the specification of f(0) . f(n) and its first three derivatives are tabulated to 5d in gaps of 0.1 in n for f(0) = -1.23849, -1.2(0.05) 0.5 (0.1) 1.5, introduction gives method of solution and physical meaning of boundary conditions, etc . lock''s (amr cussed .'), (479, 'on an equation occurring in falkner and skan''s approximate treatment of the equation of the boundary layer .', 'hartree,d.r.', 'proc. cam. phil. s. 33, 1937, 223.', 'on an equation occurring in falkner and skan''s approximate treatment of the equation of the boundary layer . the differential analyser has been used to evaluate solutions of the equation y'''''' = -yy'''' + with boundary conditions y = y'' = 0 at x = 0, as which occurs in falkner and skan''s approximate treatment of the laminar boundary layer (see abstract 1081 (1932)) . a numerical iterative method has been used to improve the accuracy of the solutions, and the results show that the accuracy of the machine solutions is about insufficient to specify a unique solution for negative values of,. a discussion of this situation is given, and it is shown that for the application to be made of the solution the appropriate condition is that from below, and as rapidly as possible, as . the condition that from below can be satisfied only for values of greater than a limiting value whose value is approximately -0.199, and which is related to the point at which the laminar boundary layer breaks away from the boundary .'), (480, 'adiabatic wall temperature due to mass transfer cooling with a combustible gas .', 'd. b. spalding', 'imperial college, london, england', 'adiabatic wall temperature due to mass transfer cooling with a combustible gas . a recent technical note by sutton (1), with the above title, discusses the influence of the burning of a transpiration coolant on the quantity of coolant necessary to maintain a given wall temperature . the present note discusses the same problem in a way which has been found useful in calculating the burning rates of solid and liquid fuels (2) . consider the transpiration cooling of a porous surface in a gas stream . then a simple modification of the general mass .'), (481, 'mass transfer cooling of a laminary boundary layer by injection of a light weight foreign gas .', 'eckert,e., schneider,p., hayday,a. and larson,r.', 'jet prop. 1958, 34.', 'mass transfer cooling of a laminary boundary layer by injection of a light weight foreign gas . analytical predictions are given for the development of the velocity, temperature and concentration fields in a laminar air boundary layer on a flat plate in high-speed dissipative flow, the plate being considered porous and cooled by injection of hydrogen from its surface . the admixture of hydrogen, having a low density and high thermal capacity relative to air, is shown to greatly diminish the skin friction and to markedly relieve the adverse thermal effects of intense aerodynamic heating under conditions of hypersonic flow .'), (482, 'a re-examination of the use of the simple concepts for prediction the shape and location of detached shock waves .', 'love,e.s.', 'naca tn.4170, 1957.', 'a re-examination of the use of the simple concepts for prediction the shape and location of detached shock waves . a reexamination has been made of the use of simple concepts for predicting the shape and location of detached shock waves . the results show that simple concepts and modifications of existing methods can yield good predictions for many nose shapes and for a wide range of mach numbers .'), (483, 'stagnation point shock detachment distance for flow around spheres and cylinder .', 'ambrosio,a. and wortman,a.', 'ars j. 32, 1962.', 'stagnation point shock detachment distance for flow around spheres and cylinder . development of an analytical relation between shock detachment distance and free-stream mach numbers . results are presented graphically for shock detachment distance of cylinders and spheres in air .'), (484, 'the influence of two-dimensional stream shear for airfoil maximum lift .', 'vidal,r.j.', 'j. ae. scs. 29, 1962, 889.', 'the influence of two-dimensional stream shear for airfoil maximum lift . the effects of stream velocity gradients on airfoil maximum lift are defined with experimental data obtained in a simulated two-dimensional slipstream . the experimental results show that when positioned near the slipstream plane of symmetry, the airfoil maximum lift varies markedly with location in the slipstream . in moving the airfoil from above to below the slipstream plane of symmetry through a total distance corresponding to the airfoil thickness, force data and boundary-layer observations show that boundary-layer separation is delayed to higher angles of attack, and the airfoil maximum lift is doubled . it is concluded that the destalling effect observed in the non-uniform slipstream is not associated with slipstream boundary interference, but stems from the influence of the large local slipstream shear on airfoil characteristics . the effects of uniform and nonuniform shear on airfoil lift and pressure distribution are discussed, within the framework of existing first-order, small-shear theory, to show that these effects of shear tend to promote stall . a pohlhausen calculation of the laminar boundary layer in a stream with shear is used to identify and to assess the effects of stream shear on boundary-layer separation criteria . it is demonstrated that these effects are negligibly small, and that the uniform-flow criterion applies . it is concluded on the basis of the experimental data that the observed destalling phenomenon stems from a shear effect of higher order than those treated in the inviscid theories . it is hypothesized that it is a second-order effect, fixed by the product of the stream shear and the derivative of the shear, which was large in the present experiments .'), (485, 'linear heat flow in a composite slab .', 'reid,w.p.', 'j.ae.scs. 29, 1962.', 'linear heat flow in a composite slab . the temperature is determined as a function of position and time in the case of linear heat conduction in a composite slab of ture throughout, and the two external surface temperatures are considered to be prescribed functions .'), (486, 'similarity laws for aerothermoelastic testing .', 'dugundji,j.', 'j.ae.scs. 29, 1962, 935.', 'similarity laws for aerothermoelastic testing . the similarity laws for aerothermoelastic testing are presented in the range . these are obtained by making nondimensional the appropriate governing equations of the individual external aerodynamic flow, heat conduction to the interior, and stress-deflection problems which make up the combined aerothermoelastic problem . for the general aerothermoelastic model, where the model is placed in a high-stagnation-temperature wind tunnel, similitude is shown to be very difficult to achieve for a scale ratio other than unity . the primary conflict occurs between the free-stream mach number reynolds number aeroelastic parameter heat conduction parameter and thermal expansion parameter . means of dealing with this basic conflict are presented . these include (1) looking at more specialized situations, such as the behavior of wing structures and of thin solid plate lifting surfaces, and panel flutter, where the aerothermoelastic similarity parameters assume less restrictive forms, (2) the use of /incomplete aerothermoelastic/ testing in which the pressure and/or heating rates are estimated in advance and applied artificially to the model, and (3) the use of /restricted purpose/ models investigating separately one or another facet of the complete aerothermoelastic problem . some numerical examples of modeling for the general aerothermoelastic case as well as for the specialized situations mentioned in (1) above are given . finally, extension of the aerothermoelastic similarity laws to higher speeds and temperatures is discussed .'), (487, 'theory for supersonic two-dimensional, laminar, base-type flows using the crocco-lees mixing concepts .', 'rom,j.', 'j.ae.scs. 29, 1962, 963.', 'theory for supersonic two-dimensional, laminar, base-type flows using the crocco-lees mixing concepts . a separated flow field, in which the incoming boundary layer is undisturbed up to the separation point, is defined as a / base-type/ flow . examples are the flows over a blunt base and over a backward-facing step . the crocco-lees theory is applied to the supersonic, two-dimensional, laminar, base-type flows defined above . the separated flow is divided into a mixing region and a recompression (or reattachment) region . calculations of base pressure show its dependence on the mach number and on two reynolds-number-dependent variables, and . it is shown that existing base-pressure data can be explained by these results .'), (488, 'a reaction-rate parameter for gasdynamics of a chemically reacting gas mixture .', 'leonard,m.', 'j.ae.scs. 29, 1962, 995.', 'a reaction-rate parameter for gasdynamics of a chemically reacting gas mixture . presented note proposes a linearized reaction rate parameter which is applicable to any reacting gas mixture provided all the pertinent reactions and their rate constants are known at the thermodynamic conditions under consideration . linearizing is achieved by expanding equation of rate of chemical reaction in a taylor series and neglecting higher-order terms . author announces that tables of linearized reaction rate parameters for dissociated and slightly ionized air are now in preparation at the space sciences laboratory, general electric co., msvo . comparison of preliminary results with exact calculations published by hall, i. g., et.al., /inviscid hypersonic air-flows with coupled non-equilibrium processes/ (ias paper 62-67, 30th annual meeting, new york, jan. 1962) indicates good agreement .'), (489, 'on calculation of the laminar separation point and results of certain flows .', 'morduchow,m.', 'j.ae.scs. 29, 1962, 996.', 'on calculation of the laminar separation point and results of certain flows . paper studies compressible laminar boundary layer in adverse pressure gradient . after mentioning mathematical instabilities in howarth''s and like solutions, authors quote equation from one of the references, based on the assumptions that zero heat transfer and y = 1.4 . thence authors compute nondimensional distances to separation, comparing with solutions by other workers . results are interesting, though reviewer feels rather unhappy about approximations leading to eq. (4),. more detailed justifications should have been given . thus we have the statement ber, as ./ surely a fuller discussion of effects of letting is warranted . typography in eqs. (2) and (3) is rather confusing and there is a typographical error in heading to table 2 .'), (490, 'normal-shock relations in magnetohydrodynamics .', 'gundersen,r.m.', 'j.ae.scs.29, 1962, 997.', 'normal-shock relations in magnetohydrodynamics . the magnetic-field vector is perpendicular to the flow direction,. thus for normal shocks there is no change of flow direction through the shock front . this class of shocks is included in investigations by several authors (five are referred to here), but the presentation here is thought to be especially convenient . all downstream quantities are given in terms of upstream flow conditions, including the upstream ratio of alfven speed to sound speed, and the shock strength (density ratio) .'), (491, 'on the close relationship between turbulent plane-couette and pressure flows .', 'burton,r.a.', 'j. ae. scs. 29, 1962, 1004.', 'on the close relationship between turbulent plane-couette and pressure flows . author studies the velocity profiles measured by others in plane and turbulent couette flow, such as is induced in parallel channels of which one of the walls moves in its own plane . he finds these profiles to be satisfactorily describable in terms of the seventh-power law, which was originally set up for plane and turbulent pressure flow in channels where both walls are stationary . further, he finds the shear law for pressure flow, to be applicable also to the couette flow, in a similar range of reynolds number, r . no attempt is made in this concise contribution to put these findings on a firmer basis through a theoretical explanation .'), (492, 'prediction of ogive-forebody pressures at angles of attack .', 'earl r. keener', 'aerodynamicist, nasa flight research center, edwards, calif.', 'prediction of ogive-forebody pressures at angles of attack . various approximations are being suggested for obtaining surface pressures on arbitrary bodies at angle of attack . this not presents a method for obtaining an approximate pressure distribution over the lower surface of an ogive forebody at angle of attack by utilizing the calculated pressures for zero angle of attack .'), (493, 'real-gas laminar boundary layer skin friction and heat transfer .', 'wilson,r.e.', 'j. ae. scs. 29, 1962, 640.', 'real-gas laminar boundary layer skin friction and heat transfer . the laminar-boundary-layer equations have been integrated for the case of a flat plate over a wide range of free-stream enthalpies and velocities and over a wide range of enthalpies of the gas at the wall . the range of free-stream velocities extended up to 25,000 ft sec at low free-stream enthalpies, corresponding to local conditions on a slender body traveling at high speeds . at low free-stream velocities, the range of free-stream enthalpies extended up to 400,000 btu slug, corresponding to the local conditions on a blunt body traveling at speeds up to 25,000 ft sec . the gas was assumed to be in thermodynamic equilibrium at each point in the boundary layer and diffusion effects were neglected . the solutions to the boundary-layer equations were carried out on a high-speed digital computing machine, both skin-friction and heat-transfer coefficients being obtained from the computations . before presenting the results, the t'' method of rubesin and johnson for computing skin-friction coefficients for the perfect-gas case is reviewed . for the real-gas case, the average temperature, t'', is replaced by the average enthalpy, h'', and the h'' method is then used to compute skin-friction coefficients . these values are in excellent agreement with the computing-machine results . it was found that the recovery factor for the real-gas case can be approximated by, the best results for the cases considered being obtained if a value of pr corresponding to the enthalpy, h'', is used . using this recovery factor and reynolds analogy, heat-transfer rates can be computed which, with a few exceptions, are within 5 percent of values obtained from computing-machine results .'), (494, 'axisymmetric viscous flow plast very slender bodies of revolution .', 'yashura,m.', 'j. ae. scs. 29, 1962, 667.', 'axisymmetric viscous flow plast very slender bodies of revolution . axisymmetric viscous flow past unyawed very slender bodies of revolution is treated within the category of the perfect gas . attention is paid especially to the effect of transverse curvature of the body . from the transformed equations, the similarity conditions are deduced, and the parameter characterizing the effect of transverse curvature is obtained . several numerical solutions of similarity equations for hypersonic flows are presented, and upon the basis of these results, the effect of the transverse-curvature parameter is discussed . a method of applying the local-similarity approximation to obtain the approximate solution for nonsimilar cases is described, as are practical applications to incompressible flow past a long cylinder and to hypersonic flow past a very slender cone . comparison with experimental results shows fair agreement with calculations using the local-similarity approximation in the present range of experimental flow conditions .'), (495, 'on similar solutions for strong blast waves and their application to steady hypersonic flow .', 'borcher,e.f.', 'j. ae. scs. 29, 1962, 694.', 'on similar solutions for strong blast waves and their application to steady hypersonic flow . the general solution of the strong blast wave is found in the newtonian approximation--i.e., neglecting terms of order the expressions obtained for the pressure, temperature, density, and velocity profiles are simple . the results are applied to power-law bodies in hypersonic flow using the equivalence principle . higher-order approximations for strong blast waves are investigated for the cases in which the shock layer is thin . a simple pressure formula is found, which constitutes an improvement upon the newton-busemann formula, and some of its applications are shown .'), (496, 'a theory of transonic aileron buzz, neglecting viscous effects .', 'eckhaus,w.', 'j. ae. scs. 29, 1962, 712.', 'a theory of transonic aileron buzz, neglecting viscous effects . usaf-sponsored analysis of the unsteady perturbations of two-dimensional transonic flow around an airfoil, where local supersonic regions terminated by shock waves are present in the vicinity of the airfoil . viscous effects are neglected, and a linearized theory of the perturbations due to harmonic oscillations of an aileron is developed . a series solution for the pressure distribution is obtained, and numerical results for the nonsteady hinge moment, from the first approximation to the solution, are presented . as a result of flutter analysis a stability boundary for transonic aileron buzz is obtained . comparison of the theoretical results with experimental observations shows satisfactory agreement .'), (497, 'theoretical and experimental investigation of thermal stresses in hypersonic aircraft wing structures .', 'tramposch,h.', 'j. ae. scs. 29, 1962.', 'theoretical and experimental investigation of thermal stresses in hypersonic aircraft wing structures . a simple and relatively accurate analytic approximation is developed to determine the temperature and thermal-stress distribution in aircraft wing structures . theoretical investigations show that the results of the existing thermal-stress theories which neglect the temperature gradient through the skin thickness may exceed, in the range of higher biot numbers, the true values by more than 30 percent . refined photothermoelastic experiments verify these results and add another significant conclusion . they indicate that thermal stresses in wing structures generated by a variable heat-transfer coefficient coincide with the theoretical predictions which are based on a constant heat-transfer coefficient, as long as the latter represents the arithmetic average over the heating cycle and the variation is in the order of 10 percent . however, even much greater variations in the order of 100 percent produce only relatively small differences .'), (498, 'calculation of potential flow about bodies of revolution having axes perpendicular to the free-stream direction .', 'hess,j.l.', 'j. ae. scs. 29, 1962.', 'calculation of potential flow about bodies of revolution having axes perpendicular to the free-stream direction . a general method is described for calculating, with the aid of an electronic computer, the potential flow about arbitrary bodies of revolution whose axes are perpendicular to the free-stream direction . when combined with the solution for the axisymmetric flow about these bodies, this method makes it possible to calculate the pressure distribution on any body of revolution at angle of attack forward of any separated region of the flow, and also to calculate the flow at points off the body surface . after the basic equations of the method have been derived, its accuracy is exhibited by comparison with analytic solutions for ellipsoids of revolution . calculated pressure distributions are then compared with experimental data for a variety of bodies . the agreement is quite satisfactory in all cases . the calculated velocities for other selected bodies are presented to exhibit certain properties of this type of flow .'), (499, 'a closed-form solution for the oscillations of a vehicle entering a planetary atmosphere .', 'greensite,a.l.', 'j. ae. scs. 29, 1962, 745.', 'a closed-form solution for the oscillations of a vehicle entering a planetary atmosphere . author considers the equation of the yawing motion of a missile, derived with a series of customary assumptions and with the distance traveled as the independent variable . his assumptions include the linearity of the aerodynamic forces, the constancy of the aerodynamic coefficients with respect to mach number, the absence of spin, and the absence of gravity . if to these assumptions one could add the common ballistic assumption of a constant air density, the coefficients of this equation would have been con-damped sinusoids . in ballistics any slow variation of these coefstant, and the solution would have been simply the exponentially-ficients is usually treated by adding an approximate correction term to the damping rate (which is spoken of as the wkb perturbation) . however, with a body entering the planetary atmosphere the variation of the air density is apparently of greater essence (this is a point not stated explicitly in this brief communication), and the equation is of the type . the author shows that with a series of further transformations the equation can be reduced to the form the solutions of which are confluent hypergeometric functions . these functions are defined as series involving gamma functions, and with a series of further assumptions can be reduced to laguerre polynomials and bessel functions . it is certainly nice to have an exact solution to a problem which has heretofore been extensively treated by approximations and by the numerical approach . this reviewer is puzzled, however, as to the practical significance of the proposed approach . an idealization is of value in that it facilitates our understanding,. and the numerical approach, in that it allows refinements of the problem, freeing us from the necessity of idealizing . but the proposed solution is certainly more difficult to refine than the original problem,. and it is certainly not simple (the solution of the original equation is not the value of z, but the various /reverse/ transformations of z) . an evaluation of a series in practice must compete with the numerical approach,. and the equation suggested is of the zero) . viewing the problem /afresh/ (in the light of the / computer revolution/ and without the constraints imposed by the prior art), it seems at least equally easy to /standardize/ the solutions of the original equation .'), (500, 'joule heating in magnetohydrodynamic free-convection flows .', 'cramer,k.r.', 'j.ae.scs., 29, 1962, 746.', 'joule heating in magnetohydrodynamic free-convection flows . the steady, fully developed, laminar, free-convection flow of an electrically conducting fluid between two fully submerged open-ended, constant-temperature vertical plates located in a constant, uniformly distributed, transverse magnetic field has been analyzed with the joule heating term retained in the energy equation . analytic results are obtained . such analytic results are useful in estimating the actual magnitude of the influence of joule heating as well as a qualitative description of the manner in which it alters the temperature and flow fields . the present result confirms the usual practice that the influence of joule heating is negligibly small .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (501, 'stagnation-point shock detachment of blunt bodies in supersonic flow .', 'ridyard,h.w.', 'j. ae. scs. 29, 1962, 751.', 'stagnation-point shock detachment of blunt bodies in supersonic flow . presentation of stagnation-point shock-detachment distances determined by the exact numerical method of gravalos, edelfelt, and emmons . the results are compared with those from the previously published methods of van dyke and gordon, li and geiger, and serbin, and with experimental data .'), (502, 'on squire''s test of the compressibility transformation .', 'mager,a.', 'j. ae. scs. 29, 1962,752.', 'on squire''s test of the compressibility transformation . discussion of a previous application, by squire, of the author''s compressibility transformation to the correlation of high-speed boundary-layer data for air and helium . squire''s suggestion that the compressibility transformation is invalid is shown to be incorrect .'), (503, 'theoretical prediction of the transonic characteristics of airfoils .', 'sinnott,c.s.', 'j. ae. scs. 29, 1962, 275.', 'theoretical prediction of the transonic characteristics of airfoils . it is shown that the author''s transonic-flow airfoil theory can be used to estimate transonic drag-rise and onset-of- separation-effects mach numbers without reference to experimental results . a simple comparative method is applied to a series of airfoils, and the results are analyzed to determine some of the design features of importance in transonic flow . an improvement to this scheme is shown to give results in good agreement with experiment for both the first appearance of shock waves and the onset of separation effects . application to finite swept wings is briefly considered and illustrated .'), (504, 'stability of compressible boundary layers induced by a moving wave .', 'ostrach,s.', 'j. ae. scs. 29, 1962, 289.', 'stability of compressible boundary layers induced by a moving wave . the problem of determining the stability of compressible viscous flows with nonzero surface velocities is formulated and is shown to be identical to that for conventional boundary layers, with only a redefinition of the mach and reynolds numbers required . specific consideration is given to the wall boundary layer behind a moving shock wave, and the minimum critical reynolds numbers are obtained for various shock velocities . the entire stability map is determined for the limiting case of a weak wave, which is analogous to the rayleigh problem . the minimum critical reynolds number is found to increase monotonically with shock velocity--i.e., with increasing surface cooling and stream mach number combined . for the ratio of wall to stream velocity of 2.92 with (shock mach number of 2.18) the flow is found to be infinitely stable to two-dimensional disturbances . experimental transition data do not follow the trends predicted by the theory . in fact, the transition reynolds numbers are orders of magnitude below the computed minimum critical reynolds numbers . the lack of correlation between theory and experiment is attributed to disturbances which are external to the boundary layer .'), (505, 'transition measurements on cones in free flight ballistics range tests .', 'lyons,w.c.', 'j. ae. scs. 29, 1962, 352.', 'transition measurements on cones in free flight ballistics range tests . navy-sponsored experimental investigation of the location of boundary-layer transition on sharp-nosed cones having 10 total angles . the ambient temperature in a portion of the aeroballistics range is varied so as to obtain different adiabatic recovery temperatures at a constant nominal mach number of 3.1 . the location of transition is expressed as a transition reynolds number, and results are presented graphically as a function of the ratio between the wall temperature and the adiabatic recovery temperature .'), (506, 'a note on havelock''s shallow-water wave-resistance curves .', 'brandmaier,h.e.', 'j. ae. scs. 29, 1962, 257.', 'a note on havelock''s shallow-water wave-resistance curves . in the continuous quest for improved means of transportation, attention is currently focused on the ground-effect machine . as there is no physical contact between the vehicle and the terrain over which it operates, its performance should be similar over land and water . however, over water there is an additional resistance to motion due to the gravity-wave system generated by the supporting or /cushion/ pressure acting on the water surface . estimates of this component can be made using the analysis of t. h. havelock . it is the purpose of this note to present an ibm 650 digital-computer solution of his equations . as shown below, these results differ from havelock''s original results .'), (507, 'energy equation approximations in fluid mechanics .', 'goldstein,a.w.', 'j. ae. scs. 29, 1962,358', 'energy equation approximations in fluid mechanics . discussion of several forms of the energy equation and of their use for the study of the flow of nearly incompressible fluids .'), (508, 'a correlation of nose-bluntness induced pressures on cylindrical and conical after-bodies at hypersonic speeds .', 'greenberg,r.a.', 'j. ae. scs. 29, 1962, 359.', 'a correlation of nose-bluntness induced pressures on cylindrical and conical after-bodies at hypersonic speeds . van hise, in his detailed study of the nose-bluntness-induced pressures on cylindrical afterbodies, shows that, starting a few nose diameters aft of the nose-afterbody junction, these pressures are correlated with the parameter as predicted by the blast-wave analogy . chernyi developed a modified form of the blast-wave analogy which takes into account the addition of energy to the flow by a thin afterbody . he showed that for thin afterbodies and hypersonic speeds, the pressure distribution, plotted as should correlate with the parameter . the purpose of this note is to show that the above correlation techniques may be combined into a form such that pressures on cylindrical and conical afterbodies are correlated by one parameter .'), (509, 'a graphical approximation for temperatures and sublimation rates at surfaces subjected to small net and large gross heat transfer rates .', 'adams, e. w.', 'j. aero. sc. v. 29, march 1962, pp. 360-1 .', 'a graphical approximation for temperatures and sublimation rates at surfaces subjected to small net and large gross heat transfer rates . considers a material, acted upon by heat of conduction, which changes its state by sublimation at the heated surface . the derived method is most suitable under conditions of severe heating such as space vehicle re-entry .'), (510, 'manoeuvring technique for changing the plane of circular orbits with minimum fuel expenditure .', 'weiss,d.c.', 'j. ae. scs. 29, 1962, 368.', 'manoeuvring technique for changing the plane of circular orbits with minimum fuel expenditure . usaf-supported discussion of the use of an intermediate elliptic orbit for changing the plane of a circular orbit . values of the perigee and apogee velocities are calculated for the following cases .. the braking impulse supplied by grazing of the atmosphere,. and (3) re-orbit with 90 of the braking impulse supplied in this manner .'), (511, 'tunnel tests on a double cascade to determine the interaction between the rotor and the nozzles of a supersonic turbine .', 'stratford,b.s.', 'ngte m359.', 'tunnel tests on a double cascade to determine the interaction between the rotor and the nozzles of a supersonic turbine . experimental confirmation has been required that in a supersonic turbine the leading edges of the rotor governs the rotor incidence and, hence, the gas exit angle from the nozzles . evidence has also been required that, once the rotor incidence has been allowed for, there is no adverse effect of the rotors on the nozzle flow, even when the rotors have a large turning angle . the present test cascade represented the stationary configuration of a turbine of 2.5 nozzle mach number and 74 swirl angle, the rotors being designed to operate at 1.9 relative mach number and to provide a turning angle of 140 . in the tests, fully supersonic flow could be established through the system, but the losses were fairly high and an increase in loss of about 25 per cent would have caused choking in the rotor .'), (512, 'quasi-cylindrical surfaces with prescribed thickness distributions .', 'moore,k.c.', 'rae tn.aero.2815, 1962.', 'quasi-cylindrical surfaces with prescribed thickness distributions . a formula for the supersonic velocity field in terms of a given surface distribution of sources is applied to points lying in the surface . an equation giving the camber shape of a quasi circular-cylindrical surface in terms of a prescribed thickness distribution is derived and the half ring wing with prescribed thickness distribution is discussed as an example .'), (513, 'pressure measurements at supersonic speeds on three uncambered conical wings of unit aspect ratio .', 'britton,j.w.', 'rae tn.aero.2821, 1962.', 'pressure measurements at supersonic speeds on three uncambered conical wings of unit aspect ratio . pressure measurements were made at mach numbers between 1.3 and 2.8 over a range of incidences on three simple models representing thick conical uncambered wings with sharp leading edges . these tests form part of an investigation into the effects of thickness and camber on slender wings . the aspect ratio of the models was unity in each case, and the spanwise cross sections were bounded by .. the measured pressure distributions are presented, along with overall lift and drag (excluding skin friction and base drag) obtained by integration .'), (514, 'pressure distributions and flow patterns on some conical shapes with sharp edges and symmetrical cross-sections at m=4 .0.', 'squire,l.c.', 'rae tn.aero.2823, 1962.', 'pressure distributions and flow patterns on some conical shapes with sharp edges and symmetrical cross-sections at m=4 .0. results are given of a wind tunnel programme made to study the pressure distributions and flow patterns over a series of simple, conical shapes at a mach number of 4.0 . the results have been compared with various approximate theories and the limitations of these theories are discussed . it is found that at this mach number leading edge separations still have an influence on the suction surface pressure, and that this surface still makes a significant contribution to the overall forces .'), (515, 'self sustained oscillations of a system with non-linear damping of a particular type .', 'neumark,s.', 'rae tn.aero.2839, 1962.', 'self sustained oscillations of a system with non-linear damping of a particular type . the paper deals with self-sustained oscillations of a dynamic system of single degree of freedom, with linear restoring force and non-linear damping force . the latter is supposed to be a function of velocity representable by a simple /polygonal/ graph, such that the damping is negative at small velocities but becomes positive at velocities above a certain value . on these assumptions, a rigorous solution is presented, including the equations of motion, amplitude, maximum velocity and period . a very simple solution is obtained for the limiting case of vanishingly small damping . an approximate solution by series in powers of damping ratio is worked out which gives a satisfactory accuracy for quite large values of .'), (516, 'free-flight measurements of the dynamic longitudinal stability characteristics of a wind tunnel interference model (m=0 .92 to 1. 35) .', 'greenwood,g.h.', 'rae tn.aero.2798, 1961.', 'free-flight measurements of the dynamic longitudinal stability characteristics of a wind tunnel interference model (m=0 .92 to 1. 35) . the dynamic longitudinal-stability characteristics of a standard wind tunnel interference model have been investigated in free flight over a mach number range of 0.92 to 1.35 . measurements of lift-curve slope and manoeuvre margin were obtained, and are compared with results from transonic-tunnel tests under low blockage conditions . the analysis was extended to obtain damping derivatives to allow comparison to be made with possible future dynamic tests in wind tunnels on the standard shape .'), (517, 'reaction-resisted shock fronts .', 'clarke,j.f.', 'coa r.150, 1961.', 'reaction-resisted shock fronts . it is shown that shock waves whose structure is determined solely by the effects of chemical reactions (reaction-resisted shock fronts) are possible and completely analogous to relaxation - resisted waves . a single dissociation reaction is considered and numerical results indicate that such waves could be observed experimentally . bulk viscosities equivalent to reaction effects are possibly 10 or more times shear viscosity values . (examples are based on lighthill''s ideal dissociating gas) .'), (518, 'heat conduction through a polyatomic gas .', 'clarke,j.f.', 'coa r.149, 1961.', 'heat conduction through a polyatomic gas . a heat conduction problem is set up which, in essence, simulates the conditions arising when a plane shock wave reflects from a co-planar solid boundary . the gas is assumed to be polyatomic, with one the quantity of primary interest is the temperature of the solid at the interface, since this can be observed experimentally without much difficulty . solutions are obtained for this quantity which cover a range of practically plausible relaxation times and ''wall effect'' parameters . it is essential to include proper temperature jump boundary conditions for both active and relaxing (or inert) energy modes . thus it is necessary to know accommodation coefficients for these modes of energy storage . the temperature jump effects are found to dominate the (interface) solid''s temperature time history, with relaxation effects playing a very secondary role . the theoretical results are compared with some experimental observations and encouraging agreement is found . as a result of this agreement it proves possible to estimate the accommodation coefficient for the active modes (in this case for the combination platinum air), the pressure being about 15 atmospheres . the pressure sensitivity of accommodation effects is commented on .'), (519, 'base pressure at supersonic speeds in the presence of a supersonic jet .', 'craven,a.h.', 'coa r.144, 1960.', 'base pressure at supersonic speeds in the presence of a supersonic jet . the effects on base pressure of jet mach number, free stream reynolds number and jet to base diameter ratio have been investigated experimentally . it was found that, for jet stagnation pressures greater than that required for the nozzle to reach its design mach number, an increase of jet mach number reduced the base pressure . similarly the base pressure increased with increase of the ratio of jet diameter to base diameter and, at high jet stagnation pressures, base pressures higher than free stream static pressure were found . the base pressure was independent of free stream reynolds numbers greater than 2 x 10 per foot but increased with reduction of reynolds number below 2 x 10 per foot . unsteady wave patterns were found when the jet mach number did not differ markedly from the free stream mach number and the jet had just reached its design conditions .'), (520, 'wing-tail interference as a cause of ''magnus'' effects on a finned missile .', 'benton,e.r.', 'j. ae. scs. 29, 1962, 1358.', 'wing-tail interference as a cause of ''magnus'' effects on a finned missile . wing-tail interference is shown to cause large /magnus/ effects on a finned missile whose wings are deflected into an aileron setting . a simple experimental method with water as the working medium is used to obtain low-speed magnus data on a rolling missile . the missile is a slender cruciform configuration with all-movable wings and fixed tail fins . magnus data are presented for angles of attack up to 15 and for the one (high) roll rate which accompanies a 30 aileron deflection angle of the wings . tests conducted at zero roll rate but with the wing deflection maintained, revealed large forces in the magnus direction, thereby providing the basis for understanding magnus effects due to wing-tail interference . a semiempirical theory is proposed to explain the experimental data . a simplified model of the wake behind the wings is introduced to predict tail-interference factors . good agreement with the data is obtained . this magnus effect is opposite in direction to the classical magnus lift on a spinning cylinder ,. it is much larger than either that effect or the one on a missile with only one set of fins . wing-tail interference is the predominant source of the effect ,. roll rate only modifies the basic interference mechanism .'), (521, 'a note on application of transonic linearization to an airfoil with a round leading edge .', 'hosokawa,i.', 'j. ae. scs. 29, 1962, 1395.', 'a note on application of transonic linearization to an airfoil with a round leading edge . the profile of a symmetric airfoil of unit length with a round leading edge can be expressed, in general, as where p(x) has a finite slope at x = 0 . it is well known that the conventional sub- and supersonic linear theories of compressible flow break down in the neighborhood of such a round leading edge due to the failure of the small-disturbance assumption . the linearized transonic flow theory has the same short-coming, but if the determination of the sonic point on the airfoil plays an important role in any more advanced theory--e.g., spreiter''s local-linearization method or hosokawa''s method of refinement--this theoretical barrier will become more serious because the sonic point is usually located in a flow region near the leading edge that may be greatly affected by the roundness .'), (522, 'laminar, transitional and turbulent heat transfer to a cone-cylinder-flare body at mach 8. 0.', 'zakkay,v. and callahan,c.j.', 'j. ae. scs. 29, 1962, 1403.', 'laminar, transitional and turbulent heat transfer to a cone-cylinder-flare body at mach 8. 0. an experimental investigation of the laminar, transitional, and turbulent heat transfer rates over a conical cylindrical flared body is presented . regions of favorable, zero, and adverse pressure gradient on the body are investigated . the experimental results are compared with the theories available in the literature . the model chosen for this investigation is a cone-cylinder-flare configuration consisting of a 20 semivertex conical nose portion smoothly blended by a shoulder radius into a long cylindrical body and terminated by a smooth large radius flare . the model was tested at a free stream mach number of 8, over a range of reynolds number from 0.3 x 10 to 1.6 x 10 per inch based on free stream conditions . various stagnation-to-wall temperature ratios were obtained by cooling the model prior to the test with liquid nitrogen . the stagnation-to-wall temperature ratios were 10 and 3.3 . the theoretical predictions gave good results for the heat transfer rates in the laminar region, and fair prediction in the transitional and turbulent regimes extending over the shoulder and forward portion of the cylindrical body . over the aft portion of the cylinder and over the flare the predictions are only qualitatively correct, and underestimate the heating rate by a factor as high as 3 . conversely, the /flat plate reference enthalpy/ over the aft portion of the body, but to increasingly overestimate the heating rates over the forward portion of the cylinder . a modified equation for the heat transfer coefficient in the transitional and fully turbulent region based on the f.p.r.e. method is then presented . this method gives good agreement with the experimental results presented over the entire range of transitional and turbulent flow . from the results the following is concluded .. cooling the wall delayed transition . by expanding the flow rapidly between the cone and the cylinder, the transition reynolds number is reached very rapidly . by making a smooth transition between the cylinder and the flare, no separation occurred at the cylindrical flare junction . the transitional and turbulent heat transfer in the presence of an adverse pressure gradient may be predicted with sufficient accuracy by the f.p.r.e. method .'), (523, 'approximate determination of position of the sonic line for a blunt body in hypersonic flow .', 'rahman,m.a.', 'j. ae. scs. 29, 1962.', 'approximate determination of position of the sonic line for a blunt body in hypersonic flow . the detached shock in front of a blunt body in hypersonic flow tends to acquire the shape of the frontal curvature of the body . thus the curvature of the shock can be assumed to be the same as that of the body, at least up to the sonic point (point a, fig. 1) . if the equation of curvature of the body is known, the equation of curvature of the shock is also known . in this paper, with this assumption, a method is described to determine the approximate position of the sonic line (ao''b, fig. 1) . the shock-detachment distance is assumed known . the method is, of course, general . this can be applied to any detached shock provided its equation of curvature is known corresponding to that of the body . for simplicity the detached shock is assumed to be circular in this paper and the procedure is outlined below with the assumption that the sonic line ao''b is parabolic .'), (524, 'stagnation point heat transfer in partially ionized air .', 'rozycki,r.c. and fenster,s.j.', 'j. ae. scs. 29, 1962.', 'stagnation point heat transfer in partially ionized air . comparison of heat-transfer rates, obtained by using transport properties recently reported by peng and pindroh, with rates based on hansen''s thermodynamic and transport properties . it is shown that the heat-transfer rates based on the peng and pindroh data are 20 to 30 lower for the velocity range of 25,000 to 40,000 ft sec .'), (525, 'on hypersonic viscous flow over an insulated flat plate with surface mass transfer .', 'tien,c.l.', 'j. ae. scs. 29, 1962, 1024.', 'on hypersonic viscous flow over an insulated flat plate with surface mass transfer . hypersonic viscous flow over an insulated flat plate with surface mass transfer is studied . the tangent-wedge approximation is used in the inviscid-flow region, and the integral method is applied to the treatment of the laminar boundary layer . the law of surface mass transfer for the present analysis is derived . a continuous transition of the pressure variation is achieved from the strong to the weak pressure-interaction region . first-order formulas for the induced surface pressure and the skin-friction coefficient are obtained for both the strong and weak pressure-interaction regions . results are compared with those calculated from other analyses .'), (526, 'leading edge attachment in transonic flow with laminar or turbulent boundary layers .', 'mabey,d.g.', 'j. ae. scs. 29, 1962.', 'leading edge attachment in transonic flow with laminar or turbulent boundary layers . the transonic flow round a two-dimensional airfoil at incidence is often determined by the type of flow in the leading-edge region . if the flow separates at the leading edge at low speeds it is liable to attach as the speed increases, often quite suddenly . a review of this change with laminar or with turbulent boundary layers re-emphasizes the importance of fixing transition when making model tests at transonic speeds in order to obtain flows closest to full-scale conditions . it is shown that similar airfoils with attached leading-edge flow show transonic similarity upstream of the terminal shock .'), (527, 'note on the three-point boundary layer problem for the blasius equations .', 'martin,e.d.', 'j. ae. scs. 29, 1962.', 'note on the three-point boundary layer problem for the blasius equations . in a recent paper a method was presented for obtaining higher accuracy in the numerical solution of the blasius equation with three-point boundary conditions . the well-known blasius equation was previously developed in an investigation of the steady two-dimensional incompressible boundary-layer flow over a flat plate, but it has been extensively used in investigating other fluid flow problems . the three-point boundary-value problem is encountered in the theory of laminar mixing and in approximate analyses of separated and wake flows as noted in ref. 1 .'), (528, 'first-order slip effects on the laminar boundary layer over a slender body of revolution with zero pressure gradient .', 'jay m. solomon', 'research aerospace engineer, u. s. naval ordnance laboratory, white oak, silver spring, md.', 'first-order slip effects on the laminar boundary layer over a slender body of revolution with zero pressure gradient . in reference 1, the analysis given by probstein and elliott for the zero-pressure-gradient, constant-wall-temperature, compressible, laminar boundary layer with transverse curvature was extended to first-order slip flow . this extension was based on a double asymptotic expansion in a transverse-curvature parameter and a slip parameter . the expansion in ref. 1, however, was carried out with the parameter held constant . for and a constant wall temperature, is constant and e varies with x due to the dependence of the local body radius on x . thus, for arbitrary body shapes, e will not be constant . in the present note, the analysis of ref. 1 is re-examined taking into account the variation of e .'), (529, 'some effects of injection of foreign gases in a decelerating laminar boundary layer in supersonic flow .', 'gouse,s.w., brown,g.a. and kaye,j.', 'j. ae. scs. 29, 1962, 1250.', 'some effects of injection of foreign gases in a decelerating laminar boundary layer in supersonic flow . the purpose of this research program was to investigate the effects of a diffusion field on a laminar boundary layer in a supersonic flow . specifically, helium, nitrogen, and argon were uniformly injected into the laminar boundary layer of a high-speed flow in a tube with the objective of determining the effects of such injection on the pressure, temperature, and recovery factor distribution along and downstream of the injection region . a continuously operating axially-symmetric wind tunnel has been designed, constructed, and operated . this tunnel consists of an air supply system, a flowmeter, an upstream stagnation tank, a supersonic nozzle (throat diameter 0.262 and exit diameter 1.400), a test section of variable length (zero to 81 diameters, test section diameter of 1.400), a downstream stagnation tank, an exhaust system, a foreign gas supply system, and all necessary instrumentation . the overall performance of this apparatus in terms of the design specifications was excellent . the tunnel was instrumented with 109 thermocouples . all temperatures except ambient temperatures were automatically measured and recorded by means of a self-balancing recording potentiometer . there was 29 pressure taps distributed along the tunnel, 23 along the test section itself . pressures were measured by means of an interconnected micromanometer and a vacuum referenced manometer system with overlapping ranges . for all of the results reported herein, the overall test section was 41 diameters in length,. composed of a porous test section approximately 7.2 diameters in length (leading edge approximately 1.8 diameters from the nozzle exit plane) and four nylon test sections of 8 diameters each .'), (530, 'an aerodynamic analysis for flutter in oseen-type viscous flow .', 'chu,wen-hwa.', 'j. ae. scs. 29, 1962, 781.', 'an aerodynamic analysis for flutter in oseen-type viscous flow . oseen''s equations for unsteady flow are employed to obtain a linearized solution based on a discontinuous-wake model . the analysis is employed to estimate the viscous correction to unsteady lift and moment at large reynolds number . if the asymptotic solution is not too slowly convergent, the correction is of the order of the ratio of the logarithm of reynolds number to the reynolds number . the theory is preliminary in nature as it is limited by the accuracy of oseen''s equations and is restricted to small angle of attack . however, it also shows that the generalized trailing-edge condition for potential flow is reasonable and might predict the essential correction in a real fluid .'), (531, 'the flow about a moving body in the upper ionosphere .', 'bird,g.a.', 'j. ae. scs. 29, 1962.', 'the flow about a moving body in the upper ionosphere . a particle approach is used to study the flow pattern around a body moving in the upper layers of the ionosphere . the effects of distant encounters between charged particles (dynamic friction) and of the earth''s magnetic field are taken into account . it is shown that, when the magnetic lines of force are parallel to the direction of motion of the body, there may be a marked concentration of charged particles in the vicinity of the body and a considerable fraction of the reflected or deflected charged particles may reimpinge on the body surface . a numerical example is given for the size and shape of the charged- particle-density contours in the flow field surrounding a circular disc, and these are compared with the corresponding neutral-particle contours .'), (532, 'pitch-yaw stability of a missile oscillating in roll via the second method of lyapunov .', 'parks,p.c.', 'j. ae. scs. 29, 1962, 874.', 'pitch-yaw stability of a missile oscillating in roll via the second method of lyapunov . the stability theory of a. m. lyapunov, a popular topic in the u.s.s.r., is receiving increasing attention elsewhere . this note describes lyapunov''s /second method/ very briefly and applies it to an aeronautical stability problem .'), (533, 'stagnation-point shock-detachment distance for flow around spheres and cylinders in air .', 'ambrosio,a. and wortman,a.', 'j. ae. scs. 29, 1962.', 'stagnation-point shock-detachment distance for flow around spheres and cylinders in air . author discusses the problem of deflection of a cantilevered bar, initially in the shape of a circular arc, subjected to an arbitrarily inclined end load .'), (534, 'consideration of energy separation for laminar slip flow in a circular tube .', 'inman,r.m.', 'j. ae. scs. 29, 1962, 1014.', 'consideration of energy separation for laminar slip flow in a circular tube . the energy separation for laminar low-density-nonunity prandtl number flow in circular cross-section tubes is the topic of this note . a conclusion is reached as to the effect of prandtl number on the velocity profiles for these flows . however, in order to reach valid quantitative conclusions the reviewer feels that more detailed analysis is in order, and that the analysis as presented here is of qualitative value only .'), (535, 'shroud design for simulating hypersonic flow over the nose of a hemisphere .', 'roger dunlap', 'associate research engineer, dept, of aeronautical and astronautical engineering, ann arbor, mich.', 'shroud design for simulating hypersonic flow over the nose of a hemisphere . following is an analytical method for designing a shroud which will generate the hypersonic pressure distribution on a hemisphere . the method was found to be successful throughout the region of subsonic flow . this shroud was designed as part of a low-turbulence wind tunnel used for investigating the effects of cooling on boundary-layer transition on a hemisphere . the design of the shroud contour was carried out in two steps . first, an approximate solution for the incompressible, irrotational flow field was found in the region, and, second, the resulting contour was corrected for compressibility near the sonic region, assuming one-dimensional flow .'), (536, 'transition in the viscous wakes of blunt bodies at hypersonic speeds .', 'hidalgo,h., taylor,r.l. and keck,j.c.', 'j. ae. scs. 29, 1962.', 'transition in the viscous wakes of blunt bodies at hypersonic speeds . transition from laminar to turbulent flow in the hypersonic wakes of spheres was detected in laboratory measurements of the radiation from the flow field . a hypervelocity gun facility was used to fire models, 0.22-in. in diameter, into a range at velocities from 10,000 to 17,000 ft sec . experiments were performed by changing .. (a) the material of the projectile ,. (b) the ambient gas in the range ,. and (c) the pressure in the range . three optical techniques were used to observe the wake radiation .. which show a turbulent viscous wake as the pressure in the range is decreased from one atmosphere to about 20 cm hg . which show the luminous flow field at pressures between 30 and ence of short luminous streaks, which disappear suddenly as the pressure is decreased below 3 cm hg for air, and below 0.8 cm hg for argon . both air and argon, which show the main features of the flow field . above the transition pressure, the intensity of radiation from the wake is always associated with fluctuations that appear to be the same phenomenon as the drum-camera streaks . the appearance of the streaks in the drum camera and photo-multiplier data is interpreted as transition from laminar to turbulent flow in the viscous wake, because experimental evidence shows that their appearance is not controlled by chemical, radiative, or ablative processes, but depends on aerodynamic effects . this conclusion is supported by other experiments based on optical and schlieren techniques . the transition in the wake at positions very close to the body is given by a local reynolds number of 10 for air, and 3 x 10 for argon . the results indicate a possible local-mach-number effect .'), (537, 'stagnation point viscous hypersonic flow .', 'stoddard,f.j.', 'j. ae. scs. 29, 1962, 1138.', 'stagnation point viscous hypersonic flow . several methods have been developed for computing the hypersonic, low reynolds number flow in the stagnation region of a blunt body . in general, these methods involve complicated numerical solutions . simultaneous iterations on several parameters are usually required in view of the boundary-value nature of the problem . the purpose of this note is to present an approximate closed-form solution to axisymmetric stagnation point hypersonic flow in the viscous layer regime .'), (538, 'the conpressibility transformation and the turbulent boundary layer equations .', 'burgraf,o.r.', 'j. ae. scs. 19, 1962.', 'the conpressibility transformation and the turbulent boundary layer equations . the compressibility transformation first introduced by dorod-nitzyn has been applied in this paper to the equations of the turbulent boundary layer on a flat plate, considering heat transfer and arbitrary prandtl numbers . assuming the shear distribution to be invariant under the transformation, the stream function and the momentum equation take the proper form for incompressible flow, allowing the use of incompressible velocity profiles in the transformed coordinates . application of crocco''s method to the transformed energy equation permits integration of the energy equation resulting in a formulism remarkably similar to that proposed by eckert . finally, the reference condition was chosen to correspond to the edge of the sublayer from considerations of the assumptions made regarding the shear-stress distribution . with this choice, the reference enthalpy is in good agreement with eckert''s formula over the ordinary range of test conditions . in view of these results, the analysis may be considered to provide a theoretical basis for the reference-enthalpy method .'), (539, 'local heat transfer to a yawed, infite, circular cylinder in laminar compressible flow .', 'weiss,d.', 'j. ae. scs. 29, 1962.', 'local heat transfer to a yawed, infite, circular cylinder in laminar compressible flow . this note presents a simplification of a method for calculating the ratio of local to stagnation-line heat-transfer coefficients on a yawed, infinite, circular cylinder in laminar compressible flow . a brief description of the method of ref. 1 is presented, followed by a discussion of the assumptions and mathematical procedure which lead to a considerable simplification .'), (540, 'use of local similarity concepts in hypersonic viscous interaction problems .', 'forbes dewey,c.', 'a.i.a.a. j. 1963, 20.', 'use of local similarity concepts in hypersonic viscous interaction problems . the problem of predicting the characteristics of a hypersonic laminar boundary layer that interacts with the external flow field is approached using the tangent wedge formulation for the inviscid flow field and the method of similar solutions for the viscous flow . it is shown that the concept of local similarity which allows the pressure gradient parameter to vary in the streamwise direction leads to an explicit relation between the viscous and inviscid flows for all values of the hypersonic interaction parameter . the conditions of /strong/ and limits of the general relations . the present theory is compared with three independent experimental investigations . in each case, the agreement is found to be excellent over the range of investigated . it is shown, using asymptotic solutions to the exact boundary layer equations, that the present theory is applicable to a wide variety of viscous interaction problems . a large number of solutions to the laminar boundary layer similarity equations for a perfect gas with cross flow and surface mass transfer are given . these numerical results, when combined with the solutions of previous authors, are sufficient to describe the range of conditions with high precision .'), (541, 'similitude of hypersonic flows over slender bodies in non-equilibrium dissociated gases .', 'inger,g.r.', 'a.i.a.a. j. 1963, 46.', 'similitude of hypersonic flows over slender bodies in non-equilibrium dissociated gases . this paper is concerned with the similitude laws governing inviscid, nonequilibrium gas flows around blunt or sharp-nosed slender bodies at zero angle of attack, based on the hypersonic small disturbance flow theory . some related features of the interaction between the effects of nose bluntness and nonequilibrium dissociation and vibration and the influence of a dissociated freestream are also discussed . the hypersonic equivalence principle and the related similitude for affinely related bodies are set forth for nonequilibrium flows in either diatomic gases or a gas mixture such as air . for a family of diatomic gases, as opposed to a given gas such as air, a generalized ambient gas state scaling condition is obtained, whereby the ambient density and temperature need not be simulated . a detailed discussion is given of blunted cylinders and slabs or sharp-nosed cones and wedges, including example nonequilibrium flow field correlations of numerical solutions available in the literature . low density nonequilibrium flows with a negligible shock layer atom recombination rate are also examined ,. as expected, a less restrictive small disturbance similitude law is obtained in this case .'), (542, 'biot''s variational principle in heat conduction .', 'lardner,t.j.', 'a.i.a.a. j. 1963, 196.', 'biot''s variational principle in heat conduction . biot''s variational principle is applied to a number of different one-dimensional heat conduction problems . these problems show the applicability of the variational principle to problems involving prescribed heat flux boundary conditions and to those with temperature-dependent material properties . a method is introduced for including boundary conditions when these are expressed as prescribed heat fluxes . the idea behind this is overall energy balance within the body, which is a constraint condition to be satisfied by the time histories of the generalized coordinates . the variational principle is then applied to the well-known problem of constant surface heat flux in order to present the technique and provide a basis for the remaining sections . the equivalence of the result obtained in applying the variational principle for a prescribed surface temperature history to that obtained for a prescribed heat flux is also pointed out . radiation cooling due to fourth power radiation from semi-infinite solids and finite slabs together with radiation according to newton''s law of cooling is then treated . finally, the introduction of temperature-dependent material properties is discussed and the determination of the temperature distribution in a semi-infinite solid with variable properties is investigated .'), (543, 'the stacking of compressor stage characteristics to give an overall compressor performance map .', 'doyle,m.d.c.', 'aero. aquart. 13, 1962.', 'the stacking of compressor stage characteristics to give an overall compressor performance map . a method of calculation is developed to compute the overall performance of a multi-stage axial compressor, from a knowledge of the individual stage characteristics, by a /stacking/ technique . compressor models are designed and their overall performance calculated . these results are compared to show, qualitatively, the effect of alterations in design and stage performance on overall performance and to find how compressors should be designed for optimum performance .'), (544, 'a theoretical and experimental study of oscillating wedge shaped aerofoils in hypersonic flow .', 'r. a. east', '', 'a theoretical and experimental study of oscillating wedge shaped aerofoils in hypersonic flow . aerodynamic stiffness and damping derivatives have been measured in a /hypersonic gun/ wind tunnel for sharp and blunt-nosed two dimensional single wedge shapes oscillating in the pitching mode in hypersonic flow . the results, which have been compared with theoretical prediction, modified to account for leading edge bluntness, show that this may increase the damping by up to 50 percent for certain axis positions . details of the experimental technique designed to measure the derivatives in the short running times available are described .'), (545, 'calculation of sideslip derivatives and pressure distribution in asymmetric flight conditions on a slender wing-fin configuration .', 'sells,c.c.l.', 'rae tn.aero.2805, 1962.', 'calculation of sideslip derivatives and pressure distribution in asymmetric flight conditions on a slender wing-fin configuration . the flow around slender wing-fin configurations having curved leading edges, whose shape is defined by polynomials, is considered . a general expression for the pressure distribution on such a configuration in asymmetric flow is derived and the derivatives due to the particular case of sideslipping motion are also given . no numerical results are given for wing-fin load distribution, but the sideslip derivatives have been evaluated in a number of cases for gothic and ogee wings .'), (546, 'measurements of aerodynamic heating on a 15 cone of graded wall thickness at a mach number of 6. 8.', 'woodley,j.g.', 'rae tn.aero.2847, 1962.', 'measurements of aerodynamic heating on a 15 cone of graded wall thickness at a mach number of 6. 8. this note describes transient wall temperature measurements made on a in an airstream of mach number 6.8 . the skin of the model was sufficiently thin to allow it to reach zero heat transfer conditions within a running time of one minute . in order to reduce effects of longitudinal heat conduction during a run the electroformed-nickel skin of the model was made with graded thickness, and as a result fairly uniform temperature distributions along the surface were obtained at all times in both the laminar and turbulent regions . values of heat transfer, calculated from the wall temperature-time histories using the thin-wall temperature are compared to theoretical estimates using the intermediate enthalpy method 10, 11 .'), (547, 'boundary layer characteristics of caret wings .', 'catherall,d.', 'rae tn.aero.2835, 1962.', 'boundary layer characteristics of caret wings . the theory of laminar boundary layers along flat surfaces has been used in conjunction with eckert''s approximations to the displacement thickness, skin friction and temperature profiles on the undersurface of a caret wing configuration . to a first approximation it has been assumed that parallel flow exits behind the shock outside the boundary layer, and the displacement of the shock by the boundary layer near the leading edge is neglected . conduction of heat within the body and along the surface is neglected but radiation is included, so that are found . examples are given for various altitudes and configurations and the effect of the skin friction on the lift drag ratio calculated, assuming the undersurfaces to be plane .'), (548, 'the contraction of satellite orbits under the influence of air drag . pt .iv with scale height dependent on altitude .', 'king-hele,d.g., and cook,g.e.', 'rae tn.space 18, 1962.', 'the contraction of satellite orbits under the influence of air drag . pt .iv with scale height dependent on altitude . the effect of air drag on satellite orbits of small eccentricity e was studied in part i (tech. note gw 533), on the assumption that atmospheric density varies exponentially with distance r from the earth''s centre, so that the ''density scale height'' h, defined as, is constant . in practice h varies with height in an approximately linear manner, and in the present note the theory is developed for an atmosphere in which h varies linearly with r . equations are derived which show how perigee distance and orbital period vary with eccentricity, and how eccentricity varies with time . expressions are also obtained for the life-time and air density at perigee in terms of the rate of change of orbital period . the results are also presented graphically . the results are formulated in two ways . the first is to specify the extra terms to be added to the constant-h equations of part i . the second the best constant value of h for use with the equations of part i . for example, it is found that the constant-h equations connecting perigee distance (or orbital period) and eccentricity can be used unchanged without loss in accuracy, if h is taken as the value of the variable h at a height above the mean perigee height during the time interval being considered, where, and decreases from to 0 as e decreases from 0.02 to 0 . similarly the constant-h equations for air density at perigee can still be used if h is evaluated at a height above perigee, where, and decreases to zero as e decreases from constant-h equations can still be used if h is evaluated at the scale height below the initial height . variation of h with altitude has a small effect on the lifetime - about 3 - and on the e-versus-time curve .'), (549, 'experimental study of the velocity and temperature distribution in a high-velocity vortex-type flow .', 'hartnett,j.p. and eckert,e.r.g.', 'asme trans. 70, 1957.', 'experimental study of the velocity and temperature distribution in a high-velocity vortex-type flow . the vortex tube represents a simple device in which a particular type of vortex motion may be studied in the laboratory in an attempt to obtain a better understanding of such flows . such an investigation has been pursued in the heat transfer laboratory of the university of minnesota . the present paper summarizes the major results of this vortex-tube investigation .'), (550, 'laminar heat transfer in tubes under slip-flow conditions .', 'sparrow,e.m. and lin,s.h.', 'asme paper 61-wa-165.', 'laminar heat transfer in tubes under slip-flow conditions . the effects of low-density phenomena on the fully developed heat-transfer characteristics for laminar flow in tubes has been studied analytically . consideration is given to the slip-flow regime wherein the major rarefaction effects are manifested as velocity and temperature jumps at the tube wall . the analysis is carried out for both uniform wall temperature and uniform wall heat flux . in both cases, the slip-flow nusselt numbers are lower than those for continuum flow and decrease with increasing mean free path . extension of the results is made to include the effects of shear work at the wall, temperature jump modifications for a moving fluid, and thermal creep .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (551, 'analysis of a loaded cantilever plate by finite difference methods .', 'livesley,r.k. and birchall,p.c.', 'rae tn. ms26, 1956.', 'analysis of a loaded cantilever plate by finite difference methods . the various difference patterns necessary for finite difference solution of rectangular plate problems, with various boundary conditions and under various transverse loads, are developed . the solution of one particular problem on deuce is also described .'), (552, 'chemical kinetics of high temperature air .', 'wray,k.l.', 'hypersonic flow research, p 181, academic press, new york, 1962.', 'chemical kinetics of high temperature air . when a hypersonic object enters earth''s atmosphere, a shock wave is formed in front of it, and the air passing through this shock wave is heated to high temperatures . the shock heated molecules equilibrate their translational and rotational degrees of freedom within a distance of a few mean free paths . to achieve equilibrium, it is necessary to excite vibration, dissociate molecules, produce new molecules and produce ions and electrons . the problem is complex, since all these phenomena occur simultaneously and because the reaction rates depend on the temperature, density and composition which are changing during the relaxation toward equilibrium . the experimental techniques used to investigate these reactions are briefly discussed along with the resulting rate expressions obtained by the various investigators . a compilation of the rate expressions for these reactions representing the author''s evaluation of all the available data is presented . several pertinent problems which are not yet completely understood and which still require theoretical and experimental investigation are outlined . computed concentration, temperature and density time histories are shown for three different shock speeds in air . the time rate of change of concentration for each chemical reaction is also shown and regimes of importance for the various processes are discussed .'), (553, 'ablation of glassy materials around blunt bodies of revolution .', 'hidalgo,h.', 'ars j. 30, 1960.', 'ablation of glassy materials around blunt bodies of revolution . the steady-state equations of motion for a thin layer of an incompressible glassy material on the surface of an ablating and radiating blunt body are reduced to a first-order ordinary differential equation which is integrated numerically . this solution is coupled with the solution of the air boundary layer for both laminar and turbulent heat transfer with or without mass vaporization of the ablating material . the distribution of the effective energy of ablation around the body is thus obtained for a cone cylinder with a hemispherical cap that re-enters the atmosphere at hypersonic flight speeds, and has quartz as the ablating material . it is found that the ablation process from turbulent heating is more efficient than from the laminar case because of increased vaporization . this solution of the equations of motion at the stagnation point has been verified by are wind tunnel experiments . the present state of development of the are wind tunnel does not permit its use for experimental investigations of ablation around blunt bodies under turbulent heating .'), (554, 'generalized heat transfer formulas and graphs .', 'detra,r.w. and kidalgo,h.', 'ars j. 1961.', 'generalized heat transfer formulas and graphs . utilizing the research results of previously reported investigations of the laminar, turbulent and radiative heat transfer in dissociated air, some generalized formulas for calculating heat transfer are given . graphs for determining the laminar heat transfer, momentum thickness reynolds number, and turbulent heat transfer distributions around an axisymmetric body are also given . these heat transfer correlations are valid for velocities between 6000 and 26,000 fps and for altitudes up to 250,000 ft . this range of velocities and altitudes covers the important re-entry regime of practical re-entry trajectories having interest today . in the last section of this report these generalized results are specialized for icbm nose cone re-entry applications . these formulas and graphs may be found useful for making rapid engineering estimates and preliminary design evaluations of the heating problems associated with re-entry into earth''s atmosphere .'), (555, 'closing reply to comments on generalized heat transfer formulas and graphs for nose cone re-entry into the atmosphere .', 'hidalgo,h.', 'ars j. 1962.', 'closing reply to comments on generalized heat transfer formulas and graphs for nose cone re-entry into the atmosphere . in a recent paper (1), detra and hidalgo have shown that, when the boundary layer is turbulent, the heat flux per unit area at the sonic point of a nose cone may exceed the corresponding laminar heat flux per unit area at the stagnation point . the ratio of turbulent sonic-point to laminar stagnation-point heat flux per unit area has been estimated (2) to vary from about 1.0 to 10 for a hemispherical nose as the reynolds number (based on nose diameter) increases from 10 to 10 . since for an axisymmetric body the surface area in the vicinity of the sonic point greatly exceeds the area in the vicinity of the stagnation point, the ratio of turbulent to laminar heat fluxes to the entire body will be much greater than the above quoted ratios of heat fluxes per unit area .'), (556, 'numerical comparison between exact and approximate theories of hypersonic inviscid flow past slender blunt nosed bodies .', 'feldman,s.', 'ars j. 30, 1960.', 'numerical comparison between exact and approximate theories of hypersonic inviscid flow past slender blunt nosed bodies . this paper presents numerical results of exact calculations of the inviscid equilibrium flow about a long hemisphere-cylinder in motion at hypersonic velocity . a comparison is made with blast wave as well as free layer theories of hypersonic flow . as a result of the comparison, it is concluded that the second-order blast wave theory can be used for the purpose of finding the shock shape and the body pressure distribution . however, this procedure is definitely empirical and cannot be justified on rational or theoretical grounds . we show that the presently calculated radial distribution of energy is radically different than that given by blast wave theory . if body shapes other than those considered here are of interest, the only reliable approach at the present time is to carry out numerical calculations . it was found that for certain flight velocities the pressure on the body does not decay to free stream pressure monotonically but overexpands .'), (557, 'a numerical comparison between exact and approximate theories of hypersonic inviscid flow past slender blunt nosed bodies .', 'cheng,k.h. and chang,a.l.', 'ars j. 31, 1961.', 'a numerical comparison between exact and approximate theories of hypersonic inviscid flow past slender blunt nosed bodies . this note refers to paper of same title by feldman in ars j. 30, validity of blast wave theory cannot be justified on rational or theoretical grounds because of different values of energy in cross flow field as calculated by this theory and by method of characteristics . present note questions this conclusion, shows reasonably good agreement when energy is calculated for points where shock location, streamline pattern, and velocity, temperature, and pressure profiles are adequately defined, and still better agreement when energy is calculated from flow quantities provided by- characteristics method . results are checked using data from independent source . conclusion is reached that blast wave theory is still valid .'), (558, 'experimental measurements of turbulent transition motion, statistics and gross radial growth behind hypervelocity object.', 'slattery,r.e. and clay,w.g.', '', 'experimental measurements of turbulent transition motion, statistics and gross radial growth behind hypervelocity object. the laminar-turbulent transition behind 0.500-in.-diameter spheres at 8500 ft sec and behind measured as a function of pressure . schlieren motion-picture techniques were used to analyze the turbulent motion and the results are described . autocorrelation functions of the density fluctuations of the turbulence have been measured . from these values has been calculated and the results are given for several positions in the turbulent trail at 30 mm hg downstream air pressure . in addition the authors'' previous measurements of the gross radial growth of the turbulent wake have been extended to pressures of 10 mm hg for the case of 0.500-in.-diameter spheres and to the trail behind'), (559, 'heat transfer at the forward stagnation point of blunt bodies .', 'reshotko,e. and cohen,c.b.', 'naca tn.3513, 1955.', 'heat transfer at the forward stagnation point of blunt bodies . relations are presented for the calculation of heat transfer at the forward stagnation point of both two-dimensional and axially symmetric blunt bodies . the relations for the heat transfer, which were obtained from exact solutions to the equations of the laminar boundary layer, are presented in terms of the local velocity gradient at the stagnation point . these exact solutions include effects of variation of fluid properties, prandtl number, and transpiration cooling . examples illustrating the calculation procedure are also included .'), (560, 'a theoretical study of the effect of upstream transpiration-cooling on the heat transfer and skin friction characteristics of a compressible laminar boundary layer .', 'rubesin,m.w. and inouye,m.', 'naca tn.3969, 1957.', 'a theoretical study of the effect of upstream transpiration-cooling on the heat transfer and skin friction characteristics of a compressible laminar boundary layer . an analysis is presented which predicts the skin-friction and heat-transfer characteristics of a compressible, laminar boundary layer on a solid flat plate preceded by a porous section that is transpiration cooled . the analysis is restricted to a prandtl number of unity and linear variation of viscosity with temperature . the local skin friction has been found to have a low value in the region of transpiration cooling and then to increase until it approaches the value for a completely nonporous surface asymptotically . the initial increase in local skin friction is rapid as half of the ultimate increase occurs in a distance beyond the porous region that is about 20 percent of the length of the porous region for all rates of injection . when the total coolant flow rate is kept constant and the porous length is varied, it is found that the average skin friction on a partially porous plate is slightly lower than that on a fully porous plate . the local heat transfer behaves in a manner similar to that of the local skin friction . it is found, in an example, that the temperature at the end of a partially porous plate could be maintained at about the same temperature as a fully porous plate by doubling the total rate of coolant flow .'), (561, 'a geometric problem related to the optimum distribution of lift on a planar wing in supersonic flow .', 'graham,e.w.', 'j. ae. scs. 1958, 771.', 'a geometric problem related to the optimum distribution of lift on a planar wing in supersonic flow . the problem studied may be regarded as a problem of geometry . its simplest form (loosely stated) is then as follows .. a mountain rises up from the x-y plane . determine the exact shape of the mountain knowing only the cross-sectional area of every possible cut which can be made through the mountain with a vertical plane . in a more complicated version of the problem, the given information might be restricted to the cross-sectional area of every cut which can be made by a vertical plane inclined less than 45 to the y-axis . this latter case has direct applications to certain minimum drag problems in supersonic flow . the shape of the mountain corresponds to the (unknown) shape of the optimum lift distribution on a planar wing . the cross-sectional area of a cut is the integrated value of the lift along a straight line crossing the wing plan form . for a restricted range of line inclinations, these optimum integrated lift values can sometimes be determined directly . here it is assumed that they are given . the problem in its simplest form was originally solved by radon, who found solutions for a large class of such problems . the derivation presented here may perhaps be more readily understood .'), (562, 'concerning the effect of compressibility on laminar boundary layers and their separation .', 'howarth,l.', 'proc. roy. soc. a, 194, 1948.', 'concerning the effect of compressibility on laminar boundary layers and their separation . the theory of compressible flow in a laminar boundary layer has been developed for the case when the viscosity is assumed to be proportional to the absolute temperature and the prandtl number is unity . (these assumptions may be compared with the empirical relations suggested by cope .) it is shown that a transformation of the ordinate normal to the layer can lead to a simplified form of equation of motion very similar to the ordinary incompressible equation but modified by a multiplicative factor g in the pressure term . this factor is greater than unity at the boundary and tends to one at the outside of the layer . several particular solutions are considered including accelerated flow with a linearly increasing velocity and retarded flow along a flat plate with a linearly decreasing velocity . the general implications of the theory are discussed and qualitative conclusions are drawn when the mainstream velocity starts from a stagnation point, rises to a maximum and subsequently falls . it is concluded that for such a velocity distribution increasing compressibility will reduce the skin friction, increase the boundary layer thickness and cause earlier separation as compared with the incompressible flow with the same mainstream velocity distribution and the kinematic viscosity corresponding to conditions at the stagnation point .'), (563, 'the law of the wake in the turbulent boundary layer .', 'coles,d.', 'j. fluid mech. 1, 1956,191.', 'the law of the wake in the turbulent boundary layer . after an extensive survey of mean-velocity profile measurements in various two-dimensional incompressible turbulent boundary-layer flows, it is proposed to represent the profile by a linear combination of two universal functions . one is the well-known law of the wall . the other, called the law of the wake, is characterized by the profile at a point of separation or reattachment . these functions are considered to be established empirically, by a study of the mean-velocity profile, without reference to any hypothetical mechanism of turbulence . using the resulting complete analytic representation for the mean-velocity field, the shearing-stress field for several flows is computed from the boundary-layer equations and compared with experimental data . the development of a turbulent boundary layer is ultimately interpreted in terms of an equivalent wake profile, which supposedly represents the large-eddy structure and is a consequence of the constraint provided by inertia . this equivalent wake profile is modified by the presence of a wall, at which a further constraint is provided by viscosity . the wall constraint, although it penetrates the entire boundary layer, is manifested chiefly in the sublayer flow and in the logarithmic profile near the wall . finally, it is suggested that yawed or three-dimensional flows may be usefully represented by the same two universal functions, considered as vector rather than scalar quantities . if the wall component is defined to be in the direction of the surface shearing stress, then the wake component, at least in the few cases studied, is found to be very nearly parallel to the gradient of the pressure .'), (564, 'local heat transfer and recovery temperature on a yawed cylinder at a mach number of 4. 15 and high reynolds numbers .', 'beckwith,i.e. and gallagher,j.j.', 'nasa memo 2-27-59l, 1959.', 'local heat transfer and recovery temperature on a yawed cylinder at a mach number of 4. 15 and high reynolds numbers . local heat transfer, equilibrium temperatures, and wall static pressures have been measured on a circular cylinder at yaw angles of 0, 10, 20, 40, and 60 . the reynolds number range of the tests was from 1x10 to 4x10 based on cylinder diameter . increasing the yaw angle from 0 to 40 increased the stagnation-line heat-transfer coefficients by 100 to 180 percent . a further increase in yaw angle to heat-transfer coefficients . at zero yaw angle the boundary layer over the entire front half of the cylinder was laminar but at yaw angles of 40 and 60 it was evidently completely turbulent, including the stagnation line, as determined by comparison of local heat-transfer coefficients with theoretical predictions . the level of heating rates and the nature of the chordwise distribution of heat transfer indicated that a flow mechanism different from the conventional transitional boundary layer may have existed at the intermediate yaw angles of 10 and 20 . at all yaw angles the peak heat-transfer coefficient occurred at the stagnation line and the chordwise distribution of heat-transfer coefficient decreased monotonically from this peak . the average heat-transfer coefficients over the front half of the cylinder are in agreement with previous data for a comparable reynolds number range . the theoretical heat-transfer distributions for both laminar and turbulent boundary layers are calculated directly from simple quadrature formulas derived in the present report .'), (565, 'similar solutions for the compressible boundary layer on a yawed cylinder with transpiration cooling .', 'beckwith,i.e.', 'naca tn.4345, 1958.', 'similar solutions for the compressible boundary layer on a yawed cylinder with transpiration cooling . heat-transfer and skin-friction parameters obtained from exact numerical solutions to the laminar compressible-boundary-layer equations for the infinite cylinder in yaw are presented . the chordwise flow in the transformed plane is of the falkner-skan type . solutions are given for chordwise stagnation flow with both a porous and a nonporous wall . the effect of a linear viscosity-temperature relation is compared with the effect of the sutherland viscosity-temperature relation at the stagnation line of the cylinder for a prandtl number of 0.7 . the effects of pressure gradient, mach number, yaw angle, and wall temperature are investigated for a linear viscosity-temperature relation and a prandtl number of 1.0 with a nonporous wall . the results indicate that compressibility effects become important at large mach numbers and yaw angles, with larger percentage effects on the skin friction than on the heat transfer . the use of the two different viscosity relations gives about the same results except when large changes in temperature occur across the boundary layer, as for a highly cooled wall . the present solutions predict that a larger amount of coolant would be required at a given large mach number and yaw angle than would be predicted from solutions of the corresponding incompressible-boundary-layer equations .'), (566, 'investigation of local heat transfer and pressure drag characteristics of a yawed circular cylinder at supersonic speeds .', 'goodwin,g., creager,m.o. and winkler,e.l.', 'naca rm.a55h31, 1956.', 'investigation of local heat transfer and pressure drag characteristics of a yawed circular cylinder at supersonic speeds . local heat-transfer coefficients, temperature recovery factors, and pressure distributions were measured on a circular cylinder at a nominal mach number of 3.9 over a range of free-stream reynolds numbers from from 0 to 44 . it was found that yawing the cylinder reduced the local heat-transfer coefficients, the average heat-transfer coefficients, and the pressure drag coefficients over the front side of the cylinder . for example, at is reduced by 34 percent and the pressure drag by 60 percent . the amount of reduction may be predicted by a theory presented herein . local temperature recovery factors were also reduced by yaw, but the amount of reduction is small compared to the reduction in heat-transfer coefficients . a comparison of these data with other data obtained under widely different conditions of body and stream temperature, mach number, and reynolds number indicates that these factors have little effect upon the dropoff of heat transfer due to yaw .'), (567, 'aerodynamic characteristics of a circular cylinder at mach number of 6. 86 and angles of attack up to 90 .', 'penland,j.a.', 'naca tn.3861, 1957.', 'aerodynamic characteristics of a circular cylinder at mach number of 6. 86 and angles of attack up to 90 . pressure-distribution and force tests of a circular cylinder have been made in the langley 11-inch hypersonic tunnel at a mach number of based on diameter, and angles of attack up to 90 . the results are compared with the hypersonic approximation of grimminger, williams, and young and with a simple modification of the newtonian flow theory . the comparison of experimental results shows that either theory gives adequate general aerodynamic characteristics but that the modified newtonian theory gives a more accurate prediction of the pressure distribution . the calculated crossflow drag coefficients plotted as a function of crossflow mach number were found to be in reasonable agreement with similar results obtained from other investigations at lower supersonic mach numbers . comparison of the results of this investigation with data obtained at a lower mach number indicates that the drag coefficient of a cylinder normal to the flow is relatively constant for mach numbers above about 4 .'), (568, 'shock wave effects on the laminar skin friction of an insulated flat plate at hypersonic speeds .', 'li,t.y. and nagamatsu,h.t.', 'j. ae. scs. 20, 1953, 345.', 'shock wave effects on the laminar skin friction of an insulated flat plate at hypersonic speeds . an approximate theory on the phenomena of interaction between the shock wave and the laminar boundary layer on an insulated flat plate at hypersonic speeds has been formulated . results on the rate of growth of the boundary-layer thickness and the rate of decay of the shock-wave strength have been found that hold for . a new set of formulas for the average skin-friction coefficient, over an insulated flat plate at hypersonic speeds has been obtained . calculations on the basis of the new formulas yield the data shown in figs. steady decrease in as increases, the present results indicate that may increase with at hypersonic mach numbers .'), (569, 'an experimental investigation of leading edge shock wave boundary layer interaction at mach 5.8 .', 'kendall, j. m.', 'j. aero. sc. v. 24, pp 47-56, 1957 .', 'an experimental investigation of leading edge shock wave boundary layer interaction at mach 5.8 . the boundary layer on a slender body tends to be very thick at hypersonic speeds . it interacts with the external flow by producing larger flow deflections near the leading edge than those due to the body alone flow around the boundary layer gives rise to an induced pressure with a negative gradient which thins the boundary layer and increases the skin friction with respect to the zero pressure gradient value . experiments on a flat plate with a sharp leading edge have been performed in the galcit 5-dash by 5-dash in. mach 5.8 hypersonic wind tunnel . the induced pressure was measured by means of orifices in the plate surface . profiles of mach number, velocity, mass flow, pressure, and momentum deficiency were calculated from impact pressure surveys normal to the plate surface made at various distances from the leading edge . the results are as follows . /1/ the induced pressures are 25 per cent higher than the weak interaction theory . /2/ the boundary layer and the external flow are distinctly separate for as low as 6,000 . /3/ the shock wave location is in good agreement with that predicted by the friedrichs theory for a body shape equivalent to the observed boundary-layer displacement thickness . /4/ expansion waves reflected from the shock are weak . /5/ the average skin-friction coefficient tends toward and nearly matches the zero pressure gradient value downstream, but increases to approximately twice that value as the leading edge is approached .'), (570, 'on the boundary layer equations in hypersonic flow and their approximate solutions .', 'lees,l.', 'j. ae. scs. 20, 1953, 143.', 'on the boundary layer equations in hypersonic flow and their approximate solutions . analytical solutions of the prandtl boundary-layer equations are obtained for the problem of the /strong/ interaction between the leading-edge shock and the viscous layer over a flat plate at hypersonic velocities . as the mach number increases and the interaction region spreads downstream over the plate, the local skin-friction coefficient increases rapidly over its conventional value . the local heat-transfer coefficient at first remains practically unaffected but then also begins to increase with mach number .'), (571, 'heat transfer to flat plate in high temperature rarefied ultra-high mach number flow .', 'nagamatsu,h.t., weil,h.a. and sheet,r.e.', 'ars j. 32, 1962, 533.', 'heat transfer to flat plate in high temperature rarefied ultra-high mach number flow . an investigation was conducted in a hypersonic shock tunnel to determine the local heat transfer rates for a sharp leading edge flat plate . the free stream mach number range was 7.95 to 25.1 with stagnation temperatures of approximately 2550 and 6500 r . for these temperature and mach number conditions, the strong interaction parameter, varied from 2.35 to 826 . the corresponding knudsen numbers, based on the ratio of the free stream mean free path and the leading edge thickness, varied from 0.38 to 85.5 . for free stream mach numbers greater than 10, knudsen numbers of approximately unity, and perfect gas conditions, the calculated heat transfer coefficients were found to vary as as predicted by the noninsulated flat plate theory of li and nagamatsu . for the case of, the leading edge slip phenomenon drastically reduced the local heat transfer coefficients as compared to the theoretical values predicted with no slip at the surface . for the extreme case of and, the measured local heat transfer rate was an order of magnitude less than the analytical value . both the knudsen number and the free stream mach number are important physical parameters that determine the extent of the slip-flow region .'), (572, 'boundary layer displacement and leading edge bluntness effects in high temperature hypersonic flow .', 'cheng, h.k., hall,j.g., golian, t.c. and hertzberg, a.', 'j. aero. sc. v. 28, pp 353-381, 410. 1961 .', 'boundary layer displacement and leading edge bluntness effects in high temperature hypersonic flow . two important features of hypersonic flow over slender or thin bodies are the displacement effect of the boundary layer and the large down-stream influence of leading-edge bluntness . the present paper contributes new theoretical and experimental results on this problem . the interaction of the two effects is treated theoretically by extending the basic shock-layer concept . in the outer inviscid flow, a model consisting of a detached shock layer and an entropy layer is introduced to account for bluntness . in the boundary layer, the approximate solution is found to be governed by a local flat-plate similarity . under the assumption of a strong bow shock and a specific heat ratio close to unity, a theory is developed for an arbitrary thin body . for flat-plate afterbodies, the theory yields a solution agreeing with blast-wave theory at one limit and strong-interaction theory at the other . within the framework of the present theory, the problems involving angle of attack are also analyzed . complementary to the above study, a hypersonic similitude involving strong shocks, but not requiring close to one a natural comparison with experimental data correlated on the basis of this similitude . flat-plate experiments in air, conducted in the c.a.l. 11 x 15-dashin. hypersonic shock tunnel under cold-wall conditions, included measurement of surface heat-transfer distributions and schlieren studies for zero and nonzero angle of attack . steady laminar heat-transfer rates were measured by means of thin-film resistance thermometers at air test-flow mach numbers around 12, free-stream reynolds numbers from 1.4 x 10 to 1. for most of the experiments, airflow stagnation temperatures ranged from ratios of about 0.15 . the range of test conditions at this stagnation temperature encompassed the limiting cases of dominant bluntness and dominant viscous-interaction effects . heat-transfer distributions were also measured on a sharp plate for air stagnation temperatures ranging from 2,000degreek up to 4,000degreek . the experimental data are quite well correlated in terms of the foregoing theoretical similitude variables characterizing combined effects of boundary-layer displacement and bluntness . the correlations obtained suggest that for the present experimental conditions, at least, the hypersonic viscous similitude is valid even with leading-edge bluntness in the paper, is generally fair .'), (573, 'viscous hypersonic similitude .', 'hayer,w.d. and probstein,r.f.', 'j. ae. scs. 26, 1959, 815.', 'viscous hypersonic similitude . an extension of classical hypersonic similitude is developed which takes into account the interaction effect of the displacement thickness of the boundary layer . a basic result of this viscous similitude is that the total drag including frictional drag obeys the classical similarity law for the pressure drag . additional similarity conditions governing viscous effects must be imposed in this similitude . underlying the similitude is a new hypersonic boundary-layer independence principle . according to this principle, the principal part of a hypersonic boundary layer with given pressure and wall temperature distributions and free-stream total enthalpy is independent of the (high) external mach number distribution outside the boundary layer . various features of viscous hypersonic similitudes are discussed . it is found, for example, that it applies to three- dimensional boundary-layer interaction effects on flat bodies, provided the concepts of strip theory may be applied, and provided the aspect ratio is an invariant .'), (574, 'inviscid flow with nonequilibrium molecular dissociation for pressure distributions encountered in hypersonic flight .', 'bloom, m.g. and steiger, m.h.', 'j. aero. sc. v. 27, pp 821-835, 1960 .', 'inviscid flow with nonequilibrium molecular dissociation for pressure distributions encountered in hypersonic flight . one-dimensional inviscid nonequilibrium flows of a two-component model gas are studied for prescribed pressure variations and an average reaction rate based on recent data for oxygen recombination . these flows are interpreted in relation to the flow along streamlines around blunt hypersonic bodies . assuming equilibrium conditions in the subsonic region, it is estimated that the flow in the initial supersonic expansion region, which is approximately of prandtl-meyer character, will be chemically frozen with respect to the molecular dissociation of the primary components under the hypersonic, high-altitude flight conditions considered . the flight conditions consist of flight velocities between furthermore, on bodies of small surface inclination beyond the nose, the flow will continue to be effectively frozen for at least 20 ft down-stream of the nose . these conclusions may lead to the simplification of procedures for theoretical calculation and testing . the problem of distinguishing a dimensionless length-reaction rate parameter, which characterizes the extent of departures from equilibrium or from frozen behavior in the flow fields of interest here, is discussed'), (575, 'atomic recombination in a hypersonic wind tunnel nozzle .', 'bray, k.n.c.', 'j. fluid mech. v.6 part 1, 1-32. july, 1959 .', 'atomic recombination in a hypersonic wind tunnel nozzle . the flow of an ideal dissociating gas through a nearly conical nozzle is considered . the equations of one-dimensional motion are solved numerically assuming a simple rate equation together with a number of different values for the rate constant . these calculations suggest that deviations from chemical equilibrium will occur in the nozzle if the rate constant lies within a very wide range of values, and that, once such a deviation has begun, the gas will very rapidly ''freeze'' . the dissociation fraction will then remain almost constant if the flow is expanded further, or even if it passes through a constant area section . an approximate method of solution, making use of this property of sudden ''freezing'' of the flow, has been developed and applied to the problem of estimating the deviations from equilibrium under a wide range of conditions . if all the assumptions made in this paper are accepted, then lack of chemical equilibrium may be expected in the working sections of hypersonic wind tunnels and shock tubes . the shape of an optimum nozzle is derived in order to minimize this departure from equilibrium . it is shown that, while the test section conditions are greatly affected by ''freezing'', the flow behind a normal shock wave is only changed slightly . the heat transfer rate and drag of a blunt body are estimated to be reduced by only about 25 per cent even if complete freezing occurs . however, the shock wave shape is shown to be rather more sensitive to departures from equilibrium .'), (576, 'viscous and inviscid stagnation flow in a dissociated hypervelocity free stream .', 'inger, g.r.', 'proc. 1962. heat transfer and fluid mech. inst .', 'limit characteristics . the analysis indicated that all these problems could be attributed to discontinuities in the performance characteristics of the front stages . such discontinuities can be due to the type of stage stall or to a deterioration of stage performance resulting blades is included .'), (577, 'on hypersonic similitude .', 'hayes,w.d.', 'q. app. math. 5, 1947, 105.', 'on hypersonic similitude . tsien in a recent paper (j. math. phys. mass. inst. tech. sonic flows around slender bodies and has pointed out that the product of mach number and fineness ratio is a basic similarity parameter . the author enlarges on this notion, indicating that the problem of hypersonic flow about a slender body in three dimensions is the same as that of a certain two-dimensional nonsteady flow (with time replacing the lengthwise spatial coordinate) characterized by essentially the same similarity parameter .'), (578, 'dissociation scaling for nonequilibrium blunt nose flows .', 'gibson, w.e.', 'ars journal. vol. 32pp. 285-287. 1962 .', 'compressor operation with one or more blade rows stalled . an analysis of the part-speed operating problems of high-pressure-ratio air .'), (579, 'further developments of new methods in heat flow analysis .', 'biot,m.a.', 'j.ae.scs.26, 1959.', 'further developments of new methods in heat flow analysis . lagrangian methods in heat-flow problems and transport phenomena were introduced by the writer in some previous work . the present paper develops further one particular aspect of the method,--i.e., the elimination of /ignorable coordinates ./ this is accomplished by a special choice of generalized coordinates, each of which is constituted by an arbitrary temperature distribution and an /associated flow field ./ the latter is a vector field which is derived from the corresponding scalar field by a variational method . the procedure is valid for a certain class of nonlinear problems, provided we replace the temperature by the heat content as the unknown . it is shown that for normal coordinates derivation of the associated flow field is immediate . the use of normal coordinates and their associated flow fields is illustrated by an example . introduction of dirac functions and associated flow fields yields a procedure which constitutes a generalization of the classical formulation by green''s functions and integral equations . this is illustrated by application to one-dimensional problems of heating of a homogeneous or composite slab and directly verified by classical methods in the appendix .'), (580, 'new thermo-mechanical reciprocity relations with application to thermal stress analysis .', 'biot,m.a.', 'j. ae. scs. 26, 1959.', 'new thermo-mechanical reciprocity relations with application to thermal stress analysis . based on the variational formulation of linear thermodynamics as developed previously by the writer, thermomechanical reciprocity relations are discussed which lead to new methods of analysis of thermal stresses . these reciprocity relations are quite different from the usual ones derived from the analogy of thermal loading with a combination of surface and body-force distribution . the results are applicable to stationary and transient temperatures in elastic and viscoelastic structures . the methods are entirely variational and do not require the evaluation of the temperature field . the stresses at one point are expressed directly in terms of any arbitrary distribution temperatures applied externally, including the effect of surface heat-transfer layer . the concepts and procedures are illustrated on a simple example . the relation is pointed out between the reciprocity property and the generalization of castigliano''s principle to thermomechanics .'), (581, 'approximate formulas for thermal-stress analysis .', 'd. j. johns', 'lecturer, the college of aeronautics, cranfield, england', 'approximate formulas for thermal-stress analysis . the basis of any thermal-stress analysis is the determination of the temperature distributions in the structure . for arbitrary flight histories, the determination of such distributions is rather tedious and not completely general . this latter fact handicaps optimization studies in the project design stage when it is desirable to be able to express the thermal-stress distributions in a general manner . in this note, general expressions are derived for the thermal-stress distributions in a typical i-section using similar assumptions to those of biot .'), (582, 'the melting of finite slabs .', 'goodman,t.r. and shea,j.j.', 'j. app. mech. 27, 1960, 16.', 'the melting of finite slabs . an approximate method, known as the heat-balance integral, is used to determine the melting rate of a finite slab which is initially at a uniform temperature below the melting point . the slab is acted upon by a constant heat input at one face and has its other face either insulated or kept at its initial temperature . the first three terms of series solutions in an intrinsically small parameter are obtained for the time histories of melting and the temperature distribution in the slab .'), (583, 'influence coefficients for real gases .', 'mario william cardullo', 'u.s. naval air rocket test station, lake denmark, dover, n.j.', 'influence coefficients for real gases . in the analysis of one-dimensional fluid-flow problems, it is often assumed that the behavior of the medium is that of a perfect gas . this assumption is justified, provided the pressure and temperature range of interest is small and near atmospheric . at higher pressures and temperatures various deviations are introduced thereby causing deviations from the results obtained by using the ideal fluid-flow equations . in this note, influence coefficients, similar to those developed by shapiro, are presented for the case of real gases . this analysis is based upon the use of various functions of the compressibility factor emmons . some of the assumptions made were as follows.. (1) the flow is one-dimensional and steady, (2) changes in the stream properties are continuous, and (3) the flow is comprised of imperfect gases'), (584, 'conduction of heat in a solid with a power law of heat transfer at its surface .', 'jaeger,j.c.', 'proc. cam.phil. s. 46, 1950, 634.', 'conduction of heat in a solid with a power law of heat transfer at its surface . the nonlinear boundary value problem, where and m are constants, is solved formally by first introducing power series in t for the unknown temperature and flux at the surface and then determining the coefficients in those series . in this manner the temperature function is determined as a series of repeated integrals of error functions . the convergence is rapid only for small values of t . the special cases and generalizations of the condition at the surface for which the same method applies, are noted . surface temperatures are also found by methods of difference equations, where t is not limited to small values . graphs of these temperatures corresponding to various laws of heat transfer at the surface are shown .'), (585, 'nonlinear heat transfer problem .', 'chambre,p.l.', 'j. app. phys. 30, 1959, 1683.', 'nonlinear heat transfer problem . a study has been made of the time-dependent heat conduction in a semi-infinite medium subject to a boundary condition which can involve the temperature in a nonlinear manner . a formulation for the determination of the surface temperature, which is often of greatest physical interest, leads to a nonlinear volterra integral equation . a simple iterative solution method, with an accuracy suitable for many practical purposes is presented . as an example, the problem of the time-dependent surface temperature of a body receiving heat according to the stefan-boltzmann law is treated . the analysis is also applicable to physical adsorption or chemisorption processes which occur at the boundary .'), (586, 'an approximate treatment of unsteady heat conduction in semi-infinite solids with variable thermal properties .', 'yang,k.t. and szewczyk,a.', 'j. heat trans. asme trans. 81, 1959, 251.', 'an approximate treatment of unsteady heat conduction in semi-infinite solids with variable thermal properties . this very short paper presents an approximate procedure for the calculation of unsteady heat conduction in semi-infinite solids with variable thermal properties . it is claimed to be an improvement over previous efforts in this area since it yields physically sensible results for cases where thermal properties have a large dependence on temperature . instead of using polynomials to represent an unsteady temperature profile an exponential form is used . good agreement is shown for several cases where the method of the paper is compared with exact solutions .'), (587, 'variational analysis of ablation .', 'm. a. biot and h. daughaday', 'consultant and principal research engineer, respectively, cornell aeronautical laboratory, inc., buffalo 21, n.y.', 'variational analysis of ablation . the variational and lagrangian thermodynamics developed in earlier publications are directly applicable to problems of heat conduction with melting boundaries . these techniques are used here in treating the problem of a half-space subjected to a constant rate of heat input at the melting surface (fig. 1) . the applicability of the lagrangian equations to this case follows from the fact that the basic variational principle is valid whether the boundaries are fixed or move as arbitrary functions of time . this can be seen if we remember that the equations govern only the instantaneous configuration of the flow rates for a given geometry and temperature field .'), (588, 'compressor operation with one or more blade rows stalled .', '', 'naca rm e56b03b, 1956 . chapter xiii', 'compressor operation with one or more blade rows stalled . an analysis of the part-speed operating problems of high-pressure-ratio ratio multistage axial-flow compressors was made by means of a simplified stage-stacking study . the principal problems considered were poor low-speed efficiency, multiple-valued performance characteristics at intermediate speeds, and poor intermediate-speed compressor surge or stall-limit characteristics . the analysis indicated that all these problems could be attributed to discontinuities in the performance characteristics of the front stages . such discontinuities can be due to the type of stage stall or to a deterioration of stage performance resulting from stall of adjacent stages . the effects of compromises of stage matching to favor part-speed operation were also considered . this phase of the study indicated that such compromises would severly reduce the complete-compressor-stall margin . furthermore, the low-speed stage stall problem is transferred from the inlet stages to the middle stages, which are more susceptible to abrupt-stall characteristics . the analysis indicates that inlet stages having continuous performance characteristics at their stall points are desirable with respect to part-speed compressor performance . these characteristics must, however, be obtained when the stages are operating in the flow environment of the multistage compressor . alleviation of part-speed operation problems may also be obtained by improvement in either stage flow range or stage- loading margin . the results of this analysis are only qualitative . the trends obtained, however, are in agreement with those obtained from experimental studies of high-pressure-ratio multistage axial-flow compressors, and the results are valuable in developing an understanding of the off-design problem . in addition to these stage-matching studies, a general discussion of variable-geometry features such as air bleed and adjustable blades is included .'), (589, 'some stall and surge phenomena in axial flow compressors .', '', '', 'some stall and surge phenomena in axial flow compressors . observations of rotating stall have shown that a wide variety of stall patterns is possible . hot-wire anemometer data on a multistage compressor have shown a progressive-type stall at low speeds . the amplitude of the flow fluctuations increases in magnitude through the first few stages and then diminishes rapidly to a small value in the latter stages . a stage-stacking analysis has shown that rotating stall will exist over a large portion of the compressor map at low speeds but will be instigated almost simultaneously with compressor surge at high speeds . blades failures attributable to resonant vibrations excited by rotating stall have been experienced in single and multistage compressors . in the stage-stacking analysis no deterioration of stage performance due to unsteady flow resulting from stall of adjacent stages was considered . in general, the pressure drop at the stall point is believed to be much larger than indicated by an analytical formulation of compressor performance . compressor surge is attributed to a limit cycle operation about the compressor stall point, and, as indicated in a few compressor tests and in jet-engine tests, a small compressor discharge receiver volume may result simply in stall of the compressor without the cyclic characteristics of compressor surge . in this event, engine operation will be limited because of the large drop in performance which accompanies compressor stall .'), (590, 'effects of stage characteristics and matching on axial flow compressor performance .', 'stone, a.', 'trans. asme, v. 80, p. 1273, 1958 .', 'effects of stage characteristics and matching on axial flow compressor performance . the use of stage characteristics obtained from test data in the performance analysis and development of an axial-flow compressor is described relative stage matching as shown by an idealized example and also by test experience . factors governing major performance parameters are discussed and certain development problems and possible solutions are reviewed .'), (591, 'an approximate equation for the /choke line/ of a compressor .', 'csanady, g.t.', 'j. aero. sc. p. 637, august 1960 .', 'an approximate equation for the /choke line/ of a compressor . discussion of a similarity between the pressure-ratio versus inlet-mass-flow-coefficient characteristic of a stream or gas turbine and the analogous characteristic of an expansion /laval/ nozzle . this idea is extended to a compressor and a compression nozzle, and an approximate expression for the /choke line/ of the compressor is developed .'), (592, 'design of axial compressors .', 'howell, a.r.', 'proc. i mech. e. v. 153, 1945 .', 'design of axial compressors . the main types of axial compressors are described, and the use of generalized design curves to make performance estimates is advocated . the different variables are weight, power, pressure ratio, temperature rise, mass flow, rotational speed, stage efficiency, blade bending stresses due to aerodynamic loading, and methods and materials of construction . air outlets, flow coefficients and different blade forms are also considered .'), (593, 'theoretical considerations of flutter at high mach number .', 'morgan,h.g., runyam,h.l. and huckell,v.', 'j. ae. scs. 25, 1958.', 'theoretical considerations of flutter at high mach number . some of the theories for two-dimensional oscillatory air forces which may be applied in flutter calculations at high mach numbers are discussed . these include linear theory, van dyke''s second-order theory, piston theory, landahl''s method, tangent-wedge and tangent-cone approximations, newtonian theory, and a new nonlinear-pressure method . a comparison of the theories is made by showing the results of flutter calculations for mach numbers up to 10, and the possibility of flutter at these higher mach numbers is pointed out . results of flutter calculations are shown to illustrate the various effects arising from a nonlinear thickness theory . the possibility of large flutter speed thickness effects which depend on frequency ratio is shown . the influence of airfoil shape is discussed and flutter speed trends with center of gravity and elastic axis locations are presented . some possible refinements of piston theory are discussed for use at very high mach numbers . these include the use of local flow conditions and the use of newtonian theory over the leading edge of a blunt-nosed airfoil .'), (594, 'wind tunnel techniques for the measurements of oscillatory derivatives .', 'bratt, j.b.', 'arc 22, 146, 1960', 'wind tunnel techniques for the measurements of oscillatory derivatives . this paper discusses the basic principles employed in techniques for the measurement of oscillatory derivatives in wind tunnels, and gives some account of the associated instrumentation . the suitability of the various techniques for different test conditions is also discussed and brief reference is made to wind tunnel effects on the measurements .'), (595, 'the equilibrium piston technique for gun tunnel operation .', 'east,r.a. and pennelegion,l.', 'arc 22, 852, 1961.', 'the equilibrium piston technique for gun tunnel operation . a modified technique for the operation of a gun tunnel is suggested based on experimental results . if the piston mass and the initial barrel pressure are chosen correctly, then the peak pressures associated with the gun tunnel may be eliminated . under these conditions the piston is brought to rest with no overswing . some measurements of the piston motion using a microwave technique are reported which confirm this idea . the wave diagram associated with this mode of operation is shown, and some calculations of the stagnation pressure are given which show that during the suggested running time, the stagnation pressure may be considerably greater than the driving pressure if the driving chamber cross-sectional area is large compared with that of the driven section . for a uniform shock tube the stagnation pressure will always be less than the driving pressure . the use of air, helium and hydrogen as driving gases has been considered . experiments in a gun tunnel are reported which show that the equilibrium piston technique enables steady stagnation pressures to be achieved over a time of approximately 15 ms using air as the driving gas . the expansion caused by the piston acceleration is shown to interact with the stationary piston, but this is found to produce only a small drop in stagnation pressure .'), (596, 'the properties of crossed flexure pivots, and the influence of the point at which the strips cross .', 'wittrick, w.h.', 'aero. quart. v. 2, 1950-1 .', 'the properties of crossed flexure pivots, and the influence of the point at which the strips cross . it is shown that the rotational stiffness of a crossed flexure pivot varies considerably when subjected to an applied force . the type of variation can be radically changed simply by moving the point at which the strips cross . the relation between torque and rotation for a given applied force is not exactly linear and the extent of the non-linearity is determined by taking into account the small movements of the centre of rotation of the pivot . finally, for design purposes, an analysis of the maximum stresses in the strips is given .'), (597, 'measurements of pitching moment derivatives for blunt-nose aerofoils oscillating in two-dimensional supersonic flow .', 'pugh, p.g. and woodgate, l.', 'arc. 23, 012, 1961 .', 'measurements of pitching moment derivatives for blunt-nose aerofoils oscillating in two-dimensional supersonic flow . direct pitching moment derivatives have been measured using the method of scruton, woodgate et al for two single wedge blunt-nosed aerofoils . these measurements were made at mach numbers of 1.75 and 2.47 and frequency parameters less than 0.02 . in general, nose blunting was found to have little effect on the derivatives although changes were observed for the thinner wedge at a mach number of 1.75 .'), (598, 'new test techniques for a hypervelocity wind tunnel .', 'stalmach, c.j. and cooksey, j.m.', 'aerospace engineering. vol. 21. march 1962 .', 'new test techniques for a hypervelocity wind tunnel . the measurement of rocket exhaust effects on vehicle stability and the measurement of aerodynamic damping were made in an arc-discharge type of hypervelocity wind tunnel . sample data are given to indicate the quality of data obtainable in this tunnel, and samples of self-luminous and shadowgraph photographs are also presented .'), (599, 'aerodynamic forces, moments and stability derivatives for slender bodies of general cross section .', 'sacks,a.h.', 'naca tn.3283, 1954.', 'aerodynamic forces, moments and stability derivatives for slender bodies of general cross section . the problem of determining the total forces, moments, and stability derivatives for a slender body performing slow maneuvers in a compressible fluid is treated within the assumptions of slender-body theory . general expressions for the total forces (except drag) and moments are developed in terms of the geometry and motions of the airplane, and formulas for the stability derivatives are derived in terms of the mapping functions of the cross sections . all components of the motion are treated simultaneously and second derivatives as well as first are obtained, with respect to both the motion components and their time rates of change . coupling of the longitudinal and lateral motions is thus automatically included . a number of general relationships among the various stability derivatives are found which are independent of the configuration, so that, at most, only 35 of a total of 325 first and second derivatives need be calculated directly . calculations of stability derivatives are carried out for two triangular wings with camber and thickness, one with a blunt trailing edge, and for two wing-body combinations, one having a plane wing and vertical fin . the influence on the stability derivatives of the squared terms in the pressure relation is demonstrated, and the apparent mass concept as applied to slender-body theory is discussed at some length in the light of the present analysis . it is shown that the stability derivatives can be calculated by apparent mass although the general expressions for the total forces and moments involve additional terms .'), (600, 'the calculation of lateral stability derivatives of slender wings at incidence including fin effectiveness, and correlation with experiment .', 'ross,a.j.', 'rae r.aero.2647, 1961.', 'the calculation of lateral stability derivatives of slender wings at incidence including fin effectiveness, and correlation with experiment . comparisons are made between low-speed experimental results and estimates based on attached-flow theory for the lateral stability derivatives of slender wings at incidence, and it is found that the flow separation has little effect on the sideslip derivatives . the reduction in due to part-span anhedral is evaluated, and a semi-empirical formula is derived to account for important second-order terms . for the rotary derivatives, an attempt is made to estimate the effect of the leading edge vortices, but no satisfactory conclusions have been reached . the fin contributions to the derivatives are evaluated on the basis of treating the wing surface as a total reflection plate . good agreement with experiment is reached for the sideslip derivatives, and for the damping-in-yaw at moderate incidences . sidewash is found to have a large effect on the rolling derivatives, and further information on the strength and position of the leading edge vortices in non-symmetric flow is required before a complete calculation of the sidewash can be given .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (601, 'calculation of the flow past slender delta wings with leading edge separation .', 'mangler,k.w. and smith,j.', 'rae r. aero.2593, 1957.', 'calculation of the flow past slender delta wings with leading edge separation . the flow past a slender delta wing with a sharp leading edge, at incidence, usually separates along this edge, i.e. a vortex layer extends from the edge and rolls up to form a /core/ (a region of high vorticity) . a potential flow model of this is constructed in which the layer is replaced by a vortex sheet which is rolled up into a spiral in the region of the /core/ . this problem is reduced to a two-dimensional one by assuming a conical field and using slender wing theory . the shape and strength of the sheet are determined by the two conditions that it is a stream surface and sustains no pressure difference . use is made of results previously obtained for the core region and the remaining finite part of the sheet is dealt with by choosing certain functions for its shape and strength . the parameters in these functions are found by satisfying the two conditions stated above at isolated points . results are obtained for the pressure distribution, chord loading and norman force coefficient as functions of the ratio of the incidence to the apex angle . the lift for a given incidence is about 15 below that found by brown and michael . flow patterns are indicated in two typical cases . the effect of separation on the drag due to lift of a wing with small thickness is discussed .'), (602, 'the 7 x 7 in . hypersonic wind tunnel at rae farnborough, part 1, design, instrumentation and flow visualization techniques .', 'crabtree,l.f. and crane,j.', 'rae tn.aero.2716.', 'the 7 x 7 in . hypersonic wind tunnel at rae farnborough, part 1, design, instrumentation and flow visualization techniques . this is the first of three parts of the calibration report on the r.a.e. some details of the design and lay-out of the plant are given, together with the calculated performance figures, and the major components of the facility are briefly described . the instrumentation provided for the wind-tunnel is described in some detail, including the optical and other methods of flow visualization used in the tunnel . later parts will describe the calibration of the flow in the working-section, including temperature measurements . a discussion of the heater performance will also be included as well as the results of tests to determine starting and running pressure ratios, blockage effects, model starting loads, and humidity of the air flow .'), (603, 'the 7 in. x 7 in. hypersonic wind tunnel at r.a.e. farnborough part ii. heater performance .', 'j. f. w. crane', '', 'the 7 in. x 7 in. hypersonic wind tunnel at r.a.e. farnborough part ii. heater performance . tests on the storage heater, which is cylindrical in form and mounted horizontally, show that its performance is adequate for operation at m=6.8 and probably adequate for flows at m=8.2 with the existing nozzles . in its present state, the maximum design temperature of 680 degrees centigrade for operation at m=9 cannot be realised in the tunnel because of heat loss to the outlet attachments of the heater and quick-acting valve which form, in effect, a large heat sink . because of this heat loss there is rather poor response of stagnation temperature in the working section at the start of a run . it is hoped to cure this by preheating the heater outlet cone and the quick-acting valve . at pressures greater than about 100 p.s.i.g. free convection through the fibrous thermal insulation surrounding the heated core causes the top of the heater shell to become somewhat hotter than the bottom, which results in /hogging/ distortion of the shell . this free convection cools the heater core and a vertical temperature gradient is set up across it after only a few minutes at high pressure . modifications to be incorporated in the heater to improve its performance are described .'), (604, 'the 7 in. x 7 in. hypersonic wind tunnel at r.a.e., farnborough part iii - calibration of the flow in the working section .', 'j. f. w. crane', '', 'the 7 in. x 7 in. hypersonic wind tunnel at r.a.e., farnborough part iii - calibration of the flow in the working section . the fused silica nozzle to give m=7 in the 7 in. x 7 in. hypersonic wind tunnel produces a flow field with an average mach number of 6.85 along the centreline of the working section . the mach number gradually decreases towards the boundary layer, and over a core of approximately mach number . the nozzle heats up during a run but this has little effect on the mach number distribution . at one station the mach number was one-third per cent greater for a run of 1 minute than for a run of 10 seconds . the temperature field in the inviscid flow has an average variation of in temperature with time throughout a run .'), (605, 'pressure measurements on a cone-cylinder-flare configuration at small incidences for m 6.8 .', 'woodley, j.g.', 'r.a.e. tn. aero. 2739 .', 'pressure measurements on a cone-cylinder-flare configuration at small incidences for m 6.8 . pressure measurements were made on a slender cone-cylinder-flare configuration, slightly blunted at the nose, for 0, 3 and 6 degrees incidence at a free-stream mach number of 6.8 . it was found that the surface pressures obtained on the cone agreed with extrapolations to m equals 6.8 of theoretical values given in m.i.t. tables /kopal/for yawed cones, and that impact theory gave a good indication of the pressure level to be expected on all parts of the body where surface incidence was sufficiently large to merit its use . the semi-angles of the conical and flared parts of the model were both the pressure level on the flare rose in all cases to approximately that developed upstream on the cone surface . no evidence of a marked over-expansion to pressures below the free-stream value was noticed at the junction between cone and cylinder .'), (606, 'formulae and approximations for aerodynamic heating rates in high speed flight .', 'monaghan, r.j.', 'rae. tn. aero. 2407 .', 'formulae and approximations for aerodynamic heating rates in high speed flight . this note gives formulae and approximations suitable for making preliminary estimates of aerodynamic heating rates in high speed flight . the formulae are based on the /intermediate enthalpy/ approximation which has given good agreement with theoretical and experimental evidence . in the general flight case they could be used in conjunction with an analogue computer or a step-by-step method of integration to predict the variations of heat flow and skin temperature with time . in the restricted case of flight at constant altitude and mach number, simple analytical methods and results are given which include the effects of radiation and can be applied to /thick/ as well as /thin/ skins where h is the aerodynamic heat transfer factor, and g, d and k are the heat capacity, thickness and thermal conductivity of the skin . if 0.1 the skin is approximately /thin/, i.e. temperature gradients across its thickness may be neglected .'), (607, 'duct flow in magnetohydrodynamics .', 'chang,c.c. and lundgen,t.s.', 'z.angew. math.phys., 12 100-114, 1961.', 'duct flow in magnetohydrodynamics . this paper is an extension of the work of hartmann (2) and shercliff transverse magnetic fields -- the simplest class of magnetohydrodynamic problems . we are concerned here mainly with the boundary value problems associated with flow in ducts with conducting walls .'), (608, 'aerodynamic noise in supersonic wind tunnels .', 'laufer, j.', 'j. aero. sc. v. 28, sept. 1961 .', 'aerodynamic noise in supersonic wind tunnels . hot-wire measurements in the free stream of a supersonic wind tunnel were made in the mach number range of 1.6 to 5.0 . it is shown that the mass-flow fluctuations increase very rapidly with increasing mach number . if the fluctuation field is assumed to consist of sound waves-dash an assumption that is consistent with the measurements-dashthe sound intensity is approximately proportional to m, within the range of the experiments . furthermore, the orientation of the field is found to be different from the mach line direction,. it corresponds to a sound-source velocity of approximately one-half the free-stream velocity for the higher mach numbers . it is shown that the turbulent boundary layer along the nozzle and the tunnel walls is responsible for this sound field .'), (609, 'on three dimensional bodies of delta planform which can support plane attached shock waves .', 'peckman,d.h.', 'rae tn. aero.2812.', 'on three dimensional bodies of delta planform which can support plane attached shock waves . this note collects together in one report available theoretical work on bodies which can support attached plane shock waves, discusses some of the possible merits of such shapes, and includes some calculations illustrating their properties . also, some preliminary results from wind tunnel tests are given, together with details of proposed future tests .'), (610, 'corner interference effects .', 'gersten,k.', 'agard r.299, 1959.', 'corner interference effects . the three-dimensional incompressible flow of fluid along the corner of two semi-infinite plates intersecting at right angles, especially the interference of the boundary layers of the two plates, is discussed . mainly, the more important case of turbulent boundary layer is treated by means of experimental studies carried out at the technical university of braunschweig . some theoretical results for laminar flow are also taken into account . in order to describe the interference effects in the boundary layer, an interference displacement thickness and an interference skin friction have been introduced . it is shown from experiments and also from theoretical considerations how these two quantities depend on reynolds number . furthermore, the influence of interference on the transition from laminar to turbulent flow is investigated . in addition, some preliminary results are given about the effect of the pressure gradient on the interference effects .'), (611, 'an approximate solution of the compressible laminar boundary layer on a flat plate .', 'monaghan,r.j.', 'rae tn. aero.2025.', 'an approximate solution of the compressible laminar boundary layer on a flat plate . following a major assumption that enthalpy and velocity are dependent only on local conditions, an enthalpy-velocity relation is obtained for the laminar boundary layer on a flat plate where subscripts p refer to the plate, 1 to the free stream and e to the equilibrium temperature condition at the plate . when compared with general results, this relation (exact for prandtl number o = 1) gives a close approximation to crocco''s numerical results for o = 0.725 and 1.25, up to . using the above relation in conjunction with the approximate viscosity-temperature relation suggested by chapman and rubesin, and with young''s suggested first approximation for shearing stress it is shown that close approximations to displacement thickness and velocity distribution are given by and where and which serves to define c . these have the advantage of being algebraic in form whereas previous results have involved complex numerical integrations for individual cases .'), (612, 'pressure distributions and flow patterns at m=4 . on some delta wings of inverted ''v'' cross section .', 'squire,l.c.', 'rae tn.aero.2838, 1962.', 'pressure distributions and flow patterns at m=4 . on some delta wings of inverted ''v'' cross section . wind tunnel tests have been made to measure pressure distributions and to study flow patterns on a series of delta wings of inverted ''v'' cross-section . each of these wings was designed to have a plane shock wave in the plane of the leading edges at a chosen mach number and incidence . it was found that for a wide incidence range about the design point the shock wave remained virtually attached to the leading edges and at each incidence the pressure was approximately constant over the lower surface .'), (613, 'the contraction of satellite orbits under the influence of air drag part i . with spherically symmetrical atmosphere .', 'king-hele, d.g., cook, g.e. and walker, d.m.c.', 'r.a.e. tn. gw. 533. 1959 .', 'the contraction of satellite orbits under the influence of air drag part i . with spherically symmetrical atmosphere . the effect of air drag on satellite orbits of small eccentricity e/0.2/ is studied analytically by a perturbation method, on the assumption that the atmosphere is spherically symmetrical . equations are derived which show/1/how orbital period and perigee distance vary with eccentricity as the orbit contracts, and/2/how each of these quantities varies with time . the equations of type/1/are nearly independent of the oblateness of the atmosphere . in all the equations, terms of order e and higher are usually neglected . the results are also presented graphically, in a manner designed for practical use . the theory is to be extended to an oblate atmosphere in part ii, and will later be compared with observation .'), (614, 'the contraction of satellite orbits under the influence of air drag .', 'part ii . with oblate atmosphere .', 'king-hele, d.g., cook, g.e. and walker, d.m.c. r.a.e. tn. gw. 565. 1960 .', 'the contraction of satellite orbits under the influence of air drag . part ii . with oblate atmosphere . the effect of air drag on satellite orbits of small eccentricity e/0.2/ was studied in part i/technical note no.g.w.533/on the assumption that the atmosphere was spherically symmetrical . here the theory is extended to an atmosphere in which the surfaces of constant density are spheroids of arbitrary small ellipticity . equations are derived which show how perigee distance and orbital period vary with eccentricity, and how eccentricity is related to time . expressions are also obtained which give lifetime and air density at perigee in terms of the rate of change of period . in most of the equations, terms of order e and higher are neglected . the results take different forms according as the eccentricity is greater or less than about 0.025, while circular orbits are dealt with in a separate section . the results are also presented graphically in a manner designed for practical application, and examples of the theory in use are given . the influence of atmospheric oblateness is difficult to summarize fairly simultaneously assume their''worst''values, some of the spherical-atmosphere results can be altered by up to 30( as a result of oblateness and 5-10( would be a more representative figure .'), (615, 'the contraction of satellite orbits under the influence of air drag . part iii . high eccentricity orbits . /0.2 e 1/ .', 'king-hele, d.g.', 'r.a.e. tn. space 1, 1962 .', 'the contraction of satellite orbits under the influence of air drag . part iii . high eccentricity orbits . /0.2 e 1/ . the effect of air drag on satellite orbits of eccentricity e less than between 0.2 and 1 is presented . equations are derived which show how perigee distance and orbital period vary with eccentricity during the satellite''s life, and how eccentricity is related to time,.and formulae are obtained for the lifetime and the air density at perigee, in terms of the rate of change of period . the results are also presented graphically and their implications and limitations are discussed .'), (616, 'determination of upper-atmosphere air density and scale height from satellite observations .', 'groves, g.v.', 'proc. roy. soc. a. 252, 16-27, 1959 .', 'determination of upper-atmosphere air density and scale height from satellite observations . a solution is obtained for the rate of change of semi-major axis and perigee distance of a satellite orbit with time due to the resistance of the atmosphere . the logarithm of air density is assumed to vary quadratically with height, and the oblateness of the atmosphere is taken into account . the calculation of perigee air density in terms of the rate of change of satellite period is dealt with,. and the method is applied to data at present available on six different satellites . the variation of air density with height is obtained as in p-28.59/0.15/-h-200//46/5/0.028/0.013//h-200///46/ for h in the range of approximately 170 to 700 km, where p is in grams/c m, h is in kilometres and standard deviations are given in brackets .'), (617, 'determination of upper-atmosphere air density profile from satellite observations .', 'groves, g.v.', 'proc. roy. soc. a. 252, 28-34, 1959 .', 'determination of upper-atmosphere air density profile from satellite observations . the theory previously developed for the changes in the perigee distance and semi-major axis of a satellite orbit due to air drag is extended to enable the air-density profile/i.e.its relative variation with height/to be derived from the motion of the orbit''s perigee . the solution is first obtained in terms of the change in perigee distance and then in terms of the change in the radius of the earth at the sub-perigee point the scale height in the 180 and 220 km altitude regions .'), (618, 'orbit decay and prediction of the motion of artificial satellites .', 'michielsen, h.f.', 'advances in astronautical science, vol 4 plenum press 1959 . pp 255-310', 'orbit decay and prediction of the motion of artificial satellites . the rate of decay of elliptic satellite orbits, due to atmospheric drag, is investigated through variation of parameters and through use of an atmospheric model involving a power function between density and altitude . this model is shown to fit actual conditions better than an exponential function . the effects of the equatorial belt and the rotation of the earth are investigated . the conclusion is reached that through these anomalies atmospheric drag substantially affects the orbit elements, especially those defining the orbit plane . an alternate approach of variation of parameters is presented, by which a direct relation between period decay and instantaneous density conditions is established . this approach, by itself specifically adequate for prediction work, also opens an avenue for systematic and unified evaluation of observed decay .'), (619, 'density of the upper atmosphere from analysis of satellite orbits .. further results .', 'king-hele, d.g.', 'nature, v184, 1267-1270, 1959 . r.a.e. rep. gw. 25. appendix y .', 'density of the upper atmosphere from analysis of satellite orbits .. further results . the method previously described has been refined by taking into account atmospheric rotation . further results are given from satellites of latitude and season and day-to-night changes are reported .'), (620, 'earth satellite observations and the upper atmosphere .', 'priester, w., martin, h.a. and kramp, k.', 'nature, 188, pp 200-204. 1960 .', 'earth satellite observations and the upper atmosphere . atmospheric densities have been derived from artificial satellites in altitudes 200-700 km. and from rockets up to about 200 km. to consolidate the two sets of data, h.k. kallmann suggested a model with a exact form of this curve has now been derived . corrections for the is excellent . very close correlation between atmospheric density variations/h180 km./ and the solar 20-cm. radiation implies that the origin of the''solar effect''may lie in the absorption of solar ultra-violet radiation . the atmospheric density curve between 180 and 200 km. shows a temperature inversion in the fl-layer . it is not yet possible to decide whether solar ultra-violet radiation as well as the solar he line and solar x-ray radiation contribute to the heating of the fl-layer . diurnal and seasonal density variations at altitudes 210, 562 and 660 km. have been derived from variations in acceleration of three satellites/sputnik 3, vanguard 1 and 2/ . group averages of diurnal variations are taken from different dates within the period may 15, 1958-october 1, 1959 . physcal conditions in the upper atmosphere are briefly summarized ..the''solar effect''originates in the fl-layer as a result of heating by the solar he line at 304 a. diurnal density variation at 210 km. is only a few per cent . absorption of solar electromagnetic radiation in the f2-layer, and large heat conductivity cause intense diurnal density and temperature variations above'), (621, 'latitude and diurnal variations of air densities from 190 to 280 km. as derived from the orbits of discoverer satellites .', 'groves, g.v.', 'proc. roy. soc. a. 263. 212-216. 1961 .', 'latitude and diurnal variations of air densities from 190 to 280 km. as derived from the orbits of discoverer satellites . variations in air density between day and night in the region 190 to 280 km are found to be small/less than about 25(/ . the presence of a possible region of local heating at about 220 km which disappears at night . the night-time density profile conforms with a constant scale height of 35/2/km.no definite variation of air density with latitude is evident apart from a possible increase of about 60(, which is indicated by rather limited polar-region data . for other latitudes and seasons a variation of less than about 20( is indicated .'), (622, 'scale height in the upper atmosphere, derived from changes in satellite orbits .', 'king-hele, d.g. and hughes, k.m.', 'r.a.e. tn space 4, 1962 .', 'scale height in the upper atmosphere, derived from changes in satellite orbits . the''density scale height''h in the upper atmosphere is a measure of the rate at which air density p varies with height y, being given by h-p//dp/dy/ . the value of h, although important because/with the molecular weight of the air/it determines the air temperature, has not as yet been well determined at heights above 200 km . this note develops methods for finding h from the decrease in a satellite''s perigee height and from the decrease in the orbital period of a satellite in a small-eccentricity orbit . these methods are then applied to all the 14 satellites found suitable for the purpose . the 44 values of h obtained, for heights of 200-450 km, represent an average over day and night and probably have errors/s.d./of 5-10( . it is found that, as solar activity declined between 1957 and 1961, h decreased greatly ..e.g.at height 275 km, h decreased from 60 km in early 1958 to height becomes much less rapid above 350 km, and are consistent with the supposition that h had low values, near 35 km, at heights near 250 km, for 1959-61 . the results could be greatly extended in scope and improved in accuracy if more accurate orbits were available for short-lifetime satellites .'), (623, 'on the coupling between heat and mass transfer .', 'tewfik,o.e.', 'j. ae. scs. 28, 1962, 1009.', 'on the coupling between heat and mass transfer . in mixtures of two different gases or liquids, one constituent will migrate spontaneously toward the warmer parts, and the other toward the colder parts . this phenomenon, known as the soret effect, and its converse the dufour effect, were discovered as early as 1856 and 1873 respectively . the two effects can also be considered as a simultaneous transport of mass and heat, or as a coupling between heat and mass transfer . the effects of this coupling have been neglected in all investigations of heat transfer in multicomponent flow systems so far, on the a priori assumption that they are small . in a recent publication however, it was shown that they can be large in laminar-boundary-layer-type flows with helium injection . turbulent-boundary-layer measurements and an analysis conducted at the heat transfer laboratory clearly showed significant effects of the coupling on heat transfer and adiabatic wall temperature . from additional measurements, the results of which are presented below, it is possible to separate the heat flux at the model wall into one part depending on the temperature gradient and a second part caused by the coupling . it is shown that the latter exceeds the former, and hence the coupling may not be neglected a priori without careful consideration .'), (624, 'cruise performance of channel-flow ground effect machines .', 'strand,t.', 'j. ae. scs. 29, 1962, 702.', 'cruise performance of channel-flow ground effect machines . the performance theory for high-speed air-cushion vehicles operating in close proximity to the ground is developed . the analysis is restricted to cruise flight of vehicles of rectangular planform employing an air pressure seal between the ground and the vehicle along the two streamwise sides . the variation of the optimum rearward deflection angle of the side jet pressure seal with speed for minimum overall power expenditure and maximum range is found . it is concluded that a mixed propulsion system (jet deflection plus propeller(s)) is required . volume flow and the corresponding fan pressure rise needed are also calculated . the maximum lift drag ratio is determined . the maximum thickness ratios of the vehicles are considered to be large compared with the ground-height vehicle-length ratio . two-dimensional airfoil theory is employed to show that close to stagnation conditions exist below the vehicles . the lower-surface lift, pitching moment, and aerodynamic-center location are determined . the flow over the upper surface is identified with flow over mounds . upper-surface lift coefficients are determined for typical mound shapes . it is shown that high total lift coefficients are theoretically obtainable with almost zero induced drag . the conventional induced-drag power penalty is replaced by a sealing-air power expenditure, which is shown not to be excessive .'), (625, 'viscous and inviscid nonequilibrium gas flows .', 'whalen,r.j.', 'j. ae. scs. 29, 1962, 1222.', 'viscous and inviscid nonequilibrium gas flows . the condition of immediate freezing of the mass fraction of dissociated species of air at the equilibrium value behind the shock envelope prevails over a major portion of the flight spectrum associated with lifting re-entry vehicles . this is observed by means of order-of-magnitude considerations within the limits of the present knowledge of chemical reaction rates for the constituents of air . accordingly, investigations of the viscous and inviscid hypersonic flow about blunt and sharp leading edge slender bodies are made . the investigations are generalized to consider an arbitrary degree of dissociation in the ambient free stream . this condition is included in order to allow comparison with the flow field about a model in the test section of a hypersonic facility with dissociated air species present in the free stream . inviscid frozen flow investigations are made for blunt and sharp leading edge slender body power-law geometries . the results indicate that the influence of a finite leading edge, in inducing a pressure field far downstream (/blast-wave/ analogy), is considerably diminished for this model . this conclusion is verified numerically by a characteristics solution for the hypersonic flow about a /sonic-wedge/ slab . the viscous investigations consider the boundary-layer interaction problem with a frozen degree of dissociation . in this case, as in the inviscid analysis, the governing parameter is observed to be the ratio of the dissociation energy to the free-stream kinetic energy . the influence of this parameter on the boundary-layer interaction mechanism for a highly cooled, noncatalytic wall is presented . the influence of a frozen flow field on skin friction and heat transfer is also discussed . finally, since higher mach number gas flows may be generated in wind tunnel nozzles where dissociation nonequilibrium effects are present, the possibility of employing expansions with a controlled degree of dissociation as a technique for aerodynamic simulation is presented .'), (626, 'some features of supersonic and hypersonic flow about blunted cones .', 'traugott,s.c.', 'j. ae. scs. 29, 1962, 389.', 'some features of supersonic and hypersonic flow about blunted cones . for a family of cones of various semiapex angles blunted by spherical caps, shock shapes and surface pressure distributions have been obtained from both the belotserkovskii method and experiment . these results are used to study convergence to conical flow . conditions leading to both overexpansion and underexpansion on the surface with respect to the asymptotic conical pressures are described as well as conditions leading to bow shock inflection points . conditions also exist for which a second shock may occur, or for which the sonic line cannot touch the body surface . the implications of these conditions for various blunt-body methods are discussed . for cones blunted in such a manner as to keep the flow entirely supersonic, the flow field is found to exhibit certain similarities with that for genuine blunting . this is related to the fact that the surface entropy layer for blunt bodies can be most influential, in determining surface pressure, in the interior of the flow field rather than near the surface .'), (627, 'flutter analysis of circular panels .', 'rattayya,j.v.', 'j. ae. scs. 20, 1962, 534.', 'flutter analysis of circular panels . the flutter problem of flat circular panels with edges elastically restrained against rotation has been formulated in terms of small-deflection plate theory . the panel is subjected to uniform all-round tension or compression in its middle plane, in addition to the supersonic compressible flow passing over its upper surface with still air below . linear piston theory is employed to predict the aerodynamic load on the vibrating panel . the problem is investigated by a rayleigh-type analysis involving chosen modes of the panel as degrees of freedom . in order to investigate the convergence of the solution, the flutter-mode shape of the clamped-edge panel has been expressed in a series form in powers of r cos o . the results of three-, four-, and five-term approximations have displayed oscillatory behavior with apparently rapid convergence of the solution .'), (628, 'thermal effects on a transpiration cooled hemisphere .', 'gollnick,a.f.', 'j. ae. scs. 29, 1962, 583.', 'thermal effects on a transpiration cooled hemisphere . an approximate method is used to obtain the injection distribution which would exist on an isothermal, transpiration-cooled hemisphere in a supersonic stream . this distribution is the same for both air and helium injection, and is independent of the blowing level . a model having this distribution was tested in the naval supersonic laboratory wind tunnel at a mach number of 3.53 . it is concluded that the design technique is reasonably accurate . data taken near the nose are compared with the theories for air and helium injection . the agreement in the case of the reduction in heat-transfer coefficient is good . the values of insulated wall temperature obtained near the nose with helium injection are 8 percent above the local stagnation temperature, and largely independent of injection rate . it is believed that this phenomenon may be attributed to the thermal diffusion of the helium within the boundary layer . air injection causes a slight reduction in the insulated wall temperature . it is shown that injection of either air or helium at the hemisphere nose considerably reduces the heat flux at the surface . the additional reduction in heat flux resulting from helium injection as opposed to air injection, and predicted by existing theory, is largely absent .'), (629, 'second-order effects in laminar boundary layers .', 'maslen,s.h.', 'a.i.a.a. j. 1963, 33.', 'second-order effects in laminar boundary layers . second-order boundary layer disturbances are due to the displacement of the main flow by the boundary layer, surface curvature, freestream vorticity, and slip . a procedure for finding these is given for compressible flow of a perfect gas having a classically similar boundary layer . solutions are given for the flat plate and circular cylinder and for the hypersonic axisymmetric stagnation point . for the latter flow, the dominant effect is that of vorticity, which increases both shear and heat flux . for the plate or cylinder, the same conclusion tends to hold for high speed flow . the vorticity effect is governed by the entire outer flow--not just the wall vorticity .'), (630, 'stagnation region in rarefied high mach number flow .', 'cheng,h.k. and chang,a.l.', 'a.i.a.a. j. 1963, 231.', 'stagnation region in rarefied high mach number flow . paper describes results of numerical solution of the viscous shock-layer equations for axisymmetric stagnation region, using the viscosity-temperature law with w=0.65, pr=0.71 and y=1.25 . purpose is to establish applicability of the simple approximation of w=1 (obtained earlier) to air at low reynolds numbers and low ratios of wall temperature to stagnation temperature . using a reference temperature (closely equal to eckert''s) to interpret the linear results, excellent agreement is found, in the limit of, over a wide range of reynolds numbers, covering fully merged shock layers as well as boundary layers with and without vorticity interaction . agreement with recent experiments of ferri et al is as good as to be expected from shock-layer approximation . paper provides valuable extension of the applicability of the reference temperature concept .'), (631, 'low speed wind tunnel tests on a two dimensional aerofoil with split flap near the ground .', 'bagley,j.a.', 'arc cp.568, 1961.', 'low speed wind tunnel tests on a two dimensional aerofoil with split flap near the ground . pressure distributions have been measured on a 10 thick two-dimensional aerofoil of r.a.e.101 section fitted with split flaps deflected at 15 and 55 . measurements were made at two distances above a ground plate, and also without the ground plate . the results have been integrated to give the sectional lift, drag and pitching-moment coefficients .'), (632, 'calculated lift distributions in incompressible flow on some sweptback wings .', 'bagley,j.a. and joyce,g.m.', 'rae tn. aero.2836, 1962.', 'calculated lift distributions in incompressible flow on some sweptback wings . in the course of a larger survey of some aerodynamic characteristics of a family of sweptback wings, the low-speed lift distributions were calculated . the 35 planforms considered cover a range of leading-edge sweep angles from 55 to 70, and aspect ratios from 2 to 3.9 . the results are given here, together with a comparison with other calculations and with experimental results on one particular wing .'), (633, 'an extension of the method of generalised conical flows for lifting wings in supersonic flow .', 'portnoy,h.', 'rae tn. aero.2849, 1962.', 'an extension of the method of generalised conical flows for lifting wings in supersonic flow . the method of generalised conical flows has previously been developed subject to the condition that the upwash divided by the streamwise co-ordinate to the power k, where k is the order of the conical flow, must have vanishing (k+1)th derivative with respect to the conical co-ordinate . in the present note this restriction is removed . the results are also used to discuss the effect of the application of the leading edge attachment condition on the wing pressure and geometry .'), (634, 'effects of leading edge bluntness on flutter characteristics of some square- planform double-wedge airfoils at a mach number of 15 .4.', 'goetz,r.c.', 'nasa tn.d1487, 1962.', 'effects of leading edge bluntness on flutter characteristics of some square- planform double-wedge airfoils at a mach number of 15 .4. results are presented from a wind-tunnel investigation in helium flow at a mach number of 15.4 . the models were square-planform, double-wedge, shaft-mounted airfoils with leading- and trailing-edge radii of 0, 1, 3, and 6 percent chord . in general, the tests indicate that bluntness effects on the model flutter characteristics are stabilizing as the leading-edge radius is increased from 0 to destabilizing with further increase in bluntness . results of flutter calculations made by using newtonian theory aerodynamics and a combination of newtonian theory and piston theory aerodynamics in conjunction with an uncoupled two-mode analysis are compared with experimental results . the piston-theory results accurately predicted flutter speeds for the models with'), (635, 'heat transfer and pressure distributions on a hemisphere-cylinder and a bluff-afterbody model in methane-air combustion products and in air .', 'irving weinstein', 'national aeronautics and space administration . technical note d-1503', 'heat transfer and pressure distributions on a hemisphere-cylinder and a bluff-afterbody model in methane-air combustion products and in air . an experimental investigation has been made to indicate the validity of using methane-air combustion products as the test medium for aerodynamic heating and loading tests . tests were conducted on a hemisphere-cylinder and on a bluff-afterbody model, both in methane-air combustion products and in air alone, and covered a range of mach numbers from 6 to the data showed that the nondimensional heating-rate distribution along a hemisphere-cylinder as obtained in combustion products was in good agreement with that obtained in air, and the results were in reasonable agreement with theory . the stagnation-point heating rates in air and in combustion products over the hemisphere-cylinder agreed within 10 percent of the theoretical values . the pressure distributions around a hemisphere-cylinder obtained from tests in combution products were in good agreement with those obtained in air and could be predicted by newtonian flow theory . the tests in combustion products of a bluff-afterbody model produced nondimensional heat-transfer coefficients which were in fair agreement with results obtained in air .'), (636, 'pressure distribution induced on a flat plate at a free-stream mach number of 1.39 by rockets exhausting upstream and downstream .', 'abraham leiss', 'national aeronautics and space administration technical note d-1507', 'pressure distribution induced on a flat plate at a free-stream mach number of 1.39 by rockets exhausting upstream and downstream . an experimental investigation was made of the pressures induced on a flat plate at a free-stream mach number of 1.39 by a supersonic rocket jet exhausting upstream and downstream . measurements of the pressure distribution on a flat plate were made at zero angle of attack for 11 different locations of the jet exhaust nozzle beneath the wing . measurements were made at ratios of rocket-exit total pressure to free-stream static pressure from 6 to 60 and at a reynolds number per foot of approximately 10 times 10 to the power of 6 . the rocket when exhausted upstream produced a strong shock that moved further upstream with increasing rocket-exit total-pressure ratio . positive incremental normal-force coefficients were obtained at all test positions . data at 11 test positions are tabulated for rocket-on and rocket-off pressure coefficients as well as for incremental pressure coefficients for the 48 orifices of the flat plate for the range of ratio of rocket-exit total pressure to free-stream static pressure of the investigation . changing the location of the model with respect to the plate had a negligible effect when the rocket was varied in the chordwise direction, but the pressure coefficients were reduced as the rocket was lowered away from the flat-plate wing .'), (637, 'an integral equation relating the general time-dependent lift and downwash distributions on finite wings in subsonic flow .', 'joseph a. drischler', 'national aeronautics and space administration technical note d-1521', 'an integral equation relating the general time-dependent lift and downwash distributions on finite wings in subsonic flow . an integral equation for obtaining the unsteady air forces on finite wings in subsonic compressible flow is presented . this equation is applicable for any arbitrary time-dependent motion and can be utilized for flexible as well as rigid wings . the approach involves the derivation of an integral equation relating the unknown pressure the form of the equation is such that it should lend itself readily to modern high-speed computers for obtaining pressure distributions . special cases of the integral equation are treated for two-dimensional incompressible flow and are presented in an appendix .'), (638, 'longitudinal aerodynamic characteristics at low subsonic speeds of a highly swept wing utilizing nose deflection for control .', 'spencer,b.', 'nasa tn.d1482, 1962.', 'longitudinal aerodynamic characteristics at low subsonic speeds of a highly swept wing utilizing nose deflection for control . an investigation has been conducted in the langley 7- by 10-foot transonic tunnel at low subsonic speeds to determine the longitudinal aerodynamic characteristics associated with deflection of the nose section of a highly swept delta wing having an aspect ratio of 1.33 . in order to illustrate the effectiveness of this forward control, the longitudinal control characteristics are also presented for the wing with upper-and lower-surface split flaps located at the trailing edge . comparison between the longitudinal aerodynamic characteristics of the wing utilizing the nose control and those of the wing utilizing the upper-surface split flap located at the trailing edge indicated similar control effectiveness for high control deflections (15) and similar values of trimmed lift-drag ratio with increasing lift coefficient . use of the nose control, however, indicated a lower value of trimmed angle of attack for a given value of trimmed lift coefficient than that realized from use of the upper-surface split flap . further reductions in trimmed angle of attack for a given value of trimmed lift coefficient may be realized from deflection of the lower-surface split flap at the wing trailing edge in combination with the nose control and would be accompanied by large reductions in lift-drag ratio .'), (639, 'analytical study of the tumbling motions of vehicles entering planetary atmospheres .', 'tobak,m.', 'nasa tn.d1549, 1962.', 'analytical study of the tumbling motions of vehicles entering planetary atmospheres . the tumbling motion of vehicles entering planetary atmospheres is analyzed . a differential equation governing the tumbling motion, its arrest, and the subsequent oscillatory motion is obtained and identified as the equation for the fifth painleve transcendant . an approximate analytical solution for the transcendant is derived . comparisons with results obtained from numerical integration of the exact equations of motion indicate that the solution for the angle-of-attack history is sufficiently accurate to be of practical use .'), (640, 'the design of structures to resist jet noise fatigue .', 'b. l. clarkson', 'j. royal aero. soc. 66, oct. 1962', 'the design of structures to resist jet noise fatigue . the design of structures to resist jet noise fatigue demands a knowledge of a wide range of subjects from pure acoustics at one hand to metal physics at the other . at the present time the various aspects of the problem are not sufficiently well know quantitatively for a purely theoretical design study to be made . never- the-less a knowledge of the behaviour of typical forms of construction in noise environments can be used with a limited amount of theoretical work to indicate tne most efficient types of structure . this approach to the problem is adopted in this lecture as it seems to be the most promising one available at the moment . it must be emphasized, however, that although some progress has been made in dicsovering the behaviour of a structure subjected to noise it is not possible to estimate the life of any component at the drawing board stage . some prototype strain measurements and proof testing are therefore essential if one is to prove the integrity of the design . within the structural limits of single skin construction set in this lecture the main conclusion to be reached is that no reasonable estimate of fatigue life can yet be made in the drawing board stage of a structure . nevertheless, a study of the form of behaviour of typical structures has led to a theoretical simplification of the problem of skin vibration . from this it has been possible to suggest an optimum deisgn for a skin stiffened by stringers . a suggestion for an optimum design of skin and rib for control surfaces to minimise stresses at the rib-skin intersection is put forward but no experience can check this yet . the most resonable basis for the future estimation of fatigue life of a component appears to be the /random/ s-n curve and consierable effort should be made to obtain the necessary test data . the life expectation of a new design will be uncertain and some proof testing is essential if the integrity of structure in high noise levels (150 db) is to be guaranteed .'), (641, 'reduction of the clamped plate to two membrane problems with an application to uniformly loaded sectors .', 'morley,l.s.d.', 'rae r.struct.277, 1962.', 'reduction of the clamped plate to two membrane problems with an application to uniformly loaded sectors . the clamped plate problem in the classical theory for the small deflection bending of flat plates is reduced to the solution by variational methods of two successive membrane problems . the first requires the least square minimisation of the average curvature of the deflected surface while the second problem concerns the integral of the gaussian curvature . there is a similar reduction for extensional problems where the boundary tractions are specified . the method is demonstrated by giving three distinct solutions to the problem of the clamped sector under a uniformly distributed load . one solution is of special interest because it is derived from a single membrane problem . numerical data are given .'), (642, 'the buckling strength of a uniform circular cylinder loaded in axial compression .', 'sobey,a.j.', 'rae r. struct,279, 1962.', 'the buckling strength of a uniform circular cylinder loaded in axial compression . the theoretical estimation of the buckling strength of a cylinder loaded in axial compression is improved by the use of a more representative deflected form for the buckled cylinder than has previously been used . kempner''s buckling strength for dead weight loading is reduced by 18 . the presentation of the magnitude and distribution of the constraint system required to maintain the mode is novel and instructive .'), (643, 'an investigation of wing-aileron flutter using ground launched rocket models .', 'gaukroger,d.r. and curran,j.k.', 'rae tn. struct.308, 1962.', 'an investigation of wing-aileron flutter using ground launched rocket models . control surface flutter of the wing torsion-control rotation type has been investigated for an unswept wing with an under-massbalanced, half span, outboard aileron . thirteen pairs of wings were tested, using ground launched rocket driven vehicles, and a range of values of aileron natural frequency was covered . the test results showed considerable scatter, but enabled upper and lower limits of a flutter boundary to be determined approximately . it was established that aileron flutter could be eliminated on the models tested provided the aileron frequency exceeded the wing torsional frequency by 20 per cent or more . in this condition the models were also free from single degree of freedom flutter'), (644, 'a study of the cantilever square plate subjected to a uniform loading .', 'leissa,a.w. and niedenfuhr,f.w.', 'j. aero. sc. 29, 1962.', 'a study of the cantilever square plate subjected to a uniform loading . plate problems involving free edges have been historically difficult to solve, particularly when two free edges are adjacent, resulting in a free corner . the cantilevered square plate subjected to a transverse loading is one such problem for which an exact solution has not been achieved . in the present paper results obtained by various approximate methods are presented for this problem for the case of a uniform loading . solutions obtained by the authors using the technique of point matching and the rayleigh-ritz method are compared with previously published finite-difference and experimental results and with bernoulli-euler beam and plane-strain approaches . numerical results for deflections, slope components, bending and twisting moments, and transverse distributed shears are presented for a relatively fine gridwork of points on the plate boundary and within the interior . the antielastic curvature is exhibited by all methods except beam theory . all methods present the interesting conclusion that the free edge deflection is greater when the plate is treated as a plate rather than a beam .'), (645, 'thermodynamic coupling in boundary layers .', 'baron,j.r.', 'ars space flight rep. to the nation, 2206-61, 1961.', 'thermodynamic coupling in boundary layers . experimental results gathered in recent years for binary mixture mass transfer models are shown to yield consistent evidence of discrepancies with analytic considerations . specifically, measured recovery temperatures are appreciably higher than those predicted ,. while heat transfer coefficients are satisfactorily reproduced . it is shown on the basis of both approximate and exact solutions for plates and stagnation points that the discrepancies in previous results are related to thermal diffusion effects, a major influence being apparent in application of the surface boundary condition for an adiabatic wall . as a result, some reexamination is necessary of past criteria for mass addition effects as they pertain to specific injected media . a prime example is the /equivalence/ of helium and air as coolants despite the heretofore suggested preference for low density injectants on a perfect gas basis . ref. 16 .'), (646, 'thermal diffusion effects on energy transfer in a turbulent boundary layer with helium injection .', 'tewfik,o.e., eckert,e.r.g. and shirtliffe,c.j.', 'to be presented at the 1962 heat transfer and fluid mech.inst. seattle,', 'thermal diffusion effects on energy transfer in a turbulent boundary layer with helium injection . a circular cylinder with two-inch diameter and with a porous wall fabricated out of woven wire material was aligned with its axis parallel to an air stream with approximately 100 ft sec velocity . helium gas was injected into the turbulent boundary layer through the cylinder walls at a uniform rate in the range 1.55 x 10 to 1.08 x 10 of the free stream mass velocity . the local energy transfer along the cylinder was measured at various values of the wall temperature level for the situation that the energy flows from the cylinder to the boundary layer and vice versa . the results showed clearly that the wall temperature for zero energy transfer - the adiabatic wall temperature - was larger than the free stream temperature by up to about 40 f, although viscous dissipation effects are negligible . this temperature excess increases with increasing injection rate and is independent of reynolds number . an analysis in which the laminar sublayer is treated as couette flow with helium injection and which includes thermal diffusion in this layer is formulated . the results show appreciable thermal diffusion effects on adiabatic wall temperature, increasing it over its value for zero injection by amounts of the same order of magnitude as found by measurements . thermal diffusion however has negligible effects on the heat transfer coefficient . its effects on the concentration and temperature distribution are discussed and are shown to produce appreciable modifications in the latter .'), (647, 'bending of a uniformly loaded rectangular plate with two adjacent edges and the others either simply supported or free .', 'huang,m.k. and conway,d.', 'j.app.mech. 1952, 451.', 'bending of a uniformly loaded rectangular plate with two adjacent edges and the others either simply supported or free . the distribution of deflection and bending moment in a uniformly loaded rectangular plate having two adjacent edges clamped and the others either simply supported or free, are obtained by a method of superposition . numerical values are given for square plates and, in one case, the results are compared with those obtained by another method .'), (648, 'the approximate analysis of certain boundary value problems .', 'conway,h.d.', 'j. app. mech. 1960, 275.', 'the approximate analysis of certain boundary value problems . a simple method is given which is suitable for the approximate analysis of certain boundary-value problems, including, for example, the small deflections of clamped plates and the torsion of prismatic bars . the analysis is particularly simple and lends itself well to the use of the digital computer . the method is applied here to four problems, the uniformly loaded, clamped square, and equilateral-triangle plates, and the torsion of bars of square and hexagonal cross section . the results agree well with the exact solutions, where these are known .'), (649, 'the hovercraft - a new concept in maritime transport .', 'crewe,p.r. and eggington,w.j.', 'trans. roy. inst. naval arch. 1960.', 'the hovercraft - a new concept in maritime transport . the hovercraft is the first operational british project in the ground-effect machine field . although there has, for a number of years, been a tentative searching after the principles underlying such machines, it is only now that their possibilities as commercial transport and service craft are beginning to be developed . since the hovercraft is a new vehicle, the appearance of the saunders-roe sr-n1, a manned experimental craft, excited considerable public attention and there have been a number of descriptive articles in the press . papers of a more technical type, on ground-effect machines, are now beginning to appear and it is to be expected that these will rapidly increase in number, especially since american interest in both the commercial and defence fields is expanding fast . the authors of the present paper have, therefore, concentrated attention upon features about which they had something personal to say, and which they consider to be of particular significance for assessing the possibility of the hovercraft becoming important in maritime transport . these features are ..- the hovercraft as a fundamentally new principle in the transport field . the powering requirements and resistance characteristics . the likely operating costs of hovercraft in comparison with other forms of maritime transport . in addition, relatively brief descriptions of the history and the current work being undertaken on the ground-effect machine and of the design, construction, and testing of the saunders-roe sr-n1 are provided . the final section discusses outstanding problems and some future possibilities .'), (650, 'some design problems of hovercraft .', 'stanton-jones,r.', 'inst. aero. sc. paper 61-45, 1961.', 'some design problems of hovercraft . analysis of the influence various aerodynamic parameters have on the performance of a simple peripheral jet system . power weight ratio, lift drag ratio, and effect of jet angles and thickness are each considered . structural requirements, optimum cushion pressure, and dynamic stability over waves are examined and then related to the economics of ground-effect machine operation .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (651, 'heat transfer to separated and reattached subsonic turbulen flows obtained downstream of a surface step .', 'seban,r.a., emery,a. and levy,a.', 'j. ae. scs. 1959, 809.', 'heat transfer to separated and reattached subsonic turbulen flows obtained downstream of a surface step . local heat-transfer coefficients and recovery factors are presented for separated and reattached turbulent flows as obtained by a downward step in an otherwise flat surface in a two- dimensional, subsonic, air flow . the region downstream of the step, the focus of this investigation, contained a region of separated flow with reattachment at about five step heights downstream, followed by a section of reattached flow . the salient feature of the results is the maximum in the local heat-transfer coefficient at the reattachment point, with values thereof diminishing in the separated region and also in the reattached region, where they tend toward values characteristic of turbulent boundary-layer flow . it is found that for most of the region the heat-transfer coefficient depends on the velocity to about the 0.8 power, though a decreased dependence may exist in the separated region . recovery factors have the characteristically low values associated with separated flows, and do not attain values typical of turbulent boundary-layer flows within the downstream lengths available .'), (652, 'pressure distribution on two dimensional wings near the ground .', 'bagley,j.a.', 'rae r. aero.2625, 1960.', 'pressure distribution on two dimensional wings near the ground . a simple method of calculating the pressure distribution in incompressible flow on two-dimensional aerofoils of arbitrary section at moderate distances from the ground is developed . comparisons with an /exact/ potential flow solution, and with measurements on a 10 thick aerofoil of rae.101 section, provide a satisfactory verification of the adequacy of the method ,. but it is shown that it is necessary to take account of the boundary layer on the aerofoil in the calculations .'), (653, 'transient magnetohydrodynamic duct flow .', 'lundgen,t.s., atabeck,b.h. and chang,c.c.', 'phys. fluids 4, 1961, 1006.', 'transient magnetohydrodynamic duct flow . parallel flow of an electrically conducting viscous incompressible fluid in a rectangular duct with transverse magnetic field is considered . the walls of the duct which are parallel and perpendicular to the imposed magnetic field are taken to be nonconducting and perfectly conducting, respectively . assuming the fluid to be at rest at the initial moment, exact solutions for the velocity and magnetic field components are obtained in the form of convolution integrals taking the longitudinal pressure gradient as an arbitrary given function of time . later, taking a step function for the pressure gradient, these expressions are integrated . for this case, the effect of the strength of the imposed magnetic field on the development behavior of the flow is studied . it is found that except for very large magnetic fields, the flows are over damped .'), (654, 'on the propagation and structure of the blast wave . part 1.', 'sakuri,a.', 'j. phys. soc. japan, 9, 1954.', 'on the propagation and structure of the blast wave . part 1. as a continuation of part 1 (j. phys. soc. japan 8 (1953) 662), the second approximation for the propagation and structure of a blast wave is now discussed . the solution for r=1.4 is obtained by a numerical method, using the results of the first approximation obtained in part 1 . by use of this solution, u-r curves, distance-time curves and the changing feature of distributions of velocity, pressure and density behind the shock front are discussed . further, the approximate solution of the equation is discussed by a refinement of the wkb method due to imai .'), (655, 'effects of boundary layer displacement and leading edge bluntness on pressure distribution, skin friction, and heat transfer of bodies at hypersonic speeds .', 'bertram, m.h. and henderson, a.', 'naca tn 4301, july 1958 .', 'effects of boundary layer displacement and leading edge bluntness on pressure distribution, skin friction, and heat transfer of bodies at hypersonic speeds . results are presented of an investigation to determine the effect of boundary-layer displacement and leading-edge bluntness on surfaces in hypersonic flow . the presence of the boundary layer and the blunt leading edge induce pressure gradients which in turn affect the skin friction and heat transfer to the surface . methods for predicting these phenomena on two-dimensional surfaces are given and a brief review of recent three-dimensional results is presented .'), (656, 'departure from dissociation equilibrium in a hypersonic nozzle .', 'bray, k.n.c.', 'a.r.c. 19, 983, march 1958 .', 'departure from dissociation equilibrium in a hypersonic nozzle . the equations of motion for the flow of an ideal dissociating gas through a nearly conical nozzle have been solved numerically, assuming a simple equation for the rate of dissociation, and a number of different values of the rate constant . the results of these calculations suggest that deviations from dissociation equilibrium will occur in the nozzle if the rate constant lies within a very wide range of values . they also suggest that once such a deviation has begun the gas will very rapidly/freeze/, so that the dissociation fraction will remain almost constant if the flow is expanded further, or even if it passes through a constant area test section . an approximate method of solution, making use of this property of sudden/freezing/of the flow, has been developed and applied to the problem of estimating the deviations from equilibrium under a wide range of conditions . if all the assumptions made in this report are accepted, then lack of dissociation equilibrium may be expected in the working sections of hypersonic wind tunnels and hypersonic shock tubes . it is shown, however, that the flow behind a normal shock wave in such a wind tunnel will not be greatly affected by any freezing that may take place in the nozzle upstream of the shock wave . even so, the stand-off distance of a shock wave in front of a blunt model may be quite sensitive to deviations from equilibrium .'), (657, 'interferometric studies of supersonic flows about truncated cones .', 'giese,j.h. and bergdolt,v.e.', 'j.app.phys. 24, 1953, 1389.', 'interferometric studies of supersonic flows about truncated cones . fringe shifts on interferograms of flows at m=2.45 about variously truncated 15 (half-angle) cone cylinders in free flight in a pressurized range have been examined for similarity of the flow fields, occurrence of scale effects, and convergence to conical flow . it was found that flows over similar objects with equal tip reynolds numbers were similar and that convergence to conical flow occurred before the disturbance at the tip had been reflected the second time along characteristics to the body . density distributions have been determined, and a number of comparisons have been made with theoretical predictions .'), (658, 'review of panel flutter and effects of aerodynamic noise part i.. panel flutter .', 'l. e. goodman and j. v. rattayya', 'university of minnesota, minneapolis, minnesota', 'review of panel flutter and effects of aerodynamic noise part i.. panel flutter . with the development of high-speed aircraft and missiles, vibration of panels has become a problem of practical significance . many of the failures of the early german rockets after attaining supersonic speed have been attributed to the development of such panel oscillations . it appears this phenomenon is not of much concern in the subsonic speed range., however, in the supersonic speed range panels may develop oscillations which cause instability of the structure . this effect has been exhibited experimentally under controlled laboratory conditions motion is limited and buckling may not be a serious design problem . in these cases panel flutter is still of importance because of its effect on the fatigue life and the allowable stresses for design of the panel material . the oscillations of panels may be due either to aerodynamic force induced by the motion of the panel, or to aerodynamic noise, or buffeting (irregular motion induced by turbulence in the flow) . the interaction between aerodynamic forces and panel motions, usually referred to as /panel flutter,/ has been investigated by several workers in recent years . since the problem is too complex to be dealt with in its entirety, simplifying assumptions have been made in these investigations . the literature is marked by a certain degree of controversy over the validity of these assumptions and the applicability of the results obtained . a brief review of the literature with reference to several of the approximations made and the results obtained follows .'), (659, 'nonuniform shear flow past cylinders .', 'murray,j.d.', 'q. j. mech. app. math. 10, 1957, 406.', 'nonuniform shear flow past cylinders . a general method is described whereby an approximation of any desired degree of accuracy to the stream functions for two types of variable shear flows past finite cylinders can be obtained . the two shear distributions in the free stream can be approximated to the linear shear distribution and the shear present in an unretarded incompressible boundary layer respectively . in every case the stagnation streamline is displaced from the position opposite the line of symmetry of the cylinder, and general expressions are obtained for this displacement . the line of symmetry may be in the direction of or perpendicular to the direction of flow . the two particular examples cited are those of a general elliptic cylinder and cylinders of the form where and being the polar coordinates, and 2p the maximum width of the cylinder .'), (660, 'the fundamental solution for small steady three dimensional disturbances to a two dimensional parallel shear flow .', 'lighthill,m.j.', 'j. fluid mech. 3, 1957, 113.', 'the fundamental solution for small steady three dimensional disturbances to a two dimensional parallel shear flow . after a brief review of methods of calculating the flow fields produced by disturbances in rotational basic flows, the author points out a fundamental difficulty in the treated as a perturbation of the disturbance field that would occur if the basic flow were uniform) .. slow attenuation of the secondary-flow disturbance with distance from the obstacle . the author conjectured (same j. 1 the trouble was caused by nonuniform validity of the approximation sequence in the region far from the obstacle . the analogy with /stokes'' and whitehead''s paradoxes/ is mentioned, and a solution analogous to oseen''s is suggested, one in which disturbances, but not the shear, are assumed to be small . in this paper, such a solution is found, and is shown to overlap with the small-shear, secondary-flow solution . the basic flow is a parallel, steady, inviscid, two-dimensional shear flow . the / fundamental solution/ due to a weak source is sought . the method of fourier transforms is used . simple solutions are found for a uniformly sheared basic flow (where the result coincides with the secondary-flow solution) and for an exponential basic-flow profile . in the general case it is assumed that the parallel basic flow becomes uniform at, where the x-axis lies in the flow direction . the character of the solution is determined by studying its hankel transform, especially for the class of flows where the total variation of the basic stream speed v(y) is small . an interpretation in terms of images, due to m. b. glauert, is given, and finally the relationship of the present work to theories of the displacement of the stagnation streamline (displacement effect of pitot tubes) is discussed .'), (661, 'summary of laminar boundary layer solutions for wedge-type flow over convection and transpiration cooled surfaces .', 'livingood,j.n.b. and donoughe,p.', 'naca tn.3588, 1955.', 'summary of laminar boundary layer solutions for wedge-type flow over convection and transpiration cooled surfaces . a summary of exact solutions of the laminar-boundary-layer equations for wedge-type flow, useful in estimating heat transfer to such arbitrarily shaped bodies as turbine blades, is presented . the solutions are determined for small mach numbers and a prandtl number at the wall of 0.7 ,. ranges of mainstream pressure gradients and rates of coolant flow through a porous wall are considered for the following cases .. (1) small temperature changes in the boundary layer along a constant- and along a variable-temperature wall, and (2) large temperature changes in the boundary layer along a constant-temperature wall . dimensionless forms of heat-transfer and friction parameters and boundary-layer thicknesses are tabulated . the results indicate that coolant emission and increased stream-to-wall temperature ratios diminished the friction and heat transfer for a constant wall temperature . for a variable wall temperature with small temperature differences in the boundary layer, the friction was unaffected, but the heat transfer was greatly increased for increased wall-temperature gradient . heat-transfer results in the literature reveal that transpiration cooling is much more effective for prandtl numbers of the order of 5.0 than for 0.7 .'), (662, 'theoretical and experimental investigation of aerodynamic-heating and isothermal heat transfer parameters on a hemisphere nose with laminar boundary layer at supersonic mach numbers .', 'stine,h.a. and wanlass,k.', 'naca tn.3344, 1954.', 'theoretical and experimental investigation of aerodynamic-heating and isothermal heat transfer parameters on a hemisphere nose with laminar boundary layer at supersonic mach numbers . the effect of a strong, negative pressure gradient upon the local rate of heat transfer through a laminar boundary layer on the isothermal surface of an electrically heated, cylindrical body of revolution with a hemispherical nose was determined from wind-tunnel tests at a mach number of 1.97 . the investigation indicated that the local heat-transfer parameter, based on flow conditions just outside the boundary layer, decreased from a value of 0.65 0.10 at the stagnation point of the hemisphere to a value of 0.43 0.05 at the junction with the cylindrical afterbody . because measurements of the static pressure distribution over the hemisphere indicated that the local flow pattern tended to become stationary as the free-stream mach number was increased to 3.8, this distribution of heat-transfer parameter is believed representative of all mach numbers greater than 1.97 and of temperatures less than that of dissociation . the local heat-transfer parameter was independent of reynolds number based on body diameter in the range from 0.6x10 to 2.3x10 . the measured distribution of heat-transfer parameter agreed within theoretical distribution calculated with foreknowledge only of the pressure distribution about the body . this method, applicable to any body of revolution with an isothermal surface, combines the mangler transformation, stewartson transformation, and thermal solutions to the falkner-skan wedge-flow problem, and thus evaluates the heat-transfer rate in axisymmetric compressible flow in terms of the known heat-transfer rate in an approximately equivalent two-dimensional incompressible flow . measurements of recovery-temperature distributions at mach numbers of 1.97 and 3.04 yielded local recovery factors having an average value of 0.823 0.012 on the hemisphere which increased abruptly at the shoulder to an average value of 0.840 0.012 on the cylindrical afterbody . this result suggests that the usual representation of the laminar recovery factor as the square root of the prandtl number is conservative in the presence of a strong, accelerating pressure gradient .'), (663, 'viscous flow along a flat plate moving at high speeds .', 'kuo,y.h.', 'j.ae.scs. 23, 1956, 125.', 'viscous flow along a flat plate moving at high speeds . by the distortion of coordinates, it is shown that, in the case of supersonic viscous flow past a flat plate, the boundary-layer and simple wave theories can be combined to give a complete representation of the velocity and pressure fields . consistent first-order solutions are considered . an expression for the induced pressure on the plate, correct to the second order, is obtained . at high mach numbers the important parameter satisfies the hypersonic similarity law ,. and for arbitrary mach and reynolds numbers and for different gases, the theoretical curve correlates closely the experimental data . asymptotic shock curve and skin-friction coefficient are also deduced, but the experimental verifications are yet to be made .'), (664, 'the boundary layer on a flat plate in a stream with uniform shear .', 'murray,j.d.', 'j. fluid mech. 11, 1961, 309.', 'the boundary layer on a flat plate in a stream with uniform shear . the incompressible laminar boundary layer on a semi-infinite flat plate is considered, when the main stream has uniform shear . a solution is obtained for the first two terms of an asymptotic solution for small viscosity . it is shown that one of the principal effects of free-stream vorticity is to introduce a modified pressure field outside the boundary-layer region .'), (665, 'on the theory of hypersonic gas flow with a power law shock wave .', 'sychev,v.v.', 'j. app. math. mech. 24, 1960, 756.', 'on the theory of hypersonic gas flow with a power law shock wave . plane and axisymmetric hypersonic gas flows are considered with shock waves of very great intensity that have a power-law form . on the basis of an investigation of the portions of the flow with high entropy adjoining the surface of the body (not necessarily for a shock wave of the given form) it is shown that the use in the flow problem of the exact solution for the corresponding unsteady self-similar gas motion requires a supplementary refinement of the thickness of the high entropy layer . a method is shown for introducing such a correction and constructing the shape of the body contour, on which is to be applied the pressure distribution obtained on the basis of the theory of small disturbances .'), (666, 'blunt body heat transfer at hypersonic speed and low reynolds numbers .', 'ferri, a. zakkay, v. and ting, l.', 'j. aero. sc. v. 28. p. 862. 1961 .', 'blunt body heat transfer at hypersonic speed and low reynolds numbers . an analytical method for the determination of effect of shock curvature on heat transfer in the region of the nose has been developed . it is shown that for practical body shape the viscous terms in the navier-stokes equations are not important in the region of the flow far from the wall, and the displacement thickness can be neglected . then the flow can be approximately represented by an inviscid-flow solution having as boundary conditions the body shape, which is not affected by the reynolds number, and by a boundary-layer type of flow near the wall, having appropriate boundary conditions . this approach permits us to determine the heat transfer in the region of the nose even at very low reynolds numbers . experimental results are presented . the experimental results agree with the values given by the analysis .'), (667, 'hypersonic shock layer theory of the stagnation region at low reynolds number .', 'cheng, h.k.', 'proc. 1961, heat transfer and fluid mech. inst. stanford. univ. press. 1961 . p.161', 'hypersonic shock layer theory of the stagnation region at low reynolds number . cheng, h.k. hypersonic flow at low reynolds number is studied utilizing the shock-layer concept . the present formulation takes into account the salient features of the transport processes within the shock layer in a manner consistent with the shock-layer approximation . the rankine-hugoniot shock relations are modified to include contributions due to heat conduction and viscous effects immediately behind the shock . the specific problem of an axisymmetric stagnation region is treated . the flow regimes for this problem can be classified according to whether or not the transport effects are important immediately behind the shock . in one regime where the ordinary rankine-hugoniot relations hold across the shock, the vorticity-interaction theory based on the boundary-layer approximation is shown to be sufficient . in the other regime where the rankine-hugoniot relations have to be modified but the continuum-flow model applies, an approximate, an analytical solution is obtained . this solution reveals a substantial reduction of the temperature behind the shock and of the shock stand-off distance in the presence of strong surface cooling . the present study is intended to provide a knowledge to bridge the gap between the free-molecule flow regime and that of the boundary layer via the continuum theory . in this respect, the solution obtained appears to be satisfactory in that it yields the correct free-molecule limits for the skin friction and surface-heat transfer rate .'), (668, 'measurements of stagnation point heat transfer at low reynolds number .', 'ferri,a. and zakkay,v.', 'j. ae. scs. 28, 1962, 847.', 'measurements of stagnation point heat transfer at low reynolds number . measurements of stagnation point heat transfer are presented in the reynolds number range between the free molecular flow and the range where modified boundary layer theory still applies . the measurements are compared with the analytical methods set forth by ferri, zakkay, and ting . the results show smooth transition between the two regions and indicate that the predicted reynolds number for which the modified boundary layer theory can be used is in agreement with experiments . in the lower range of reynolds number the ratio of decreases and reaches a value of 1 at a reynolds number of 40 .'), (669, 'subsonic potential flow past a sphere inside a cylindrical duct .', 'william l. haberman', 'david taylor model basin, carderock, md.', 'subsonic potential flow past a sphere inside a cylindrical duct . the subsonic potential flow of a compressible fluid past a sphere in an infinite medium was first determined by rayleigh . subsequently, caplan and tamada extended the solution to include the fourth power of the mach number . to the author''s knowledge, no solution for subsonic flow past a sphere in a finite medium has been published . it is the purpose of this note to present a solution for subsonic potential flow past a sphere inside a circular cylindrical duct .'), (670, 'on blunt-body heat transfer at hypersonic speed and low reynolds number .', 'ferri, a. zakkay, v. and ting, l.', 'j. aero. sc. v. 29. p. 882, 1962 .', 'on blunt-body heat transfer at hypersonic speed and low reynolds number . a discussion of differences arising between experimental and analytical results, in particular those due to inconsistencies introduced in the presentation of data and the way the comparison is made .'), (671, 'pressure and boundary-layer measurements on a two dimensional wing at low speed .', 'brebner, g.g. and bagley, j.a.', 'a.r.c. r + m 2886, july, 1962 .', 'pressure and boundary-layer measurements on a two dimensional wing at low speed . results are given of pressure measurements and boundary-layer traverses on a two-dimensional wing with 10 per cent rae 101 section at reynolds numbers of 1.6x10 and 3.2x10 . these results which have been integrated to give lift, drag and aerodynamic-centre characteristics, are used to check some calculation methods for the growth of the turbulent boundary layer and for the effect of a known boundary layer on the pressure distribution . it is concluded that the calculation of the boundary layer still needs a little refinement before it is accurate enough to predict viscosity effects on pressure distribution, lift, drag and aerodynamic center,.but that these effects can be calculated if the actual boundary-layer characteristics are known .'), (672, 'tunnel interference effects .', 'pankhurst, r.c. and holder, d. w.', 'wind tunnel techniques, chapter 8 pitman. 1952 .', 'tunnel interference effects . the problems of solid blockage, wake blockage, lift effect, and the influence of boundary constraint at high mach number are considered in detail . corrections are given for various open and closed tunnels, rectangular, circular and octagonal, and different speeds, two and three dimensional flows, with several aerofoils and wings . other interferences include the wall boundary layer, gradient of static pressure and problems with the working fluid used .'), (673, 'investigation of full scale split trailing edge wing flaps with various chords and hinge locations .', 'wallace,r.', 'naca r.539, 1935.', 'investigation of full scale split trailing edge wing flaps with various chords and hinge locations . an investigation was conducted in the n. a. c. a. full-scale wind tunnel on a small parasol monoplane equipped with three different split trailing-edge wing flaps . the object of the investigation was to determine and correlate data on the characteristics of the airplane and flaps as affected by variation in flap chord, flap deflection, and flap location along the wing chord . the chords of the flaps were 10, 20, and 30 percent of the wing chord and each flap was tested at deflections from 0 to 75 when located successively at 68, 80, and 88.8 percent of the wing chord aft of the leading edge . the investigation included force tests, pressure-distribution tests, and downwash surveys . the results give the lift, the drag, and the pitching-moment characteristics of the airplane, the flap forces and moments, the pressure distribution over the flaps and wing at one section, and the downwash characteristics of the flap and wing combinations . an increase in flap chord or distance of the flap from the leading edge of the wing increased the lift of the airplane but had an adverse effect on the wing pitching moment . the ld ratio of the airplane decreased with increase in flap deflection or flap chord . flap normal-force coefficients were primarily a function of flap deflection and were relatively independent of flap chord, hinge-axis location, and airplane attitude . the location of the flap center of pressure in percentage of flap chord aft of the hinge axis remained practically constant irrespective of airplane attitude and of flap deflection, chord, or location . flap hinge-moment coefficients varied with a power of flap chord greater than the square so that with regard to hinge moments narrow flaps were the most efficient in producing a given increase in lift . split trailing-edge flaps materially affected the magnitude and distribution of pressures over the entire wing profile . at low angles of attack the predominant effect of the flaps was to increase positively the lower-surface pressures ,. at high angles of attack, to increase negatively the upper-surface pressures . downwash surveys indicated that horizontal tail planes located above the wing chord line would be more effective than those below the chord in counteracting the increased diving moment of the airplane with flaps deflected .'), (674, 'the shapes and lift-dependent drags on some sweptback wings designed for m= 1. 2.', 'bagley,j.a. and beasley,j.a.', 'rae r. aero.2620, 1959.', 'the shapes and lift-dependent drags on some sweptback wings designed for m= 1. 2. the camber and twist distributions needed to produce a constant span-wise -distribution and certain linear chordwise load distributions have been calculated by linearised supersonic theory at for a set of 34 thin sweptback wings . the wing planforms cover a range of aspect ratios from 2.0 to 3.5 and leading-edge sweep angles from 55 to 70 . both leading and trailing edges are subsonic at the design mach number, and the slenderness parameter is between 0.19 and 0.40 . the lift-dependent vortex and wave drags associated with these loadings have also been calculated, and appear not to be excessive in almost all the cases considered .'), (675, 'pressure distribution and surface flow on 5( and 9( thick wings with curved tip and 60degree sweepback .', 'garner, h.c. and walshe, d.e.', 'a.r.c. 20,982, r + m 3244. may 1959 .', 'pressure distribution and surface flow on 5( and 9( thick wings with curved tip and 60degree sweepback . extensive tables are given of pressure coefficients measured at reynolds numbers from 1.3x10 to 3.9x10 on two half-models of identical planform with 5( rae 101 and 9( rae 101 streamwise sections . the planform of aspect ratio 3.899 has a straight trailing edge with 60degree of sweepback, constant chord over most of the span and a parabolic outer portion of the leading edge curving to a pointed tip . the overall wing characteristics are obtained from integrated normal pressures and are compared with lifting-surface theory . the low-speed experimental pressure distributions and surface oil-flow patterns are analysed and discussed in relation to the onset of separation and the distinct vortex flows that develop at high incidence . series of contrasting upper-surface isobars illustrate some features of the different stalling processes of the two wings . the direct influence of the main vortex on local surface pressures is assessed in general terms . a fuller appraisal of secondary surface flow is obtained from the oil patterns, observations in water and measurements of high suction near the trailing edge . studies of the extent of leading-edge stall and location of part-span vortices, in particular two simultaneous leading-edge vortices on the thinner wing, follow from further analysis of local surface pressures . after a detailed discussion of the effect of reynolds number and the distinct types of separated flow, a few results with leading-edge roughness are considered in relation to scale effect on separation and the extensive influence of part-span roughness .'), (676, 'a simple method for calculating the span and chordwise loading on straight and swept wings of any aspect ratio at subsonic speeds .', 'kuchemann, d.', 'r + m 2935, r.a.e. rep. aero. 2476, a.r.c. 15,633. august 1962 .', 'a simple method for calculating the span and chordwise loading on straight and swept wings of any aspect ratio at subsonic speeds . the methods of the classical aerofoil theory are used to derive a general theory for wings of any given planform . the load over the whole surface of a given wing can be calculated at a given subcritical mach number, and the procedure is as simple and rapid as that of the classical aerofoil theory . the calculated results are confirmed by experiments .'), (677, 'methods for calculating the lift distribution of wings /subsonic lifting surface theory/ .', '', 'r + m 2884, r.a.e. rep. aero. 2353. a.r.c. 13,439. january 1950 .', 'methods for calculating the lift distribution of wings /subsonic lifting surface theory/ . this report contains some fairly simple and economic methods for calculating the load distribution on wings of any plan form based on the conceptions of lifting surface theory . the computer work required is only a small fraction of that of existing methods with comparable accuracy . this is achieved by a very careful choice of the positions of pivotal points, by plotting once for all those parts of the downwash integral which occur frequently and by a consequent application of approximate integration methods similar to those devised by the author for lifting line problems . the basis of the method is to calculate the local lift and pitching moment at a number of chordwise sections from a set of linear equations satisfying the downwash conditions at two pivotal points in each section . interpolation functions of trigonometrical form are used for spanwise integration both in setting up the downwash equations and in getting the resultant forces on the wing from the local forces . the preliminary chordwise integrations for the downwash are predigested in a series of charts/figs.1-6/,.it is these which make the method a practical computing proposition . the theory is outlined in sections 2-5,.section 6 deals with the solution of the linear equation and section 7 with the resultant forces on the wing . some examples are worked out in section 8 to compare with other methods,. one solution is given in full detail in tables 8-30 as a guide for computers . appendices i-vi discuss more carefully some salient points of the mathematical theory, and appendix vii is intended to instruct the computer how to carry out the steps of the calculation .'), (678, 'the effect of end plates on swept wings .', 'kuchemann,d. and kettle,d.j.', 'rae r.aero.2429, 1952.', 'the effect of end plates on swept wings . existing methods of calculating the effect of endplates on straight wings are modified so as to apply to swept wings . the changes in overall lift and drag, and also the spanwise distribution of the additional load, can be calculated . the theoretical results are compared with experimental results obtained on swept wings, including new measurements of lift, drag and pitching moment, made on an untapered the method of calculation is also extended to cover the effect of the tip vortex which is formed on wings without endplates .'), (679, 'low speed tests on 45 sweptback wings .', 'weber,j., brebner,g.g. and kuchemann,d.', 'rae r. aero.2374, 1958.', 'low speed tests on 45 sweptback wings . this report contains the results of pressure measurements on three and aspect ratio 5, over an incidence range up to 10 . chordwise and spanwise lift distributions are given, mostly near the centre where, on two of the wings, modifications had been made to the section shape . it was found that altering the thickness distribution in the centre did not affect the loading but that approximately straight isobars could be obtained at values of below about 0.1 . by the incorporation of twist and camber in the central part the distortion of the lift distribution in the centre could be avoided at one particular incidence, and thus the same chordwise distribution obtained over most of the span . twist and camber alone do not improve the isobar pattern and therefore a thickness modification would be needed to give the desired lift distribution and isobar pattern at one particular incidence . the results of experimental investigations of the boundary layer and of the effect of aspect ratio will be given in a later report .'), (680, 'generalized conical flow fields in supersonic wing theory .', 'lomax, h. and heaslett, m.a.', 'naca tn 2497, september 1951 .', 'generalized conical flow fields in supersonic wing theory . linearized, compressible-flow analysis is applied to the study of quasi-conical supersonic wing theory . single-integral equations are derived which relate either the loading to the shape of a lifting surface or the thickness of a symmetrical wing to the pressure distribution for triangular wings with subsonic leading edges . the forms of these equations and their inversions are simplified through the introduction of the finite part and the generalized principal part of an integral . applications of the theory, in the lifting case, include previously known results . in the nonlifting case, it is shown that for a specified pressure distribution the theory does not always predict a unique thickness distribution . this is demonstrated for a triangular plan form having a constant pressure gradient in the stream direction .'), (681, 'integrals and integral equations in linearized wing theory .', 'lomax, h. haslet, m.a. and fuller, f.b.', 'naca rep. 1054, 1951 .', 'integrals and integral equations in linearized wing theory . the formulas of subsonic and supersonic wing theory for source, doublet, and vortex distributions are reviewed, and a systematic presentation is provided which relates these distributions to the pressure and to the vertical induced velocity in the plane of the wing . it is shown that care must be used in treating the singularities involved in the analysis and that the order of integration is not always reversible . concepts suggested by the irreversibility of order of integration are shown to be useful in the inversion of singular integral equations when operational techniques are used . a number of examples are given to illustrate the methods presented, attention being directed to supersonic flight speeds .'), (682, 'the lift of twisted and cambered wings in supersonic flow .', 'lance, g.n.', 'aero. quart. v. 6/2/, may, 1955 .', 'the lift of twisted and cambered wings in supersonic flow . a generalised conical flow theory is used to deduce an integral equation relating the velocity potential on a delta wing/with subsonic leading edges/to the given downwash distribution over the wing . the complete solution of this integral equation is derived . this complete solution is composed of two parts, one being symmetric and the other antisymmetric with respect to the spanwise co-ordinate,. each part represents a velocity potential . for example, if y is the spanwise co-ordinate and x is measured in the free stream direction, then a downwash of the form w-a ux/y/is symmetric and will give rise to a symmetric potential, whereas w-a ux/y/sgn y is anti-symmetric and gives rise to an anti-symmetric potential . the velocity potentials of such flows are given in the form of tables for all downwashes up to and including homogenous cubics in the spanwise and streamwise co-ordinates . table iii gives similar formulae in the limiting case were used over a cycle of the tumbling motion . the analytical expression was in good agreement with numerical solutions of the complete non-linear equations of motion .'), (683, 'the use of conical camber to produce flow attachment at the leading edge of a delta wing and to minimize the lift-dependent drag at sonic and supersonic speeds .', 'smith,j.h.b. and mangler, k.w.', 'r.a.e. rep. aero. 2584. arc 19,961, september 1957 .', 'the use of conical camber to produce flow attachment at the leading edge of a delta wing and to minimize the lift-dependent drag at sonic and supersonic speeds . in an attempt to avoid flow separation at the leading edge of a thin delta wing with subsonic leading edges, an attachment line is prescribed there . this is done by requiring the load, as predicted by attached flow theory, to vanish along the leading edge at the design lift coefficient . for sonic speed, a complete account of this flow is given in terms of slender wing theory and the load distributions corresponding to arbitrary conical camber are calculated . for supersonic speeds load distributions arising in the slender wing theory are considered and the corresponding conical camber distributions are found by linearized theory . the lift-dependent drag for a given lift is then minimized with respect to the coefficients of a linear combination of these load distributions . it is found that the lift-dependent drag factor for these conically cambered wings approaches the value it takes for the attached flow/in which leading edge suction occurs/past the uncambered wing at the same mach number, as more terms are included in the linear combination . however, when the leading edge is almost sonic an appreciable reduction is predicted . the corresponding load distributions and wing shapes are calculated and drawn . the optimum shapes for a fixed number of terms resemble flat plates drooped downwards near their edges, so that the localised leading edge suction is replaced by a distributed force on a forward-facing surface, producing an effect of similar magnitude .'), (684, 'tables of complete elliptic integrals .', 'heuman,c.', 'j. maths. and phys. v. 20, 1941, pp 127-206 .', 'tables of complete elliptic integrals . the present paper contains a set of tables of complete elliptic integrals computed and collected especially for applications to certain dynamical problems . the tabulated functions are four in number and are denoted by f/a/, g/a/, e/a/, and/a,b/respectively . the definitions of these functions and their connections with the functions of legendre will be discussed in the following .'), (685, 'aerodynamic effects of some configuration variables on the aeroelastic characteristics of lifting surfaces at mach numbers from 0. 7 to 6. 86 .', 'hanson,p.w.', 'nasa tn.d984, 1961.', 'aerodynamic effects of some configuration variables on the aeroelastic characteristics of lifting surfaces at mach numbers from 0. 7 to 6. 86 . results of flutter tests on some simple all-movable-control-type models are given . one set of models, which had a square planform with double-wedge airfoils with four different values of leading- and trailing-edge radii from 0 to 6 percent chord and airfoil thicknesses of 9, 11, at mach numbers from 0.7 to 6.86 . the bending-to-torsion frequency ratio was about 0.33 . the other set of models, which had a tapered planform with single-wedge and double-wedge airfoils with thicknesses of 3, 6, 9, and 12 percent chord, was tested at mach numbers from 0.7 to 3.98 and a frequency ratio of about 0.42 . the tests indicate that, in general, increasing thickness has a destabilizing effect at the higher mach numbers but is stabilizing at subsonic and transonic mach numbers . double-wedge airfoils are more prone to flutter than single-wedge airfoils at comparable stiffness levels . increasing airfoil bluntness has a stabilizing effect on the flutter boundary at supersonic speeds but has a negligible effect at subsonic speeds . however, increasing bluntness may also lead to divergence at supersonic speeds . results of calculations using second-order piston-theory aerodynamics in conjunction with a coupled-mode analysis and an uncoupled-mode analysis are compared with the experimental results for the sharp-edge airfoils at supersonic speeds . the uncoupled-mode analysis more accurately predicted the flutter characteristics of the tapered-planform models, whereas the coupled-mode analysis was somewhat better for the square-planform models . for both the uncoupled- and coupled-mode analyses, agreement with the experimental results improved with increasing mach number . in general, both methods of analysis gave unconservative results with respect to the experimental flutter boundaries .'), (686, 'flutter tests of some simple models at a mach number of 7. 2 in helium flow .', 'morgan,h.g. and miller,r.w.', 'nasa memo 4-8-59l, 1959.', 'flutter tests of some simple models at a mach number of 7. 2 in helium flow . results of hypersonic flutter tests on some simple models are presented . the models had rectangular plan forms of panel aspect ratio 1.0, no sweepback, and bending-to-torsion frequency ratios of about . two airfoil sections were included in the tests ,. double wedges of 5-, 10-, and 15-percent thickness and flat plates with straight, parallel sides and beveled leading and trailing edges . the models were supported by a cantilevered shaft . the double-wedge wings were tested in helium at a mach number of 7.2 . an effect of airfoil thickness on flutter speed was found, thicker wings requiring more stiffness to avoid flutter . a few tests in air at a mach number of 6.9 showed the same thickness effect and also indicated that tests in helium would predict conservative flutter boundaries in air . the data in air and helium seemed to be correlated by piston-theory calculations . piston-theory calculations agreed well with experiment for the thinner models but began to deviate as the thickness parameter approached and exceeded 1.0 . a few tests on flat-plate models with various elastic-axis locations were made . piston-theory calculations would not satisfactorily predict the flutter of these models, probably because of their blunt leading edges .'), (687, 'oscillating airfoils at high mach number .', 'lighthill, m.j.', 'j. aero. sc. v. 20. june 1953. pp 402-406 .', 'oscillating airfoils at high mach number . a simple formula is given for the pressure distribution on an oscillating airfoil in two-dimensional flow at high mach number . the formula is expected to be reasonably accurate if the pressure on the surface remains within the range 0.2 to 3.5 times the mainstream pressure . to illustrate the application of the formula, some results for symmetrical airfoils performing pitching oscillations are obtained and compared with results obtained from existing theories in the case of high mach number .'), (688, 'tables of aerodynamic coefficients obtained from developed newtonian expressions for complete and partial conic and spheric bodies at combined angles of attack and sideslip with some comparisons with hypersonic experimental data .', 'wells, w.r. and armstrong, w.o.', 'nasa tr r -dash 127, 1962 .', 'tables of aerodynamic coefficients obtained from developed newtonian expressions for complete and partial conic and spheric bodies at combined angles of attack and sideslip with some comparisons with hypersonic experimental data . closed-form expressions and tables composed from these expressions are presented for complete and partial conic and spheric bodies at combined angles of attack and sideslip in newtonian flow . aerodynamic coefficients of these bodies are tabulated for various body segments over a range of angles of attack from 1degree to 85degree and angles of sideslip from 0degree to 15degree . some comparisons between newtonian predictions and hypersonic experimental aerodynamic characteristics were made for conic bodies having various surface slopes, nose bluntnesses, and body cross sections to indicate the range of validity of the theory . in general, the theory is shown to agree quite well with experimental results for sharp-nose complete cones and for configurations having large blunted noses and steep surface slopes . however, agreement between theory and experiment generally is poor for the more slender, slightly blunted complete or half conic bodies and also for sharp-nose half conic bodies where real-flow phenomena such as forebody interference, viscous forces, leeward surface contributions, or leading-edge pressure reductions may have significant effect . the agreement between theory and experiment for the bodies considered can be improved by using the stagnation pressure coefficient behind a normal shock rather than 2 as the newtonian coefficient, although for the sharp-nose half conic bodies there is no theoretical justification for this modification .'), (689, 'investigation of the laminar aerodynamics heat transfer characteristics of a hemisphere cylinder in the langley 11-inch hypersonic tunnel at a mach number of 6. 8.', 'crawford,d.h. and mccauley,w.d.', 'naca r.1323, 1957.', 'investigation of the laminar aerodynamics heat transfer characteristics of a hemisphere cylinder in the langley 11-inch hypersonic tunnel at a mach number of 6. 8. a program to investigate the aerodynamic heat transfer of a nonisothermal hemisphere-cylinder has been conducted in the langley 11-inch hypersonic tunnel at a mach number of 6.8 and a reynolds number from approximately 0.14x10 to experimental heat-transfer coefficients were slightly less over the whole body than those predicted by the theory of stine and wanlass (naca technical note 3344) for an isothermal surface . for stations within 45 of the stagnation point the heat-transfer coefficients could be correlated by a single relation between local stanton number and local reynolds number . pitot pressure profiles taken at a mach number of 6.8 on a hemisphere-cylinder have verified that the local mach number or velocity outside the boundary layer required in the theories may be computed from the surface pressures by using isentropic flow relations and conditions immediately behind a normal shock . the experimental pressure distribution at a mach number of velocity gradients calculated at the stagnation point by using the modified newtonian theory vary with mach number and are in good agreement with those obtained from measured pressures for mach numbers from 1.2 to 6.8 . at the stagnation point the theory of sibulkin, in which the diameter and conditions behind the normal shock were used, was in good agreement with the experiment when the velocity gradient at the stagnation point appropriate to the free-stream mach number was used .'), (690, 'investigaion of the flow over a spiked-nose hemisphere cylinder at a mach number of 6. 8.', 'crawford,d.h.', 'nasa tn.d118, 1959.', 'investigaion of the flow over a spiked-nose hemisphere cylinder at a mach number of 6. 8. the shape and nature of the flow over a spiked-nose hemisphere-cylinder was studied in detail at a nominal mach number of 6.8 and in a reynolds number range (based on diameter and stream conditions ahead of the model) of 0.12 x 10 to 1.5 x 10 . schlieren photographs showed the effect of varying the spike length and reynolds number upon the shape of the separated boundary and upon the location of transition . the heat transfer and pressure distribution over the body were then correlated with the location of the start of separation, the location of reattachment, and the location of the start of transition .'), (691, 'calculation procedure for thermodynamic transport, and flow properties of the combustion products of a hydrocarbon fuel mixture burned in air with results for ethylene-air and methane-air mixtures .', '', 'nasa tn d-914, 1962 .', 'calculation procedure for thermodynamic transport, and flow properties of the combustion products of a hydrocarbon fuel mixture burned in air with results for ethylene-air and methane-air mixtures . a procedure is presented whereby the composition, thermodynamic properties, and transport properties of the dissociated combustion products of a fuel consisting of a mixed hydrocarbon compound burned in air may be calculated . equations and procedures for determining supersonic nozzle ordinates and flow properties for the dissociated combustion products are presented in an appendix . results are presented for the respective hydrocarbon fuels, methane and ethylene, at the equivalence ratios of 1.0, 0.9, 0.8, and 0.7 for pressures varying between 10 and 8 x 10 atmospheres and temperatures from 200degree k to'), (692, 'investigation of the jet effects on a flat surface downstream of the exit of a simulated turbojet nacelle at a free-stream mach number of 2.02 .', 'bressette, w.e.', 'naca rm l54e05a, 1954 .', 'investigation of the jet effects on a flat surface downstream of the exit of a simulated turbojet nacelle at a free-stream mach number of 2.02 . an investigation at a free-stream mach number of 2.02 was made to determine the effects of a propulsive jet on a wing surface located in the vicinity of a choked convergent nozzle . static-pressure surveys were made on a flat surface that was located in the vicinity of the propulsive jet . the nozzle was operated over a range of exit pressure ratios at different fixed vertical distances from the flat surface . within the scope of this investigation, it was found that shock waves, formed in the external flow because of the presence of the propulsive jet, impinged on the flat surface and greatly altered the pressure distribution . an integration of this pressure distribution, with the location of the propulsive jet exit varied from 1.450 propulsive-jet exit diameters to 3.392 propulsive-jet exit diameters below the wing, resulted in an incremental lift for all jet locations that was equal to the gross thrust at an exit pressure ratio of 2.86 . this incremental lift increased with increase in exit pressure ratio, but not so rapidly as the thrust increased, and was approximately constant at any given exit pressure ratio .'), (693, 'investigation of jet effects on a flat surface downstream of the exit of a simulated turbojet nacelle at a free-stream mach number of 1.39 .', 'bressette, w.e. and leiss, a.', '', 'investigation of jet effects on a flat surface downstream of the exit of a simulated turbojet nacelle at a free-stream mach number of 1.39 . an investigation at a free-stream mach number of 1.39 utilizing a blowdown-type tunnel was made to determine the effects of a propulsive jet on a zero angle-of-attack wing surface located in the vicinity of both a choked convergent nozzle and a convergent-divergent nozzle . staticpressure surveys were made on a flat surface that was located in the vicinity of the propulsive jet . the nozzles were operated over a varied range of both exit static- and total-pressure ratios at different within the scope of this investigation, it was found that shock waves, formed in the external flow because of the presence of the jet exhaust, impinged on the flat surface and greatly altered the pressure distribution . an integration of this pressure distribution for the choked convergent nozzle, with the location of the propulsive-jet exit varied from 1.747 jet-exit diameters to 4.981 jet-exit diameters below the wing surface, resulted in a positive incremental normal force on the wing at all positions .'), (694, 'pressure distribution induced on a flat plate by a supersonic and sonic jet exhaust at a free-stream mach number of 1.80 .', 'leiss, a. and bressette, w.e.', 'naca rm l56106, 1957 .', 'pressure distribution induced on a flat plate by a supersonic and sonic jet exhaust at a free-stream mach number of 1.80 . as a continuation of previous research at mach numbers of 2.02 and 1.39, an experimental investigation was made of the pressures induced on a flat plate by a propulsive jet exhausting from sonic and supersonic nozzles at a free-stream mach number of 1.80 . measurements of the pressure distribution on a flat-plate wing were made at zero angle of attack for four different locations of the jet exhaust nozzle beneath the wing . both a choked convergent nozzle and a convergent-divergent nozzle on the nacelle were used . the nozzles were operated at nacelle-exit total-pressure ratios from 2 to 16 and the reynolds number per foot was approximately 13 x 10 . two distinct shock waves impinged on the wing surface and greatly altered the pressure distribution at all nozzle positions . positive incremental normal force resulted on the wing at all positions . comparisons are presented for two free-stream mach numbers .'), (695, 'some experiments relating to the problem of simulation of hot jet engines in studies of jet effects on adjacent surfaces at a free-stream mach number of 1.80 .', 'bressette, w.e.', 'naca rm l56e07. 1956 .', 'some experiments relating to the problem of simulation of hot jet engines in studies of jet effects on adjacent surfaces at a free-stream mach number of 1.80 . an investigation at a free-stream mach number of 1.80 in a blowdown type tunnel was made to study the effect on the pressure distribution of a zero angle of attack wing surface when certain exhaust parameters of a hot turbojet engine are varied . static-pressure surveys were made on a wing surface that was located in the vicinity of a small-scale propulsive jet . this propulsive jet was operated with four types of jet exhausts . these jet exhausts were a hot jet /hydrogen burned in air/, a cold air jet, a cold helium jet, and a jet composed of a mixture of two cold gases /hydrogen and carbon dioxide/ . the hot jet, because of its high exhaust temperature /3,300degreer/ and because combustion was performed in air, was believed reasonably able to simulate the exhaust parameters of an actual afterburning turbojet engine . the cold jets used were selected in order that the effects of a variation in the exhaust parameters of jet-exit static-pressure ratio, ratio of specific heats, density, and velocity, could be obtained by comparing each cold jet with the hot jet or with another cold jet . the tests were made over a range of jet-exit staticpressure ratios from 1 to 9 with values of the ratio of specific heats of 1.27, 1.40, and 1.66 and at variations in density and velocity of the order of approximately 8 and 3 times, respectively . within the scope of this investigation, it was found that jet-exit static-pressure ratio and the ratio of specific heats affected the pressure distribution on the wing associated with jet interference while a variation in exit velocity and density did not . the jet-exit staticpressure ratio affected the wing pressure distribution in a major way while the ratio of specific heats had only a minor effect . the addition of temperature in the propulsive jet exhaust at a jet-exit staticpressure ratio of 4 had little or no effect on the pressure distribution associated with jet interference on the wing .'), (696, 'pressure loads produced on a flat-plate wing by rocket jets exhausting in a spanwise direction below the wing and perpendicular to a free-stream flow of mach number 2.0 .', 'falangan, r.a. and janos, j. j.', 'nasa tn d-893, 1961 .', 'pressure loads produced on a flat-plate wing by rocket jets exhausting in a spanwise direction below the wing and perpendicular to a free-stream flow of mach number 2.0 . an investigation at a reynolds number per foot of 14.4 x 10 was made to determine the pressure loads produced on a flat-plate wing by rocket jets exhausting in a spanwise direction beneath the wing and perpendicular to a free-stream flow of mach number 2.0 . the ranges of the variables involved were /1/ nozzle types - one sonic /jet mach number of two-dimensional supersonic /jet mach number of 1.71/,. /2/ vertical nozzle positions beneath the wing of 4, 8, and 12 nozzle-throat diameters,. and /3/ ratios of rocket-chamber total pressure to free-stream static pressure from 0 to 130 . the incremental normal force due to jet interference on the wing varied from one to two times the rocket thrust and generally decreased as the pressure ratio increased . the chordwise coordinate of the incremental-normal-force center of pressure remained upstream of the nozzle center line for the nozzle positions and pressure ratios of the investigation . the chordwise coordinate approached zero as the jet vertical distance beneath the wing increased . in the spanwise direction there was little change due to varying rocket-jet position and pressure ratio . some boundary-layer flow separation on the wing was observed for the rocket jets close to the wing and at the higher pressure ratios . the magnitude of the chordwise and spanwise pressure distributions due to jet interference was greatest for rocket jets close to the wing and decreased as the jet was displaced farther from the wing . the design procedure for the rockets used is given in the appendix .'), (697, 'effects on adjacent surfaces from the firing of rocket jets .', 'bressette, w.e. and leiss, a.', 'naca rm l 57d19a, 1957 .', 'effects on adjacent surfaces from the firing of rocket jets . this paper is a preliminary and brief account of some research currently being conducted to determine the jet effects on adjacent surfaces from the firing of rocket jets . measurements of jet-effect pressures on a flat plate as well as shadowgraphs are presented that were obtained when a rocket jet at a mach number of 3 was exhausted downstream and upstream into free-stream flow at a mach number of 2 located from 2 to 4.7 rocket-jet-exit diameters from the plate . the jet effects on the flat plate with the rocket jet exhausting downstream are of the same order of magnitude as those previously obtained from sonic exits with a total pressure 10 times lower . a maximum pressure coefficient on the plate of rocket-jet-exit diameters below the plate, and an integration of the measured jet-effect pressures at this position resulted in a normal force on the plate equal to 2.3 times the thrust output of the rocket jet .'), (698, 'the unsteady lift of a wing of finite aspect ratio .', 'jones, r.t.', '1940, naca rep. 681 .', 'the unsteady lift of a wing of finite aspect ratio . unsteady-lift functions for wings of finite aspect ratio have been calculated by correcting the aerodynamic inertia and the angle of attack of the infinite wing . the calculations are based on the operational method . the starting lift of the finite wing is found to be only slightly less than that of the infinite wing,. whereas the final lift may be considerably less . the theory indicates that the initial distribution of lift is similar to the final distribution . curves showing the variation of lift after a sudden unit change in angle of attack, during penetration of a sharpedge gust, and during a continuous oscillation are given . operational equivalents of these functions have been devised to facilitate the calculation of lift under various conditions of motion . as an application of these formulas, the vertical acceleration of a loaded wing caused by penetrating a gust has been calculated .'), (699, 'approximate indical lift functions for several wings of finite span in incompressible flow as obtained from oscillatory lift coefficients .', 'drischler j.a.', 'naca tn 3639, 1956 .', 'approximate indical lift functions for several wings of finite span in incompressible flow as obtained from oscillatory lift coefficients . the unsteady-lift functions for a wing undergoing a sudden change in sinking speed have been presented for delta wings having aspect ratios of 0, 2, and 4 and for rectangular and elliptical wings having aspect ratios of 0, 3, and 6 . for the elliptical and rectangular wings the spanwise lift distributions were also presented . these functions were calculated from the lift coefficients associated with a wing oscillating harmonically in pure translational motion, as obtained from several sources . the results of these calculations indicate that the normalized unsteady-lift functions are substantially independent of the shape of the plan form for elliptical, rectangular, or moderately tapered wings,. however, for delta wings the increase of lift toward the steady-state value is much more rapid than that for the aforementioned wings of the same aspect ratio . these results also corroborate the results of other investigations in that the rate of growth of lift tends to increase with a decrease in aspect ratio . the shape of the spanwise distributions of the indicial lift seems to be, for all practical purposes, independent of time for rectangular and elliptical wings .'), (700, 'two and three-dimensional unsteady lift problems in high speed flight .', 'lomax et al.', 'naca rep. 1077, 1952 .', 'two and three-dimensional unsteady lift problems in high speed flight . the problem of transient lift on two- and three-dimensional wings flying at high speeds is discussed as a boundary-value problem for the classical wave equation . kirchhoffs formula is applied so that the analysis is reduced, just as in the steady state, to an investigation of sources and doublets . the applications include the evaluation of indicial lift and pitchingmoment curves for two-dimensional sinking and pitching wings flying at mach numbers equal to 0, 0.8, 1.0, 1.2, and triangular wings in both forward and reversed flow are presented and compared with the two-dimensional values .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (701, 'numerical determination of indical lift of a two-dimensional sinking airfoil at subsonic mach numbers from oscillatory lift coefficients with calculations for mach number 0.7 .', 'mazelsky, b.', 'naca tn 2562, 1951 .', 'numerical determination of indical lift of a two-dimensional sinking airfoil at subsonic mach numbers from oscillatory lift coefficients with calculations for mach number 0.7 . the reciprocal equations for relating the incompressible circulatory indicial lift to the lift due to harmonic oscillations have been modified to include the noncirculatory lift associated with apparent-mass effects . although the apparent-mass effects are impulsive in nature in incompressible flow, the lift due to apparent-mass effects in compressible flow is a time-dependent function . the corresponding reciprocal equations for the total compressible lift are given . by use of the reciprocal equations for compressible flow, the indicial lift and moment functions due to an airfoil''s experiencing a sudden acquisition of vertical velocity are determined numerically for mach number 0.7 . lack of sufficient flutter coefficients prevents the calculation of these functions at other mach numbers . although the indicial lift and moment functions due to penetration of a sharp-edge gust may be obtained from the oscillatory tab or aileron coefficients by a similar analysis, sufficient coefficients are not available at the present . however, an approximate method is shown for determining a portion of this unsteady-lift function . when a comparison is made of the indicial lift functions at mach numbers appears to be less rapid for the compressible case than for the incompressible case . consequently, the calculation of the gust load factor at high subsonic mach numbers utilizing the two-dimensional incompressible indicial lift functions and an over-all correction for compressibility such as the prandtl-glauert factor might be conservative .'), (702, 'numerical determination of indical lift and moment functions for a two dimensional sinking and pitching airfoil at mach numbers 0.5 and 0.6 .', 'mazelsky, b. and drischler, j.a.', 'naca tn 2739, 1952 .', 'numerical determination of indical lift and moment functions for a two dimensional sinking and pitching airfoil at mach numbers 0.5 and 0.6 . the indicial lift and moment functions are determined approximately for sinking and pitching motion at mach numbers m of 0.5 and 0.6 . these functions are determined from a knowledge of the existing oscillatory coefficients at the low reduced frequencies and from approximate expressions of these coefficients at the high reduced frequencies . the beginning portion of the indicial lift function associated with an airfoil penetrating a sharp-edge gust in subsonic flow is evaluated by use of an exact method . by use of an approximate method for determining the remaining portion, the complete indicial gust function is determined for m 0.5, m 0.6, and m 0.7 . all the indicial lift and moment functions are approximated by an exponential series,. the coefficients which appear in the exponential approximations for each indicial function are tabulated for m 0.5, m 0.6, and m 0.7 .'), (703, 'general airfoil theory .', 'kussner, h.g.', 'naca tm 979, 1941 .', 'general airfoil theory . on the assumption of infinitely small disturbances the author develops a generalized integral equation of airfoil theory which is applicable to any motion and compressible fluid . successive specializations yield various simpler integral equations, such as possio''s, birnbaum''s, and prandtl''s integral equations, as well as new ones for the wing of infinite span with periodic downwash distribution and for the oscillating wing with high aspect ratio . lastly, several solutions and methods for solving these integral equations are given .'), (704, 'a systematic kernel function procedure for determining aerodynamic forces on oscillating or steady finite wings at subsonic speeds .', 'watkins, c.e., woolston, d.s. and cunningham, h.j.a.', 'nasa tr r-48, 1959 .', 'a systematic kernel function procedure for determining aerodynamic forces on oscillating or steady finite wings at subsonic speeds . a detailed description is given of a method of approximating solutions to the integral equation that relates oscillatory or steady lift and downwash distributions on finite wings in subsonic flow . the method of solution is applicable to general plan forms with either curved or straight leading and trailing edges . moreover, it is directly applicable to control surfaces such as all-movable tails but modifications are needed to apply it to controls in general . applications of the method involve evaluations of numerous integrals that must be handled by numerical procedures but systematic schemes of evaluations have been adopted that are well suited to the routines of automatic digital computing machines . these schemes of evaluation have been incorporated in a program for an ibm 704 electronic data processing machine . with this machine, a pressure distribution together with such quantities as section or total lift and moment coefficients or generalized forces can be determined for a given value of frequency and mach number and for several /four or five/ modes of oscillation in about 4 minutes of machine time . in the case of steady downwash conditions corresponding quantities can be obtained in about 2 minutes of machine time . in order to illustrate applications of the method, results of several calculations are presented . in these illustrations total forces and moments are compared /1/ with results of analytic procedures for a circular plan form with steady downwash conditions, /2/ with results of other theories and with experiment for a rectangular plan form of aspect ratio 1 at a uniform angle of attack, and /3/ with some experimental results for a rectangular plan form of aspect ratio 2 undergoing pitching and flapping oscillations . also included in the illustrations are results of flutter calculations compared with experimental results for an allmovable control surface of aspect ratio 3.50 and for a cantilevered rectangular plan form of aspect ratio 5.04 .'), (705, 'on the kernel function of the integral equation relating the lift and downwash distributions of oscillating finite wings in subsonic flow .', 'watkins, c.e. and runyal, h.l. and woolston, d.s.', 'naca rep. 1234, 1955 .', 'on the kernel function of the integral equation relating the lift and downwash distributions of oscillating finite wings in subsonic flow . this report treats the kernel function of an integral equation that relates a known or prescribed downwash distribution to an unknown lift distribution for a harmonically oscillating finite wing in compressible subsonic flow . the kernel function is reduced to a form that can be accurately evaluated by separating the kernel function into two parts .. a part in which the singularities are isolated and analytically expressed and a nonsingular part which may be tabulated . the form of the kernel function for the sonic case /mach number of 1/ is treated separately . in addition, results for the special cases of mach number of o /incompressible case/ and frequency of o /steady case/ are given . the derivation of the integral equation which involves this kernel function, originally performed elsewhere /see, for example, naca technical memorandum 979/, is reproduced as an appendix . another appendix gives the reduction of the form of the kernel function obtained herein for the three-dimensional case to a known result of possio for two-dimensional flow . a third appendix contains some remarks on the evaluation of the kernel function, and a fourth appendix presents an alternate form of expression for the kernel function .'), (706, 'on som reciprocal relations in the theory of nonstationary flows .', 'garrick, i.e.', 'naca rep. 629. 1938 .', 'on som reciprocal relations in the theory of nonstationary flows . in the theory of nonstationary flows about airfoils, the /indicial lift/ function k /s/ of wagner and the /alternating lift/ function c /k/ of theodorsen have fundamental significance . this paper reports on some interesting relations of the nature of fourier transforms that exist between these functions . general problems in transient flows about airfoils may be given a unified broad treatment when these functions are employed . certain approximate results also are reported which are of notable simplicity, and an analogy with transient electrical flows is drawn .'), (707, 'thermal analysis of stagnation regions with emphasis on heat-sustaining nose shapes at hypersonic speeds .', 'hanawalt, a.j., blessing, a.h. and schmidt, c.m.', 'j.aero. sc. may 1959. p. 257-263 .', 'thermal analysis of stagnation regions with emphasis on heat-sustaining nose shapes at hypersonic speeds . the leading edges and noses of hypersonic vehicles are subjected to severe aerodynamic heating and must be cooled in some manner-dash e.g., internal convection, transpiration, or radiation . it is this latter mode of handling the problem that is discussed in this paper . neglecting conduction in the leading-edge region, the maximum temperature for long-range hypersonic gliders is of the same order as the melting point of refractory materials, with a corresponding large temperature gradient away from the leading edge . inclusion of conduction in the aft direction reduces the maximum temperature and distributes the heat to a location that will radiate it out from the surface . for either steady-state or transient conditions, the temperature at the leading edge is reduced by conduction, while the temperature aft of the leading-edge shoulder is increased, thus setting up a heat transmission balance between the convective influx of heat, the redistribution of heat by conduction, and the radiation of heat from the surface . the feasibility of such a mechanism can be enhanced by suitably choosing leading-edge shapes and materials . the philosophy behind the choice of leading-edge shapes is discussed and the effects of varying parameters, such as shape, diameter, emissivity, conductivity, thickness, etc., are shown .'), (708, 'aerodynamic characteristics of two winged reentry vehicles at supersonic and hypersonic speeds .', 'ladson, c.l. and johnston, p.j.', 'nasa tm x 346, 1961 .', 'aerodynamic characteristics of two winged reentry vehicles at supersonic and hypersonic speeds . tests were conducted at the langley research center on two winged lifting hypersonic reentry glider configurations . performance, stability, and control data are presented at mach numbers of 1.62 and 2.91 for angles of attack up to 15degree and at mach numbers of 6.8 and 9.6 for angles of attack up to 25degree .'), (709, 'static longitudinal aerodynamic characteristics at transonic speeds and angles of attack up to 99degree of a reentry glider having folding wingtip panels .', 'olstad, w.b.', 'nasa tm x-610, 1961 .', 'static longitudinal aerodynamic characteristics at transonic speeds and angles of attack up to 99degree of a reentry glider having folding wingtip panels . data are presented which were obtained from a transonic wind-tunnel investigation of a reentry glider having folding wing-tip panels . the tests were conducted at angles of attack from -4degrees to 99degrees . the reynolds number based on the mean geometric chord of the fixed planform varied from 2.35 x 10 to 2.99 x 10 . the maximum lift-drag ratio for the model with the folding wing-tip panels fully extended decreased from a maximum value of 7.8 at a mach number of 0.60 to about 3.4 at mach numbers from 1.03 to 1.20 . the model with the folding wing panels fully extended was stable for values of the lift coefficient from 0 up to at least 0.8 . above this lift coefficient pitch-up tendencies were observed, followed by an unstable or neutrally stable region which extended up to values of angle of attack of 50degrees or 60degrees . deflecting the folding wing panels between ducing a significant change in the trim angle of attack or in any of the force or moment coefficients in the angle-of-attack range from 49degree to 99degree .'), (710, 'the smallest height of roughness capable of affecting boundary-layer transition .', 'smith, a.m.o. and clutter, d.w.', 'j. aero. sc. april, 1959. p.229-245, 256 .', 'the smallest height of roughness capable of affecting boundary-layer transition . an investigation was made to determine the smallest size of isolated roughness that will affect transition in a laminar-boundary layer . critical heights for three types of roughness were found in a low-speed wind tunnel . the types were /1/ two-dimensional spanwise wires, /2/ three-dimensional discs, and /3/ a sandpaper type . in addition to type of roughness, test variables included the location of roughness, pressure distribution, degree of tunnel turbulence, and length of natural laminar flow . the most satisfactory correlation parameter was found to be the roughness reynolds number, based on the height of roughness and flow properties at this height . the value of this critical reynolds number was found to be substantially independent of all test variables except the shape of roughness . this parameter also correlates well other published data on critical roughness in low-speed flow . the value of the roughness reynolds number necessary to move transition forward to the roughness itself was also determined for the three types of roughness and was found to be approximately constant for a given type of roughness . an investigation of the limited amount of available data on critical roughness in supersonic flow indicates that the effects of roughness may still be correlated by the roughness reynolds number . the value of this reynolds number depends primarily on the mach number at the top of the roughness . when this mach number is greater than 1.0, the roughness reynolds number based on conditions behind a shock is probably the characteristic parameter .'), (711, 'an investigation at subsonic speeds of aerodynamic characteristics at angles of attack from -dash 4degrees to 100degrees of a delta-wing reentry configuration having folding wingtip panels .', 'spencer, b.', 'nasa tm x-288, 1960 .', 'an investigation at subsonic speeds of aerodynamic characteristics at angles of attack from -dash 4degrees to 100degrees of a delta-wing reentry configuration having folding wingtip panels . an investigation was made at subsonic speeds in the langley highspeed lifting reentry configuration having folding wingtip panels . the configuration is of the type used in a high angle-of-attack /near 90degree/ reentry to minimize aerodynamic heating . by unfolding the wingtip panels into the airstream, a moderate angle-of-attack glide is used for a controlled landing . the basic configuration tested utilized a whose area was 25 percent of the total wing area . the effects of varying the plan form and size of the wingtip panels was studied as well as the effects of unfolding the wingtip panels in a high angle- of-attack attitude . tests were made at mach numbers of 0.40, 0.60, and'), (712, 'low-speed longitudinal aerodynamic characteristics associated with a series of low-aspect ratio wings having variations in leading-edge contour .', 'spencer, b. and hammond, a.d.', 'nasa tn d-1374, 1962 .', 'low-speed longitudinal aerodynamic characteristics associated with a series of low-aspect ratio wings having variations in leading-edge contour . an investigation has been conducted at various reynolds numbers and low subsonic speeds to determine the longitudinal aerodynamic characteristics associated with a series of low-aspect-ratio wings having variations in leading-edge contours . the planforms included a highly swept triangular wing, a rectangular wing, and intermediate wings including planforms having elliptic and parabolic leading-edge contours, all having an aspect ratio of 1.33 . the effects of changing aspect ratio for a given leading-edge contour were investigated for two of the wings presented,. also included are the longitudinal characteristics associated with various fuselage sizes . an effort has been made to estimate the lift variation with angle of attack for the wing planforms of the present investigation . improvements in the lifting capabilities at low subsonic speeds associated with a basic triangular planform of low aspect ratio are possible by slight alterations in leading-edge design, which should still conform to possible design requirements at hypersonic speeds . these changes in planform resulted in increases in lift-curve slope, lift at high angles of attack, and in the maximum untrimmed lift-drag ratio, provided the fuselage was sufficiently small . the longitudinal stability characteristics of the majority of planforms indicate more desirable stability characteristics at high lifts than either a triangular wing or rectangular wing of the same aspect ratio . the effects of increasing reynolds number for each of the planforms investigated generally resulted in slight reductions in the lift at high angles of attack . a method is presented for estimating the subsonic-lift variation with angle of attack for the low-aspect-ratio wings of the present investigation and indicated good agreement with experimental data throughout the angle-of-attack range of this investigation .'), (713, 'static longitudinal stability characteristics of a blunted glider re-entry configuration having 79.5degree sweepback and 45degree dihedral at a mach number of 6.2 and angles of attack up to 20degree .', 'mayo, e.e.', 'nasa tm x-222. 1959 .', 'static longitudinal stability characteristics of a blunted glider re-entry configuration having 79.5degree sweepback and 45degree dihedral at a mach number of 6.2 and angles of attack up to 20degree . an experimental investigation was conducted at a mach number of 6.2 to determine the static longitudinal stability characteristics of a model of a blunted glider reentry configuration having 79.5degree sweepback and 45degree dihedral . the free-stream reynolds number for the investigation was 3.0 x 10 based on the basic model length of 7.5 inches . tests were made through an angle-of-attack range from 0degrees to investigation showed that incorporating 10degree nose incidence in the basic model resulted in a lower lift-curve slope, a lower lift-drag ratio, a higher value of trim lift coefficient, and a decrease in static longitudinal stability . in comparison, the effect of extending the configuration length and incorporating 10degrees and 20degrees boattail angles resulted in smaller changes in the longitudinal stability characteristics of the model .'), (714, 'blockage corrections for three-dimensional flow closed throat wind tunnels, with considerations of the effect of compressibility .', 'herriot,j.g.', 'naca r.995, 1950.', 'blockage corrections for three-dimensional flow closed throat wind tunnels, with considerations of the effect of compressibility . theoretical blockage corrections are presented for a body of revolution and for a three-dimensional unswept wing in a circular or rectangular wind tunnel . the theory takes account of the effects of the wake and of the compressibility of the fluid, and is based on the assumption that the dimensions of the model are small in comparison with those of the tunnel throat . formulas are given for correcting a number of the quantities, such as dynamic pressure and mach number, measured in wind-tunnel tests . the report presents a summary and unification of the existing literature on the subject .'), (715, 'motion of a ballistic missile angularly misaligned with the flight path upon entering the atmosphere and its effect upon aerodynamic heating, aerodynamic loads and miss distance .', 'allen,h.j.', 'naca tn.4048, 1957.', 'motion of a ballistic missile angularly misaligned with the flight path upon entering the atmosphere and its effect upon aerodynamic heating, aerodynamic loads and miss distance . an analysis is given of the oscillating motion of a ballistic missile which upon entering the atmosphere is angularly misaligned with respect to the flight path . the history of the motion for some example missiles is discussed from the point of view of the effect of the motion on the aerodynamic heating and loading . the miss distance at the target due to misalignment and to small accidental trim angles is treated . the stability problem is also discussed for the case where the missile is tumbling prior to atmospheric entry .'), (716, 'study of the oscillatory motion of manned vehicles entering the earth''s atmosphere .', 'sommer,s.c. and tobak,m.', 'nasa memo 3-2-59a, 1959.', 'study of the oscillatory motion of manned vehicles entering the earth''s atmosphere . an analysis is made of the oscillatory motion of vehicles which traverse arbitrarily prescribed trajectories through the atmosphere . expressions for the oscillatory motion are derived as continuous functions of the properties of the trajectory . results are applied to a study of the oscillatory behavior of re-entry vehicles which have decelerations that remain within limits of human tolerance . it is found that a deficiency of aerodynamic damping for such vehicles may have more serious consequences than it does for comparable ballistic missiles .'), (717, 'motions of a short 10degree blunted cone entering a martian atmosphere at arbitrary angles of attack and arbitrary pitching rates .', 'peterson, v.l.', 'nasa tn-d 1326 .', 'motions of a short 10degree blunted cone entering a martian atmosphere at arbitrary angles of attack and arbitrary pitching rates . the dynamic behavior of two probe vehicles entering a martian atmosphere in a passive manner with arbitrary initial angles of attack and pitching rates to 12degree per second has been determined . results for an entry velocity of 21,700 feet per second and an entry angle of -40degree were obtained from machine calculated solutions of the six-degree- of-freedom rigid-body equations of motion using experimental aerodynamic characteristics for the vehicles . one of the vehicles had a flat base and was statically stable in two attitudes /nose forward and base forward/ . the other vehicle, derived from the first by adding a conical afterbody, was statically stable in only one attitude /nose forward/ . a 10-rpm vehicle spin rate, believed ample for the purpose of distributing solar and aerodynamic heating over the vehicle surface, and model atmospheres encompassing the probable extremes for the planet were also considered . it was found that while the motion of the flat-based vehicle could be oscillatory about either the nose-forward or base-forward stable trim attitudes when aerodynamic heating rates were high, the range of initial angles of attack resulting in base-forward orientation was reduced by more than a factor of 3. when initial pitch rates were increased from body having only nose-forward stability showed that oscillatory angles of attack at maximum heating-rate conditions probably would not exceed about 25degrees although angles of attack when heating rates were 50 percent of maximum could be as high as 40degree . values of these upper bound angles of attack were essentially independent of initial pitch rates for the range considered . furthermore, the envelope of maximum probable angles of attack was increased only slightly when the vehicle was given a 10-rpm spin rate . the relationship between maximum amplitudes of oscillation and heating rates through high heating portions of the trajectories was preserved when model atmospheres believed to encompass the extreme possibilities for mars were used in the calculations'), (718, 'means and examples of aeronautical research in france at onera .', 'maurice roy', 'the twenty-second wright brothers lecture office national d''etudes et de recherches aeronautiques', 'means and examples of aeronautical research in france at onera . cosmonautics is currently very much to the forefront in the news . it embraces and extends aeronautics, and i would like to propose including both, at least on certain occasions, under a general denomination of /aerocosmonautics/ . in your country, the sciences and technology of space are subjects which have been backed by initial advances and abundantly treated . since france has not yet launched any artificial satellite or built any circumlunar space vehicle, i propose to confine myself here to the field of aeronautics, where there is still so much progress of manifest utility to accomplish . i shall accordingly content myself with presenting some examples of aeronautical research and experiments undertaken in my country by onera, a body whose mission is akin to that of the illustrious naca, now nasa, but bearing in mind the considerable difference between the scales of the respective resources .'), (719, 'tumbling bodies entering the atmosphere .', 'remmler, k.l.', 'ars jnl. v. 32, january 1962. pp 92-95 .', 'tumbling bodies entering the atmosphere . the equations of motion of a tumbling flat plate entering an exponential atmosphere were linearized and solved analytically to obtain a simple expression for the altitude at which tumbling would cease and libration would commence . the plate had only three degrees of freedom, and aerodynamic forces were derived from newtonian impact theory . in the linear analysis, mean values of the drag and pitch damping coefficients so that flutter occurs in the range of a low-speed wind tunnel . a particular type of construction for supersonic flutter models is described in detail . methods of vibration testing, static testing, and flutter testing are discussed . particular emphasis is placed on the technique of varying flow parameters rather than model parameters to precipitate flutter . the tool for varying flow parameters is the variable mach number supersonic test section of the massachusetts institute of technology blowdown wind tunnel . the aerodynamic features of the supersonic test section are presented .'), (720, 'a note on the use of sandwich structures in severe acoustic environments .', 'd. j. mead, d. c. ae.', 'd. j. mead, d.c.ae.', 'a note on the use of sandwich structures in severe acoustic environments . this paper reviews some of the experience to date of using sandwich type structures in severe acoustic pressure environments . the methods used for testing sandwich structures for acoustic fatigue are described and their limitations considered . experimental and theoretical work relating to the damping and mode-frequency relationships of certain sandwich configurations is also reviewed . special attention is given to the estimation of the stress in the bond of a honeycomb sandwich panel subjected to sudden pressure fluctuations . a /uni-modal/ theory is presented, relating the mean-square bond-stress to the random exciting pressure and panel dynamic characteristics . this theory indicates that tensile bond stresses may be encountered of up to six times the local r.m.s. exciting pressure . these must be combined with bending and shear stresses to obtain the principal stresses which precipitate bond fatigue failures . finally, an outline is given of some of the lines of future research which should lead to the achieving of the maximum possible fatigue resistance from sandwich configurations .'), (721, 'near noise field of a jet engine exhaust .', 'callaghan,e.e., howes,w.l. coles,w.d. and mull,r.h.', 'naca r.1338.', 'near noise field of a jet engine exhaust . aircraft structures located in the near noise field of a jet engine are subjected to extremely high fluctuating pressures that may cause structural fatigue . studies of such structures have been limited by lack of knowledge of the loadings involved . the acoustic near field produced by the exhaust of a stationary turbojet engine having a high pressure ratio was measured for a single operating condition without afterburning . the maximum over-all sound pressure without afterburning was found to be about 42 pounds per square foot along the jet boundary in the region immediately downstream of the jet-nozzle exit . with afterburning the maximum sound pressure was increased by 50 percent . the largest sound pressures without afterburning were obtained on a constant percentage band width basis in the frequency range from 350 to 700 cps . additional tests were made at a few points to find the effect of jet velocity on near-field sound pressures and to determine the difference in value between sound-pressure levels at rigid surfaces and corresponding free-field values . near the jet nozzle, over-all sound pressures were found to vary as a low power (approx. unity) of the jet velocity . over-all sound-pressure levels considerably greater than the corresponding free-field levels were recorded at the surface of a rigid plate placed along the jet boundary . the downstream locations of the maximum sound pressure at any given frequency along the jet-engine-exhaust boundary and the longitudinal turbulent-velocity maximum of the same frequency along a small cold-air jet at 1 nozzle-exit radius from the jet axis were found to be nearly the same when compared on a dimensionless basis . also, the strouhal number of the corresponding spectra maximums was found to be nearly equal at similar distances downstream . in addition to the magnitude and frequency distribution of the acoustic pressures, it is necessary to know the cross correlation of the pressure over the surface area . cross-correlation measurements with microphones were made for a range of jet velocities at locations along the jet and at a distance from the jet . free-field correlations of the over-all sound pressure and of the sound pressure in frequency bands from 100 to 1000 cps were obtained both longitudinally and laterally . in addition, correlations were obtained with microphones mounted at the surface of a rigid plate that was large compared with the distance over which a positive correlation existed . the region of positive correlation was generally found to increase with distance downstream of the engine to 6.5 nozzle-exit diameters, but remained nearly constant thereafter . in general, little change in the correlation curves was found as a function of jet velocity or frequency-band width . the distance from unity correlation to the first zero correlation was greater for lateral than for longitudinal correlations for the same conditions and locations . the correlation curves obtained in free space and on the surface of the plate were generally similar . the results are interpreted in terms of pressure loads on surfaces .'), (722, 'random excitation of a tailplane section by jet noise .', 'clarkson,b.l. and ford,r.d.', 'univ. southampton r. a.a.s.u.171.', 'random excitation of a tailplane section by jet noise . the response of a section of tailplane structure to both discrete and random noise pressures has been studied in detail . initially the specimen was mounted behind a jet engine and the induced strains were analysed with the object of determining both the resonant frequencies and the corresponding modes of vibration . during these tests a survey was made of the spectrum and correlation pattern of the jet noise on the surface of the model . secondly the specimen was mounted in front of a loudspeaker in an acoustics laboratory and the structural resonances were excited by means of discrete frequency sound . the mode shapes were studied in detail with the aid of a stroboscope . it is concluded that the tailplane skin on this particular piece of structure only responds to any significant degree in one structural mode . although reasonable comparison has been obtained between the random and discrete tests, it was not possible to calculate the induced stresses using the observed mode shapes and measured pressure excitation .'), (723, 'on the fatigue failure of structures due to vibrations excited by random pressure fields .', 'alan powell', 'the university, southampton', 'on the fatigue failure of structures due to vibrations excited by random pressure fields . on the assumption that the forced modes of vibration of a structure, subjected to pressure fluctuations random in time and space, can be approximated by the composition of the motions of the uncoupled natural modes, a general analysis is made using the ideas of vibration theory and spectrum analysis . the power spectrum, and hence the rms value, of any quantity depending linearly upon structural distortions is derived and it involves a quantity (called the /joint acceptance/) concerning the spacewise structure of the pressure field and of the geometry of the modes of vibration . it is shown how this result may be used (on assuming /normal/ randomness) to estimate the fatigue life on the hypothesis of cumulative damage .'), (724, 'structural acoustic proof testing .', 'schjeldrup,h.c.', 'air eng. 1959.', 'structural acoustic proof testing . with the introduction of high-powered propulsion systems, and paralleling their continued development, an accompanying increase in acoustical problems has arisen . of these acoustical problems, that of acoustical fatigue failures has become paramount in the eyes of the structural engineer . aircraft designed to normal strength requirements have been known literally to fall apart under acoustical loading . this problem has required much endeavour to produce a solution, and considerable structural research, based upon results of siren or other testing, have proved inadequate . this failure to find a satisfactory solution has resulted in the conviction that the final proof of a design can be found only in proof testing . proof testing, in the acoustic fatigue sense, is the testing of a design structure in a simulated acoustical environment for a period of time long enough to assure equality with design life .'), (725, 'the response of a typical aircraft structure to jet noise .', 'clarkson,b.l. and ford,r.d.', 'j. roy. aero. soc. 1962.', 'the response of a typical aircraft structure to jet noise . an analysis is made of experimentally determined mode shapes excited on the rear structure of a modern airliner by jet noise from a pod-mounted turbojet engine . power spectra of stresses determined from strain-gage measurements are obtained and cross correlated . extensive measurements were made on skin panels of the fuselage and elevator and limited ones were made on fuselage stringers and frames . the skin-panel results are compared with theoretical predictions . reviewer believes that this paper is of considerable value for those concerned with response of aircraft-type structures to jet-induced noise .'), (726, 'on structural fatigue under random loading .', 'miles,j.w.', 'j. ae. scs. 21, 1954, 753.', 'on structural fatigue under random loading . experience has shown that the fluctuating loads induced by a jet may cause fatigue failure of aircraft structural components . in order to throw some light on this and similar problems, the stress spectrum and the /equivalent fatigue stress/ of an elastic structure subjected to random loading are studied . the analysis is simplified by assuming the structure to have only a single degree of freedom and by using the concept of cumulative damage, the results being expressed in terms of quantities that can be directly measured . as an example, a similarity expression for the probable value of the equivalent fatigue stress of a panel subjected to jet buffeting is derived .'), (727, 'a study of the acoustic fatigue characteristics of some flat and curved aluminium panels exposed to random and discrete noise .', 'hess,r.w., herr,r.w. and mayes,w.h.', 'nasa tn.d1.', 'a study of the acoustic fatigue characteristics of some flat and curved aluminium panels exposed to random and discrete noise . a study was made of the fatigue life of simple 2024-t3 aluminum-alloy panels measuring 11 by 13 inches and exposed to both discrete-frequency noise from a siren and random noise from an air jet . noise levels varied from approximately panel variables included thickness, edge conditions, curvature, and static-pressure differential . no significant differences were noted in the nature of failures experienced for the two types of loadings . at a given root-mean-square stress level, the failure times were generally shorter for the random loading than for the discrete-frequency loading . these differences in failure times were noted to be a function of stress level, the larger differences occurring at the lower stress levels . increases in time to failure were obtained as a result of increased panel thickness, increased panel curvature, and particularly for increased static-pressure differential across curved panels . for the discrete-type loading, the location of weak points in these simplified structural designs can be satisfactorily accomplished but quantitative predictions of fatigue life are much more difficult .'), (728, 'free vibrations of continuous skin stringer panels .', 'lin, y.k.', 'j. appl. mech. december 1960 .', 'free vibrations of continuous skin stringer panels . the determination of the natural frequencies and normal modes of vibration for continuous panels, representing more or less typical fuselage skin-panel construction for modern airplanes, is discussed in this paper are considered . a numerical example is presented, and analytical results for a particular structural configuration agree favorably with available experimental measurements .'), (729, 'stresses in continuous skin stiffener panels under random loading .', 'lin, y.k.', 'j. aero. sc. january 1962 .', 'stresses in continuous skin stiffener panels under random loading . theoretical aspects involved in the prediction of stress levels for continuous skin-stiffener panels subjected to a random pressure field are considered in the light of powell''s general theory for statistical superposition of modal response . the choice of structural model is dictated by the prevalence of skin-stiffener construction in modern flight vehicle design . the present study clearly demonstrates that any truly adequate prediction of stress levels in actual aircraft structures requires a much better representation of structural characteristics than can be provided by single panel idealizations . in an example considering fuselage panels exposed to jet engine noise, essential agreement is shown with experimental data, although better correlation is shown for rms stress than for power spectrum . it is shown that reduction of stress level by increasing damping is effective only in the higher frequency range .'), (730, 'on the bending of a clamped plate .', 'weinstein,a., rock,d.h.', 'q. app. math. 2, 1944, 262.', 'on the bending of a clamped plate . the present paper contains an application of a recently developed variational method to the boundary value problem of the bending of a clamped plate of arbitrary shape . it will be shown that this problem can be linked to the simpler problem of the equilibrium of a membrane by a chain of intermediate problems, which can be solved explicitly and in finite form in terms of the membrane problem . in the intermediate problems, the deflection converges uniformly in the domain of the plate of the clamped plate, and the derivatives of all orders of the deflection converge uniformly in every domain completely interior to the plate . (in the ritz method, not even the convergence of the slopes can be guaranteed .) the method yields numerical results for plates of all shapes for which the membrane problem (which we shall call the base problem) admits an explicit solution . as an example we shall consider a clamped square plate under a uniform load . this problem has been the object of numerous investigations, some of which are theoretical, while others are purely numerical, use infinite simple and double series, and operate with an infinite number of linear equations and an infinite number of unknowns . an inspection of the general formulae derived in the present paper, formulae which become simple in numerical applications, would show how some of the numerical methods might be rendered rigorous . the convergence of higher derivatives is of great practical interest for the approximate computation of the stresses .'), (731, 'upper and lower bounds for the solution of the first biharmonic boundary value problem .', 'diaz,g.b., and greenberg,h.g.', 'j. math. phys. 27, 1948, 193.', 'upper and lower bounds for the solution of the first biharmonic boundary value problem . let w(x,y) be a solution of the boundary value problem where r is a plane domain with the boundary c . the authors obtain upper and lower bounds for, the value of w at a point in r, by a method which is applicable to many other problems . if u is a function satisfying the boundary conditions and v is a function satisfying the partial differential equation, then the authors obtain by applying green''s classical identity and schwarz''s inequality a pair of inequalities of the form where . together with the function w the authors consider a function the solution of the boundary value problem on c, and in analogy with the functions u and v associated with the function w a pair of functions and associated with the function . in the expression for derived from green''s classical identity appears an unknown line integral containing the values of w and on c . but the same line integral appears also in the expressions for to which the above inequalities are applicable . in this way the authors obtain two inequalities of the form where b and b'', respectively, are approximate values of . in order to improve these bounds one may add to u a linear set of functions and to v a linear set of functions and then minimize h(u-v) in order to determine the coefficients of the best linear combinations . if the sequences and are complete in a certain sense defined by the authors the approximations will converge to the value .'), (732, 'on the analogues relating flexure and extension of flat plates .', 'southwell,r.v.', 'q. j. mech. app. math. 3, 1950, 257.', 'on the analogues relating flexure and extension of flat plates . the displacement of a flat plate bent by transverse loading, and the extensional or in ''plane stress'', are governed by equations of identical form ,. and the boundary conditions have identical form when edge-displacements are specified in the flexural, edge-tractions in the extensional problem, so mathematically, in these circumstances, only a single problem is presented . this, the ''first analogue'' relating flexure and extension, is well known . a ''second analogue'', relating the flexural problem when edge-tractions with the extensional problem when edge-displacements are specified, is believed to have been first propounded in 1941 . by introducing two quantities u and v, analogous with the components u and v of extensional displacement, it permits a treatment of the flexural problem by any method--e.g. which yields extensional solutions of this second type . in this paper both analogues are combined in an inclusive statement covering the perforated (multiply connected) plates which were discussed in 1948 . reasons are stated for believing that ''two-diagram technique'' is preferable in problems governed by ''mixed'' boundary conditions .'), (733, 'the bending of a sectorial plate .', 'carrier,g.f.', 'j. app. mech. 11, 1944, 134.', 'the bending of a sectorial plate . the problem of evaluating the bending moments, existing in a uniformly loaded clamped plate having the form of a sector of a ring, is one which arises in connection with the stress analysis of reinforced piston heads and in other design problems . in this paper, expressions are derived for the bending moments along the edges of such a plate . similar problems, i.e., those of the clamped rectangular plate under uniform pressure, under a central concentrated load, and that of the simply supported sector of a disk under uniform pressure, have been discussed by previous authors . the general approach used in the foregoing problems is adopted in the present case ,. a considerable reduction in the computational work is achieved, however, by the use of an integral-equation method of solving the boundary-condition equations . numerical results are obtained for plates of various dimensions, and the edge moment distributions are plotted for these cases . curves are also plotted which indicate the relationship existing between the maximum bending moments derived for sectorial plates and those previously obtained for clamped rectangular plates of similar size .'), (734, 'the bending of uniformly loaded clamped plate in the form of a circular sector .', 'hasse,h.r.', 'q. j. mech. app. math. 3, 1950, 271.', 'the bending of uniformly loaded clamped plate in the form of a circular sector . the deflexion of a uniformly loaded plate in the form of a semicircle clamped along its boundary is obtained by a method due to weinstein . this problem requires the solution of the biharmonic equation where z is given, subject to the conditions that w = 0 and on the boundary, n being the direction of the outward normal . the solution is expressed in the form where, writing is found by solving (in succession) two harmonic equations of the forms where z may be zero, and where f and have to satisfy certain boundary conditions . the constants are then determined to satisfy the boundary condition . numerical calculations show that five or six terms of the series give a good approximation to the accurate value as judged by the closeness with which the approximate solution satisfies the boundary condition . the procedure to be adopted in the case of the general circular sector and for non-uniform loading is indicated briefly . the connexion between the deflexion problem and that of plane strain in which the stress function satisfies the equation, where and have given values on the boundary, is discussed as a preliminary to the further consideration of the latter problem by a method of the same type .'), (735, 'the bending of uniformly loaded sectorial plates with clamped edges .', 'conway,h.d. and huang,m.k.', 'j. app. mech. 19, 1962, 5.', 'the bending of uniformly loaded sectorial plates with clamped edges . this paper analyzes the bending of a sectorial plate, clamped on all edges and subjected to uniformly distributed load, by using two different methods of superposition on the elementary solution for a uniformly loaded circular plate with a clamped edge .'), (736, 'the bending of a wedge shaped plate .', 'woinowsky-krieger,s.', 'j. app. mech. 20, 1953, 77.', 'the bending of a wedge shaped plate . a general method of solution is given in this paper for the problem of bending of a wedge-shaped thin elastic plate with arbitrary boundary conditions on the radial edges in the case of a single load . the solution is carried out for a plate with clamped edges and a single load on the bisector radius of the plate . stress distribution along the edges is shown and the behavior of the solution near the corner point is discussed for several opening angles of the plate .'), (737, 'on the analysis of elastic plates of variable thickness .', 'mansfield,e.h.', 'q. j. mech. app. math. 15, 1962, 167.', 'on the analysis of elastic plates of variable thickness . the extensional and flexural equations governing the elastic behaviour of a plate of variable thickness are expressed in terms of the laplacian operator . temperature variations in the plane of the plate and across the thickness of the plate are taken into account . general solutions are given for a rectangular plate whose thickness varies exponentially along the length, and for a circular, or annular, plate whose thickness varies as a power of the radius . the large-deflexion equations, including effects of initial irregularities, are also discussed .'), (738, 'finding zero''s of arbitrary functions .', 'frank,w.l.', 'j. assoc. comput. mach. 5, 1958, 154.', 'finding zero''s of arbitrary functions . a method for finding real and complex roots of polynomial equations, due to d. muller, is applied to finding roots of general equations of the form f(z) = 0, where f(z) is analytic in the neighborhood of the roots . the procedure does not depend on any prior knowledge of the location of the roots nor on any special starting process . all that is required is the ability to evaluate f(z) for any desired value of z . multiple roots can also be obtained . a general purpose program, prepared for the univac scientific 1103 and 1103a, is described and numerical results are presented for the following applications .. finding eigenvalues of differential operators ,. finding eigenvalues of arbitrary matrices ,. finding zeros of the generalized eigenvalue problem ,. finding roots of a number of transcendental equations .'), (739, 'the buckling of thin cylindrical shells under axial compression .', 'von karman,t. and tsien,h.s.', 'j. ae. scs. 8, 1941, 303.', 'the buckling of thin cylindrical shells under axial compression . in two previous papers the authors have discussed in detail the inadequacy of the classical theory of thin shells in explaining the buckling phenomenon of cylindrical and spherical shells . it was shown that not only the calculated buckling load is 3 to 5 times higher than that found by experiments, but the observed wave pattern of the buckled shell is also different from that predicted . furthermore, it was pointed out that the different explanations for this discrepancy advanced by l. h. donnell and w. flugge are untenable when certain conclusions drawn from these explanations are compared with the experimental facts . by a theoretical investigation on spherical shells the authors were led to the belief that in general the buckling phenomenon of curved shells can only be explained by means of a non-linear large deflection theory . this point of view was substantiated by model experiments on slender columns with non-linear elastic support . the non-linear characteristics of such structures cause the load necessary to keep the shell in equilibrium to drop very rapidly with increase in wave amplitude once the structure started to buckle . thus, first of all, a part of the elastic energy stored in the shell is released once the buckling has started,. this explains the observed rapidity of the buckling process . furthermore, as it was shown in one of the previous papers the buckling load itself can be materially reduced by slight imperfections in the test specimen and vibrations during the testing process . in this paper, the same ideas are applied to the case of a thin uniform cylindrical shell under axial compression . first it is shown by an approximate calculation that again the load sustained by the shell drops with increasing deflection . then the results of this calculation are used for a more detailed discussion of the buckling process as observed in an actual testing machine .'), (740, 'the behaviour of a cylindrical shell under axial compression when the buckling load has been exceeded .', 'leggett,d.m.a. and jones,r.p.n.', 'arc r + m.2190, 1942.', 'the behaviour of a cylindrical shell under axial compression when the buckling load has been exceeded . the value of the compressive stress at which a thin circular cylindrical shell becomes unstable has been worked out theoretically by southwell (1914) . subsequent experimental results, however, have indicated that this value is appreciably too high and that the form of distortion which occurs in practice differs from that assumed in theory . in recent years much work has been done on this problem in america . lundquist (1933) and donnell (1934) have concluded that the buckling of a cylindrical shell is greatly influenced by initial irregularities,. von karman and tsien (1941) have indicated that a thin cylindrical shell can be maintained in a buckled state by a compressive load considerably smaller than that previously predicted by theory . the present paper is an extension of the work of von karman and tsien . it shows that the smallest load which will keep a thin cylindrical shell in a buckled condition is about one-third of that given by southwell, a result in very fair agreement with experiment, and that once the cylinder has buckled, and so long as the stresses remain within the elastic range of the material, the cylinder has only about one-quarter of its original stiffness .'), (741, 'the behaviour of thin cylindrical shells after buckling under axial compression .', 'michielsen,h.f.', 'j. ae. scs. 15, 1948, 738.', 'the behaviour of thin cylindrical shells after buckling under axial compression . the fundamental investigations of von karman and tsien on the buckling of cylindrical shells under axial compression are continued . the energy expression is simplified and minimized with respect to the axial and circumferential wave-length parameters . solution of the equations obtained yields curves of the reduced average stress and of the wave dimensions plotted against the reduced average strain . they illustrate the behavior of the cylinder during the buckling process . the minimum buckling stress is found to be 0.195e(tr) .'), (742, 'post-buckling behaviour of axially compressed circular cylinder shells .', 'kempner,j.', 'j. ae. scs. 21, 1954, 329.', 'post-buckling behaviour of axially compressed circular cylinder shells . the postbuckling characteristics of an axially compressed thin-walled circular cylindrical shell loaded either by dead weights or by a rigid testing machine are determined . it is shown that for either loading condition the minimum applied stress in the postbuckling region is 0.182(er) and that the region of stable equilibrium corresponding to loading by the rigid testing machine includes and extends beyond that obtained with dead weight loading . the work here described is a continuation of work done earlier by von karman and tsien, by michielsen, and by leggett and jones .'), (743, 'new developments in the nonlinear theories of the buckling of thin cylindrical shells .', 'w. f. thielemann', 'deutsche versuchsanstalt fur luftfahrt mulheim (ruhr), germany', 'new developments in the nonlinear theories of the buckling of thin cylindrical shells . in the present paper a short survey will be given first of the buckling and postbuckling behavior of isotropic cylindrical shells subjected to different loading conditions as obtained by the nonlinear theory of finite deflections of shells during the last twenty years . next a report will be given on new investigations carried out in the structures department of the dvl concerning the elastic stability of isotropic and orthotropic cylindrical shells loaded in axial compression and internal pressure . these studies are based on the nonlinear theory of finite deformations . the theoretical rsults will be compared with new experimental results obtained with a series of axially loaded pressurized isotropic and orthotropic cylindrical shells .'), (744, 'lower buckling load in the non-linear buckling theory of thin shells .', 'tsien,h.s.', 'q. app. math. 5, 1947, 236.', 'lower buckling load in the non-linear buckling theory of thin shells . for thin shells the relation between the load p and the deflection beyond the classical buckling load is very often non-linear . for instance, when a uniform thin circular cylinder is loaded in the axial direction, the load p when plotted against the end-shortening has the characteristic shown in fig. 1 . if the strain energy s and the total potential are calculated, their behavior can be represented by the curves shown in figs. 2 and 3 . it can be demonstrated that the branches oc and ab corresponds to stable equilibrium configurations and the branch bc to unstable equilibrium configurations . the point b is then the point of transition from stable to unstable equilibrium configurations .'), (745, 'an automatic method for finding the greatest or least value function .', 'rosenbrock,h.h.', 'computer jnl. 1960.', 'an automatic method for finding the greatest or least value function . the greatest or least value of a function of several variables is to be found when the variables are restricted to a given region . a method is developed for dealing with this problem and is compared with possible alternatives . the method can be used on a digital computer, and is incorporated in a program for mercury .'), (746, 'aeroelastic problems in connection with high speed flight .', 'broadbent,e.g.', 'j. roy. aero. soc. 1956.', 'aeroelastic problems in connection with high speed flight . a review is given of developments in the field of aeroelasticity during the past ten years . the effect of steadily increasing mach number has been two-fold .. on the one hand the aerodynamic derivatives have changed, and in some cases brought new problems, and on the other hand the design for higher mach numbers has led to thinner aerofoils and more slender fuselages for which the required stiffness is more difficult to provide . both these aspects are discussed, and various methods of attack on the problems are considered . the relative merits of stiffness, damping and massbalance for the prevention of control surface flutter are discussed . a brief mention is made of the recent problems of damage from jet efflux and of the possible aeroelastic effects of kinetic heating .'), (747, 'bodt freedom flutter of ground launched rocket models at supersonic and high subsonic speeds .', 'gaukroger,d.r.', 'rae r. struct.237, 1957.', 'bodt freedom flutter of ground launched rocket models at supersonic and high subsonic speeds . a theoretical investigation of symmetric body freedom flutter of a rocket model is described . the results confirm that structural failures of models were caused by this type of flutter, and an extension of the investigation indicates the parameters that are of importance . a high ratio of body to wing mass and a well forward position of the overall centre of gravity are conditions under which flutter may occur . increase of body pitching radius of gyration and tailplane volume are beneficial . it is concluded that this type of flutter may be significant in some aircraft designs, and that the canard has no advantage in this respect over the conventional lay-out of wing and tailplane .'), (748, 'subsonic aerodynamic flutter derivatives for wings and control surfaces, /compressible and incompressible flow/ .', 'minhinnick, i. t.', 'r.a.e. rep. structs. 87. july 1950 .', 'subsonic aerodynamic flutter derivatives for wings and control surfaces, /compressible and incompressible flow/ . this report gives tables of the two-dimensional subsonic flutter derivatives,. where possible the values given are based on the published work of various authors, but some have been specially calculated for this report . wing derivatives are given for mach numbers 0, 0.5, 0.6 and 0.7 for the frequency parameter range 0 /0.04/ 0.2 /0.2/ 1.6 and mach numbers 0 and 0.7 for frequency parameter 5.0 . control surface derivatives are given for mach numbers 0 and 0.7 for control surface/ wing chord ratios 0.02 /0.02/ 0.10 /0.05/ 0.50 and frequency parameters are also given for mach numbers 0, 0.5, 0.6 and 0.7 for frequency parameter 0 /0.04/ 0.2 /0.2/ 1.4 . control surface-tab derivatives are given for some particular values of the variables and methods of obtaining approximate values of these derivatives for other values of the variables are suggested . control surface and tab derivatives are in all cases for no aerodynamic balance .'), (749, 'the aerodynamic effects of aspect ratio and sweepback on wing flutter .', 'molyneux,w.g. and hall,h.', 'arc r + m.3011.', 'the aerodynamic effects of aspect ratio and sweepback on wing flutter . the report describes tests to obtain direct measurements of the aerodynamic effects of aspect ratio and sweepback on wing flutter . the tests were made on rigid wings with root flexibilities . it is shown that measured effects of aspect ratio and sweepback on the flutter of these wings can be represented quite closely in flutter calculations based on two-dimensional flow theory by multiplying the two-dimensional aerodynamic coefficients by appropriate factors . the effect of sweepback is represented by multiplying all aerodynamic coefficients by cos, where is the wing leading-edge sweepback, and the effect of aspect ratio is represented by multiplying the aerodynamic damping coefficients by 1f(a) and, the stiffness coefficients by 1(f(a)) where a is the aspect ratio . for the wings tested an average value for f(a) is f(a) = (1 + (0.8a)) .'), (750, 'transonic flow in two dimensional and axially symmetrical nozzles .', 'hall,i.m.', 'arc 23,347, 1961.', 'transonic flow in two dimensional and axially symmetrical nozzles . by means of suitable expansions in inverse powers of r, the radius of curvature of the nozzle profile at the throat measured in throat half-heights, the velocity components in the throat region of a convergent-divergent nozzle can be calculated . the first three terms of the series solution have been obtained both for two-dimensional and for axially-symmetric nozzles . the numerical accuracy of the solution is confirmed by comparison with the known exact solution along the branchline .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (751, 'a note on the use of end plates to prevent three dimensional flow at the ends of bluff cylinders .', 'cowdrey,c.f.', 'npl aero. r.1025, 1962.', 'a note on the use of end plates to prevent three dimensional flow at the ends of bluff cylinders . the results are given of some observations of the effects of end plates on the three-dimensional separated flow at the ends of cylindrical models . while these are by no means exhaustive, it is felt that they are of sufficient interest to merit putting on record .'), (752, 'slender not-so-thin wing theory .', 'cooke,j.c.', 'rae r.aero.2660, 1962.', 'slender not-so-thin wing theory . a method for making an approximate thickness correction to slender thin-wing theory is presented . the method is tested by applying it to cones with rhombic cross-sections and the agreement is found to be good if the cones are not too thick . it is then suggested that the thickness correction to slender thin-wing theory may be applied unchanged to linear thin-wing theory . this suggestion is compared with some experiments on delta wings and it is found that there is considerable improvement over thin-wing theory near the centre line, but that this improvement is not maintained as the wing tips are approached .'), (753, 'development of a quasi-steady approach to flutter and correlation with kernel-function results .', 'gravitz,s.i., laidlaw,w.r., bryce,w.w. and cooper,r.e.', 'j.ae.scs. 29, 1962, 445.', 'development of a quasi-steady approach to flutter and correlation with kernel-function results . the quasi-steady approach to flutter utilizes experimental or theoretical steady-state aerodynamic data to arrive at increased understanding of the flutter mechanism, and also, in many cases, acceptably accurate quantitative flutter predictions . circulation lag effects are neglected, but aerodynamic damping is included in the evaluation of the air forces . situations requiring the inclusion of rate aerodynamics for accurate flutter estimation are specified . a quasi-unsteady approach is also discussed, in which the approximate magnitude of the circulation lag function at flutter is included in simple modifications of quasi-steady parameters . closed-form solutions are derived for the flutter characteristics of a typical section with and without rate aerodynamics . application is then made to the rational flutter analysis of three-dimensional multi-degree-of-freedom lifting surfaces . a specific planform is evaluated in the mach-number range from zero to two . quasi-steady, quasi-unsteady, and kernel-function results are compared subsonically . quasi-steady results are utilized supersonically . primary applications of the quasi-steady approach are in the areas of preliminary design and parameter-variation studies, modification of more sophisticated flutter theories to force compatibility with available steady-state data, and flutter evaluation of complex configurations which can be rationally analyzed by steady-state aerodynamic theories, but for which no complete unsteady aerodynamic theories are presently available .'), (754, 'heat transfer through laminar boundary layers on semi-infinite cylinders of arbitrary cross section .', 'bourne,d.e. and wardle,s.', 'j. ae. scs. 29, 1962, 460.', 'heat transfer through laminar boundary layers on semi-infinite cylinders of arbitrary cross section . this paper shows how to calculate the rate of heat transfer through a laminar boundary layer on a semi-infinite cylinder of arbitrary cross section . the cylinder is placed in a stream of incompressible fluid, the flow at infinity being parallel to the generators, and is maintained at a uniform temperature . a series solution for small downstream distances and an asymptotic formula for large downstream distances are given . to cover the intermediate range an approximate pohlhausen solution is obtained,. a correction of the error involved in the pohlhausen solution is suggested which, it is believed, will lead to final errors of at most 2 percent . the calculations are applied to elliptic cylinders, and illustrate the effect on the local rate of heat transfer of varying the ratio of the major and minor axes of cross section, the length of perimeter being held fixed .'), (755, 'oscillatory derivative measurements on sting-mounted wind tunnel models method of test and results for pitch and yaw on a cambered ogee wing at mach numbers up to 2. 6.', 'thompson,j.s.', 'rae r.aero.2668, 1962.', 'oscillatory derivative measurements on sting-mounted wind tunnel models method of test and results for pitch and yaw on a cambered ogee wing at mach numbers up to 2. 6. this report describes a method which has been developed for measuring oscillatory derivatives on sting-mounted models in the 8 ft by 8 ft supersonic tunnel at r.a.e. bedford . direct and cross derivatives with respect to angular displacements and velocities in pitch and yaw have been measured satisfactorily, and results are given of tests on a cambered ogee wing at six mach numbers from 0.2 to 2.6 . some tests were made on this model in the course of the preliminary development work in the 13 ft by 9 ft low speed wind tunnel, and results of these are included .'), (756, 'further comments on the inversion of large structural matrices .', 'charles h. samson, jr.', 'professor, departments of aeronautical and civil engineering, a. and m. college of texas', 'further comments on the inversion of large structural matrices . in a recent note, klein referred to a paper co-authored by the writer, and to ref. 3 . regarding the subject of inversion of large-order matrices, klein stated that he would show ''that the situation is not as hopeless as the anove-mentioned authors intimate'' . the purpose of this note is not to take exception to klein/s conclusions, but rather to disagree with his implication that the authors of ref. 2 were pessimistic with respect to large-matrix inversions . two general methods of analysis were treated.. the method of consistent distortion and the method of transfer matrices . the first method leads directly to a relatively large matrix of structural coefficients of both internal forces and displacements . this matrix must be inverted to solve the problem . the second method ultimately produces a relatively small matrix requiring inversion., however, to arrive at this point one must perform a number of matrix multiplications .'), (757, 'an investigation of the flow about a plane half-wing of cropped delta planform and 6( symmetrical section at stream mach numbers between 0. 8 and 1. 41 .', 'rogers,e.w.e., hall,i.m. and berry,c.j.', 'arc r + m.3286, 1960.', 'an investigation of the flow about a plane half-wing of cropped delta planform and 6( symmetrical section at stream mach numbers between 0. 8 and 1. 41 . a study has been made of the flow development over the wing as the incidence and stream mach number vary and this is illustrated by surface pressure distributions and oil-flow patterns . the growth and movement of the two main surface shocks (the rear and forward shocks) is discussed, and conditions for flow separation through these shocks are considered . for the rear shock, which has little sweep, these conditions are similar to those for shock-induced separation on two-dimensional aerofoils . the forward shock is comparatively highly swept and separation seems to correspond to two rather different but simultaneously-attained conditions, one related to the component mach number normal to the shock front and the other to the position of the reattachment line . the flow in the region between the leading edge and the forward shock is shown to have certain characteristics analogous to those found upstream of the shock on two-dimensional aerofoils . to the rear of the forward shock, but ahead of the rear shock, the flow at low supersonic speeds resembles in some respects that about a simple cone . the general flow development is related in the text to the wing lift and pitching moment, and the drag . the first two are most affected by the aft movement of the rear shock, which also stimulates the transonic drag rise . the lift-dependent drag is shown to be influenced by the appearance of leading-edge separation and possibly also by some stage in the development of the forward shock . the flow over the cropped-delta planform is noteworthy for the absence of the strong outboard shock and this is attributed partly to the cropped tip and partly to the unswept trailing edge . a comparison is made with results obtained during preliminary tests in which the wing planform closely resembled that of a true delta .'), (758, 'the lower bound of attainable sonic-boom over-pressure and design methods of approaching this limit .', 'carlson,h.w.', 'nasa tn.d1494, 1962.', 'the lower bound of attainable sonic-boom over-pressure and design methods of approaching this limit . from a study of existing sonic-boom theory it has been possible to establish an approximate lower bound of attainable sonic-boom overpressure, which depends only on the airplane length, weight, and volume and on the flight conditions . this lower bound may be approached over a narrow range of flight conditions through the application of appropriate design considerations . in general, for intermediate values of lift coefficient the major portion of the lift generating surfaces must be located aft of the maximum cross-sectional area, whereas for higher values of lift coefficient the maximum area must be well forward and or the lift-producing surfaces must extend well toward the airplane nose .'), (759, 'stability investigation of a blunted cone and a blunted ogive with a flared cylinder afterbody at mach numbers from 0. 30 to 2. 85 .', 'coltrane,l.c.', 'nasa tn.d1506, 1962.', 'stability investigation of a blunted cone and a blunted ogive with a flared cylinder afterbody at mach numbers from 0. 30 to 2. 85 . a cone with a blunt nose tip and a blunt nose tip and a 20 flared cylinder afterbody have been tested in free flight over a mach number range from 0.30 to 2.85 and a reynolds number range from 1 x 10 to 23 x 10 . time histories, cross plots of force and moment coefficients, and plots of the longitudinal-force coefficient, rolling velocity, aerodynamic center, normal-force-curve slope, and dynamic stability are presented . with the center-of-gravity location at about models were both statically and dynamically stable throughout the mach number range . for the cone, the average aerodynamic center moved slightly forward with decreasing speeds and the normal-force-curve slope was fairly constant throughout the speed range . for the ogive, the average aerodynamic center remained practically constant and the normal-force-curve slope remained practically constant to a mach number of approximately 1.6 where a rising trend was noted . maximum drag coefficient for the cone, with reference to the base area, was approximately 0.6, and for the ogive, with reference to the area of the cylindrical portion, was approximately 2.1 .'), (760, 'inelastic buckling of initially imperfect cylindrical shells subject to axial compression .', 'lee,l.h.n.', 'j. ae. scs. 29, 1962, 87.', 'inelastic buckling of initially imperfect cylindrical shells subject to axial compression . an analytical and experimental study is made for inelastic instability of initially imperfect cylindrical shells subject to axial compression . donnell''s equations and the principle of virtual work are adapted to determine the effects of initial imperfections on the buckling modes and the critical buckling stresses . the deformation theory and the incremental theory of plastic stress-strain relationships are both considered . the experimental results of ten tests on specimens made of aluminum alloy 3003-0 are presented . comparison of experimental with theoretical results indicates that the application of the deformation theory provides a fairly accurate prediction of buckling strength, but fails in this case to yield a correct description of post-buckling behavior . on the other hand, the application of the incremental theory, which is mathematically and physically more rigorous, leads to an overestimation of buckling strength, even though initial imperfections are considered . this paradox has existed for years, and remains to be resolved .'), (761, 'buckling of sandwich under normal pressure .', 'yao,j.c.', 'j. ae. scs. 29, 1962, 264.', 'buckling of sandwich under normal pressure . a theoretical study is made of the buckling of a sandwich sphere comprised of a core layer of low-modulus material and two thin facing layers of higher modulus material . the solution for the buckling resistance of the sphere under normal external pressure is obtained by linearized theory, and is reducible to the classical solution for monocoque spherical shells . critical buckling pressures are calculated for various radius-thickness ratios and sphere materials .'), (762, 'allowable axial loads and bending moments for inelastic structures under nonuniform temperature distribution .', 'b. e. gatewood and r. w. gehring', 'the ohio state university and north american aviation, inc.', 'allowable axial loads and bending moments for inelastic structures under nonuniform temperature distribution . a strain-analysis method is derived and demonstrated for the calculation of design allowable load-strain curves for the cross section of a structure supporting axial loads and bending moments . the temperature effects of thermal stresses and changed material properties and all inelastic effects are included in the calculations so that the final curve is a design curve for the applied stresses as calculated by room-temperature elastic procedures . the method allows for sequence application and removal of load and temperature, as well as cycling of load and/or temperature . applications are shown for a rectangular bar under temperature cycling with axial loads and/or bending moments and for a box beam with one bending-moment temperature cycle . interaction curves beyween axial load and bending moment with inelastic effects included are given, the calculations being done on a digital computer . a procedure is given for using the method to construct design curves .'), (763, 'effects of internal pressure on the buckling of circular-cylindrical shells under bending.', 'weingarten,v.i.', '', 'effects of internal pressure on the buckling of circular-cylindrical shells under bending. the effect of internal pressure on the small-deflection buckling of thin-walled cylinders under bending is investigated by means of a modified donnell equation . the results indicate that the maximum critical stress due to bending increases with internal pressure, unlike the case of pressurized cylinders under compression . these results represent the moment at which significant deformations appear in the cylinder, rather than the maximum moment able to be carried, but may be a good approximation to the latter for metal cylinders .'), (764, 'breathing vibrations of a circular shell with an internal liquid .', 'lindholm,u.s., kana,d.d. and abramson,h.n.', 'j. ae. scs. 29, 1962, 1052.', 'breathing vibrations of a circular shell with an internal liquid . resonant breathing frequencies and mode shapes are determined experimentally for a thin-walled, circular cylindrical shell containing a nonviscous incompressible liquid . the resonant frequencies determined for the full shell are in good agreement with those predicted by reissner''s shallow-shell vibration theory with the inclusion of an apparent-mass term for the liquid . the effect of the internal liquid on the shell mode shapes is significant only for the partially full shell . in this case the circumferential node lines tend to shift toward the bottom or filled portion of the shell . excitation of low-frequency liquid-sloshing motion by high-frequency forced oscillation of a partially filled shell occurred in many cases . this low-frequency liquid response is tentatively explained as being excited by a beat frequency in the forced oscillation . a similar type of response has been reported by yarymovych in axially excited rigid tanks .'), (765, 'clamped short oval cylindrical shells under hydrostatic pressure .', 'vafakos,w.p.', 'j. ae.scs. 29, 1962, 1347.', 'clamped short oval cylindrical shells under hydrostatic pressure . the principle of the minimum of the total potential is employed to obtain stresses and displacements for clamped, short, oval cylindrical shells under hydrostatic pressure . classical shell theory, in which buckling effects are not considered, was used . a fourier series is assumed for the deflections in the closed circumferential direction so that the partial differential equations of equilibrium are replaced by a set of ordinary differential equations . the energy solution is compared with a simplified approximation which can be considered an equivalent circular cylinder solution . graphs of the significant stresses and displacements are presented for oval cylinders having major to minor axis ratios of 1.10, 1.30, and 1.50 . it is shown that the maximum stresses and displacements increase significantly as the major to minor axis ratio is increased .'), (766, 'experimental investigation at mach number of 3. 0 of effects of thermal stress and buckling on flutter characteristics of flat single-bay panels of length-width ratio 0. 96 .', 'dixon,s.c.', 'nasa tn.d1485, 1962.', 'experimental investigation at mach number of 3. 0 of effects of thermal stress and buckling on flutter characteristics of flat single-bay panels of length-width ratio 0. 96 . flat, single-bay, skin stiffener panels with length-width ratios of 0.96 were tested at a mach number of 3.0, at dynamic pressures ranging from 1,500 to stagnation temperatures from 300 f to effects of thermal stress and buckling on the flutter of such panels . the panel supporting structure allowed partial thermal expansion of the skins in both the longitudinal and lateral directions . panel skin material and skin thickness were varied . a boundary faired through the experimental flutter points consisted of a flat-panel portion, a buckled-panel portion, and a transition point, at the intersection of the two boundaries, where a panel is most susceptible to flutter . the flutter region consisted of two fairly distinct sections, a large-amplitude flutter region and a small-amplitude flutter region . the results show that an increase in panel skin temperature flutter . the flutter trend for buckled panels is reversed . use of a modified temperature parameter, which approximately accounts for the effects of differential pressure and variations in panel skin material and skin thickness, reduced the scatter in the data which resulted when these effects were neglected . the results are compared with an exact theory for clamped panels for the condition of zero midplane stress . in addition, a two-mode /transtability/ solution for clamped panels is compared with the experimentally determined transition point .'), (767, 'mathematical techniques applying to the thermal fatigue behaviour of high temperature alloys .', '', '', 'mathematical techniques applying to the thermal fatigue behaviour of high temperature alloys . during thermal fatigue testing of a specimen with a thin edge, or during rapid temperature changes in the gas flow past turbine blades, the thin edges are deformed plastically in compression during heating and subsequently creep in tension as the bulk of the specimen or blade heats up . the plastic deformation is determined from temperature distributions, which are calculated by biot''s variational method . the creep deformation is determined as a function of time by a differential equation, which expresses the balance between increasing elastic stress and reduction of stress due to creep relaxation, and which is solved to a riccati equation soluble in terms of bessel functions, or (iii) by transformation to a second-order differential equation with a periodic coefficient . using the thermal stresses obtained from the solution of the differential equation, the theoretical thermal fatigue endurance is determined from cyclic (mechanical) stress endurance data . agreement between theoretical and experimental thermal fatigue endurances is obtained, over ranges of temperature, strain, and strain rate, or equivalently, over ranges of temperature-edge radius and heat transfer coefficient . this agreement supports the use of the theoretical methods in wider contexts . the accuracy of the temperature distributions is better than the accuracy of other factors entering into the correlation between theoretical and experimental endurances . improvement in the interpretation of experimental results requires consideration of the alteration of the stress cycles during the course of thermal fatigue testing . this requirement is catered for partially by the various solutions of the differential equation for thermal stress .'), (768, 'formulae for use with the fatigue load meter in the assessment of wing fatigue life .', 'phillips,j.', 'rae tn. struct.279, 1960.', 'formulae for use with the fatigue load meter in the assessment of wing fatigue life . this note gives a method for the derivation of suitable constants which, when multiplied by the readings recorded at each appropriate acceleration level on a fatigue load meter and then added together, give directly the proportion of fatigue life used up in the wing . it is suggested that when the estimated proportion is of order 80, then a more detailed assessment of fatigue life should be made .'), (769, 'local circumferential buckling of thin circular cylindrical shells .', 'johns,d.j.', 'nasa tn.d1510, 1962, 267.', 'local circumferential buckling of thin circular cylindrical shells . the problem of circumferential buckling of a thin circular cylindrical shell due to compressive hoop stresses which vary in the axial direction is examined . for extremely localised compressive hoop stress distributions resulting from thermal discontinuity effects, or from a uniform, radial line loading, the buckle pattern should also be localised . simplified analyses into these two types of problem are considered which show that only a limited number of buckle deflection modes needs to be assumed .'), (770, 'the flow of a compressible fluid past a sphere .', 'caplan,c.', 'naca tn.762, 1940.', 'the flow of a compressible fluid past a sphere . the flow of a compressible fluid past a sphere fixed in a uniform stream is calculated to the third order of approximation by means of the janzen-rayleigh method . the velocity and the pressure distributions over the surface of the sphere are computed and the terms involving the fourth power of the mach number, neglected in rayleigh''s calculation, are shown to be of considerable importance as the local velocity of sound is approached on the sphere . the critical mach number, that is, the value of the mach number at which the maximum velocity of the fluid past the sphere is just equal to the local velocity of sound, is calculated for both the second and the third approximations and is found to be, respectively, and .'), (771, 'on the flow of a compressible fluid past a sphere .', 'tamada,k.', 'proc. physico-math. soc japan, 21, 1939, 743.', 'on the flow of a compressible fluid past a sphere . it was shown by raleigh (philos. mag. 32, 1 (1916)) that the velocity potential for the subsonic flow of a compressible fluid past a sphere can be expressed as a power series in terms of mach''s number m (which is the ratio of the undisturbed velocity u, divided by the velocity of sound for the undisturbed flow) . the equation in question is and boundary conditions are prescribed for raleigh himself computed the first two terms of this series,. the author finds the third term . he gives some graphs showing numerical differences between raleigh''s and his approximation .'), (772, 'an experimental study of jet-flap compressor blades .', 'clark, e.l. /jnr./ and ordway, d.e.', 'j. aero. sc. november 1959. p. 698-702, 738 .', 'an experimental study of jet-flap compressor blades . the results of a preliminary experimental investigation to determine the feasibility of using the jet flap to improve the section characteristics of an axial-flow compressor blade are presented and discussed trailing edge . internal design of the blade is described and details of the resulting jet flow are given . also included are wind-tunnel design and test procedures for the two-dimensional cascade used in the test . test results are presented in the form of the measured turning angle, pressure rise, and lift coefficient . they are examined with particular reference to the prevention of rotating stall .'), (773, 'q . app . math . 7, 1950, 381 . experiments on porous-wall cooling and flow separation control in a supersonic nozzle .', 'green,l. and nall,k.l.', 'j. ae. scs. 1959, 689.', 'q . app . math . 7, 1950, 381 . experiments on porous-wall cooling and flow separation control in a supersonic nozzle . control of flow separation by fluid injection at one diverging boundary of a two-dimensional, transparent-walled de laval nozzle was investigated by spark schlieren photography of dry nitrogen flows expanded from two stagnation temperatures injection conditions at the permeable boundary were varied by the use of three grades of porous stainless steel with nominal pore diameters of 10, 20, and 30 microns, through which nitrogen was forced by coolant reservoir pressures of 25, 50, and 100 psig, in addition to the case of no forced injection . pressure distribution measurements were made along the nonpermeable diverging boundary . it was found that flow separation at expansion ratios approaching the optimum value for maximum thrust coefficient could be induced at the porous wall by a local injection mass velocity of the order of a few per cent of the local main-stream mass velocity . separation at the solid boundary was not noticeably influenced by injection at the opposite wall, and the asymmetrical separation thus effected jet deflections of up to 10 degrees at the lower stagnation-pressure levels . variation of the wall heat-transfer condition by changing the stagnation temperature did not significantly influence separation behavior . temperature measurements at the reservoir face of the porous section, together with use of published correlations and of the rube-sin analysis for estimation of stream-side stanton numbers under noninjection and injection conditions, respectively, permitted heat-transfer calculations which indicated that the effectiveness of the transpiration technique in controlling nozzle wall temperatures derives primarily from intimate fluid-solid contact in a porous material of high specific surface .'), (774, 'general characteristics of the flow through nozzles at near critical speeds .', 'sauer,r.', 'naca tm.1147, 1944.', 'general characteristics of the flow through nozzles at near critical speeds . the characteristics of the position and form of the transition surface through the critical velocity are computed for flow through flat and round nozzles from subsonic to supersonic velocity . corresponding considerations were carried out for the flow about profiles in the vicinity of sonic velocity .'), (775, 'studies on two dimensional flows of compressible fluid.', '', '', 'studies on two dimensional flows of compressible fluid. it is well-known that when the flow is everywhere subsonic in a field of flow, the nature of the two-dimensional isentropic flow of a compressible perfect fluid differs only slightly from that of the corresponding flow of an incompressible perfect fluid . thus, in such a case, we can calculate the field of flow by any of the well-known methods of approximation . on the other hand, if the flow is supersonic throughout the field, we can determine the flow pattern by the method of characteristics .'), (776, 'force measurements on square and dodecagonal sectional cylinders at high reynolds numbers .', 'cowdrey,c.f. and lawes,j.a.', 'npl. aero.351.', 'force measurements on square and dodecagonal sectional cylinders at high reynolds numbers . results are given of measurements in the compressed air tunnel of the forces on two cylinders, one of square cross-section and the other dodecagonal . the tests were carried out at various reynolds numbers ranging from approximately 0.1 x 10 to 5.5 x 10, based on the distance between parallel faces .'), (777, 'a technique for rendering approximate solutions to physical problems uniformly valid .', 'lighthill,m.j.', 'phil. mag. 40, 1949, 1179.', 'a technique for rendering approximate solutions to physical problems uniformly valid . a method is described for treating some of the characteristically non-linear problems of physics, in particular those involving a non-linear partial differential equation for which an approximate linearization is permissible everywhere except in a limited region, such as the neighbourhood of (5) a singular characteristic of the approximate solution, or of approximation is valueless . the method involves a transformation of an independent variable, which is determined progressively with successive approximations to the solution .. only one step being necessary if a first approximation valid uniformly be obtained . the method is most easily understood in its application to simple first order ordinary differential equations, which are studied in detail in 2 and 3 as a preparation for the extension to more complicated problems in 4, 5 and 6 . physically, the longest section, 6, concerns the /spread/ of a progressive wave at infinity, an important and essentially non-linear process .'), (778, 'an integral related to the radiation integrals .', 'powell,e.o.', 'phil. mag. 34, 1943, 600.', 'an integral related to the radiation integrals . the author points out the relation of the integral to problems in astrophysics and quantum mechanics, and because of its importance supplies a table of values . the table gives rl(x) to seven places of decimals from x=0 to x=6.00 at intervals of 0.01 . second order central differences are tabulated to assist in interpolation .'), (779, 'calculation and compilation of the unsteady lift functions for a rigid wing subjected to sinusoidal gusts and to sinusoidal sinking oscillations .', 'drischler, j.a.', 'naca tn 3748, october 1956', 'calculation and compilation of the unsteady lift functions for a rigid wing subjected to sinusoidal gusts and to sinusoidal sinking oscillations . the total lift responses of wings to sinusoidal gusts and to sinusoidal vertical oscillations are calculated from the response to gust penetration and to a sudden change in sinking velocity through use of the well-established reciprocal relations for unsteady flow . the cases considered are two-dimensional wings in incompressible, subsonic compressible, sonic, and supersonic flow,. elliptical and rectangular wings in incompressible flow,. wide rectangular and delta wings in supersonic flow,. and delta wings of vanishingly low aspect ratio in incompressible and compressible flow . for most of the cases considered, closed-form expressions are given and the final results are presented in the form of plots of the square of the modulus of the lift coefficient for wings in a sinusoidally oscillating gust and in the form of the real and imaginary parts of the lift component for wings undergoing sinusoidal sinking oscillations . a summary table is presented as a guide to the scope and results of this paper,. this table contains the figure and equation numbers for the types of flow and plan forms considered .'), (780, 'the transonic characteristics of 38 cambered rectangular wings of varying aspect ratio and thickness as determined by transonic-bump technique .', 'nelson,w.h. and krumm,w.j.', 'naca tn.3502, 1955.', 'the transonic characteristics of 38 cambered rectangular wings of varying aspect ratio and thickness as determined by transonic-bump technique . an investigation to determine the effects of camber on the aerodynamic characteristics of a series of rectangular wings having various aspect ratios and thickness-to-chord ratios was conducted in the ames the transonic-bump method . the mach number range of the investigation was from 0.6 to 1.12, with a corresponding reynolds number range of 1.7 to 2.2 million . the lift, drag, and pitching-moment data are presented for wings having aspect ratios of 4, 3, 2, 1.5, and 1, and naca 63a2xx and 63a4xx sections with thickness-to-chord ratios of'), (781, 'use of subsonic kernel function in an influence-coefficient method of aeroelastic analysis and some comparisons with experiment .', 'john l. sewall, robert w. herr, and charles e. watkins', 'technical note d-515', 'use of subsonic kernel function in an influence-coefficient method of aeroelastic analysis and some comparisons with experiment . this paper illustrates the development and application of an influence-coefficient method of analysis for calculating the response of a flexible wing in an airstream to an oscillating disturbing force and for treating such aeroelastic instabilities as flutter and divergence . aerodynamic coefficients are derived on the basis of lifting-surface theory for subsonic compressible flow by use of the method presented in nasa technical report r-48 . application of the analysis is made to a uniform cantilever wing-tip tank configuration for which responses to a sinusoidal disturbing force and flutter speeds were measured over a range of subsonic mach numbers and densities . calculated responses and flutter speeds based on flexibility influence coefficients measured at nine stations are in good agreement with experiment, provided the aerodynamic load is distributed over the wing so that local centers of pressure very nearly coincide with these nine influence stations . the use of experimental values of bending and torsional structural damping coefficients in the analysis generally improved the agreement between calculated and experimental responses . some calculations were made to study the effects of density on responses near the flutter conditions, and linear response trends were obtained over a wide range of densities .'), (782, 'calculated subsonic span loads and resulting stability derivatives of unswept and 45degree sweptback tail surfaces in sideslip and steady roll .', 'queiijo, m.j. and riley, d.r.', 'naca, tn 3245, 1954 .', 'calculated subsonic span loads and resulting stability derivatives of unswept and 45degree sweptback tail surfaces in sideslip and steady roll . subsonic span loads and the resulting stability derivatives have been calculated for a systematic series of vertical- and horizontal-tail combinations in sideslip and in steady roll in order to provide information embracing a wide range of probable tail configurations . all calculations were made by application of the discrete-horseshoe-vortex method to the problem of estimating loads on intersecting surfaces . the investigation covered variations in vertical-tail aspect ratio, the ratio of horizontal-tail aspect ratio to vertical-tail aspect ratio, the effects of horizontal-tail dihedral angle /for the sideslip case/, and the effects of vertical position of the horizontal tail for surfaces having their quarter-chord lines swept back 0degrees and 45degrees . the results of the investigation are presented in charts from which the span loads for the various conditions can be obtained . the resulting stability derivatives are presented as vertical- and horizontal-tail contributions as well as total-tail-assembly derivatives . the results of this investigation, which was made for a wider range of geometric variables than previous studies, showed trends which were in general agreement with the results of previous investigations . also presented in this paper and used in the computations is an extensive table of values of sidewash due to a rectangular vortex .'), (783, 'a method for calculating the subsonic steady-state loading on an airplane with a wing of arbitrary planform and stiffness .', 'gray, w.l.', 'naca tn 3030, 1953 .', 'a method for calculating the subsonic steady-state loading on an airplane with a wing of arbitrary planform and stiffness . a method for computing the steady-state span load distribution on an elastic airplane wing for specified airplane weights and load factors is given . the method is based on a modification of the weissinger l-method and applies at subcritical mach numbers . it includes the effects of external stores and fuselage on the spanwise loading . modifications are outlined for treating tail-boom and tailless airplane configurations and for calculating the divergence dynamic pressure of a swept wing with a large external store . a method is also outlined for reducing wind-tunnel data to obtain effective aerodynamic coefficients which are free of model flexibility effects . the effects of mach number can readily be evaluated from the aerodynamic coefficients thus obtained .'), (784, 'heat transfer through the laminar boundary layer on a circular cylinder in axial incompressible flow .', 'bourne,d.e. and davies,d.r.', 'q. j. mech. app. math. 11, 1958, 52.', 'heat transfer through the laminar boundary layer on a circular cylinder in axial incompressible flow . this paper presents a method of calculating the distribution of rate of heat transfer into a laminar incompressible boundary layer from the exterior surface of a long thin circular cylinder, when the surface of the cylinder is maintained at a constant temperature and the flow is parallel to the cylinder axis,. the temperature difference between the surface and the main stream is taken to be small enough to neglect buoyancy effects . a series solution, valid for small downstream distances from the nose, has been obtained already by seban, bond, and kelly . this is now extended by deriving an asymptotic series solution, valid at large downstream distances, and bridging the gap between these two series solutions by an approximate solution, based on the method used recently by davies and bourne to calculate heat transfer from a flat plate . the calculation is used to demonstrate the effect of curvature and of prandtl number on the local rate of heat transfer at various downstream distances by comparing with the corresponding flat plate results .'), (785, 'the flow of fluid along cylinders .', 'cooke, j.c.', 'quart. j. mech. app. math, vol.x /3/, 1957, p. 312-321 .', 'the flow of fluid along cylinders . the boundary layer equations for uniform flow parallel to the generators of any cylinder without corners are put into the form of a series of linear third-order differential equations . the first three of these are the same as those obtained by seban and bond /1/ for a circular cylinder and solved by kelly /2/ . the rest have additional terms depending on the radius of curvature of the cylinder and its derivatives . the problem is also attacked by a pohlhausen method as far as four terms of the series . for large distances from the front, rayleigh''s method, as given by hasimoto /3/, gives the first two terms of an asymptotic expansion for the drag . explicit calculations are made of the drag of an elliptic cylinder of eccentricity 1/2 3 . there is evidence that the drag is everywhere less than that of a circular cylinder of the same perimeter .'), (786, 'the skin friction on infinite cylinders moving parallell to their length .', 'batchelor, g.k.', 'quart. j. mech. app. math. vol. vii, /2/, 1954, p. 179-192 .', 'the skin friction on infinite cylinders moving parallell to their length . the frictional force on a cylinder moving steadily parallel to its length through a viscous liquid which is initially at rest is determined with reasonable accuracy over the whole range of values of the duration of the motion and for a wide variety of shapes of the cylinder cross-section . when the time t is small, the first approximation gives a force per unit area which is the same as that for a flat plate of infinite width . the second approximation takes the shape of the cylinder into account and the force on unit length of cylinder is determined in terms of the number of corners, and their angles, in the cylinder cross linder is the same, to this approximation, as that on a circular cylinder of the same perimeter . for large values of t the determination of the frictional force is reducible to that of a potential problem, the solution of which is known for a number of different shapes . the approximations for small and large values of t for any one cylinder do not overlap but can be joined without much ambiguity . for no value of t do the forces on cylinders of different shape /excluding those whose curvature is not everywhere inwards/ differ by more than about 25 per cent .'), (787, 'rayleigh''s problem for a cylinder of arbitrary shape .', 'hasimoto, h.', 'j. physical soc. of japan, vol. 9 /4/, 1954, p. 611-619 .', 'rayleigh''s problem for a cylinder of arbitrary shape . the motion of an incompressible viscous fluid generated by a cylinder of arbitrary cross-sectional form which is started to move suddenly from rest with uniform velocity in the direction of its length is considered formulae in powers of are derived for the velocity distribution /valid in the vicinity of the cylinder/ and for the frictional drag on the cylinder, correct to the order of a, where a is the characteristic length of the cross section, v is the kinematic viscosity, and t is the time . these formulae are given in terms of only the analytic function which maps conformally the region outside the cross section of the cylinder onto the region outside the unit circle, and of certain integrals e which are common to any arbitrary cylinder . in particular, when a is sufficiently small, the total frictional drag on the cylinder per unit length is expressed as, irrespective of the cross-sectional form, where b 2 and y 0.5772.../euler''s constant/ .'), (788, 'an approximate boundary layer theory for semi-infinite cylinders of arbitrary cross-section .', 'varley, e.', 'j. fluid mech. vol. 3. 1958, p. 601-614 .', 'an approximate boundary layer theory for semi-infinite cylinders of arbitrary cross-section . an estimate is given of the distribution of skin frictional force per unit length, and of displacement area, on the outside of a semi-infinite cylinder, of arbitrary cross-section, moving steadily in a direction parallel to its generators . a pohlhausen method is employed with a velocity distribution chosen to yield zero viscous retarding force on the boundary layer approximations . /the smallness of the fluid acceleration far from the leading edge has been pointed out by batchelor reasonable results atlarge distances from the leading edge . however, for a large class of cross-sections, which includes all convex cross-sections and locally concave cross-sections with re-entrant angles greater than 1/2, the method yields the expected square root growth of the boundary layer at the leading-edge, with a fairly close approximation to the coefficient, and it is supposed that the skin-frictional force and displacement area are given with reasonable accuracy along the whole length of the cylinder . results for the elliptic cylinder and the finite flat plate are given in closed form, valid for the whole length of the cylinder, and are expected to be in error by at most 20 per cent . in addition, some estimate is given of the effect of corners on skin frictional force and displacement area .'), (789, 'a further note on the calculation of heat transfer through the axisymmetrical laminar boundary layer on a circular cylinder .', 'bourne,d.e., davies,d.r. and wardle,s..', 'q. j. mech. app. math. 12, 1959, 257.', 'a further note on the calculation of heat transfer through the axisymmetrical laminar boundary layer on a circular cylinder . by using a karman-pohlhausen method the distribution of local rate of heat transfer is ovaluated for the case of air flow in an axisymmetrical laminar boundary layer on a heated circular cylinder, the temperature of the cylinder being independent of downstream distance . this calculation serves to link the numerical values obtained by seban, bond, and kelly for small downstream distances to those obtained by bourne and davies for large downstream distances .'), (790, 'a wind-tunnel test technique for measuring the dynamic rotary stability derivatives at subsonic and supersonic speeds .', 'report 1258', 'benjamin h. beam', 'a wind-tunnel test technique for measuring the dynamic rotary stability derivatives at subsonic and supersonic speeds . a method is described for measuring the dynamic stability derivatives of a model airplane in a wind tunnel . the characteristic features of this system are that single-degree-of-freedom oscillations were used to obtain combinations of rolling, yawing and pitching motions., that the oscillations were excited and controlled by velocity feedback which permitted operation under conditions unfavorable for more conventional types of oscillatory testing., and that data processing was greatly simplified by using analog computer elements in the strain-gage circuitry . the system described is primarily for measurement of the damping derivatives damping in roll damping in pitch, damping in yaw, and the cross derivatives rolling moment due to yawing and yawing moment due to rolling . the method of testing also permits measurement under oscillatory conditions of the static derivatives rolling moment due to sideslip, yawing moment due to sideslip, and pitching moment due to angle of attack . all these derivatives are of particular importance in estimating the short-period oscillatory motions of a rigid airplane . a small number of experimental data are included to illustrate the general scope of results obtainable with this system .'), (791, 'measurements at mach numbers up to 2. 8 of the longitudinal characteristics of one plane and three cambered slender ''ogee'' wings .', 'taylor,c.r.', 'rae r. aero.2658, 1961.', 'measurements at mach numbers up to 2. 8 of the longitudinal characteristics of one plane and three cambered slender ''ogee'' wings . measurements have been made of the longitudinal characteristics of one plane and three cambered slender ogee wings (p = 0.45,) at two subsonic and eight supersonic mach numbers up to 2.8 . the tests also included measurements of the zero-lift pressure drag and support interference of the plane wing . the results have been analysed to give data for estimating the performance of supersonic transport aircraft .'), (792, 'some low speed problems of high speed aircraft .', 'spence, a. and clean, d.', 'j. royal aero. soc. v. 66, april 1962, pp 211-225 .', 'some low speed problems of high speed aircraft . the first part of the paper deals with the low speed aerodynamics of aircraft shapes suggested by kuchemann, at the second international congress in aeronautical sciences at zurich in 1960, as suitable for achieving a required range at supersonic speeds, namely wingbody arrangements with sweepback angles of 55degrees or 60degrees and streamwise thickness-chord ratio of about 5 per cent suitable for low supersonic speed, and slender near-triangular wings with sharp leading edges suitable for mach numbers of about 2 or more . no attention is given to /slewed/ wings, powered lift or variable geometry . in dealing briefly with swept wings, the need for avoiding separation of flow from the leading edge is demonstrated, with the conclusion that it is desirable to use leading edge flaps with blowing or suction at the knee together with blown trailing edge flaps . wind tunnel tests are described on a simplified model with these boundary layer control methods applied . mention is made of the possibility of adverse ground effect on maximum lift . more attention is given to the case of slender wings because their use involves a new type of flow with separation from all edges . this flow and its steadiness are therefore discussed from the point of view of the possibility of buffeting,. the effect of plan form on static longitudinal stability and pitch-up is analysed,. and a short summary of available results on damping in pitch is given . large rolling moments due to sideslip are shown to give rise to serious problems of control, and the present state of knowledge of static lateral and directional stability and rolling and yawing rotary derivatives is discussed . finally the effects of proximity to the ground are summarised . the second part of the paper is concerned with work aimed at clarifying some of the requirements for handling qualities of future aircraft . it is not so much concerned with forecasts of the dynamic behaviour of these future aircraft as with determining what the pilot wants . two aspects of control in the vertical plane are discussed in some detail namely speed control and glide path holding . flight tests on an avro 707a aircraft, with artificially worsened characteristics, are described, and it is shown that substantially constant performance in the piloting task can be achieved at the expense of increased pilot effort . some tentative conclusions on desirable levels of speed stability and phugoid damping are, nevertheless, drawn . a brief review of the present status of lateral/directional handling requirements, using mainly american data, is also included .'), (793, 'the flow pattern on a tapered sweptback wing at mach numbers between 0.6 and 1.6 .', 'hall, i.m. and rogers, e.w.e.', 'a.r.c. r + m 3271, part 1, july 1960 .', 'the flow pattern on a tapered sweptback wing at mach numbers between 0.6 and 1.6 . the development of the flow pattern on a swept wing with incidence and stream mach number is described . the wing, of aspect ratio 2 828, taper ratio 0 333 and leading-edge sweep 53 5 deg, was tested at mach numbers between 0 6 and 1 6 at incidences up to about 12 deg . the test reynolds number varied with mach number, being typically 2 3 x 10 at m leading edge . it is shown that the flow pattern at moderate incidences develops smoothly from a subsonic type involving leading-edge separation to a supersonic type where the flow is attached near the leading edge and with shock-induced separation further aft . the formation and movement of the shock-wave system and the vortices near the wing surface are briefly discussed .'), (794, 'experiments with a tapered swept-back wing of warren 12 planform at mach numbers between 0.6 and 1.6 .', 'hall, i.m. and rogers, e.w.e.', 'a.r.c. r + m 3271, part ii, july 1960 .', 'experiments with a tapered swept-back wing of warren 12 planform at mach numbers between 0.6 and 1.6 . the development of the flow pattern on a wing of aspect ratio 2 828, taper ratio 0 333, leading-edge sweepback 53 5 deg and 6 per cent thickness/chord ratio in the streamwise direction has been described in part 1, which discussed oil-flow patterns obtained on the surface of the wing . the complete programme of tests also included pressure plotting at four spanwise stations and force measurements . these are discussed in relation to the flow development in this part of the report . the wing was tested at mach numbers between 0 6 and 1 6 for incidences up to about 14 deg . the tunnel stagnation pressure was held constant at a value near atmospheric pressure during the tests, so that the reynolds number varied with mach number .. at m 1 0 it was 2 3 x 10 based on the mean aerodynamic chord . boundary-layer transition was fixed by a roughness band at the leading edge . a detailed analysis has been made of the pressure distributions on the surface of the wing and the chordwise distributions integrated to determine the spanwise loading . the overall lift and pitching moment of the wing were also obtained from these data, as well as from direct measurements using a strain-gauge balance, by means of which the wing drag was also determined . these results are considered in some detail to illustrate the effects of mach number and incidence on the flow about the model . a preliminary analysis is also made of the conditions for boundary-layer separation due to shock waves on the wing surface . the principal factor appears to be the component of mach number normal to the shock front .'), (795, 'the operation of the npl 18in x 14in. wind tunnel in the transonic speed range .', 'hall, i.m.', 'a.r.c. c.p. 338, january 1957 .', 'the operation of the npl 18in x 14in. wind tunnel in the transonic speed range . a brief description of the slotted liners used is given together with the power requirements and some flow surveys . some observations are made on wall interference on a half-model of a swept wing tested in the wind tunnel .'), (796, 'an investigation at transonic speeds of the performance of various distributed roughness bands used to cause boundary layer transition near the leading edge of a cropped delta half-wing .', 'rogers, e.w.e. and hall, i.m.', 'a.r.c., c.p. 481, may 1959 .', 'an investigation at transonic speeds of the performance of various distributed roughness bands used to cause boundary layer transition near the leading edge of a cropped delta half-wing . distributed roughness bands of no.320 and no.500 carborundum were found to be effective in causing boundary-layer transition if they extended over the first 5( and 10( respectively of the local chord . use of larger grain sizes, or increases in the band width for a given grain size resulted in a drag penalty . with very large particle sizes /about between the particles . the drag penalty was constant over the test mach number range /0.80 to 1.15/ and decreased slowly with incidence . the wing lift and pitching moment were only slightly modified by the presence of any of the roughness bands tested, but this result would not of course necessarily apply to wings of other planforms or section shapes . the test reynolds number was about 2.7 million . in the appendix, the structure of the roughness bands is discussed, as well as the details of the materials used and the techniques used to apply the band .'), (797, 'a study of the effect of leading-edge modifications on the flow over a 50degree sweptback wing at transonic speeds .', 'rogers, e.w e., townsend, j.e.g. and berry, c.j.', 'a.r.c., r + m 3270, may 1960 .', 'a study of the effect of leading-edge modifications on the flow over a 50degree sweptback wing at transonic speeds . summary . an investigation has been made in the n.p.l. 18 in. x 14 in . tunnel of the effects of leading-edge modifications on the flow and forces on an untapered wing of 50 deg leading-edge sweep, at stream mach numbers between 0 60 and 1 20 . seven leading-edge profiles were tested, ranging from a drooped extension of 18 per cent of the chord of the basic sharp-nosed section to a round-nosed section with a leading-edge radius of 1 0 per cent of the basic chord . leading-edge droop was found to increase the wing drag near zero lift but to reduce appreciably the lift-dependent drag component, except at the highest test mach numbers . droop also increased the lift coefficient at which leading-edge separation occurred on the upper surface at moderate subsonic speeds, but in addition reduced the mach number for transonic flow attachment . the appearance of the forward shock /but not the rear shock/ is considerably delayed when the leading edge is drooped . with the undrooped sections an increase in leading-edge radius was accompanied by successively earlier appearances of the forward shock, and hence the outboard shock with its attendant separation . the conditions at which the rear shock first appeared changed only slowly as the section was changed . the variations in wing flow pattern as the leading edge is modified are discussed and related to measured changes in the wing lift and drag . an attempt is also made to estimate the local mach numbers on some parts of the wing from the oil-flow patterns,. this material is used to assess the flow conditions appropriate to shock-induced separation . the main section of the report concludes with a tentative discussion of the significance of the present results to the design of swept wings . in an appendix results obtained with the wing in a sweptforward configuration are briefly considered .'), (798, 'interaction between shock waves and boundary layers, with a note on the effects of the interaction of the performance of supersonic intakes .', 'holder, d.w., pearcey, h.h. and gadd, g.e.', 'a.r.c., c.p. 180, february 1954 .', 'interaction between shock waves and boundary layers, with a note on the effects of the interaction of the performance of supersonic intakes . the interaction between shock waves and boundary layers has important effects in many problems of high-speed flow . this paper has been written as a guide to the literature on the subject, and as a critical review of the present state of knowledge concerning both the underlying physical processes and the practical applications . it will be clear to the reader that, although substantial progress has been made, our knowledge is still far from complete and that more work both of a fundamental nature and on specific applications is needed before the problem is understood sufficiently well for design purposes . part i of the paper describes experiments on comparatively simple types of flow designed to provide fundamental information and to assist in the development of the theory . these experiments show that the interaction depends mainly on the mach and reynolds numbers and on the strength of the shock wave . in particular, the interaction of a shock wave with a laminar boundary layer is shown to produce much larger effects than if the boundary layer is turbulent . for most cases where the effects of the interaction are large enough to have serious practical consequences it is found that the boundary layer separates from the surface, and the difference between the interaction with laminar and turbulent layers arises mainly because the laminar layer separates much more readily in an adverse pressure gradient . the details of the interaction downstream of the separation point thus depend critically on the behaviour of the separated layer, and on the conditions under which it reattaches to the surface . many of the features found in the fundamental experiments appear also in practical applications and these are considered in parts ii and iii of the paper . although the emphasis hero is on the performance of aerfoils and wings moving at high subsonic speeds, the importance of the interaction in other examples such as at supersonic trailing edges and in supersonic intakes is also discussed briefly . the differences between the interaction with laminar and turbulent boundary layers are often a source of serious discrepancy between model experiments and full-scale conditions . for small-scale models it is, therefore, frequently essential to make the boundary layer turbulent by artificial means . some of the difficulties involved in doing this, and certain of the more promising methods are briefly discussed . it is shown that experiments on models with transition fixed can be used to explain a number of aerodynamic effects encountered in transonic flight, and connected with the occurrence of shock-induced separation of the turbulent boundary layers . for both two-dimensional aerofoils and straight and sweptback wings, turbulent separation occurs for shocks above a certain strength which applies for both model and full-scale conditions full-scale conditions,. differences in magnitude would be expected if the pressure recovery along the separated layer between the shock and the trailing edge is affected by reynolds number, but little information is at present available on this point . most of the repercussions of turbulent separation on the steady-motion characteristics of aerofoils and wings can be traced to the associated reduction in the pressure recovery over the roar of the surface . this is because the pressure at the trailing edge controls the inter-relation between the two surfaces /so long as the flow at the trailing edge remains subsonic/, and in particular the relative movements of the shock waves and the extents of the local regions of supersonic flow . certain unsteady-flow characteristics such as buffeting and control surface separation . some evidence is presented on the influence of section shape on the occurrence and effects of separation, but in this, as in many other respects, information relevant to turbulent boundary layers is scarce . some notes on the further work which is required are given in part iv of the paper .'), (799, 'some effects of wind-tunnel interference observed in tests on two-dimensional aerofoils at high subsonic and transonic speeds .', 'pearcey, h.h., sinnott, c.s. and osborne, j.', 'n.p.l. aero. 373. february 1959 .', 'some effects of wind-tunnel interference observed in tests on two-dimensional aerofoils at high subsonic and transonic speeds . in the high-speed research on two-dimensional aerofoils at the national physical laboratory the need to keep model size above a certain minimum, in order to reproduce correctly the boundary layer separation effects experienced at full scale, has been considered paramount even at the risk of incurring significant tunnel interference effects . this report discusses the interference effects for the slotted working sections now in use . the magnitudes of the blockage and lift effect corrections are deduced for the ratio of model chord to tunnel height normally used . it is shown that a simple adaptation to reduce the open area of the walls would reduce both corrections to insignificant proportions simultaneously, but would give a reduced choking mach number separated flows . the observed trends in the variation of the blockage effects for other ratios of model chord to tunnel height differ from those predicted theoretically, and so the results cannot be applied more generally until these trends have been checked by further investigations . it is suggested that wake interference effects can significantly influence the manner in which separated flows develop with increasing incidence or mach number, particularly for walls of small open area . examples are also given of effects of distortions in the local supersonic flow, which are most noticeable for walls with relatively large open areas .'), (800, 'wall interference at transonic speeds on a hemisphere cylinder model .', 'rogers, e.w.e. and hall, i.m.', 'a.r.c., c.p. 510, september 1959 .', 'wall interference at transonic speeds on a hemisphere cylinder model . tests have been made in three n.p.l. wind tunnels on a pressure-plotting model consisting of a long cylinder with a hemispherical nose . the surface pressure distributions were measured for stream mach numbers between 0.7 and 1.1 at zero model incidence, and schlieren photographs were taken . the blockage ratios were 0.211(, 0.117( and 0.120( . the principal feature of the flow is the effect of working section size on the rate at which the terminal shock wave moves back along the model with increasing stream mach number . this is thought to depend mainly on the distance from the model to the slotted walls of the tunnel, and not necessarily on the blockage ratio . the distance of the solid sidewall is important in influencing the local mach number ahead of the terminal shock, by reflecting the expansion-wave system originating near the model nose .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (801, 'experimental study of the equivalence of transonic flow about slender cone-cylinders of circular and elliptic cross section .', 'page,w.a.', 'naca tn.4233, 1958.', 'experimental study of the equivalence of transonic flow about slender cone-cylinders of circular and elliptic cross section . this report describes an experimental investigation of the equivalence relationship and the related theory for lifting forces proposed by transonic slender-body theory . the models chosen for this study are a flat, winglike, elliptic cone-cylinder and its equivalent body of revolution, a circular cone-cylinder . it is determined that the flows about the two models are closely related in the manner predicted by the theory, the relationship persisting over a mach number range of 0.92 to cone-cylinder vary linearly only over the small angle-of-attack range of approximately 1 and that the aerodynamic loading at sonic speed compares favorably with jones'' slender-wing theory . the results of the investigation suggest that at transonic speeds and at small angles of attack the calculation of all aerodynamic characteristics of slender, three-dimensional shapes can be made by use of transonic slender-body theory when the pressures on the equivalent body of revolution are known, either by experiment, or by an adequate nonlinear theory . from transonic slender-body theory it is deduced that the slenderness required for this application is the same as that required for the successful application of the transonic area rule .'), (802, 'the behaviour of supersonic flow past a body of revolution far from the axis .', 'whitham,g.b.', 'proc. roy. soc. a, 201, 1950, 89.', 'the behaviour of supersonic flow past a body of revolution far from the axis . a theory is developed of the supersonic flow past a body of revolution at large distances from the axis, where a linearized approximation is valueless owing to the divergence of the characteristics at infinity . it is used to find the asymptotic forms of the equations of the shocks which are formed from the neighbourhoods of the nose and tail . in the special case of a slender pointed body, the general theory at large distances is used to modify the linearized approximation to give a theory which is uniformly valid at all distances from the axis . the results which are of physical importance are summarized in the conclusion (9) and compared with the results of experimental observations .'), (803, 'the shock pattern of a wing-body combination far from the flight path .', 'walkden,f.', 'aero. quart. 9, 1958, 164.', 'the shock pattern of a wing-body combination far from the flight path . the position and strength of the front shock wave at large distances from a wing-body combination, are deduced from the linear theory for the combination, using a method developed by whitham . the combination consists of a body of revolution and a wing which has thickness and is lifting . the effects of interference between the flow over the body and the flow over the wing are included . in any direction the flow far from the wing-body combination is equivalent to the flow past a body of revolution determined from the configuration of the combination . the modified formulae for unsteady flow are given and some results are evaluated for the combination of a body of revolution and a delta wing with subsonic leading edges .'), (804, 'a flight test investigation of the sonic boom .', 'mullens,m.e.', 'afftc-tn-56-20, air res. and dev. command, u.s.a.f., 1956.', 'a flight test investigation of the sonic boom . the /sonic boom/ as it is now popularly called, has become the center of considerable interest during the past few years because of widespread public disturbance and possible damage that can result from it . in the hopes of minimizing this disturbance and to extend the general knowledge of the shock waves which produce the booming noise, the aeronautical research laboratory, wright air development center, has initiated an extensive research program to study the sonic boom phenomenon . this report presents the results of flight tests undertaken as one phase of this program . the tests had as their objective the determination and measurement of the shock wave pressure pattern surrounding an f-100 aircraft in level supersonic flight . the flight tests were conducted at the air force flight test center, edwards air force base, california, under the authority of air research and development command test directive no. 5524-f1 .'), (805, 'ground measurements of the shock wave noise from airplanes in level flight at mach numbers to 1. 4 and at altitudes to 45,000 feet .', 'maglieri,d.j., hubbard,h.h. and lansing,d.l.', 'nasa tn.d48, 1959.', 'ground measurements of the shock wave noise from airplanes in level flight at mach numbers to 1. 4 and at altitudes to 45,000 feet . time histories of noise pressures near ground level were measured during flight tests of fighter-type airplanes over fairly flat, partly wooded terrain in the mach number range between 1.13 and 1.4 and at altitudes from 25,000 to 45,000 feet . atmospheric soundings and radar-tracking studies were made for correlation with the measured noise data . the measured and calculated values of the pressure rise across the shock wave were generally in good agreement . there is a tendency for the theory to overestimate the pressure at locations remote from the track and to underestimate the pressures for conditions of high tailwind at altitude . the measured values of ground-reflection factor averaged about 1.8 for the surfaces tested as compared to a theoretical value of 2.0 . two booms were measured in all cases . the observers also generally reported two booms,. although, in some cases, only one boom was reported . the shock-wave noise associated with some of the flight tests was judged to be objectionable by ground observers, and in one case the cracking of a plate-glass store window was correlated in time with the passage of the airplane at an altitude of 25,000 feet .'), (806, 'ground measurements of airplane shock wave noise at mach numbers to 2, and at altitudes of 60,000 feet .', 'lina,l.j. and maglieri,d.j.', 'nasa tn.d235, 1960.', 'ground measurements of airplane shock wave noise at mach numbers to 2, and at altitudes of 60,000 feet . the intensity of shock-wave noise at the ground resulting from flights at mach numbers to 2.0 and altitudes to 60,000 feet was measured . measurements near the ground track for flights of a supersonic fighter and one flight of a supersonic bomber are presented . level cruising flight at an altitude of 60,000 feet and a mach number of 2.0 produced sonic booms which were considered to be tolerable, and it is reasonable to expect that cruising flight at higher altitudes will produce booms of tolerable intensity for airplanes of the size and weight of the test airplanes . the measured variation of sonic-boom intensity with altitude was in good agreement with the variation calculated by an equation given in nasa technical note d-48 . the effect of mach number on the ground overpressure is small between mach numbers of 1.4 and 2.0, a result in agreement with the theory . no amplification of the shock-wave overpressures due to refraction effects was apparent near the cutoff mach number . a method for estimating the effect of flight-path angle on cutoff mach number is shown . experimental results indicate agreement with the method, since a climb maneuver produced booms of a much decreased intensity as compared with the intensity of those measured in level flight at about the same altitude and mach number . comparison of sound pressure levels for the fighter and bomber airplanes indicated little effect of either airplane size or weight at an altitude of 40,000 feet .'), (807, 'ground measurements of the shock wave noise from supersonic bomber airplanes in the altitude range from 30,000 to 50,000 feet .', 'maglieri,d.j. and hubbard,h.h.', 'nasa tn.d880, 1961.', 'ground measurements of the shock wave noise from supersonic bomber airplanes in the altitude range from 30,000 to 50,000 feet . shock-wave ground-pressure measurements have been made for supersonic bomber airplanes in the mach number range from 1.24 to 1.52, for altitudes from about 30,000 to 50,000 feet, and for a gross-weight range from about 83,000 to 120,000 pounds . the measured overpressures were generally higher than would be predicted by the theory which accounts only for volume effects . there is thus a suggestion that lift effects on sonic-boom intensity may be significant for this type of airplane for the altitude range of the present tests .'), (808, 'an investigation of some aspects of the sonic boom by means of wind tunnel measurements of pressures about several bodies at a mach number of 2. 01 .', 'carlson,h.w.', 'nasa tn.d161, 1959.', 'an investigation of some aspects of the sonic boom by means of wind tunnel measurements of pressures about several bodies at a mach number of 2. 01 . an investigation of some aspects of the sonic boom has been made with the aid of wind-tunnel measurements of the pressure distributions about bodies of various shapes . the tests were made in the langley at a mach number of 2.01 and at a reynolds number per foot of 2.5 x 10 . measurements of the pressure field were made at orifices in the surface of a boundary-layer bypass plate . the models which represented both fuselage and wing types of thickness distributions were small enough to allow measurements as far away as 8 body lengths or 64 chords . the results are compared with estimates made using existing theory . to the first order, the boom-producing pressure rise across the bow shock is dependent on the longitudinal development of body area and not on local details . nonaxisymmetrical shapes may be replaced by equivalent bodies of revolution to obtain satisfactory theoretical estimates of the far-field pressures .'), (809, 'an investigation of the influence of lift on sonic-boom intensity by means of wind tunnel measurements of the pressure fields of several wing-body combinations at a mach number of 2. 01 .', 'carlson,h.w.', 'nasa tn.d881, 1961.', 'an investigation of the influence of lift on sonic-boom intensity by means of wind tunnel measurements of the pressure fields of several wing-body combinations at a mach number of 2. 01 . an investigation of the effect of lift on sonic-boom intensity has been performed by means of wind-tunnel measurements of the pressure fields surrounding small wing-body combinations . the tests were conducted in the langley 4- by 4-foot supersonic pressure tunnel at a mach number of per foot . effects of lift were found to be real and significant . measured bow-shock intensities agreed fairly well with, but were consistently less than, shock intensities estimated by theoretical methods . available flight data were examined for correlation with wind-tunnel test results .'), (810, 'the shock wave noise problem of supersonic aircraft in steady flight .', 'maglieri,d.j. and carlson,h.w.', 'nasa memo 3-4-59l, 1959.', 'the shock wave noise problem of supersonic aircraft in steady flight . data are presented which provide an insight into the nature of the shock-wave noise problem, the significant variables involved, and the manner in which airplane operation may be affected . flight-test data are also given, and a comparison with the available theory is made . an attempt is also made to correlate the subjective reactions of observers and some associated physical phenomena with the pressure amplitudes during full-scale flight . it is indicated that for the proposed supersonic transport airplanes of the future, booms on the ground will most probably be experienced during the major portion of the flight plan . the boom pressures will be most severe during the climb and descent phases of the flight plan . during the cruise phase of the flight, the boom pressures are of much lesser intensity but are spread laterally for many miles . the manner in which the airplane is operated appears to be significant,. for example, the boom pressures during the climb, cruise, and descent phases can be minimized by operating the airplane at its maximum altitude consistent with its performance capabilities .'), (811, 'an investigation of lifting effects on the intensity of sonic booms .', 'morris,j.', 'j. roy. aero. soc. 64, 1960, 610.', 'an investigation of lifting effects on the intensity of sonic booms . this paper is a brief summary of an investigation made to check the effect of lift on the shock noise of aircraft flying at supersonic speeds . the method of hayes has been combined with the theory of whitham to predict the asymptotic shock strength of wings carrying lift and of combinations of bodies and lifting wings . (a similar, but not quite as general, method was derived by walkden in ref. 6 .) whitham''s formula, including only the volume term, has been used extensively to predict the boom intensity of aeroplane type bodies and the agreement with experiment has, so far, been quite reasonable . the test data obtained to date extends only up to about 40,000 ft. altitude and the calculations of this paper show that under those conditions the shock noise of the aircraft tested so far will, in most cases, be dominated by the volume term . it is shown that at higher altitudes lifting effects will dominate for even the small fighter and they will dominate over most of the altitude range for large bomber and supersonic transport aircraft . the boom intensity due to lift decreases with altitude as which compares to in the volume case (=pressure at altitude h) . it is insensitive to mach number, wing loading, wing plan shape and lift distribution . a simple rule for calculating the shock noise due to combined volume and lifting effects is proposed which is applicable to configurations with wings located towards the rear of the fuselage . the rule states that the shock noise of an aircraft carrying lift is equal to the shock noise due to volume (neglecting lift) or the shock noise due to lift (neglecting volume), whichever is the greater . a chart is presented from which rapid estimates can be made of the shock noise of lifting wing-body combinations .'), (812, 'investigation of two bluff shapes in axial free flight over a mach number range from 0. 35 to 2. 15 .', 'coltrane,l.c.', 'naca rm l58a16, 1958.', 'investigation of two bluff shapes in axial free flight over a mach number range from 0. 35 to 2. 15 . a fineness-ratio-2.71 right circular cylinder and a fineness- ratio-been tested in free flight over a mach number range of 0.35 to 2.15 and a reynolds number range of 1 x 10 to 12 x 10 . time histories, cross plots of force coefficients, rolling velocity, and longitudinal-force coefficient are presented for both cylinders . in addition, cross plots of moment coefficients and plots of the normal-force curve slope and the aerodynamic center are presented for the fineness-ratio-2.71 cylinder . the average aerodynamic center of the right circular cylinder moved rearward with decreasing speeds until at the subsonic mach numbers it remained approximately constant and comparisons of the drag data of this test with wind tunnel and other free-flight data show good agreement . an appreciable decrease in drag was observed when the data of the present test of the rounded nose cylinder were compared with data of a right circular cylinder of a similar configuration .'), (813, 'the motion of rolling symmetrical missiles referred to a body-axis system .', 'nelson,r.l.', 'naca tn.3737, 1956.', 'the motion of rolling symmetrical missiles referred to a body-axis system . the linearized equations of motion have been derived for a rolling missile having slight aerodynamic asymmetries . time histories of rolling-missile motions referred to a body-axis system have been prepared to show the types of missile motions that can be encountered . the motions resulting from a trim change and a pulse-rocket disturbance are shown to be determined mainly by the ratio of rolling velocity to pitching frequency . finally, the derived equations are used in establishing a technique for the reduction of rolling-missile oscillation data . it is shown that the aerodynamic derivatives can be obtained from flight data if four accelerations are measured . the method is applied to the results obtained from a flight test of a missile configuration .'), (814, 'stability derivatives of cones at supersonic speeds .', 'tobak,m. and wehrend,w.r.', 'naca tn.3788, 1956.', 'stability derivatives of cones at supersonic speeds . the aerodynamic stability derivatives due to pitching velocity and vertical acceleration are calculated by use of potential theory for circular cones traveling at supersonic speeds . the analysis is based on two theoretical techniques used successfully previously in application to the case of uniform axial and inclined flow . in the first, potential solutions for axial flow and crossflow are derived from the first-order wave equation but in application to calculations for the forces no approximations are made either to the tangency condition or to the isentropic pressure relation . the second method consists in combining the first-order crossflow potential with an axial-flow potential correct to second order . closed-form solutions by both methods are found for a cone, and numerical results for the stability derivatives are presented as a function of mach number for cones having semivertex angles of 10 and 20 . in addition, expressions for the forces, moments, and stability derivatives of arbitrary bodies of revolution are obtained using newtonian impact theory . numerical results for cones compare well with those obtained from the combined first- and second-order potential theory at the highest mach number for which the latter theory is applicable .'), (815, 'investigation of several blunt bodies to determine trans- onic aerodynamic characteristics including effects of spinning and of extendible afterbody flaps and some measurements of unsteady base pressures .', 'fisher,l.r. and dicamillo,j.r.', 'nasa memo 1-21-59l, 1959.', 'investigation of several blunt bodies to determine trans- onic aerodynamic characteristics including effects of spinning and of extendible afterbody flaps and some measurements of unsteady base pressures . several blunt bodies having shapes that may be suitable for atmospheric reentry vehicles were tested to determine the aerodynamic characteristics of such shapes for angles of attack up to 34 . the tests were conducted through the transonic mach number range and at reynolds numbers from 1.74 x 10 to 2.78 x 10, based on body diameter . a full-skirted rather than a short-skirted type of shape developed the greatest amount of static stability and the largest lift-curve slopes . the angle of attack for maximum lift for such bodies appears to be subject to mach number effects . spinning a full-skirted body about its longitudinal axis generally increased the lift and reduced the pitching moment at angles of attack and reduced the aerodynamic static stability parameter through the transonic mach number range . the extension of segmented clamshell-shaped flaps from the afterbody of a short-skirted model served to increase the lift and static stability only if the flaps extended into the airstream . some evidence was found of oscillatory base pressures on two dissimilar shapes at certain high angles of attack and the highest mach number in these tests . there is doubt, however, that these pressures can induce any significant oscillatory motion for a reentry vehicle because of their small amplitude and phasing .'), (816, 'experimental investigation at a mach number of 3. 11 of the lift, drag and pitching moment characteristics of a number of blunt low-fineness-ratio bodies .', 'letko,w.', 'nasa memo 1-18-159l, 1959.', 'experimental investigation at a mach number of 3. 11 of the lift, drag and pitching moment characteristics of a number of blunt low-fineness-ratio bodies . a number of blunt bodies having shapes that may be suitable for atmospheric reentry were tested to determine the lift, drag, and pitching-moment characteristics at a mach number of 3.11 and a reynolds number of 6 x 10 based on maximum body diameter of 2 inches . the results of the tests showed that all the bodies were statically stable about a point located one-third of the body length from the nose . the results also showed that high-drag bodies which have a large portion of their afterbodies negatively sloped (decrease in cross-sectional area from nose to base) may have a negative lift-curve slope . this negative slope results from the large negative lift component of the axial force obtained with those bodies and the fact that with negatively sloped afterbodies only small normal forces are developed .'), (817, 'loading paths and the incremental stress law .', 'hankelman,g.h. and warner,w.h.', 'j. math. and phys. 33, 1954, 157.', 'loading paths and the incremental stress law . this paper will be concerned with some properties of the stress-strain law for work hardening materials introduced by w. prager incremental strain or plastic flow law by which it is meant that the differentials of strain are expressed as functions of the stresses, strains, and differentials of the stress . we shall also have occasion to refer to total strain or plastic deformation laws, in which the strains are given directly as functions of the stress .'), (818, 'a quantitative comparison of flow and deformation theories of plasticity .', 'hodge,p.g. and white,g.n.', 'j. app. mech. 17, 1950.', 'a quantitative comparison of flow and deformation theories of plasticity . the stresses and displacements in a partly plastic, infinitely long, hollow cylinder are obtained according to the flow type of stress-strain law of prandtl-reuss and to the deformation law of hencky . in both cases the mises yield condition is used and the compressibility of the material is taken into account . it is shown that under these assumptions the two theories yield substantially the same results for this particular problem, but that one theory or the other may be preferable for computing purposes in certain cases . the results are compared with those of other investigations in which different combinations of stress-strain law, yield condition, compressibility, and end loading were assumed .'), (819, 'stresses in the plastic range around a normally loaded circular hole in an infinite sheet .', 'magasarian,o.l.', 'j. app. mech. 27, 1960, 65.', 'stresses in the plastic range around a normally loaded circular hole in an infinite sheet . the stresses in the plastic range around a normally loaded circular hole in an infinite sheet are found numerically on the basis of both deformation and incremental theories . the results of deformation theory are quantitatively assessed in the light of a criterion, recently developed by budiansky, for the acceptability of deformation theories . the criterion is completely satisfied . moreover, the results obtained by using these two different theories of plasticity do not differ greatly despite the fact that the stress paths are far from being radial .'), (820, 'theories of plastic buckling .', 'batdorf,s.b.', 'j. ae. scs. 16, 1949, 405.', 'theories of plastic buckling . the theory for the plastic buckling of columns which appears finally to have achieved a satisfactory form, rests upon the well-established uniaxial stress-strain relation . the development of a correspondingly satisfactory theory for the plastic buckling of plates has been hampered by the nonexistence of an established polyaxial stress-strain relation in the plastic range . present theories for the polyaxial stress-strain relation beyond the elastic range can be divided into two types, often called flow and deformation theories . theories of plastic buckling based on deformation theories are in better agreement with experiment than those based on flow theories . on the other hand, tests in which a material is compressed into the plastic range and then subjected to shear at constant compressive stress are in better agreement with flow than with deformation theories . legitimate doubt therefore has existed as to the validity of any theory for the plastic buckling of plates . as a result of studying these apparent contradictions, a new theory of plasticity has been developed which is of neither the flow nor the deformation type . it is based upon the concept of slip, and its formulation was guided more by physical, and less by mathematical, considerations than previous theories . experimental evidence of limited scope but of crucial character is in better agreement with the new theory than with either flow or deformation theories . the new theory accounts for the apparent contradictions previously alluded to and justifies the use of deformation theory in the analysis of the plastic buckling of plates .'), (821, 'inelastic column theory .', 'shanley,f.r.', 'j. ae. scs. 14, 1949, 261.', 'inelastic column theory . the action of a column in the plastic range is analyzed on the basis that bending may proceed simultaneously with increasing axial load . this leads to a new column formula that includes both the tangent-modulus (engesser) and the reduced-modulus the tangent-modulus load and that the column load increases with increasing lateral deflection, approaching the reduced-modulus load as a limit if the tangent modulus is assumed to remain constant .'), (822, 'effects of imperfections on buckling of thin cylinders and columns under axial compression .', 'donnell,l.h. and wan,c.c.', 'j. app. mech. 1950.', 'effects of imperfections on buckling of thin cylinders and columns under axial compression . von karman and tsien have shown that under elastic conditions the resistance of perfect thin cylinders subjected to axial compression drops precipitously after buckling . it is considered that this indicates that this type of buckling is very sensitive to imperfections or disturbances . in this paper the effects of certain imperfections of shape turbances combined) are studied by the large-deflection shell theory developed in a previous paper (2) . it is found that two types of buckling failure may occur . one is of a purely elastic type which occurs when the peak of the average stress versus average strain curve is reached, while the other type is precipitated by yielding, which for thicker cylinders or lower-yield material may occur before such a peak is reached . curves are derived giving the dependence of each type of failure upon the dimensions and elastic and yield properties of the specimen and also upon an /unevenness factor/ u which determines the magnitude of the initial imperfections and is assumed to depend on the method of fabrication . the relations derived are in line with test results, and similar studies of the buckling of struts indicate that the magnitude of the initial imperfections which have to be assumed to explain test strengths are reasonable .'), (823, 'plastic torsional buckling strength of cylinders including the effects of imperfections .', 'lee,l.h.n. and ades,c.s.', 'j. ae. scs. 24, 1957, 241.', 'plastic torsional buckling strength of cylinders including the effects of imperfections . the torsional buckling strength of a cylinder in the plastic range has been determined . an energy solution and a more exact solution, both based on a plastic stress-strain relationship given by the simple deformation theory, are presented . close agreement between the two solutions is found . the effects of large deflections and imperfections on buckling strength are analyzed . for two groups of experimental results used for comparison, the effects of geometrical imperfections in the plastic range are negligible . the theoretical results are found to be in good agreement with the experimental results .'), (824, 'on the concept of stability of inelastic systems .', 'drucker,d.c. and onat,e.t.', 'j.ae.scs., 21, 1954, 543.', 'on the concept of stability of inelastic systems . simple models are employed to bring out the large and important differences between buckling in the plastic range and classical elastic instability . static and kinetic criteria are compared and their interrelation discussed . nonlinear behavior in particular is often found to be the key to the physically valid solution . the nonconservative nature of plastic deformation in itself or in combination with the nonlinearity requires concepts not found in classical approaches . conversely, the classical linearized condition of neutral equilibrium is really not relevant in inelastic buckling . plastic buckling loads are not uniquely defined but cover a range of values and are often more properly thought of as maximum loads for some reasonable initial imperfection in geometry or dynamic disturbance . the models indicate that basically the same information is obtained from essentially static systems by assuming initial imperfection in geometric forms as by assuming dynamic disturbances . one approach complements the other and both are helpful in obtaining an understanding of the physical phenomena .'), (825, 'inelastic instability and incremental theories of plasticity .', 'onat,e.t. and drucker,d.c.', 'j. ae. scs. 20, 1953, 181.', 'inelastic instability and incremental theories of plasticity . a most troublesome paradox has existed for a number of years with respect to buckling in the plastic range . theoretical considerations and all direct experimental evidence show conclusively that an incremental or flow type of mathematical theory of plasticity is valid . however, the results of plastic buckling tests are well correlated by a simple total or deformation theory and bear no resemblance to published predictions of incremental theory . the suggestion was made that initial imperfections of shape or loading might well explain this most peculiar result . however, subsequent investigations by several authors seem to have given the impression that excessively large imperfections would be needed and that the answer would be overly sensitive to the magnitude of such imperfections . it is the purpose of this paper to demonstrate that extremely small, and therefore unavoidable, imperfections of shape do account for the paradox in a simple manner . the buckling load is shown to be extremely insensitive to the amount of imperfection . the example chosen is a simplified version of the long rectangular plate hinged along one edge and free on the other under uniform compressive stress at the ends . this is the equivalent of the case of the cruciform column, which has been so disturbing in the past because incremental theory applied to a perfect cruciform column did lead to an entirely incorrect result .'), (826, 'small bending and stretching of sandwich type shells .', 'reissner,e.', 'naca tn.1832, 1949.', 'small bending and stretching of sandwich type shells . a theory has been developed for small bending and stretching of sandwich-type shells . this theory is an extension of the known theory of homogeneous thin elastic shells . it was found that two effects are important in the present problem, which have not been considered previously in the theory of curved shells .. (1) the effect of transverse shear deformation and (2) the effect of transverse normal stress deformation . the first of these two effects has been known to be of importance in the theory of plates and beams . the second effect was found to occur in a manner which is typical for shells and has no counterpart in flat-plate theory . the general results of this report have been applied to the solution of problems concerning flat plates, circular rings, circular cylindrical shells, and spherical shells . in each case numerical examples have been given, illustrating the magnitude of the effects of transverse shear and normal stress deformation . the results of this investigation indicate the necessity of taking account of transverse shear and normal stress in sandwich-type shells, as soon as there is an order-of-magnitude difference between the elastic constants of the core layer and of the face layers of the composite shell . it was found that the changes due to transverse shear and normal stress deformation in the core may be so large as to be no mere corrections to the results of the theory without transverse core flexibility . the actual magnitude of the changes is greatly dependent on the geometry and loading condition of the structure under consideration so that no general rules may be given which indicate for which elastic modulus ratio the changes begin to be significant . solutions of problems in the present theory may in general be obtained by mathematical methods which are similar to those employed in the theory of plates and shells without the effect of transverse shear and normal stress deformation included . the present work does not include consideration of buckling and finite deflection effects .'), (827, 'a nonlinear theory of bending and buckling of thin elastic shallow spherical shells .', 'kaplan,a. and fung,y.c.', 'naca tn.3212, 1954.', 'a nonlinear theory of bending and buckling of thin elastic shallow spherical shells . a shallow spherical dome subjected to lateral pressure is a structure for which the deformation departs appreciably from the linear theory at relatively small values of the deflection amplitude . it is also one for which the buckling process is characterized by a rapid decrease in the equilibrium load once the buckling load has been surpassed . for structures having this type of buckling characteristics the question arises as to whether the proper buckling criterion to apply is the classical criterion, which considers equilibrium with respect to infinitesimal displacements, or the finite-displacement /energy criterion/ proposed by tsien . in this paper the problem of the finite displacement and buckling of a shallow spherical dome is investigated both theoretically and experimentally . in the theoretical approach the nonlinear equations are converted into a sequence of linear equations by expanding all of the variables in powers of the center deflection and then equating the coefficients of equal powers . the basic parameter for the shallow dome is proportional to the ratio of the central height of the dome h to its thickness t . for small values of this ratio the expansions converge rapidly and enough terms are computed to determine the buckling load according to the classical criterion . for higher values of h t, convergence deteriorates rapidly and it was not possible to determine the buckling load with the number of terms which were computed . however even for these higher values of h t the deflection shapes are determined for deflection amplitudes below the amplitude at which buckling occurs . these deflection shapes are characterized by their rapid change as h t increases and by the fact that, over most of the range of h t studied, the maximum deflection does not occur at the center of the dome . experimental results seem to indicate that the classical criterion of buckling is applicable to very shallow spherical domes for which the theoretical calculation was made . a transition to energy criterion for higher domes is also indicated .'), (828, 'stresses and small displacements of shallow spherical shells .', 'reissner,e.', 'j. math. phys. 25, 1946, 80.', 'stresses and small displacements of shallow spherical shells . the purpose of the present paper is to derive a system of equations which can be used for the analysis of shallow segments of thin, elastic, spherical shells . a segment will be called shallow if the ratio of its height to base diameter is less than, say . the results obtained on the basis of this assumption will often also be applicable to shells which are not shallow, namely then, when the loads are such that the stresses are effectively restricted to shallow zones . the problem of the spherical elastic shell has been the subject of numerous researches . for the rotationally symmetric case the fundamental results were obtained in 1912 (1) and have been the starting point of many applications . while it is possible to deduce from these results approximate equations equivalent to part of what follows, it is believed that the present approach to the problem of the shallow shell may be of some interest even for rotationally symmetric cases . a number of investigations have been concerned with the shell loaded in a non-rotationally symmetric manner (2,3,4) . in its general form this problem is quite difficult and the results so far obtained are not easy to apply . restricting attention to the shallow shell in the manner of the present paper brings with it a very considerable simplification of the analysis .'), (829, 'stability of thin-walled tubes under torsion .', 'donnel,l.h.', 'naca r.479, 1933, 12.', 'stability of thin-walled tubes under torsion . in this paper a theoretical solution is developed for the torsion on a round thin-walled tube for which the walls become unstable . the results of this theory are given by a few simple formulas and curves which cover all cases . the differential equations of equilibrium are derived in a simpler form than previously found, it being shown that many items can be neglected . the solution obtained is length ratio is zero and infinite, and is a good approximation for intermediate cases . the theory is compared with all available experiments, including about 50 tests made by the author . the experimental-failure torque is always smaller than the theoretical-buckling torque, averaging about 75 percent of it, with a minimum of 60 percent . as the form of the deflection checks closely with that predicted by theory and the experiments cover a great range of shapes and materials, this discrepancy can reasonably be ascribed largely to initial eccentricities in actual tubes .'), (830, 'nonlinear deflections of shallow spherical shells .', 'reiss,e.l., greenberg,h.j. and keller,h.b.', 'j. ae. scs. 24, 1957, 533.', 'nonlinear deflections of shallow spherical shells . the equations obtained by chien for the nonlinear deflection of shallow spherical shells under uniform external pressure are solved by means of power series expansions, following procedures introduced by friedrichs and stoker in their treatment of buckling of circular plates . these equations depend upon two parameters . one of these parameters is related to the external pressure, while the other depends upon the dimensions of the shell . the equations are solved for several ranges of the parameters under boundary conditions corresponding to a fixed edge . the solution, carried out numerically on the aec univac at new york university, yields a complete description of the stresses and deflections as functions of the polar angle over a wide range of values of the loading parameter and the dimensional parameter . prediction of the upper buckling load is then made by means of a numerical criterion based on the load vs. deflection curve . for some cases, the postbuckling behavior is investigated . the results agree well with existing experimental and theoretical studies and cover a wide range of cases not previously treated .'), (831, 'buckling of shallow shells under external pressure .', 'reiss,e.l.', 'j. app. mech. 25, 1958,556.', 'buckling of shallow shells under external pressure . a formula for the initial buckling loads for clamped, shallow spherical shells under uniform external pressure is obtained by combining the solutions of two linearized versions of the original nonlinear problem . one of these versions is a linear eigenvalue problem while the other is the bending problem for a shallow cap in the linear theory of elasticity . the formula, which is obtained in a simple manner, yields buckling loads that are in better agreement with experiments than previous approximate solutions to the nonlinear problem .'), (832, 'accelerating convergence of iteration processes .', 'wegstein,j.h.', 'comm. of assoc. for computing mach. 1958.', 'accelerating convergence of iteration processes . a technique is discussed which, when applied to an iterative procedure for the solution of an equation, accelerates the rate of convergence if the iteration converges and induces convergence if the iteration diverges . an illustrative example is given .'), (833, 'a simple method of matric structural analysis, part iv, non-linear problems .', 'klein,b.', 'j. ae. scs. 26, 1959.', 'a simple method of matric structural analysis, part iv, non-linear problems . the method presented in the previous parts is employed to solve various kinds of nonlinear problems, such as problems concerning large deflections or buckling, or thermal creep, or inelastic stress redistribution involving thermal gradients, or design . the procedure used in each case is one of direct iteration--i. e., after one assumes a starting point all subsequent cycles are self-generating . simple numerical examples are worked out .'), (834, 'limit design for economical missile structures .', 'l. a. riedinger', 'assistant manager-rand d structures dept., missile systems div., lockheed aircraft corp.', 'limit design for economical missile structures . a special safety factor alone won''t do the trick in the design of lightweight, high temperature missile structures . if you really want to end up with the most efficient structure you can get, an entirely new design approach is needed .'), (835, 'the problem of strain accumulation under thermal cycling .', 'b. e. gatewood', 'research coordinator, air force institute of technology, wright-patterson air force base, ohio', 'the problem of strain accumulation under thermal cycling . parkes and sprague and huang have shown that it is possible for strain growth to occur in a beam structure under temperature-load cycling . the various aspects of this problem as to criteria for convergence and divergence of the strain accumulation can be simply demonstrated by thermal cycling one element of a two-element structure .'), (836, 'analytical and experimental investigation of stress distributions in long flat plates subjected to lingitudinal loads and transverse temperature gradients .', 'sprague, g.h. and huang, p.c.', 'wadc tr 55-350, 1955 .', 'analytical and experimental investigation of stress distributions in long flat plates subjected to lingitudinal loads and transverse temperature gradients . stress and strain distributions were studied in long flat plates in order to develop practical analytical procedures for the design analysis of aircraft structures at elevated temperatures . various load-temperature conditions are presented . these include .. methods of analysis for calculation of stress distributions under and plastic range,. ture . experimental verification of the analytical procedures is shown with comparisons between the use of constant room temperature or temperature dependent values of modulus of elasticity and coefficient of thermal expansion . the test specimen, equipment, instrumentation, and experimental program are discussed in detail . experimental data obtained from the specimen and associated material control coupon tests are presented .'), (837, 'inelastic behaviour of structures subjected to cyclic thermal and mechanical stressing conditions .', 'padlog,j., huff,r.d. and holloway,g.f.', 'wadc tr 60-271, 1960.', 'inelastic behaviour of structures subjected to cyclic thermal and mechanical stressing conditions . a general analytical procedure is outlined for structures subjected to varying thermal and mechanical stressing conditions . consideration is given to the accumulation of time-independent plastic strains and creep strains . stress-strain-temperature-time relations for uniaxial and multiaxial stresses are defined, based on various material behavior assumptions . several of the assumptions are compared with a limited number of time-varying temperature and uniaxial stress tests . the procedure is illustrated by its application to uniaxial stress problems in which /planes originally plane remain plane/ and to plane stress plate problems . a solution, based on the influence coefficient approach to the plane stress plate problem, is obtained which is applicable to all plate plan forms, edge boundary conditions, and inplane thermal and mechanical loading conditions . from the predicted inelastic behavior of a three-bar structure subjected to cyclic thermal and mechanical loading conditions, it is shown that eventual failure could result from large permanent deformation accumulations, tensile rupture, or thermal-stress-fatigue . a sample plate with a centrally located hole was analyzed for two cycles of a time-varying temperature and edge stress condition . both plastic strain reversals and plastic strain growths were predicted at the hole . however, a test-theory comparison indicated failure by creep-rupture .'), (838, 'bending and compression tests of pressurised ring-stiffened cylinders .', 'dow,m.b. and peterson,j.p.', 'nasa tn.d360, 1960.', 'bending and compression tests of pressurised ring-stiffened cylinders . the results of tests on pressurized ring-stiffened cylinders subjected to compression and bending are presented and discussed . the results obtained at high values of internal pressure differ from those obtained by previous investigators in that the theoretical small-deflection compressive buckling coefficient of 0.6 was nearly achieved in each test . small amounts of internal pressure had a greater stabilizing effect in the bending tests than in the compression tests .'), (839, 'the bending stability of thin-- walled unstiffened circular cylinders including the effects of internal pressure .', 'suer,h.s., harris,l.a., skene,w.t. and benjamin,r.j.', 'j. ae. scs. 25, 1958.', 'the bending stability of thin-- walled unstiffened circular cylinders including the effects of internal pressure . in a recent paper, the authors presented a statistical, semiempirical design procedure for the determination of the buckling strength of unpressurized and pressurized cylinders under axial compression . this procedure has been extended in the present paper to the bending of unpressurized and pressurized cylindrical shells and allows the calculation of the critical bending stress with a knowledge of the cylinder geometry and the internal pressure only . because no published data could be found, an extensive series of bending tests of pressurized cylinders has been performed . these new data for pressurized cylinders are treated semiempirically together with all of the other known test data for unpressurized cylinders . best-fit curves are presented using applicable theoretical parameters . design curves for determining the critical buckling stress for unpressurized and pressurized cylinders in bending are then developed as 90 per cent probability curves from the test data .'), (840, 'analysis of partly wrinkled membrane .', 'stein,m. and hedgepeth,j.m.', 'nasa tn.d813, 1961.', 'analysis of partly wrinkled membrane . a theory is derived to predict the stresses and deformations of stretched-membrane structural components for loads under which part of the membrane wrinkles . rather than studying in detail the deformations in the wrinkled region, the present theory studies average displacements of the wrinkled material . specific solutions of problems in flat and curved membranes are presented . the results of these solutions show that membrane structures retain much of their stiffness at loads substantially above the load at which wrinkling first occurs .'), (841, 'on the bending of circular cylindrical shells under pure bending .', 'seide,p. and weingarten,v.i.', 'j. app. mech. 28, 1961.', 'on the bending of circular cylindrical shells under pure bending . the stability of circular cylindrical shells under pure bending is investigated by means of batdorf''s modified donnell''s equation and the galerkin method . the results of this investigation have shown that, contrary to the commonly accepted value, the maximum critical bending stress is for all practical purposes equal to the critical compressive stress .'), (842, 'an improvement on donnell''s approximation for thin-walled circular cylinders .', 'morley,l.s.d.', 'q. j. mech. app. math. 12, 1959.', 'an improvement on donnell''s approximation for thin-walled circular cylinders . donnell''s equation for thin-walled circular cylinders is replaced by where w is a non-dimensional form of the radial displacement and q is the distributed radial loading . this equation retains the essential simplicity of the original but, unlike donnell''s equation, the accuracy does not decrease as the wavelength of circumferential distortion increases .'), (843, 'a simplified method of elastic stability analysis for thin cylindrical shells .', 'batdorf,s.b.', 'i. donnell''s equation. naca tn.1341, 1947.', 'a simplified method of elastic stability analysis for thin cylindrical shells . the equation for the equilibrium of cylindrical shells introduced by donnell in naca report no. 479 to find the critical stresses of cylinders in torsion is applied to find critical stresses for cylinders with simply supported edges under other loading conditions . it is shown that by this method solutions may be obtained very easily and the results in each case may be expressed in terms of two nondimensional parameters, one dependent on the critical stress and the other essentially determined by the geometry of the cylinder . the influence of boundary conditions related to edge displacements in the shell median surface is discussed . the accuracy of the solutions found is established by comparing them with previous theoretical solutions and with test results . the solutions to a number of problems concerned with buckling of cylinders with simply supported edges on the basis of a unified viewpoint are presented in a convenient form for practical use .'), (844, 'flexural vibrations of the walls of thin cylindrical shells having freely supported ends .', 'arnold,r.n. and warburton,g.b.', 'proc. roy. soc. a, 197, 1949, 238.', 'flexural vibrations of the walls of thin cylindrical shells having freely supported ends . the paper deals with the general equations for the vibration of thin cylinders and a theoretical and experimental investigation is made of the type of vibration usually associated with bells . the cylinders are supported in such a manner that the ends remain circular without directional restraint being imposed . it is found that the complexity of the mode of vibration bears little relation to the natural frequency,. for example, cylinders of very small thickness-diameter ratio, with length about equal to or less than the diameter, may have many of their higher frequencies associated with the simpler modes of vibration . the frequency equation which is derived by the energy method is based on strain relations given by timoshenko . in this approach, displacement equations are evolved which are comparable to those of love and flugge, though differences are evident due to the strain expressions used by each author . results are given for cylinders of various lengths, each with the same thickness-diameter ratio, and also for a very thin cylinder in which the simpler modes of vibration occur in the higher frequency range . it is shown that there are three possible natural frequencies for a particular nodal pattern, two of these normally occurring beyond the aural range .'), (845, 'the flexural vibrations of thin cylinders .', 'arnold,r.n. and warburton,g.b.', 'j. proc. i. mech. e. 167, 1953, 62.', 'the flexural vibrations of thin cylinders . the flexural vibrations of the walls of thin cylinders are considered . in this type of vibration many forms of nodal pattern may exist owing to the combination of circumferential and axial nodes . theoretical expressions are developed for the natural frequencies of cylinders with freely-supported and fixed ends and a comparison is made with the frequencies obtained experimentally . in practice, the ends of cylinders are subjected to a certain degree of fixing by end-plates, flanges, etc., and the natural frequencies thus lie between the corresponding values for freely-supported and fixed ends . to make possible the estimation of such frequencies, a method is devised in which an equivalent wavelength factor is used . this factor represents the wavelength of the freely-supported cylinder that would have the same frequency as the cylinder under consideration when vibrating in the same mode . the results of experimental investigations with various end thicknesses and flange dimensions are recorded, and from these the equivalent factors are derived . sets of curves calculated for cylinders with freely-supported ends and covering a range of cylinder thicknesses are given . from these it is possible to obtain close approximation to the frequencies of cylinders under other end conditions by the use of an appropriate factor . an example is given of frequency calculations for a large air-receiver for which two frequencies were identified by experiment .'), (846, 'on the vibration of thin cylindrical shells under internal pressure .', 'fung,y.c., sechler,e.e. and kaplan,a.', 'j. ae. scs. 27, 1957, 650.', 'on the vibration of thin cylindrical shells under internal pressure . the frequency spectra and vibration modes of thin-walled circular cylinders subjected to internal pressure are considered . it is shown that for very thin cylinders the internal pressure has a significant effect on the natural vibration characteristics . for these cylinders, particularly those having smaller length to diameter ratios, the mode associated with the lowest frequency is in general not the simplest mode . the exact number of circumferential nodes, n, which occur in the mode associated with the lowest frequency, depends on the internal pressure p . if this number n is large, it decreases rapidly with increasing p when p is small, and the /fundamental/ frequency--the lowest frequency at each p--increases rapidly with increasing internal pressure . at higher values of internal pressure the frequency spectrum tends to be arranged in the regular manner, the frequency increases with the increasing number of circumferential nodes, and the lowest frequency rises slowly with the internal pressure . experimental results on the frequency spectra, vibration modes, and structural damping of a series of thin-walled cylinders subjected to internal pressure are briefly described . these results show agreement with the features predicted by reissner''s the effect of slight deviation of the cylinder from perfect circular symmetry is discussed .'), (847, 'experimental study of the vibrations of a circular cylindrical shell .', 'gottenberg,w.g.', 'j. acoust. soc. america, 32, 1960, 1002.', 'experimental study of the vibrations of a circular cylindrical shell . an apparatus is described which permits the mode shape of a vibrating circular cylindrical shell to be obtained quite easily . these measurements are made without contacting the cylinder and can be converted to actual lineal values . a representative number of results obtained with such a system are shown to illustrate the relationship between the nodal pattern and frequency in a cylinder as well as the effect of internal pressure on these frequencies . finally, comparisons are made between these results and timoshenko theory and an appropriate shell theory .'), (848, 'the effect of an internal compressible fluid column on the breathing vibrations of a thin pressurised cylindrical shell .', 'berry,j.g. and reissner,e.', 'j. ae. scs. 25, 1958,288.', 'the effect of an internal compressible fluid column on the breathing vibrations of a thin pressurised cylindrical shell . the free oscillations of a thin pressurized cylindrical shell containing a compressible fluid are studied here . the use of an approximate set of shell equations (shallow shell theory) leads to a relatively simple formula for the natural frequencies of the coupled fluid-cylinder system . the results of some computations are presented .'), (849, 'a theory of imperfection for the vibrations of elastic bodies of revolution .', 's. a. tobias, ph.d.', '', 'a theory of imperfection for the vibrations of elastic bodies of revolution . various observations and preliminary experiments have shown that the effect of imperfections upon the vibrations of bodies of revolution cannot be neglected . owing to the possibility of applying the lagrange equation, the influence of the imperfections could be traced through the kinetic energy, the potential energy and the dissipation function . although the fundamental difficulty of the uncertainty of certain variables was not eliminated, this procedure permitted at least the making of general qualitative statements as to the behaviour of the system if imperfections are present .'), (850, 'remarks on donnell''s equations .', 'kempner,j.', 'j. app. mech. 22, 1955.', 'remarks on donnell''s equations . flugge''s set of differential equations of equilibrium for circular cylindrical shells is expressed in a form analogous to the donnell equations . the results of solutions of the two sets of equations for a simply supported cylinder under a centrally applied, uniformly distributed radial line load over a generator segment, as well as under sinusoidally applied line loads, are in very good agreement for the particular geometry investigated .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (851, 'energy expressions and differential equations for stress and displacement analysis of arbitrary cylindrical shells .', 'kempner,j.', 'j. ship res. 1958, 8.', 'energy expressions and differential equations for stress and displacement analysis of arbitrary cylindrical shells . energy expressions and the related equilibrium equations and natural boundary conditions for the determination of the stresses in and displacements of uniform, thin-walled cylinders of arbitrary cross section loaded in an arbitrary manner by surface and edge forces and moments are presented . the derivations are based upon the kirchhoff-love assumptions of the classical theory of shells and are performed to within a degree of accuracy employed by flugge in his derivation of the equilibrium equations applicable to circular cylindrical shells,. hence, in terms of stress resultants, the exact, small-deflection equilibrium equations are obtained . methods of simplification of the relations derived and of solution of the differential equations presented are indicated .'), (852, 'stress and displacement analysis of simply supported non- circular cylindrical shell under lateral pressure .', 'romano,f. and kempner,j.', 'pibal r.415, 1958.', 'stress and displacement analysis of simply supported non- circular cylindrical shell under lateral pressure . this paper presents an analysis of the deflections of and stresses in a short noncircular cylindrical shell of uniform wall thickness whose median-surface cross section is described analytically by a simple expression corresponding to a family of doubly symmetric ovals . the cylinder is under a uniform lateral load and is simply supported at its edges . the small deflection analysis considered is based upon a series solution of appropriate differential equations of shell theory which leads ultimately to infinite sets of algebraic equations, truncated forms of which are considered . numerical values of the significant stresses and displacements for points of the oval cylinder, which are 5 percent of the axial length and 2.5 percent of the circumferential length apart, have been calculated for an oval cross section with a major-minor axis ratio of 1.10 .'), (853, 'the accuracy of donnell''s equations .', 'hoff,n.j.', 'j. app. mech. 22, 1955.', 'the accuracy of donnell''s equations . solutions of donnell''s equations of the small deformations of the perfectly elastic thin-walled circular cylindrical shell are compared with those obtainable from flugge''s equations . the range of the basic parameters is found within which the two solutions are approximately equal .'), (854, 'boundary-value problems of the thin-walled circular cylinder .', 'hoff,n.j.', 'j. app. mech. 21, 1954.', 'boundary-value problems of the thin-walled circular cylinder . the homogeneous differential equations of donnell''s theory of thin cylindrical shells are integrated . expressions are obtained in closed form for the displacements, membrane stresses, moments, and shear forces .'), (855, 'simplified formulas for boundary value problems of the thin-- walled circular cylinder .', 'pohle,f.v. and nardo,s.w.', 'j. app. mech. 22, 1955.', 'simplified formulas for boundary value problems of the thin-- walled circular cylinder . n. j. hoff has presented formulas which can be used in the solution of boundary-value problems of circular cylinders . the purpose of this note is to express these results in exact simplified form,. a more detailed investigation appears elsewhere . the notation will be that of hoff unless otherwise stated .'), (856, 'some experimental studies of panel flutter at mach 1 .3.', 'sylvester,m.a. and baker,j.e.', 'naca tn.3914, 1957.', 'some experimental studies of panel flutter at mach 1 .3. experimental studies of panel flutter were conducted at a mach number of 1.3 to verify the existence of this phenomenon and to study the effects of some structural parameters on the flutter characteristics . thin rectangular metal plates were used in these studies and were mounted as a section of the tunnel wall . most of the data were obtained by using aluminum-alloy panels, although a few steel, magnesium, and brass panels were also used . different materials with various thicknesses and lengths were used to determine the effect of these parameters on panel flutter . the experimental program consisted of three phases .. panels clamped front and rear, and (3) buckled panels clamped on all four edges . panel flutter was obtained under controlled laboratory conditions and it was found that, at the flow conditions of these tests, increasing tensile forces were effective in eliminating flutter, as were shortening the panels or increasing the bending stiffness . no apparent systematic trends in the flutter modes or frequencies could be observed, and it is significant that the panel flutter sometimes involved higher modes and frequencies . the presence of a pressure differential between the two surfaces of a panel was observed to have a stabilizing effect . initially buckled panels were more susceptible to flutter than panels without buckling . buckled panels with all four edges clamped were much less prone to flutter than buckled panels clamped front and rear .'), (857, 'experimental studies of flutter of buckled rectangular panels at mach numbers from 1. 2 to 3. 0 including effects of pressure differential and of panel width-length ratio .', 'sylvester,m.a.', 'nasa tn.d833, 1961.', 'experimental studies of flutter of buckled rectangular panels at mach numbers from 1. 2 to 3. 0 including effects of pressure differential and of panel width-length ratio . experimental panel flutter data have been obtained at mach numbers from 1.2 to 3.0 for buckled rectangular panels and the effect of a pressure differential has been determined . increasing the pressure differential was effective in eliminating flutter on most of the panels tested . the effects of the variables in the panel flutter parameter sure, e is young''s modulus, and t and l are the panel thickness and length, respectively) were investigated for buckled panels clamped on the front and rear edges and a critical value of this parameter of 0.44 is indicated at zero pressure differential when the panel width-length ratio is 0.69 . an estimated flutter boundary is presented for buckled panels clamped on four edges, with width-length ratios of 0.21 to 4.0 . this boundary shows that the panel width is more significant than the panel length when the ratio of width to length is less than approximately 0.5 . panels clamped on four edges and buckled in two half waves in the direction of flow were found to be particularly susceptible to flutter . the results of limited tests on panels with applied damping, curvature, and lengthwise stiffeners are also presented and discussed .'), (858, 'experimental investigation at mach numbers 3. 0 of the effects of thermal stress and buckling on the flutter of four-bay aluminium alloy panels with length-width ratios of 10 .', 'dixon,s.c., griffith,g.e. and bohon,h.k.', 'nasa tn.d921, 1961.', 'experimental investigation at mach numbers 3. 0 of the effects of thermal stress and buckling on the flutter of four-bay aluminium alloy panels with length-width ratios of 10 . skin-stiffener aluminum alloy panels consisting of four bays, each bay having a length-width ratio of 10, were tested at a mach number of 3.0 at dynamic pressures ranging from 1,500 psf to 5,000 psf and at stagnation temperatures from 300 f to 655 f . the panels were restrained by the supporting structure in such a manner that partial thermal expansion of the skins could occur in both the longitudinal and lateral directions . a boundary faired through the experimental flutter points consisted of a flat-panel portion, a buckled-panel portion, and a transition point at the intersection of the two boundaries . in the region where a panel must be flat when flutter occurs, an increase in panel skin temperature (or midplane compressive stress) makes the panel more susceptible to flutter . in the region where a panel must be buckled when flutter occurs, the flutter trend is reversed . this reversal in trend is attributed to the panel postbuckling behavior .'), (859, 'flutter of aerodynamically heated aluminium-alloy and stainless steel panels with length-width ratio of 10 at mach 3. 0.', 'guy,l.d. and bohon,h.l.', 'nasa tn.d1353, 1962.', 'flutter of aerodynamically heated aluminium-alloy and stainless steel panels with length-width ratio of 10 at mach 3. 0. an investigation of the effects of aerodynamic heating on the flutter of multibay external-skin panels has been carried out at a mach number of 3.0 in the langley 9- by 6-foot thermal structures tunnel . both aluminum-alloy and 17-7 ph stainless-steel panels with a length-width ratio of 10 for each bay were tested at dynamic pressures between addition, a few tests were made on the lower vertical stabilizer of the x-15 airplane which has external-skin panels unsupported for a length all panels showed flutter boundaries characterized by an increase in panel thickness required to prevent flutter with increasing thermally induced stress prior to buckling . after buckling the panels showed flutter boundaries characterized by a decrease in thickness required to prevent flutter with further increases in thermal stress . the largest thickness required to prevent flutter in the presence of aerodynamic heating occurred at the transition between the flat-panel boundary and the buckled-panel boundary . this peak value (for aluminum-alloy panel) was as much as 60 percent greater than the extrapolated value for an unheated, unloaded panel . values of the modified-thickness-ratio flutter parameter for the unstressed panels (obtained by extrapolation) were in fair agreement for the aluminum, steel, and x-15 stabilizer panels . peak values at transition, however, showed large differences due to apparently minor changes in panel-support construction and or changes in panel-skin material .'), (860, 'test of an aerodynamically heated multi-- web wing structure (mw-1) in a free jet at mach number 2.', 'heldenfels,r.s., rosecrans,r. and grifith,g.e.', 'naca rm l53e27, 1953.', 'test of an aerodynamically heated multi-- web wing structure (mw-1) in a free jet at mach number 2. a multiweb wing structure, representing an airplane or missile wing, was tested under simulated supersonic flight conditions to determine the transient temperature distribution . the aerodynamic loads played an important and unanticipated role, however, in that the model experienced a dynamic failure near the end of the test . the test is discussed and the conclusion reached that the model failed as a result of the combined action of aerodynamic heating and loading . the temperature data collected are analyzed and are shown to be in reasonable agreement with calculated values .'), (861, 'charts adapted from van driest''s turbulent flat-plate theory for determining values of turbulent aerodynamic friction and heat transfer coefficients .', 'lee,d.b. and faget,m.a.', 'naca tn.3811, 1956.', 'charts adapted from van driest''s turbulent flat-plate theory for determining values of turbulent aerodynamic friction and heat transfer coefficients . a modified method of van driest''s flat-plate theory for turbulent boundary layer has been found to simplify the calculation of local skin-friction coefficients which, in turn, have made it possible to obtain through reynolds analogy theoretical turbulent heat-transfer coefficients in the form of stanton number . a general formula is given and charts are presented from which the modified method can be solved for mach numbers 1.0 to 12.0, temperature ratios 0.2 to 6.0, and reynolds numbers 0.2 x 10 to 200 x 10 .'), (862, 'the phenomenon of change in buckle pattern in elastic structures .', 'stein,m.', 'nasa tr r39, 1959.', 'the phenomenon of change in buckle pattern in elastic structures . a model is analyzed which exhibits the important properties associated with change in buckle pattern of plates . the analysis includes a rigorous study of stability in its various modes . a discussion of how the present results may be applied to plates and other elastic structures is given .'), (863, 'loads and deformations of buckled rectangular plates .', 'stein,m.', 'nasa tr r40, 1959.', 'loads and deformations of buckled rectangular plates . the nonlinear large deflection equations of von karman for plates are converted into a set of linear equations by expanding the displacements into a power series in terms of an arbitrary parameter . the post-buckling behavior of simply supported rectangular plates subjected to longitudinal compression and to a uniform temperature rise is investigated in detail by solving the first few of the equations . experimental data are presented for the compression problem . comparisons are made for total shortening and local strains and deflections which indicate good agreement between experimental and theoretical results .'), (864, 'status of flutter of flat and curved panels .', 'national advisory committee for aeronautics', '', 'status of flutter of flat and curved panels . representative results are presented to show the current status of the panel flutter problem . the discussion includes flat panels with and without midplane stresses, buckled panels, and both unstiffened and stiffened infinitely long circular cylinders .'), (865, 'a study of the thermal fatigue behaviour of metals . the effect of test conditions on nickel-base high temperature alloys .', 'glenny,e. and taylor,t.a.', 'j.inst.metals, 88, 1960, 449.', 'a study of the thermal fatigue behaviour of metals . the effect of test conditions on nickel-base high temperature alloys . an attempt has been made to identify the significant factors governing the thermal-fatigue behaviour of nickel-base high-temperature alloys, mainly by using a laboratory technique with hot and cold fluidized beds as the heating and cooling media . a succession of heating shocks is generally more damaging than a succession of cooling shocks between the same temperature limits . the duration of the heating shock and the upper temperature of the cycle are dominant factors . the thermal-fatigue cracks are initiated at the surface and are intercrystalline in origin and propagation . surface oxidation, which is intergranular in nature for nickel-base alloys, has a significant effect on thermal-fatigue life .'), (866, 'regularities in creep and hot fatigue data .', 'graham,a. and walles,k.f.a.', 'arc cp.379, 1958.', 'regularities in creep and hot fatigue data . published experimental results are assembled to support a previously-given theory of uniaxial deformation, and the theory is then used to analyse published data on the creep-rupture and hot-fatigue of engineering materials . the theory enables data for different times and temperatures to be classed together, thereby providing information over a much greater range of times than could practicably be covered by experiments at a single temperature . an underlying numerical pattern common to all the widely different group 8 materials considered then shows through the experimental scatter . data for further engineering materials is considered in these terms in part 2 .'), (867, 'low frequency fatigue of nimonic 90 . low frequency fatigue - a rheological approach .', 'landau,c.s.', 'engineer, 213, 1962, 911.', 'low frequency fatigue of nimonic 90 . low frequency fatigue - a rheological approach . an alloy of nimonic 90 type has been tested under cyclic loads at temperatures of 800 deg., 850 deg. and 900 deg. cent . graham''s descriptive theory of deformation has been developed in detail for the tests reported here, and shown to provide a satisfactory description of the experimental results . the variation of cumulative strain with number of cycles can be described by the sum of powers of cycle number, n, n, n, and n . there is qualitative agreement between the observed and the calculated stress-strain loops . the effect of variation of maximum stress per cycle can be described by the sum of power terms with simple exponents . within the scatter of observation, the total time to fracture is independent of the frequency .'), (868, 'design and operation of the n. g. t. e. thermal shock analogue .', 'stanworth,c.g. and paine,d.s.c.', 'arc cp.557, 1960.', 'design and operation of the n. g. t. e. thermal shock analogue . a description is given of the n.g.t.e. thermal shock analogue which is suitable for estimating the temperature in a turbine blade section as a function of position and time when the blade is subjected to a step change in gas temperature . the method of operating the analogue and obtaining results has also been described . the limitations of the analogue have been stated, but they are considered a small penalty in view of the essential simplicity of the design .'), (869, 'the calculation of transient temperature in turbine blades and tapered discs using biot''s variational method .', 'howe,p.w.h.', 'arc cp.617, 1961.', 'the calculation of transient temperature in turbine blades and tapered discs using biot''s variational method . transient temperatures in aerofoil sections and tapered discs are calculated taking advantage of simplifications in heat flow analysis achieved in biot''s variational method . cross-sections are represented by a line of adjacent squares of various sizes suitable for the local dimensions, e.g. small squares near the leading and trailing edges . the potential, dissipation and surface dissipation functions of biot''s method are set up, and the lagrange equations lead, by automatic procedures, to an eigenvalue formulation in matrix form for the temperatures and their first time derivatives . solutions are sums of exponentials in time, and are evaluated by digital computer, requiring about five minutes for each cross-section and heat transfer coefficient . transient temperatures in a particular aerofoil section for variation of heat transfer coefficient and for external temperature depending exponentially on time agree with results obtained on an analogue computer . maximum transient temperature differences are evaluated for tapered discs by a simple electrical analogue) with variation of edge radius and heat transfer coefficient . peculiarities in the solution for cyclic temperature external to an aerofoil over a range of frequencies indicate limitations in the mathematical formulation . a successful solution for cyclic external temperature might enable eigenvalues to be separated out in experimental measurements using electronic equipment, and this might be extended to exponential external temperature if a relationship between cyclic and exponential external temperature could be established . eigenvalues and eigenvectors as discrete values arise fictitiously from the sub-division into squares and the possibility of an integral formulation is mentioned . there is a possible, but not immediate, extension to cooled blades, whose cross-sections are multiply-connected regions . transient stresses due to creep, and viscoelasticity might be included .'), (870, 'effect of rheological behaviour on thermal stresses .', 'freudenthal,a.m.', 'j.app.phys. 25, 1954.', 'effect of rheological behaviour on thermal stresses . since the conventional elastic analysis of thermal stress problems coupled with limiting creep rates and time-dependent fracture stresses as (inelastic) design criteria, results in design procedures for thermal stresses (in heat exchangers, nuclear reactors, flight structures at supersonic speeds, etc.) of considerable unreality, the effect of various types of rheological behavior (viscoelastic, plastic, work hardening) on the level of thermal stresses is analyzed under simplified assumptions, such as uniaxial stress and polar or cylindrical symmetry . the effect on the thermal stress intensity of the rheological behavior of the material is shown to be very significant, particularly with respect to stress relaxation and the development of residual stresses .'), (871, 'steady-state creep through dislocation climb .', 'weertman,j.', 'j.app. phys. 28, 1957, 362.', 'steady-state creep through dislocation climb . a dislocation climb creep model is considered which does not require the production of immobile dislocations . the creep equation that results from the analysis is where a and b are constants, is the stress, q is the activation energy of creep and kt has its usual meaning . this equation is quite similar to one previously proposed .'), (872, 'fundamentals of boundary layer heat transfer with streamwise temperature variations .', 'biot,m.a.', 'j. ae. scs. 1962, 558.', 'fundamentals of boundary layer heat transfer with streamwise temperature variations . boundary-layer heat transfer is analyzed for the case of a sinusoidal distribution of temperature in the direction of flow . it is shown that for both laminar and turbulent flow the spatial distribution of heat transfer is generally out of phase with the wall temperature by an angle of 30 to 45 . this leads to the conclusion that in some areas the heat flow is opposite to the temperature difference as used in the definition of the heat-transfer coefficient, and points to the basic shortcomings of this concept . the physical explanation for this behavior is found to be the temperature-field distortion by the fluid motion . the distortion is measured by the peclet number . approximate equations representing a /conduction analogy/ were used in this analysis and the validity of these equations for unsteady flow is examined with reference to limitations in frequency and wavelength . a solution of these equations is given for the case of a velocity profile which is not a straight line . the use of previously developed variational principles for the evaluation of convective heat transfer including cases of three-dimensional unsteady flow, turbulence, and nonparallel streamlines is also discussed .'), (873, 'lagrangian thermodynamics of heat transfer in systems including fluid motion .', 'biot,m.a.', 'j. ae. scs. 29, 1962, 568.', 'lagrangian thermodynamics of heat transfer in systems including fluid motion . the lagrangian thermodynamic equations of irreversible processes are extended to convective heat transfer . this generalization provides equations for the unified analysis of transient heat flow in complex systems comprising solid structures and moving fluids in either laminar or turbulent flow . the concept of a surface-heat-transfer coefficient is eliminated from the formulation . the theory is developed along two different lines . in one approach a new concept referred to as the /trailing function/ is introduced . it represents the surface-heat-transfer properties and may be evaluated by quite simple but remarkably accurate variational procedures . the method of /associated fields/ is also generalized to convective phenomena . the second line of approach extends to convective heat transfer the thermodynamic concept of entropy production for both laminar and turbulent flow . the theory amounts to an extension of the thermodynamics of irreversible processes to systems for which onsager''s relations are not valid .'), (874, 'the use of models for the determination of critical flutter speeds .', 'duncan, w.j.', 'r + m 1425, july 1931 .', 'the use of models for the determination of critical flutter speeds . the use of model tests in the prediction of full-scale critical flutter speeds is now well established, and the technique of such tests is therefore worthy of discussion . in order to obtain critical speeds for the model within the speed range of ordinary wind tunnels it is necessary that the model should differ in some respect from a mere small suggested by mckinnon wood the modification of the model consists in a reduction of its effective stiffnesses . this method has the defect /in most cases probably not serious/ that the model experiment is conducted at a reynolds number much below that for full-scale . in the present paper it is pointed out that an alternative method of reducing the critical speed is to increase the mass loading of the model and to make the flutter tests in compressed air . * it is then quite feasible to reach the full-scale reynolds number . this method of reducing the critical speeds by a proportionate increase of all effective densities may also be combined with a reduction of the elasticity of the model . the relation of model and full-scale stresses at the critical flutter speeds is considered . where the reduction in critical speed is effected by increase of density only, the model and full-scale stresses are equal . in a model of reduced elasticity the stresses in the wires are the same as for full-scale, whereas, the stresses in the spars are less than for full-scale . this is in accord with the usual experience that the wires of such a model are the first parts of the structure to fail in a flutter . lastly, the influence of gravity on flutter is considered . this is negligibly small for full-scale, but not necessarily so for the model . gravitational effects can sometimes be corrected by suitable orientation of the model .'), (875, 'models for aeroelastic investigation .', 'templeton,h.', 'arc cp.255, 1955.', 'models for aeroelastic investigation . this addendum provides a short note on two aspects omitted from the original paper, viz. gravitational effects and structural damping . a short list of references to earlier papers dealing with the subject is also added .'), (876, 'on flutter testing in high speed wind tunnels .', 'lambourne,n.c. and scruton,c.', 'arc r + m.3054, 1956.', 'on flutter testing in high speed wind tunnels . the requirements for simulating in a wind tunnel flutter conditions appropriate to high-speed flight are discussed, and an assessment is made of the desirable features of a wind tunnel suitable for flutter testing at transonic and supersonic speeds . it is concluded that such a tunnel should have either the mach number or the stagnation pressure variable during the tunnel run, and that it is of considerable advantage, and for some purposes essential, for high stagnation pressures to be available . the stagnation pressure required to allow flight conditions to be simulated with a flutter model is considered to range from at least 2 atmospheres for transonic speeds to about 15 atmospheres for m = 4 . no attempt to simulate kinetic heating is envisaged, although its effect on stiffness should be allowed for in the design of the model . to minimise uncertainties due to the variation of the model stiffness with temperature it is desirable that means for controlling the stagnation temperature should be incorporated in the tunnel .'), (877, 'the influence of aerodynamic heating on the flexural rigidity of a thin wing .', 'mansfield,e.h.', 'arc r + m.3115, 1957.', 'the influence of aerodynamic heating on the flexural rigidity of a thin wing . this report considers the loss of flexural rigidity of a thin wing due to the presence of middle-surface stresses resulting from aerodynamic heating . the spanwise properties of the wing are assumed constant but the wing section is arbitrary . the loss of flexural rigidity is comparable with the corresponding loss of torsional rigidity .'), (878, 'experimental model techniques and equipment for flutter investigations .', 'molyneux,w.g.', 'air. eng. 1958.', 'experimental model techniques and equipment for flutter investigations . an outline is given of the uses of flutter models as an aid to the designer in the avoidance of flutter . details are given of the different types and methods of construction that are used for flutter models and of the various test facilities that are available for high speed and low speed tests . the procedure followed in the u.k. for flutter clearance of the full scale aircraft is described, and the value of the electronic flutter simulator in this field is discussed .'), (879, 'flutter model testing at transonic speeds .', 'targoff, w.p. and white, r.p.', 'inst. aero. sc. perp. 706, may 1957 .', 'flutter model testing at transonic speeds . flutter research on reflection plane models of straight, swept, and delta wings in a 3 x 4 foot transonic test facility . techniques of model construction and testing developed .'), (880, 'the design and testing of supersonic flutter models .', 'mccarthy, j.f. and halfman, r.l.', 'j. aero. sc. june, 1956 .', 'the design and testing of supersonic flutter models . the basic problems of flutter testing in the low supersonic speed range simulate full-scale airplanes when mach number is included as a parameter are reviewed and are compared with those where velocity is scaled when the leading edges become transonic,. these are compared with results given elsewhere and serve as a check on the results of tables i and ii .'), (881, 'cumulative damage in fatigue .', 'miner,m.a.', 'j. app. mech. 12, 1945, a159.', 'cumulative damage in fatigue . the phenomenon of cumulative damage under repeated loads was assumed to be related to the net work absorbed by a specimen . the number of loading cycles applied expressed as a percentage of the number to failure at a given stress level would be the proportion of useful life expended . when the total damage, as defined by this concept, reached 100 per cent, the fatigue specimen should fail . experimental verification of this concept for an aluminum alloy, using different types of specimens, various stress ratios, and various combinations of loading cycles is presented . these data are also analyzed to provide information on different stress ratios when an s-n curve for any one ratio is known . results of a sample analysis based on experiments are given . it is concluded that a simple and conservative analysis is possible using the concept of cumulative fatigue damage .'), (882, 'the variation of gust frequency with gust velocity and altitude .', 'bullen,n.i.', 'arc cp.324, 1956.', 'the variation of gust frequency with gust velocity and altitude . information on atmospheric turbulence obtained from counting accelerometer records is examined and relations giving the variation of gust frequency with gust velocity and altitude are obtained . the results are summarized in a form convenient for use in estimating the fatigue life of an aircraft .'), (883, 'correlated fatigue data for aircraft structural joints .', 'heywood,r.b.', 'arc cp.227, 1955.', 'correlated fatigue data for aircraft structural joints . results of fatigue tests carried out at r.a.e. on typical aircraft wing structural joints are correlated to give an indication of general fatigue behaviour . the results are plotted in the form of s - log n curves, and these indicate that the mode of behaviour cannot be attributed to any single factor, such as the type of aluminium alloy, the ultimate tensile strength, or the mean stress of the fatigue cycle . the detailed method of design undoubtedly has a predominant influence on behaviour, but this quality is not revealed by a broad classification according to the proportion of load transmitted at holes .'), (884, 'the estimation of fatigue damage on structural elements .', 'chilver,a.h.', 'rae tn.struct.125, 1954.', 'the estimation of fatigue damage on structural elements . a method is presented for the estimation of fatigue damage to aircraft structural elements . the gust spectrum to which the aircraft is subjected is analysed in terms of infinitesimal loading intervals . gust data supplied by j. taylor for flying below 15,000 ft are used to study fatigue damage in a number of aircraft structural joints and one whole structure .'), (885, 'buckling of thin cylindrical shells under hoop stresses varying in axial direction .', 'hoff,n.j.', 'j. app. mech. 24, 1957, 405.', 'buckling of thin cylindrical shells under hoop stresses varying in axial direction . the buckling of a thin cylindrical shell simply supported along the perimeter of its end sections is analyzed under hoop compressive stresses varying in the axial direction . the thermal stresses arising from a uniform increase in the temperature of the cylinder are determined . it is found that such thermal stresses are not likely to cause elastic buckling . simple approximate formulas are developed for buckling stress and thermal stress .'), (886, 'thermal buckling of clamped cylindrical shells .', 'zuk,w.', 'j. ae. scs. 24, 1957, 359.', 'thermal buckling of clamped cylindrical shells . the problem of thermal buckling of shells arises in connection with air-frame bodies subject to aerodynamic heating at supersonic speeds . the case of the shell with clamped edges is presented, as this case typifies all structures with a tubular shell stiffened at intervals with stiffening rings . the shell is assumed to be unrestrained longitudinally and fully restrained laterally at the edges .'), (887, 'buckling due to thermal stress of cylindrical shells subjected to axial temperature distributions .', 'johns,d.j.', 'coa r.147, 1961.', 'buckling due to thermal stress of cylindrical shells subjected to axial temperature distributions . thermal stress distributions in uniform circular cylindrical shells due to axial temperature distributions are investigated . the discontinuity effect due to the presence of a cooler stiffening bulkhead is considered, and the possibility of thermal buckling of the shell due to the circumferential discontinuity stress is examined . the buckling analysis is based on donnell''s shell equation, and particular attention is given to shells having clamped edges . an experimental investigation of this buckling problem is discussed, and the results obtained are seen to agree reasonably well with theory .'), (888, 'combinations of temperature and axial compression required for buckling of a ring-stiffened cylinder .', 'anderson,m.s.', 'nasa tn.d1224, 1962.', 'combinations of temperature and axial compression required for buckling of a ring-stiffened cylinder . a theory is presented to predict the buckling temperature of an axially compressed, uniformly heated ring-stiffened cylinder . the cylinder buckles because of the interaction of the axial stress due to applied compressive loads and the circumferential stress resulting from restraint of thermal expansion by the rings . buckling charts covering a wide range of cylinder proportions are presented for both clamped and simply supported cylinders . the buckling temperature for a given axial loading is determined from a simple equation involving a coefficient given in the buckling charts and the radius-thickness ratio of the cylinder .'), (889, 'a simplified method of elastic stability analysis for thin cylindrical shells .', 'batdorf,s.b.', 'naca r.874, 1947.', 'a simplified method of elastic stability analysis for thin cylindrical shells . this paper develops a new method for determining the buckling stresses of cylindrical shells under various loading conditions . for convenience of exposition, it is divided into two parts . in part 1, the equation for the equilibrium of cylindrical shells introduced by donnell in naca report no. 479 to find the critical stresses of cylinders in torsion is applied to find critical stresses for cylinders with simply supported edges under other loading conditions . it is shown that by this method solutions may be obtained very easily and the results in each case may be expressed in terms of two nondimensional parameters, one dependent on the critical stress and the other essentially determined by the geometry of the cylinder . the influence of boundary conditions related to edge displacements in the shell median surface is discussed . the accuracy of the solutions found is established by comparing them with previous theoretical solutions and with test results . the solutions to a number of problems concerned with buckling of cylinders with simply supported edges on the basis of a unified viewpoint are presented in a convenient form for practical use . in part 2, a modified form of donnell''s equation for the equilibrium of thin cylindrical shells is derived which is equivalent to donnell''s equation but has certain advantages in physical interpretation and in ease of solution, particularly in the case of shells having clamped edges . the solution of this modified equation by means of trigonometric series and its application to a number of problems concerned with the shear buckling stresses of cylindrical shells are discussed . the question of implicit boundary conditions also is considered .'), (890, 'comments on ''thermal buckling of clamped cylindrical shells'' .', 'david j. johns', 'lecturer, college of aeronautics, cranfield, england', 'comments on ''thermal buckling of clamped cylindrical shells'' . in the recent paper by zuk, an expression was presented for the critical buckling temperature of a clamped cylindrical shell in terms of the material and geometrical properties of the shell . restraint at the edges of the shell was assumed to be provided by rigid frames experiencing no temperature rise . the circumferential stress induced in the shell when it experienced a temperature rise, t, may be approximated by the function . in other words, there is a compressive circumferential stress along the entire length, l, of the shell . it is well known, however, that the discontinuity stresses introduced at the junction of a shell and a rigid frame (or bulk-head) are extremely localized, and the circumferential stresses induced in the shell decrease rapidly away from the joint .'), (891, 'buckling of a finite length cylindrical shell under a circumferential band of pressure .', 'almroth,b.o. and bruch,d.o.', 'j. ae. scs. 28, 1961, 573.', 'buckling of a finite length cylindrical shell under a circumferential band of pressure . this paper is concerned with buckling of a circular cylinder of finite length subjected to a symmetrical band of external pressure . both experimental and theoretical results are presented . the experimental data were obtained from tests of three thin-walled steel cylinders subjected to external pressure by a pneumatic tube encircling the test cylinder at mid-length . the theory is based on the principle of minimum potential energy, and the rayleigh-ritz procedure is used to expand the displacement components in trigonometric series . theoretical results are given in the form of graphs which show buckling pressure as a function of the following ratios .. cylinder radius thickness cylinder length radius pressure bandwidth cylinder length theoretical results are in close agreement with existing solutions to special cases in which (1) the pressure is applied over the entire lateral surface, and (2) the pressure is concentrated along a circumferential line . the theoretical results are also in agreement with the test results .'), (892, 'research on unsteady flow .', 'jones,w.p.', 'j. ae. scs. 29, 1962, 249.', 'research on unsteady flow . this is a survey of certain recent advances made in the study of aerodynamic unsteady flow and of some of the new problems arising which require further investigation . no attempt is made to reproduce classical theory, but emphasis rather is laid on validity and general usefulness, particular attention being given to unsteady boundary-layer effects, especially when there is flow separation . coverage is broad and author thus provides a useful review for those interested in this field .'), (893, 'a new design of pitot-static tube with a discussion of pitot-static tubes and their calibration factors .', 'salter,c.', 'npl. aero. r.1013, 1962.', 'a new design of pitot-static tube with a discussion of pitot-static tubes and their calibration factors . the report describes experiments devised to investigate some of the previously unexplained peculiarities of normal types of pitot-static tube . in the process use was made of what was as nearly as could be a standard static pressure tube . the experiments led to a new alternative design of instrument having a nose of modified ellipsoidal shape and for which the main characteristics have been investigated . it has been found to be necessary as well as desirable to include a discussion of the implications of the term /calibration factor/ and (although not in this respect comprehensive) of the special features and limitations of various pitot-static tubes .'), (894, 'flutter of a two dimensional simply supported buckled panel with elastic restraint against edge displacement .', 'smith,g.e.', 'rae r. struct.269, 1961.', 'flutter of a two dimensional simply supported buckled panel with elastic restraint against edge displacement . the critical flutter speed is evaluated for a two-dimensional thin buckled panel with one surface exposed to a supersonic airstream and the other to still air at the same static pressure . the panel is simply supported along the leading and trailing edges by rigid edge members separated by an elastic member represented by a compression spring . the whole system is acted upon by a constant compressive force uniformly distributed along the edge members . the aerodynamic forces acting on the deflected panel are found from two-dimensional /quasi-steady/ theory, valid for slow oscillations where the downwash velocity is small compared with the speed of flow and provided that the mach number is sufficiently greater than . the elastic behaviour of the panel is given by von karman''s large deflection equations modified to cover initially curved plates . the solution of the equations is carried out by means of galerkin''s method, which has been shown to give valid results for a panel with a non-zero bending rigidity . the influence of the midplane compressive force carried by the panel itself, the initial buckle amplitude and the elastic restraint against edge displacements is investigated, and curves are presented giving the critical dynamic pressure ratio as a function of these variables .'), (895, 'the airforces on the low aspect ratio rectangular wing oscillating in sonic flow .', 'davies,d.e.', 'rae tn. struct.311, 1962.', 'the airforces on the low aspect ratio rectangular wing oscillating in sonic flow . approximate expressions for the generalised airforces acting on a rectangular wing of low aspect ratio oscillating harmonically in sonic flow at low frequencies are derived in this paper . the modes of oscillation considered are rigid modes and a small selection of flexible modes . results are presented as the first few terms of infinite expansions . a brief description of the modes of oscillation and of the generalised airforces is given towards the end of the paper so that the results may be used without the main text of the paper having to be read .'), (896, 'the calculation of loads on a supersonic weapon in the steady circling case .', 'seal,d.m.', 'rae tn.struct.275, 1960.', 'the calculation of loads on a supersonic weapon in the steady circling case . the economy of a design depends on the accuracy with which it is possible to solve the various structural problems as this has a direct bearing on structure weight . this paper describes the calculation of the bending moments on a specific weapon for the high-g steady circling case . a hybrid method is used to obtain the aerodynamic loads . the results presented show the effect of a number of parameters, such as - altitude, weight, acceleration and mach number - on the magnitude of the maximum bending moment .'), (897, 'some results on buckling and postbuckling of cylindrical shells .', 'kempner,j.', 'nasa tn.d1510, 1962, 173.', 'some results on buckling and postbuckling of cylindrical shells . in this summary paper, the effects of initial deformations on the buckling and postbuckling characteristics of circular cylindrical shells under hydrostatic pressure is determined in an approximate manner . the influence of initial axisymmetric deformations is stressed . also, the classical buckling of an axially compressed, noncircular the results show that the major-minor axis ratio of the cross section has a marked effect on the critical load, and that use of the maximum radius of curvature in the formula for the classical buckling stress of a circular cylindrical shell leads to good results for moderate eccentricities .'), (898, 'a survey of buckling theory and experiment for circular conical shells of constant thickness .', 'seide,p.', 'nasa tn.d1510, 1962, 401.', 'a survey of buckling theory and experiment for circular conical shells of constant thickness . a survey of the state-of-the-art for the stability of thin-walled conical shells is presented . known theoretical results are summarized and compared with experiment . the shortcomings of present knowledge and recommended work for the future are discussed .'), (899, 'aerodynamic effects on boundary layer unsteadiness .', 'moore,f.k.', '6th a.a.aero.conf. 1957.', 'aerodynamic effects on boundary layer unsteadiness . with a view to the study of aerodynamic problems, a review is made of boundary layer theory for a flat plate moving with a time-dependent velocity . unsteady effects are shown to enter according to the magnitude of the ratio of time for diffusion to act throughout the boundary layer to the characteristic time of the imposed unsteadiness . it is concluded that a boundary layer may be considered quasi-steady even during extreme flight manocuvres . generation of acoustic noise purely by boundary layer unsteadiness is generally small . thermal and heat-transfer effects are cited . unsteady boundary layer considerations are important in damping or amplifying certain instabilities, such as flutter of panels and stalling flutter of aerofoils . in connection with the aerofoil problem, laminar separation concepts and the stagnation-point boundary layer are described for unsteady flow . an analysis of aerofoil lift hysteresis is described, using unsteady laminar boundary layer considerations, which leads to a prediction of counter-clockwise hysteresis at maximum lift .'), (900, 'some measurements in the vortex flow generated by a sharp leading edge having 65 sweep .', 'lambourne,n.c. and bryer,d.w.', 'arc cp.477, 1959.', 'some measurements in the vortex flow generated by a sharp leading edge having 65 sweep . the report is concerned with the vortex flow which arises when separation occurs at a highly swept leading edge . measurements were made in the flow over flat plates at 15 incidence each having a sharp leading edge of 65 sweep . the pressure and velocity distributions both along the axis of the vortex and for one cross section of the flow are presented together with a preliminary discussion of their significance .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (901, 'long slender delta wings with leading edge separation .', 'brown,c.e. and michael,w.h.', 'naca tn.3430.', 'long slender delta wings with leading edge separation . the slender-body approximation of linearized compressible flow is applied to the problem of a delta wing in which flow separation occurs at the leading edges . the vortex sheets found in the real flow are approximated by concentrated vortices with feeding lattices, and a plausible adaptation of kelvin''s theorem is applied to simulate the force-free nature of the vortex sheet . the computations show that leading-edge separation produces an increase in lift over that given by the jones slender-wing theory and that the lift does not vary linearly with angle of attack . computed pressure distributions and span loadings are presented and the theoretical lift results are compared with the results of simple force tests made at a mach number of 1.9 .'), (902, 'some current and proposed investigations into the flow for slender delta and other wings in unsteady motion .', 'lambourne,n.c.', 'arc 21,844, 1960.', 'some current and proposed investigations into the flow for slender delta and other wings in unsteady motion . the practical need for research into the aerodynamics of slender delta wings in unsteady motion has been emphasized in a recent paper by zbrozek . two important aspects are .. - formation and presence of leading-edge vortices . with oscillatory or transient modes of longitudinal (or chordwise) bending . the first of the aspects above, has already been briefly discussed in ref. 2 . one feature of the flow with leading-edge vortices which seems to be of particular significance to the dynamic behaviour of a wing is the shedding of vorticity at the leading edge as well as at the trailing edge . any time-dependent motion, or distortion, of the wing leads to a change in the rate at which vorticity is shed . with more conventional types of flow, the free vorticity being shed only from the trailing edge has diminishing influence on the wing, but when the free vorticity is shed from the leading edge, in passing downstream, it remains close to the upper surface of the wing . it might be expected then, that, although the magnitudes of the unsteady forces may not be greatly affected for a slender delta, the time delays associated with the forces may be significantly different for the attached and separated regimes of leading-edge flow .'), (903, 'two dimensional transonic unsteady flow with shock waves .', 'eckhaus,w.', 'm.i.t. fluid dynamics res. group r.59-3, 1959.', 'two dimensional transonic unsteady flow with shock waves . a study is made of the unsteady flow around an airfoil at transonic mach numbers, the situation being such that local supersonic regions terminated by shock-waves are present in the vicinity of the airfoil . for the unsteady part of the flow, small perturbations technique is employed and the interaction with the shock wave is taken into account . the case of an oscillating aileron is considered first, and a solution is derived for the pressure distribution on the aileron . it is found that the solution has a simple form when the shock-wave is well ahead of the hinge axis of the aileron . as the shock approaches the hinge-axis a correction must be added to the solution . an interpretation of these results is given . the results are compared with results of a theory which neglects the presence of the shock and it is found that both agree for m = 1 . for m 1, however, neglecting the presence of the shock waves introduces errors of the order of magnitude (1 - m), where m is the local mach number behind the shock . the theory is finally extended to include the case in which the whole airfoil oscillates, but only the solution for the subsonic region behind the shock is treated . the role of the unsteady shock-boundary layer interaction is discussed and it is shown that this mechanism can be included in the results of the present theory .'), (904, 'calibration of the standard pitot-static head used in the rae low speed wind tunnels .', 'kettle,d.j.', 'rae tech.memo 222, 1951.', 'calibration of the standard pitot-static head used in the rae low speed wind tunnels . recent results of tests in the r.a.e. wind tunnels concerned with the measurement of pressure distributions have shown slight discrepancies between the readings of various static pressure tubes and calculated pressure distributions . as a consequence some doubt was felt concerning the calibrations of tunnel static pressure and upon the validity of the reading given by the standard pitot-static head . it was therefore decided to check the standard pitot-static head used in the r.a.e. wind tunnels, against an instrument similar to the measurements of static pressure were also made using a long tube where the interference from head and support is calculated to be small . this note gives the results of tests made in the 5 ft open jet wind tunnel and the no. 1 11 ft wind tunnel in order to determine the necessary correction to the reading of static pressure given by the r.a.e. pitot-static head . the tests were made during september and october, 1951 .'), (905, 'comparative tests of pitot-static tubes .', 'merriam,k.g. and spaulding,e.r.', 'naca tn.546.', 'comparative tests of pitot-static tubes . comparative tests were made on seven conventional pitot-static tubes to determine their static, dynamic, and resultant errors . the effect of varying the dynamic opening, static openings, wall thickness, and inner-tube diameter was investigated . pressure-distribution measurements showing stem and tip effects were also made . a tentative design for a standard pitot-static tube for use in measuring air velocity is submitted . this report covers an investigation conducted under the auspices of the national research council .'), (906, 'review of the pitot tube .', 'folson,r.g.', 'asme trans. 70, 1956.', 'review of the pitot tube . this paper is an attempt to bring together the important information regarding pitot tubes and their use,. to summarize the available data on the application of various types of impact and velocity probes for the guidance of engineers and research workers,. and to aid them in the design of flow instruments for specific applications .'), (907, 'cavitation and pressure distribution head forms at zero angle of yaw .', 'hunter rouse and john s. mcnown', 'iowa institute of hydraulic research, state university of iowa iowa city', 'cavitation and pressure distribution head forms at zero angle of yaw . early in the fall of 1943 the iowa institute of hydraulic research undertook the design and fabrication of a variable-pressure water tunnel . as the tunnel neared completion, however, its immediate use for the study of the pressure distribution around various body forms was requested . the original request for this investigation was a natural out-growth of the need for systematic data on the distribution of pressure in flow around various bodies, particularly under conditions leading to cavitation, information which is desirable for the design of a wide variety of navy equipment . ultimately the study is to include data for two- and three-dimensional head and tail forms at various angles of yaw . the first phase of the study, namely the investigation of three-dimensional head forms at zero angle of yaw, is described herein . three general geometric series have been studied.dash rounded, ellipsoidal, and conical.dash together with other related forms . the data obtained have been systematized to yield information for a wide variety of geometrical forms either directly or by interpolation . whenever possible, analytical methods have been used to corroborate the experimental data and to provide a reliable means of generalizing the results .'), (908, 'random vibration .', 'crandal,s.', 'app. mech. rev. 12, 1959, 739.', 'random vibration . random vibration is vibration which results from an excitation which is not well represented by any simple function (sinusoid, step, etc.) or any simple combination of such functions but which is satisfactorily modeled by a stochastic process . it is perhaps not too much of an exaggeration to say that /all vibration is random vibration ./ every vibration record contains /hash/ at some level . nevertheless, until recently, engineering vibration theory has been able to get along without including the consideration of random excitations . now in several fields simultaneously there has occurred a burst of activity in the application of random processes . the response of aircraft to buffeting from atmospheric turbulence and the response of ships to confused seas have been put on reasonably firm footing . possibly the most dramatic problems have been posed by the development of large jet and rocket engines which produce spectacular amounts of random vibrational energy . the high level of random vibration in a jet plane or a missile provides a severe environment with respect to fatigue failure of structural members and with respect to malfunctions of sensitive equipment .'), (909, 'the effect of jet noise on aircraft structures .', 'clarkson,b.l.', 'aero. quart. 10, 1959,103.', 'the effect of jet noise on aircraft structures . the present state of knowledge on the problem of fatigue failure due to vibrations excited by jet noise is reviewed . it is concluded that it should currently be possible to make reasonable estimates of the stress levels set up in a structure by jet noise but, in general, the resultant fatigue life of the components cannot be estimated with any confidence .'), (910, 'natural frequencies of continuous beams of uniform span length .', 'ayre,r.s. and jacobson,l.s.', 'j. app. mech. 17, 1950, 391.', 'natural frequencies of continuous beams of uniform span length . a simple graphical network is used to determine the natural frequencies of flexural vibration of continuous beams having any number of spans of uniform length . the network is based upon a relatively few calculated values .'), (911, 'experimental study of the random vibrations of an aircraft structure excited by jet noise .', 'clarkson,b.l. and ford,r.d.', 'u.s.a.a. r.128, univ. of southampton, 1960.', 'experimental study of the random vibrations of an aircraft structure excited by jet noise . recordings have been made of the strains induced in a full scale rear fuselage test structure of the caravelle air-liner when one jet engine is running at maximum take-off thrust . the analysis has been concentrated on the strains in the centres of panels . correlation measurements indicate that the larger panel strains occur above resonance peak in each panel has been identified with the fundamental stringer-twisting mode but the mode-shapes for the two smaller peaks have not been completely determined . an attempt has been made to calculate the panel resonant frequencies theoretically .'), (912, 'the axisymmetric free-convection temperature field along a vertical thin cylinder .', 'hama,p.r., recesso,j.v. and christiaens,j.', 'j. ae. scs. 1959, 335.', 'the axisymmetric free-convection temperature field along a vertical thin cylinder . with a view to studying the effect of strong transverse curvature on boundary-layer problems, the axisymmetric free- convection problem along a vertical thin cylinder is investigated theoretically as well as experimentally . a theory is developed as an extension of the pohlhausen solution of a thick axisymmetric laminar boundary layer by mark and by glauert and lighthill . experiments consist of a thermocouple survey of the temperature field over an electrically-heated brass cylinder of diameter and 10 ft. height and an interferometric study of the density field over a bare tungsten wire of 0.02-in. diameter and 5 ft. height . the thermal-layer thicknesses are about five and fifty times the radii of the cylinders, respectively . experimental results of the local heat-transfer coefficient are in excellent agreement with the theory . this, in turn, justifies the theories of laminar boundary layer along a thin cylinder, at least indirectly .'), (913, 'vibrations of beams on many supports .', 'miles,j.w.', 'proc. a.s.c.e., 82, 1956.', 'vibrations of beams on many supports . the natural frequencies of a continuous beam resting on an arbitrary number of uniformly spaced supports are determined from a difference equation formulation . these frequencies fall in periodically spaced groups that are separated by spectral gaps of widths equal to approximately half the interval between the natural frequencies of a single beam on a square root frequency scale . these groups tend to uniform spectra as the number of supports tends to infinity, but the gaps remain, giving a band-pass character to the entire spectrum . wave propagation along an infinite, periodically supported beam is discussed and the phase and group velocities evaluated as functions of frequency .'), (914, 'transtability flutter of supersonic aircraft panels .', 'r. p. isaacs', 'rand corp.', 'transtability flutter of supersonic aircraft panels . for certain aero-elastic configurations it is possible to ascertain critical flutter conditions from static considerations alone . the idea is simply one of negation.. when the air speed exceeds a certain value statically stable equilibrium - and sometimes equilibrium itself take place . there are times when the dynamics of a situation are complex enough to defy a tractable analysis . the value of being able to indicate a flutter criterion from the simpler statics is clear . we will suppose flutter begins when some critical value of the air speed (or some parameter simply related the to) is exceeded . here we will show that there is a critical value which, when exceeded, precludes static equilibrium . underlying our work is the premise that these two critical values are the same . this assumption begs discussion . we will call the lowest value of our air speed parameter to preclude statically stable equilibrium of the system the transtability value . in some cases, excess of this value will ban all possibility of static equilibrium - stable or not., we will then call it a strong transtability value .'), (915, 'a buckled plate in a supersonic stream .', 'hayes, w.', 'naaa inc. r al1029 (abstract by e.m.keen)', 'a buckled plate in a supersonic stream . the forcible buckling of an external skin plate from a guided missile, with a pinned end, is considered in a supersonic air flow . conidtions of dynamic stability orthogonality and expansion of the buckling mode, the degree of freedom and cases of small deflection are considered . without a 50 half angle conical afterbody in a pressurized ballistic of range at nominal mach numbers of 3.5 and of 90000 and 220000, respectively . it'), (916, 'the flow around oscillating low aspect ratio wings at transonic speeds .', 'landahl, m.t.', 'kth aero. t.n.40, 1954 .', 'the flow around oscillating low aspect ratio wings at transonic speeds . when certain conditions are fulfilled for thickness ratio, aspect ratio, and reduced frequency for a three-dimensional wing, it can be shown that the partial differential equation for the non-steady perturbation potential can be reduced to a comparatively simple linear equation . the solution is then obtained by applying a fourier transformation in the free-stream direction and then using an iterative process developed by adams and sears for steady flow . the method gives solutions valid for low combinations of aspect ratio and reduced frequency . the method is applied to a delta wing oscillating in some selected rigid and elastic modes . from the results it can be seen that the special non-steady forces in the potential equation, which are neglected in slender-body theory, are very important . stability derivatives can also be obtained by the method and it is seen that the damping in pitch may be negative at m 1 for delta wings of too high aspect ratio .'), (917, 'a method of calculating the short period longitudinal stability derivatives of a wing in linearised unsteady compressible flow .', 'mangler,k.w.', 'rae r. aero.2468.', 'a method of calculating the short period longitudinal stability derivatives of a wing in linearised unsteady compressible flow . a method is developed for the calculation of the pressure distribution and the aerodynamic forces and moments on a wing performing harmonic pitching and heaving oscillations . the calculation is based on the assumption of inviscid potential flow without shock waves and is restricted to small incidence, so that the linearized theory is valid . in contrast to other work in the field the theory applies to all mach numbers . it is restricted to small values of the reduced frequency and should be valid for the usual range of short periods occurring at present in flight . the formal solution yields two integral equations for the parts of the load, which are in phase and go out of phase with the oscillation,. these are of the same form as the corresponding equation in steady flow . the way is thus opened for solutions over the whole mach number range at small frequencies, if the corresponding steady solutions can be found . the calculation is in fact easiest for m = 1 and has been done here for delta-wings to supplement a previous supersonic calculation, made on different frequency assumptions, which broke down near m = 1 . it appears from the two sets of results that the short period oscillation will be unstable near m = 1, if the apex angle of the delta wing is greater than about 60 . this confirms a now generally recognised trend . such results near m = 1 must of course be invalidated to an unknown extent by thickness viscosity and shock waves at their maximum effect . nevertheless it is unlikely that these factors will remove the critical nature of the transonic damping as calculated by this method . with all its obvious limitations this method, when extended to other planforms, should provide a useful tool in studying the effect of geometrical parameters on the stability of an aircraft at transonic speeds .'), (918, 'on the low aspect ratio oscillating rectangular wing in supersonic flow .', 'miles,j.w.', 'aero. quart. 4, 1953, 231.', 'on the low aspect ratio oscillating rectangular wing in supersonic flow . the laplace transform of the lift distribution on an oscillating rectangular wing in a supersonic flow is obtained by separating the linearised equation for the velocity potential in elliptic (cylindrical) co-ordinates . the results for the case of no spanwise distortion are expanded in ascending powers of the aspect ratio in order to compare with the slender body theory, and the longitudinal stability derivatives are calculated . it is found that at either supersonic or transonic speeds single-degree- of-freedom instability in pitch is impossible insofar as the fourth power of the aspect ratio is neglected .'), (919, 'theoretical studies of unsteady transonic flow . part iii . the oscillating low aspect ratio rectangular wing .', 'landahl, m.t.', 'aero. res. inst. of sweden /f.f.a./ rep. 79, 1958 .', 'theoretical studies of unsteady transonic flow . part iii . the oscillating low aspect ratio rectangular wing . by expanding the velocity potential in an asymptotic series, the aerodynamic forces on an oscillating low aspect ratio rectangular wing are calculated . the approximate theory is valid for small values of ko /o semi-span-to-chord ratio,. k reduced frequency/ and complements an earlier low-aspect-ratio-wing theory by the author valid only for pointed wings like delta wings . the present report gives formulas for the calculation of generalized forces for any smooth, flexible or rigid mode of oscillation with spanwise symmetry . comparisons with the slender-wing theory show that, except for wings of very low aspect ratio, unsteady-flow effects are appreciable even at fairly low reduced frequencies . near the upper limit in ko for the applicability of the present theory good agreement is obtained with a recent theory for high aspect ratios .'), (920, 'supersonic flow over an inclined wing of zero aspect ratio .', 'stewartson, k.', 'proc. camb. phil. soc. v. 46, pt. 2, p307, april 1950 .', 'supersonic flow over an inclined wing of zero aspect ratio . an asymptotic expression is found for the lift distribution on a long, narrow, laminar wing, at incidence in a supersonic stream . the approximations of the linearized potential theory are used .'), (921, 'slender-body theory-review and extension .', 'adams, m.c. and sears, w.r.', 'j. aero. sc. v. 20, february 1953 .', 'slender-body theory-review and extension . the approximate theory of flow about slender bodies and wings originated by munk and jones is reviewed . it is presented here in a form that emphasizes the relation to the source-sink methods of von karman and others . the extension to noncircular bodies is made for subsonic flow, paralleling ward''s extension for supersonic flow . the calculation of pressures and forces and the extension of the theory to unsteady flows are reviewed, and some discrepancies in the published literature are explained . finally, interpreting the jones slender-wing result as the first term of an expansion in powers of a breadth parameter /e.g., aspect ratio/, it is shown how a more accurate theory can be developed by carrying additional terms for both subsonic and supersonic speeds . this theory of not-so-slender wings is applied to some practical wing problems, including direct problems of flow past given wings and problems of wing design for minimum drag . the accuracy of the new results is assessed by comparison with linearized supersonic-airfoil theory for the special case of a flat delta wing .'), (922, 'supersonic flow past slender bodies of revolution, slope of whose median section is discontinuous .', 'lighthill,m.j.', 'q. j. mech. app. math. 1948, 90.', 'supersonic flow past slender bodies of revolution, slope of whose median section is discontinuous . the theory of supersonic flow around slender bodies of revolution, yawed or unyawed, with pointed or open bows, based on the linearized equation, is extended to the case when the meridian section of the outer surface has discontinuities in slope . expressions for the pressure distribution on the surface are obtained . it is found that the drag coefficient is no longer independent of mach number, and tends to zero more slowly than the square of the thickness of the body . the large pressure change behind a discontinuity is made up remarkably rapidly . the first approximation to the lift coefficient is unchanged .'), (923, 'methods for estimating lift interference of wing-body combinations at supersonic speeds .', 'nielsen,j.n. and kaattari,g.e.', 'naca til.2950.', 'methods for estimating lift interference of wing-body combinations at supersonic speeds . the modified slender-body method used by nielsen, katzen, and tang in rm a50f06, 1950, to predict the lift and moment interference of triangular wing-body combinations has been adapted to combinations with other than triangular wings . that part of the method for predicting the effect of the body on the wing has been retained, but a new method for predicting the effect of the wing on the body has been presented . these methods have been applied to the prediction of the lift-curve slopes of nearly 100 triangular, rectangular, and trapezoidal wing-body configurations . the estimated and experimental values for the lift-curve slopes agree for most of the cases within 10 percent . some of the higher-order effects that must be taken into account in a theory that is to give greater accuracy than the present one are discussed . a numerical example illustrating the method is included .'), (924, 'a method for calculating the lift and centre of pressure of wing-body-tail combinations at subsonic, transonic speeds .', 'nielsen,j.n., kaattari,g.e. and anastasio,r.f.', 'naca til.3959.', 'a method for calculating the lift and centre of pressure of wing-body-tail combinations at subsonic, transonic speeds . a method is presented for calculating the lift and pitching-moment characteristics of circular cylindrical bodies in combination with triangular, rectangular, or trapezoidal wings or tails through the subsonic, transonic, and supersonic speed ranges . the method covers unbanked wings, sweptback leading edges or sweptforward trailing edges, low angles of attack, and the effects of wing and tail incidence . the wing-body interference is handled by the method presented in naca rm''s a51j04 and a52b06, and the wing-tail interference is treated by assuming one completely rolled-up vortex per wing panel and evaluating the tail load by strip theory . a computing table and set of design charts are presented which reduce the calculations to routine operations . comparison is made between the estimated and experimental characteristics for a large number of wing-body and wing-body-tail combinations . generally speaking, the lifts were estimated to within 10 percent and the centers of pressure were estimated to within effect of wing deflection on wing-tail interference at supersonic speeds was not correctly predicted for triangular wings with supersonic leading edges .'), (925, 'factors affecting loads at hypersonic speeds .', 'henderson,a. and bertram,m.h.', 'naca til.5576.', 'factors affecting loads at hypersonic speeds . this paper gives a brief summary of current loads information at hypersonic speeds . several methods which the designer can employ in estimating the loads on various aircraft components are discussed . the paper deals with the characteristics of both slender and blunt configurations and touches upon the effects of boundary-layer and aerodynamic interference .'), (926, 'post buckling behaviour of circular cylinderical shells under hydrostatic pressure .', 'kempner,j.', 'j. ae. scs. 24, 1957, 253.', 'post buckling behaviour of circular cylinderical shells under hydrostatic pressure . the postbuckling behavior of initially perfect, thin-walled, circular cylindrical shells under hydrostatic pressure is investigated with the aid of the principle of stationary potential energy together with appropriate approximate deflection functions . calculations show that postbuckling equilibrium configurations exist for loads greater than as well as loads slightly less than the critical load calculated from small-deflection theory . loads less than the critical load are obtained only for a finite range of a parameter indicative of shell geometry . for loads corresponding to radial displacements of the order of the shell thickness, it is found that the number of circumferential waves remain essentially constant with increasing deflection and equal to the number of waves developed at buckling .'), (927, 'investigation of normal force distributions and wake vortex characteristics of bodies of revolution at supersonic speeds .', 'mello,j.f.', 'j. ae. scs. 1959, 155.', 'investigation of normal force distributions and wake vortex characteristics of bodies of revolution at supersonic speeds . the supersonic aerodynamic characteristics of inclined bodies of revolution at high angles of attack have been investigated in order to provide a more basic understanding of the body vortex wake flow and its relation to the problem of body-wing interference . the results of wind-tunnel tests, whereby the normal force, pitching moment, normal force distributions, and the local flow properties in the vicinity of the body were determined, are discussed and analyzed . comparisons of experimental normal force coefficient and center of pressure data with values calculated in accordance with theories which include methods for estimating the effects of viscosity show that the accuracy of these estimates is strongly dependent on the body fineness ratio and the angle of attack . further comparisons of the distributions of theoretical and experimentally derived cross-flow drag coefficients clearly show that, in general, the disagreement between experiment and existing theories is due to the inadequate prediction of the magnitude and distribution of the forces resulting from flow separation . the circulation strengths of the concentrated vortices and the circulation strengths of the vortex feeding sheets in the body vortex wake are determined by closed-contour velocity-perimeter integrations for paths enclosing the vortex or the feeding sheet . the values of vortex strength calculated in this manner are in close agreement with the values predicted by vortex strength formulas written for a simple theoretical model for which it is assumed that the cross-flow in any plane along the cylindrical portion of the body is represented by the steady incompressible potential flow about a cylinder, two symmetrical vortices of equal strength, and the attendant image vortices . however, in computing these strengths it is necessary to use the vortex locations and the viscous normal force distributions determined from experiment . the experimentally determined values of vortex strength are, in turn, used to calculate--by means of the aforementioned incompressible cross-flow potential--the local flow inclination angles which are in good agreement with the measured values, except in the vortex core, in the vicinity of the feeding sheet, and in regions for which transonic cross-flow velocities are expected . a consideration of these various regions with simple methods which account for the observed phenomena leads to substantial improvement in the agreement between theory and experiment . it is indicated that the complete vortex wake flow may be adequately predicted for a body of revolution (for conditions represented by the theoretical flow model), provided that the distribution of the viscous normal force and the vortex locations are accurately known .'), (928, 'a new theory for the buckling of thin cylinders under axial compression and bending .', 'donnell,l.h.', 'asme trans. 56, 1934, 795.', 'a new theory for the buckling of thin cylinders under axial compression and bending . the results of experiments on axial loading of cylindrical shells (thin enough to buckle below the elastic limit and too short to buckle as euler columns) are not in good agreement with previous theories, which have been based on the assumptions of perfect initial shape and infinitesimal deflections . experimental failure stresses range from 0.6 to 0.15 of the theoretical . the discrepancy is apparently considerably greater for brass and mild-steel specimens than for duralumin and increases with the radius- thickness ratio . there is an equally great discrepancy between observed and predicted shapes of buckling deflections . in this paper an approximate large-deflection theory is developed, which permits initial eccentricities or deviations from cylindrical shape to be considered . true instability is, of course, impossible under such conditions,. the stress distribution is no longer uniform, and it is assumed that final failure takes place when the maximum stress reaches the yield point . the effect of initial eccentricities and of large deflections is much greater than for the case of simple struts . measurements of initial eccentricities in actual cylinders have not been made,. however, it is shown that most of these discrepancies can be explained if the initial deviations from cylindrical form are assumed to be resolved into a double harmonic series, and if certain reasonable assumptions are made as to the magnitudes of these components of the deviations . with these assumptions the failing stress is found to be a function of the yield point as well as of the modulus of elasticity and the radius-thickness ratio . on the basis of this a tentative design formula (5) is proposel, which involves relations suggested by the theory but is based on experimental data . it is shown that similar discrepancies between experiments and previous theories on the buckling of thin cylinders in pure bending can be reasonably explained on the same basis, and that the maximum bending stress can be taken as about 1.4 times the values given by equation buckling problems can probably be explained by similar considerations, and it is hoped that this discussion may help to open a new field in the study of buckling problems . the large-deflection theory developed in the paper should be useful in exploring this field, and may be used in other applications as well . the paper presents the results of about a hundred new tests of thin cylinders in axial compression and bending, which, together with numerous tests by lundquist, form the experimental evidence for the conclusions arrived at .'), (929, 'stability of the cylindrical shell of variable curvature .', 'marguerre,k.', 'naca tm.1302,1951.', 'stability of the cylindrical shell of variable curvature . the report is a first attempt to devise a calculation method for representing the buckling behavior of cylindrical shells of variable curvature . the problem occurs, for instance, in dimensioning wing noses, the stability behavior of which is decisively influenced by the variability of curvature . the calculation is made possible by simplifying the stability equations (permissible for the shell of small curvature) and by assuming that the curvature as a function of the arc length s can be represented by a very few fourier terms . we evaluated the formulas for the special case of an ellipse-like half oval with an axis ratio under compression in longitudinal direction, shear, and a combination of shear and compression . however, the results can also be applied approximately to an unsymmetrical oval-shell segment under compression, shear, and bending so that the numerical values contained in the diagrams 10 to 12 represent directly dimensioning data for the wing nose .'), (930, 'general theory of large deflections of thin shells with special applications to conical shells .', 'dill,e.h.', 'nasa tn.d826, 1961.', 'general theory of large deflections of thin shells with special applications to conical shells . a general theory is developed for the case of large deflections but with rotations of the elements negligible compared to unity . the derivation is carried out in tensor form and therefore any coordinate system on the surface of the shell can be used . the effect of initial imperfections is included . it is shown that for shells of negligible gaussian curvature (shallow shells and developable surfaces), the problem can be reduced to the solution of two fourth-order partial differential equations in a stress function and the deflection normal to the shell . for shells forming a surface of revolution the results are indicated in terms of the equation of the generating curve . the differential equations for the conical shell are then listed .'), (931, 'stability equations for conical shells .', 'kempner,j.', 'j. ae. scs. 25, 1958, 137.', 'stability equations for conical shells . the author rewrites v. s. vlasov''s equations for (linear) stability theory of shells (prikl. mat. meh. 8(1944), 109-placements . the result is a single eighth-order equation for normal deflection, and two fourth-order equations relating the displacement components in the shell middle surface to the normal displacement .'), (932, 'buckling of circular cones under axial compression .', 'lackman,l. and penzien,j.', 'j. app. mech. 27, 1960, 458.', 'buckling of circular cones under axial compression . presented are the results of an experimental investigation to determine the buckling strength of right circular cones under axial compression . correlation of these data is made with existing theory and with previously published experimental data on circular cylinders,. thus a recommended procedure for predicting the buckling load of right circular cones under the foregoing loading condition is presented .'), (933, 'the characteristics of roughness from insects as observed for two-dimensional, incompressible flow past airfoils .', 'coleman,w.s.', 'j. ae. scs. 1959, 264.', 'the characteristics of roughness from insects as observed for two-dimensional, incompressible flow past airfoils . advances in the practical development of boundary-layer control for the maintenance of extensive laminar flow have drawn attention to the problem of surface roughness, due not only to artificial irregularities such as rivet heads, lap joints, window panels, etc., but also to the kind generated in flight from impact with insects . this natural form of roughening, the effects of which have been noted, though not investigated previously, is the subject of the present paper . the phenomenon may be divided into two parts--namely, and (2) its effect upon the stability of the laminar boundary layer . wind-tunnel experiments with the fruit fly, drosophila, and the common housefly for the investigation of both (1) and airfoils are fully described . the former problem has also been treated mathematically in a separate paper, not yet published, agreement between theory and experiment being satisfactory in all essentials . the characteristics of the roughness profile consist principally of a pronounced peak near the leading edge, followed by an extensive area of surface over which there is a much reduced and gradually diminishing value of the excrescence height . further, it is shown that, if the severe leading-edge roughness, or its effect upon the boundary layer, can be eliminated, then the down-stream roughness causes no disturbance to the passage of a laminar layer--i.e., the surface, though roughened, is aerodynamically smooth . moreover, it appears that the conditions defining the upstream boundary to this region of insignificant roughness are fundamentally the same as those which determine the critical state for transition at an artificial disturbance of a three- dimensional character .'), (934, 'stability of cylindrical and conical shells of circular cross section, with simultaneous action of axial compression and external normal pressure .', 'mushtari,k.m. and sachenkov,a.v.', 'naca tm.1433, 1958.', 'stability of cylindrical and conical shells of circular cross section, with simultaneous action of axial compression and external normal pressure . we consider in this report the determination of the upper limit of critical loads in the case of simultaneous action of a compressive force, uniformly distributed over plane cross sections, and of isotropic external normal pressure on cylindrical or conical shells of circular cross section . as a starting point we use the differential equations for neutral equilibrium of conical shells (ref. 1) which have been used for the solution of the problem of stability of conical shells under torsion and under axial compression (ref. 2),. upon solution of the problem it is possible to satisfy all boundary conditions, in contrast to the report (ref. 3) where no attention is paid to the fulfillment of the boundary conditions and to the report (ref. 4) where only part of the boundary conditions are satisfied by solution of the problem according to galerkin''s method . approximate formulas are used for the determination of the critical external normal pressure with simultaneous action of longitudinal compression . let us note that the formulas suggested in reference 5 are not well founded and may lead, in a number of cases, to a substantial mistake in the magnitude of the critical load .'), (935, 'buckling of thin single- and multi-layer conical and cylindrical shells with rotationally symmetric stresses .', 'p. p. radkowski', 'avco manufacturing corporation', 'buckling of thin single- and multi-layer conical and cylindrical shells with rotationally symmetric stresses . the buckling of simply supported, thin, single- and multi-layer conical shells under axially symmetrical loading is analyzed in this paper . the results are presented in a compact manner so that they may be easily used for design and/or experimental purposes . the results are compared with known experimental values .'), (936, 'a donnell-type theory for asymmetrical bending and buckling of thin conical shells .', 'seide,p.', 'j. app. mech. 24, 1957, 547.', 'a donnell-type theory for asymmetrical bending and buckling of thin conical shells . equations, somewhat more accurate than those recently presented by n. j. hoff, are derived for bending and buckling of thin circular conical shells under arbitrary loading . these equations reduce to donnell''s equations for thin cylindrical shells when the cone semivertex angle becomes very small and the minimum radius of curvature of the median surface approaches a constant value . at the other end of the scale the equations reduce to the well-known equations for flat circular plates when the cone semivertex angle approaches a right angle . in addition, for the entire range of cone semivertex angles the equations reduce to the known equations for axisymmetrical bending when variations of the displacements around the circumference vanish . the problem of bending is reduced to the solution of a single fourth-order partial differential equation with variable coefficients .'), (937, 'on the buckling of truncated conical shells in torsion .', 'seide,p.', 'j.app.mech.28, 1962.', 'on the buckling of truncated conical shells in torsion . the problem of the buckling of thin circular conical frustums in pure torsion is solved in a manner similar to that employed previously by the author for buckling under uniform hydrostatic pressure . synthesis of the numerical results indicates that the critical torsion of a truncated cone is equal to that of an equivalent cylinder whose length and thickness are the axial length and wall thickness of the cone and whose radius is a function of the semivertex angle and the taper ratio of the cone . curves and equations to aid in the analysis of conical frustums are given . it is shown that a previous recommendation for the analysis of truncated cones in torsion may be seriously unconservative in some cases .'), (938, 'calculations for the stability of thin conical frustums subjected to external uniform hydrostatic pressure and axial loads .', 'seide,p.', 'j. ae. scs. 29, 1962, 951.', 'calculations for the stability of thin conical frustums subjected to external uniform hydrostatic pressure and axial loads . calculations are presented for the problem of the stability of conical shells subjected to combined external uniform hydrostatic pressure and axial tension or compression . stress interaction curves are found to vary only slightly as a function of the ratio of the end radii of the cone .'), (939, 'some explicit solutions for constant-temp . magnetogas dynamic channel flow .', 'berlot,r.r.', 'j. ae. scs. 29, 1962, 748.', 'some explicit solutions for constant-temp . magnetogas dynamic channel flow . in order to simplify the process of estimating the aerodynamic loading on the after portions of slender vehicles, it is frequently assumed that there is no nose-tail interaction . it is the purpose of this note to show that, aside from boundary-layer effects, this assumption is not warranted when the nose hypersonic-similarity parameter, tan, is of the order of unity, or greater . physically speaking, the entropy change associated with a strong bow wave reduces the stagnation pressure down-stream of the shock, and hence, lowers the dynamic pressure in the vicinity of the tail .'), (940, 'of a turbulent free shear layer .', 'nash,j.f.', 'npl. aero. r1019, 1962.', 'of a turbulent free shear layer . the problem of predicting the mean velocity on streamlines through the pre-asymptotic turbulent free shear layer in two-dimensional incompressible flow is resolved into two parts . the linearized momentum equation in terms of a generalized axial co-ordinate is solved in the usual way . a relation between and the distance from the separation point is then established analytically in contrast to the previous use of empirical expressions . it is shown that except in the region close to separation the velocity on the streamlines can be predicted by the simple approximation proposed by kirk .'), (941, 'viscous compressible and incompressible flow in slender channels .', 'williams,j.c.', 'a.i.a.a. j. 1963, 186.', 'viscous compressible and incompressible flow in slender channels . an analytical study is made of viscous flow in slender channels . similar solutions to the approximate equations of motion, valid for flow at moderate or high reynolds numbers in slender channels, are found for incompressible two-dimensional and axisymmetric flows and for compressible flows through two-dimensional channels with adiabatic walls . a study of compressible flows in convergent-divergent channels yields results regarding the effect of viscosity on the location of the sonic line, on the pressure ratio at the geometric throat and on the discharge coefficient for such channels .'), (942, 'secondary gas injection in a conical rocket nozzle .', 'walker,r.e., stone,a.r. and shandor,m.', 'aiaa jnl. u, 1963, 334.', 'secondary gas injection in a conical rocket nozzle . data are presented on side forces generated by secondary gas injection in a 15 conical rocket exhaust nozzle . the side force was measured directly with a force transducer and the data examined in terms of an amplification factor, where is the measured effective specific impulse of injectant, and is the specific impulse of injectant for sonic flow into a vacuum . injection was normal to the axis of the nozzle through a single circular orifice at a fixed point in the diverging portion of the nozzle . a variety of ambient temperature gaseous injectants and orifice diameters were carefully studied . injectant flow rate was varied for each configuration . the main propellant was hot gas ( catalytically decomposed), and motor conditions were held essentially constant .'), (943, 'compressible free shear layer with finite initial thickness .', 'denison,m.r. and baum,e.', 'aiaa jnl. 1, 1963, 342.', 'compressible free shear layer with finite initial thickness . the momentum equation was uncoupled from the other conservation equations for the case of a finite initial profile in a laminar free shear layer . the equation was solved numerically, in the crocco coordinate system, using an implicit finite difference method . profiles of velocity and shear function were obtained as a function of streamwise distance . the initial profiles as the flow separates from the rear of the body correspond to the blasius profile in transformed coordinates . for large distances downstream, the profiles approach the chapman distribution, corresponding to the case of zero initial free shear layer thickness . the effect of these results on calculations of base pressure and wake angle is discussed . a method for the calculation of finite chemical kinetic effects on the profiles of temperature and chemical composition in the free shear layer with finite initial thickness is outlined .'), (944, 'one dimensional heat conduction through the skin of a vehicle upon entering a planetary atmosphere at constant velocity and entry angle .', 'wells,w.r. and mclellan,c.h.', 'nasa tn.d1476, 1962.', 'one dimensional heat conduction through the skin of a vehicle upon entering a planetary atmosphere at constant velocity and entry angle . closed-form solutions of the one-dimensional heat-conduction equations for the flow of heat into a plate with a laminar boundary layer have been obtained for a configuration entering a planetary atmosphere with constant velocity and negative entry angle . the atmospheric density was assumed to obey an exponential law and the temperature was assumed constant initially . the solution is in the form of a fourier series expansion which, for most practical applications, can be approximated by retaining only one term of the expression . the solution applies to the initial part of the entry before the maximum heating conditions are encountered .'), (945, 'method for design of pump impellers using a high speed digital computer .', 'stockman,n.o. and kramer,j.l.', 'nasa tn.d1562, 1963.', 'method for design of pump impellers using a high speed digital computer . a method of designing pump impellers is derived from the equations of motion and continuity for incompressible nonviscous relative flow . the flow is assumed to follow a known stream surface (representing blade shape) that extends from hub to shroud . equations are also derived for approximate blade-surface velocities and pressures . a detailed numerical procedure and block diagram are given for use on a digital computer . a numerical example that illustrates limited use of the method is presented and further uses are indicated .'), (946, 'exploratory investigation of the effect of a forward facing jet on the bow shock of a blunt body in a mach number 6 free stream .', 'romeo,d.j. and sterrett,j.r.', 'nasa tn.d1605, 1963.', 'exploratory investigation of the effect of a forward facing jet on the bow shock of a blunt body in a mach number 6 free stream . the effect of a forward-facing jet on the bow shock of a blunt body in a mach 6 free stream was investigated experimentally . the models tested had forward-facing jets using air and helium exhausting at mach numbers from 1 to 10.3 and were run through a range of the ratio of jet total pressure to free-stream total pressure of 0.03 (jet off) to 2.5 . the ratio of body diameter to jet-exit diameter varied from 1.12 to 55.6 and the angle of attack was varied from 0 to 35 . the experimental results show that the main-stream shock can be affected by the jet in two significantly different ways . one way is simply to move the strong shock away from the body without altering its shape . the second and perhaps more interesting case occurs when the jet causes a large displacement of the main shock and considerably changes its shape . it was found that the ratio of jet total pressure to free-stream total pressure necessary to obtain the large displacements of the main-stream shock depended on the ratio of body diameter to jet-exit diameter and also on the jet-exit mach number . the maximum amount the shock could be displaced in percent of body diameter was seen to increase with increasing jet-exit mach number and also with decreasing ratio of body diameter to jet-exit diameter . for the models that were investigated through an angle-of-attack range, the displacement became very unsteady and fell off sharply as the angle of attack was increased . simplified theoretical considerations applied to the shock-displacement phenomena provide a possible explanation for the two different types of main-stream shock displacement . theoretical curves show the regions where these types of displacement would occur for different exit mach numbers and pressure ratios for a forward-facing jet in a mach 6 stream .'), (947, 'static aerodynamic characteristics of a short blunt 10 semi-vertex angle cone at a mach number of 15 in helium .', 'fohrman,m.j.', 'nasa tn.d1648, 1963.', 'static aerodynamic characteristics of a short blunt 10 semi-vertex angle cone at a mach number of 15 in helium . axial force, normal force, pitching moment, and shock-wave shape were determined for a body of revolution consisting of a short blunt 10 semivertex angle cone with a flat base and also with a conical afterbody having a semi-vertex angle of 50 . measurements were made in helium at a free-stream mach number of 15 and a free-stream reynolds number of 2.25x10 based on maximum body diameter over an angle-of-attack range from the configuration with the conical afterbody was statically stable in the nose-forward attitude only, whereas the configuration with no afterbody was statically stable in both the nose-forward and base-forward attitudes . the force and moment data of both shapes were predicted reasonably well by modified newtonian theory at all angles of attack, except the pitching-moment coefficient for the model without afterbody near 180 angle of attack . in this region, measurements indicated static stability, whereas theory indicated static instability . the helium data agreed reasonably well with a limited amount of force and moment data obtained in a ballistic range at small angles of attack in air at a mach number of 15 and also with force and moment data obtained in air over a complete angle-of-attack range at a mach number of 5.5 . the value of axial-force coefficient and the shape of the bow shock wave at zero angle of attack for both models obtained from a numerical flow field calculation agreed very well with the data . the value of the axial force coefficient at 180 angle of attack for the model with afterbody agreed reasonably well with the theoretical value for a cone . the position and shape of the shock envelope near the stagnation point also could be predicted accurately by an approximate method over an angle-of-attack range from'), (948, 'panel flutter tests on full scale x-15 lower vertical stabilizer at mach number of 3. 0.', 'bohon,h.l.', 'nasa tn.d1385, 1962.', 'panel flutter tests on full scale x-15 lower vertical stabilizer at mach number of 3. 0. panel flutter tests were conducted on two full-scale vertical stabilizers of the x-15 airplane at a mach number of 3.0 in the langley at dynamic pressures from 1,500 psf to 5,000 psf and stagnation temperatures from 300 f to 660 f . flutter boundaries were obtained for four of the five distinct types of panels which make up the vertical sides of the stabilizers . the boundaries consisted of a flat-panel boundary and a thermally buckled-panel boundary . the flat-panel boundaries were characterized by a reduction in dynamic pressure with increasing skin temperature,. whereas, after thermal buckling the trend was reversed . the minimum dynamic pressure for flutter occurred at the intersection of the flat-panel and buckled-panel boundaries and represented a large reduction in the dynamic pressure over the extrapolated, unstressed value . as a result of panel flutter, three of the five distinct types of panels were modified to provide the required flutter margin on the design flight dynamic pressure of the aircraft .'), (949, 'charts for equilibrium flow properties of air in hyper-velocity nozzles .', 'jorgensen,l.h. and baum,g.m.', 'nasa tn.d1333, 1962.', 'charts for equilibrium flow properties of air in hyper-velocity nozzles . for initial stagnation pressures up to 1,000 atmospheres and stagnation enthalpies up to 10,000 btu per pound, nozzle-flow properties for equilibrium air have been computed and plotted on charts . the work of nasa tn d-693 has been extended to include flow properties for closer intervals of specified stagnation enthalpies . properties which have been charted as a function of mach number are as follows .. temperature, pressure, density, velocity, area ratio, dynamic pressure, reynolds number, isentropic exponent, and molecular weight ratio . ratios of temperature, pressure, and density across normal shock waves are also charted, and weight-flow rate is plotted as a function of stagnation enthalpy .'), (950, 'comparison of theoretical and experimental creep buckling times of initially straight, centrally loaded columns .', 'jahsman,w.e. and field,f.a.', 'j. ae. scs. 29, 1962, 431.', 'comparison of theoretical and experimental creep buckling times of initially straight, centrally loaded columns . the creep-buckling times of initially straight, centrally loaded columns as predicted by the hypotheses of shanley, gerard, and rabotnov and shesterikov are compared with appropriate experimental data . it is found that the theoretical predictions are generally conservative, due possibly to the fact that the hypotheses predict initial instability times while the experiments normally record final collapse times . of the three hypotheses, that of gerard generally gives predictions which agree best with the experimental data .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (951, 'a unified theory of creep buckling under normal loads .', 'stowell,e.z. and thein wah.', 'j. ae. scs. 29, 1962, 658.', 'a unified theory of creep buckling under normal loads . a general theory of creep buckling, with the initial imperfection as a parameter, is developed for the case of normal loading . a hyperbolic-sine law is used to describe the process of creep . the theory is believed to be applicable to, among other structures, columns, tubes, and possibly conical shells . the wall of the structure is idealized as a sandwich in order to simplify the integration of the equations . experimental data on columns and tubes, from two different sources, are compared with the predictions of the theory .'), (952, 'study of creep collapse of a long circular cylindrical shell under various distributed force systems .', 'serpico,j.c.', 'j. ae. scs. 29, 1962, 1316.', 'study of creep collapse of a long circular cylindrical shell under various distributed force systems . an analysis is presented for determining the collapse of circular rings and long cylinders subjected to primary and secondary creep conditions at elevated temperatures . the types of loading considered for the present investigation are dead loading and hydrostatic pressure-type forces . the method of solution is based on an application of the variational theorem for creep described in ref. 1 with some additional terms being introduced for the pressure-type loading case . the general results are reduced to a relatively simple form for the theoretical predictions of collapse time and are graphically illustrated for a typical sample material .'), (953, 'vibrations of infinitely long cylindrical shells under initial stress .', 'armenakas,a.e.', 'aiaa jnl. 1, 1963, 100.', 'vibrations of infinitely long cylindrical shells under initial stress . the general bending theory of shells under the influence of initial stress presented recently by herrmann and armenakas is applied in this investigation to study the effect of initial uniform circumferential stress, uniform bending moment and uniform radial shear on the dynamic response of an infinitely long cylindrical shell .'), (954, 'analysis of stress at several junctions in pressurized shells .', 'johns,r.h., morgan,w.c. and spera,d.a.', 'aiaa jnl. 1, 1963, 455.', 'analysis of stress at several junctions in pressurized shells . theoretical and experimental results are presented for the discontinuity stresses arising at a change of wall thickness in a cylinder, a cylinder-hemisphere junction, and a cone-spherical torus junction in pressure vessels . the effect of mismatch of nonconcurrence of the middle surfaces of two joined cylinders is considered . in addition, a cylinder with a special closure which has considerably reduced stresses is described, and curves with theoretical and experimental stresses are presented .'), (955, 'the membrane approach to bending instability of pressureized cylindrical shells .', 'mccomb,h.g., zender,g.w. and mikulas,m.m.', 'nasa tn.d1510, 1962, 229.', 'the membrane approach to bending instability of pressureized cylindrical shells . recent theoretical and experimental research is briefly described to trace the development of deformation and the occurrence of collapse in pressurized circular cylindrical membranes under applied moment loading . the collapse of pure membrane cylinders is then compared with instability of pressurized cylindrical shells . this approach leads to a better understanding of the behavior of pressurized cylinders under bending loads . the results suggest possibilities for further research utilizing the membrane approach .'), (956, 'elastic stability of simply supported corrugated core sand- wich cylinders .', 'harris,l.a. and baker,e.g.', 'nasa tn.d1510, 1962, 331.', 'elastic stability of simply supported corrugated core sand- wich cylinders . theoretical buckling coefficients are obtained for the general instability of simply supported, corrugated core sandwich circular cylinders under combined loads with the core oriented parallel to the longitudinal axis of the cylinder . buckling curves are presented for axial compression, external lateral pressure, torsion, and some typical interactions . the differential equations of equilibrium used to obtain the buckling equations were derived from the small deflection equations of stein and mayer which include the effect of deformation due to transverse shear . these equations are solved by galerkin''s equation . remarks are made concerning the probable validity of the results of the small deflection theory for sandwich shells .'), (957, 'axisymmetric snap buckling of conical shells .', 'newman,m. and reiss,e.l.', 'nasa tn.d1510, 1962, 451.', 'axisymmetric snap buckling of conical shells . the authors give a brief account of some of their recent analytical and numerical studies of cone buckling, limiting the discussion to axisymmetric deformations . pertinent numerical results for the relaxation buckling of full cones subjected to uniform external pressure and belleville springs deformed by axial edge loads are presented . in addition, bifurcation buckling problems are discussed . for a specific case, the existence of friedrichs'' intermediate buckling load, as applied to cones, is established . upper and lower bounds for its value are given .'), (958, 'air scooping vehicle .', 'berner,f. and camac,m.', 'avco-everett res. lab. r.76, 1959.', 'air scooping vehicle . a satellite vehicle is described which collects gases from the upper atmosphere and stores them in liquid form . such a vehicle could serve as a filling station in space, furnishing liquid oxygen or air to other spacecraft . the vehicle represents an alternative to launching these liquids into orbit from the surface of the earth . the two methods are compared on an economic basis, and it is shown that the proposed vehicle permits substantial savings when operated beyond about one year . the feasibility of developing such a system for long-time operation is investigated . several practical designs are discussed .'), (959, 'heat transfer in separated flows .', 'larson,h.k.', 'j. ae. scs. 1959, 731.', 'heat transfer in separated flows . results of an experimental heat-transfer investigation in regions of separated flow are presented and compared with the theoretical analysis of naca tn 3792 . the average heat transfer for both laminar and turbulent separated boundary layers was found to be from 35 to 50 per cent less than that for equivalent attached boundary layers . the overall scope of the measurements included mach numbers from 0.3 to 4.0 and reynolds numbers from 10 to 4 x 10 . the results for laminar boundary layers agree well with the analysis of tn 3792 . the results for turbulent boundary layers, however, disagree considerably . results of velocity and temperature surveys in the separated turbulent boundary layer are presented and partially explain the discrepancy between the experiments and analysis . the maximum local heat-transfer rates were found to occur in the reattachment region of the separated boundary layers investigated . the effect of transition on heat transfer in the separated laminar boundary layers is described and data showing effects of mach number and wall temperature on the transition reynolds number of separated laminar flows are also included .'), (960, 'investigation of free turbulent mixing .', 'liepman,h.w. and laufer,j.', 'naca tn.1257, 1947.', 'investigation of free turbulent mixing . a discussion of the integral relations for flow of the boundary-layer type is presented . it is shown that the characteristic laws of spread of jets, wakes, and so forth, can be obtained directly for the laminar case and, with the help of dimensional reasoning, for the turbulent case as well . measurements of the mean velocity, the intensity and scale of the turbulent fluctuations, and of the turbulent shear in a two-dimensional mixing zone are presented . the results of these measurements are compared with the mixing-length theories . it is shown that both mixing length and exchange coefficient vary across the mixing zone . the theories based on the assumption of constant mixing length or exchange coefficient are thus in error . a discussion of the energy balance of the fluctuating motion is given and the triple correlation is estimated .'), (961, 'compressible two dimensional jet mixing at constant pressure .', 'korst,h.h. and page,r.h.', 'univ. of illinois dept. mech. eng. me-tn-392-1, 1954.', 'compressible two dimensional jet mixing at constant pressure . an analysis is made of turbulent constant pressure mixing for a compressible jet boundary, taking into consideration effects of the initial boundary layer . velocity profiles in the mixing region are represented in a transformed plans by one-parameter families of curves, with no specification for the mixing mechanism beyond that of an exchange coefficient concept being made . the exchange coefficient is represented by the bornel function of an integral transform for the x coordinate of an intrinsic system of coordinates . this intrinsic system and the physical coordinate system are related by means of a momentum integral . satisfactory correlation of theory and experimental low-speed data is obtained with a simple form of kernal function . an asymptotic solution, corresponding to a fully developed velocity profile in the jet boundary, allows the calculation of the mechanical energy level along the separating streamline in the jet boundary without the use of empirical information .'), (962, 'contributions to the theory of heat transfer through a laminar boundary layer .', 'm. j. lighthill', 'department of mathematics, university of manchester communicated by s. goldstein, f.r.s.', 'contributions to the theory of heat transfer through a laminar boundary layer . an approximation to the heat transfer rate across a laminar incompressible boundary layer, for arbitrary distribution of main stream velocity and of wall temperature, is obtained by using the energy equation in von mises''s form, and approximating the coefficients in a manner which is most closely correct near the surface . the heat transfer rate to a portion of surface of length l breadth is given as where k is the thermal conductivity of the fluid, o its prandtl number, p its density, u its viscosity, r(x) is the skin friction, and t(x) the excess of wall temperature over main stream temperature . a critical appraisement of the formula indicates that it should be very accurate for large, but that for of order 0.7 (for most gases) the constant should be replaced by 0.73, when the error should not exceed this yields a formula for nusselt number in terms of the reynolds number r and the mean square root of the skin friction coefficient c, in the case of uniform wall temperature . however, for the boundary layer with uniform main stream, the original formula is accurate to within 3 percent even for . by known transformations an expression is deducted for heat transfer to a surface, with arbitrary temperature distribution along it, and with a uniform stream outside it at arbitrary mach number (equation (42)) . from this the temperature distribution along such a surface is deduced in the case (of importance at high mach numbers) when heat transfer to it is balanced entirely by radiation from it . this calculation, which includes the solution of a non-linear integral equation, gives higher temperatures near the nose, and lower ones farther back (figure 2), than are found from a theory which assumes the wall temperature uniform and averages the heat transfer balance . this effect will be considerably mitigated for bodies of high thermal conductivity., the author is not in a position to say whether or not it will be appreciable for metal projectiles . but for stony meteorites at a certain stage of their flight through the atmosphere it indicates that melting at the nose and re-solidification farther back may occur, for which the shape and constitution of a few of them affords evidence . an appendix shows how the method for approximating and solving von mises''s equation could be used to determine the skin friction as well as heat transfer rate, but this line seems to have no advantage over established approximate methods .'), (963, 'a variational principle for convection of heat .', 'agrawal, h.c.', 'j. math. and mech. v.9, no. 6, 1960 .', 'a variational principle for convection of heat . authors extend variational principle of biot to various cases of heat transfer due to forced convection . numerical results are given for one-dimensional problems of fluid flowing between parallel walls with uniform or parabolic velocity profiles . agreement with exact solutions is excellent .'), (964, 'on the theory of discharge coefficients for round entrance flowmeters and venturis .', 'rivas, m.a. and shapiro, a.h.', 'trans. a.s.m.e., v. 78, april 1956, pp 489-497 .', 'on the theory of discharge coefficients for round entrance flowmeters and venturis . a theory of rounded-entrance flowmeters, based on a consideration of the potential and boundary-layer flows in a converging nozzle, is constructed . curves are presented showing the discharge coefficient as a function of diameter reynolds number, with the /total equivalent length tional length-diameter ratio of the contraction section of the asme long-radius nozzle is presented . the theoretical curves of discharge coefficient versus diameter reynolds number are in good agreement with experiment over a range of reynolds number from 1 to 10 . the theory provides a rational framework for correlating and extrapolating experimental results,. it shows the effects of contraction shape and location of pressure taps,. it furnishes values of discharge coefficient for untested designs,. and it suggests precautions to be taken in design, installation, and operation .'), (965, 'analytic determination of discharge coefficients of flow nozzles .', 'simmons, f.s.', 'naca tn 3449, 1955 .', 'analytic determination of discharge coefficients of flow nozzles . integration of the velocity profile at the throat of a flow nozzle yields the discharge coefficient as a function of the ratio of boundary solution of the approximate momentum equation for the boundary layer . the resulting expression for the discharge coefficient is then a function of the reynolds number based on nozzle diameter and of the geometry of the nozzle . good agreement is shown between this expression and published experimental data on flow nozzles for reynolds numbers between'), (966, 'on fully developed channel flows,. some solutions and limitations, and effects of compressibility, variable properties, and body forces .', 'maslen, s.h.', 'nasa tr r-34, 1959', 'on fully developed channel flows,. some solutions and limitations, and effects of compressibility, variable properties, and body forces . an examination of the effects of compressibility, variable properties, and body forces on fully developed laminar flows has indicated several limitations on such streams . in the absence of a pressure gradient, but presence of a body force liquid this follows also for the case of a constant streamwise pressure gradient . these motions are exact in the sense of a couette flow . in the liquid case two solutions /not a new result/ can occur for the same boundary conditions . an approximate analytic solution was found which agrees closely with machine calculations . in the case of approximately exact flows, it turns out that for large temperature variations across the channel the effects of convection /due to, say, a wall temperature gradient/ and frictional heating must be negligible . in such a case the energy and momentum equations are separated, and the solutions are readily obtained . if the temperature variations are small, then both convection effects and frictional heating can consistently be considered . this case becomes the constant-property incompressible case /or quasi-incompressible case for free-convection flows/ considered by many authors . finally, there is a brief discussion of cases wherein streamwise variations of all quantities are allowed but only in such form that the independent variables are separable . for the case where the streamwise velocity varies inversely as the square root of distance along the channel, a solution is given .'), (967, 'a study of laminar compressible viscous pipe flow accelerated by an axial body force, with application to magnetogasdynamics .', 'martin, d.e.', 'nasa tn d-855, april 1961 .', 'a study of laminar compressible viscous pipe flow accelerated by an axial body force, with application to magnetogasdynamics . a study is made of the steady laminar flow of a compressible viscous fluid in a circular pipe when the fluid is accelerated by an axial body force . the application of the theory to the magnetofluidmechanics of an electrically conducting gas accelerated by electric and magnetic fields is discussed . constant viscosity, thermal conductivity, and electrical conductivity are assumed . fully developed flow velocity and temperature profiles are shown, and detailed results of the accelerating flow development, including velocity and pressure as functions of distance, are given for the case where the axial body force is constant and for the case where it is a linear function of velocity . from these results are determined the pipe entry length and the pressure difference required .'), (968, 'rocket propulsion systems for interplanetary flight .', 'sutton, g.p.', 'j. aero. sc. v. 26, october 1959 .', 'rocket propulsion systems for interplanetary flight . a comparison is made of several different propulsion systems for interplanetary flight . liquid and solid propellant rockets, propulsion systems which use nuclear energy sources, are heating rockets, magneto-plasma devices, ion rocket propulsion, solar heating rockets, and solar sails are briefly described and their current status reviewed . engine performance requirements for different interplanetary missions are established . these several propulsion systems are then compared on the basis of several performance criteria, environmental characteristics, vehicle requirements, reliability, current status, growth potential, and efficiency . predictions on various propulsion system capabilities and an analysis of multiple rocket engine reliability is included . it is concluded that electrical rockets are superior for long-time inter-planetary flight applications, and that chemical rockets are satisfactory for most of the immediate applications in /near/ space . none of the several propulsion schemes discussed can be rejected until further technical work has been accomplished .'), (969, 'on the use of side-jets as control devices .', 'liepman,h.p.', 'ars jnl. 29, 1959, 453.', 'on the use of side-jets as control devices . wind tunnel experiments with side-jets, issuing laterally near the base of slender bodies in a supersonic stream, have suggested the existence of a sizable and usable interaction . with this interaction force, the use of jet reaction controls may be as attractive for flight within the atmosphere as it obviously is for flight outside the atmosphere . this note indicates the altitude regime of interest and the order of magnitude of the interaction bonus for a lateral control jet located near the base of a body of revolution .'), (970, 'loads induced on a flat plate wing by an air jet exhausting perpendicularly through the wing and normal to a free-stream flow of mach number 2.0 .', 'janos, j.j.', 'nasa tn d-649, march 1961 .', 'loads induced on a flat plate wing by an air jet exhausting perpendicularly through the wing and normal to a free-stream flow of mach number 2.0 . measurements were made of loads induced on a flat-plate wing by an air jet exhausting perpendicularly through the wing and normal to the free-stream flow . the investigation was conducted at a free-stream mach number of 2.0 and a reynolds number per foot of 14.4 x 10 . an axially symmetric sonic nozzle and two supersonic nozzles were employed for the jets . the supersonic nozzles consisted of an axially symmetric nozzle with exit mach number of 3.44 and a two-dimensional nozzle with exit mach number of 1.76 . the ratio of nozzle total pressure to free-stream static pressure was varied from 20 to 110 . negative loads were induced on the flat-plate wing by all the jets . as the nozzle pressure ratio was increased the magnitude of interference loads due to jet thrust decreased . the chordwise center-of-pressure location generally moved toward the nozzle center line as the pressure ratio was increased .'), (971, 'surface pressure distributions with a sonic jet normal to adjacent flat surfaces at mach 2.92 to 6.4 .', 'cubbison, r.w., anderson, b.h. and ward, j.j.', 'nasa tn d-580, february 1961 .', 'surface pressure distributions with a sonic jet normal to adjacent flat surfaces at mach 2.92 to 6.4 . an investigation was made to determine the interference effects on surface pressure distributions caused by a sonic jet exiting normal to the surface . two configurations, a flat plate and an arrow-wing reentry-type vehicle, with sonic nozzles near the leading edge were tested over a range of pressure ratios and reynolds numbers for mach numbers from the data indicate that jet pressure ratio had considerable effect on the pressure levels and distributions on both configurations . also, for a constant jet pressure ratio, the free-stream mach number effect on the distributions and levels was quite large . over the limited range investigated, the effect of reynolds number at constant mach number and pressure ratio was small compared to the mach number and pressure ratio effect .'), (972, 'aerodynamic interaction effects ahead of a sonic jet exhausting perpendicularly form a flat plate into a mach number 6 free stream .', 'romeo, d.j. and sterrett, j.r.', 'nasa tn d-743, april 1961 .', 'aerodynamic interaction effects ahead of a sonic jet exhausting perpendicularly form a flat plate into a mach number 6 free stream . an investigation of the effects of the interaction ahead of a two-dimensional sonic jet exhausting perpendicularly into a mach number were made at an angle of attack of 0degree at a reynolds number per foot of approximately 6 x 10 and with conditions of both transitional and turbulent separation on the flat plate . the ratio of jet stagnation pressure to free-stream static pressure was varied from 8 to 460 and the jet slot width was varied from 0.001 to 0.05 inch . the force ratio due to reaction of jet/, calculated ahead of the jet, was sizable and varied from 0.5 to 9 . in general, the ratio increased with increasing pressure ratio and decreasing slot width . for the turbulent boundary-layer separation tests it was found that the first peak pressure and the chordwise pressure distribution of the separated boundary layer ahead of the jet were similar to those for a separation caused by a forward-facing step at the same test conditions .'), (973, 'interaction effects produced by jet exhausting laterally near base of ogive-cylinder model in supersonic main stream .', 'vinson, p.w., amick, j.l. and liepman, h.p.', 'nasa memo 12-5-58w, february, 1959 .', 'interaction effects produced by jet exhausting laterally near base of ogive-cylinder model in supersonic main stream . the experimentally determined interaction effects of a side jet exhausting near the base of an ogive-cylinder model are presented and discussed . the interaction force appears to be independent of main-stream mach number, boundary-layer condition /laminar or turbulent/, angle of attack, and forebody length . the ratio of interaction force to jet force is found to be inversely proportional to the square root of the product of jet stagnation-to-free-stream pressure ratio and jet-to- body diameter ratio .'), (974, 'approximate analysis of thrust vector control by fluid injection .', 'wu, j.m., chapkis, r.l. and mager, a.', 'ars journal, 1677-1685, 1961 .', 'approximate analysis of thrust vector control by fluid injection . a study has been made of the side force generated by injection of secondary material into the main stream of a rocket nozzle . two cases have been analyzed .. gas injection and liquid injection . for the gas injection case, it is assumed that the turbulent boundary layer ahead of the injection point separates from the wall . the pressure in the separated region and the extent of the separated region are determined by a consideration of turbulent boundary layer-shock wave interaction and the accommodation height of the injected gas stream . equations are derived for calculating the side force, and the side forces predicted by the theory are compared with experimental data . the agreement between theory and experiment is fair . for the case of liquid injection, it is assumed that the liquid flows along the nozzle wall and evaporates into the main stream . the resulting side force on the nozzle wall is determined on the basis of linearized theory, thus restricting the analysis to small rates of liquid injection . the effects of small rates of heat addition are also included in the analysis . a very simple equation for calculating the side force is obtained .'), (975, 'one dimensional flows of an imperfect diatomic gas .', 'eggers,a.j.', 'naca r.959, 1950.', 'one dimensional flows of an imperfect diatomic gas . with the assumptions that berthelot''s equation of state accounts for molecular size and intermolecular force effects, and that changes in the vibrational heat capacities are given by a planck term, expressions are developed for analyzing one-dimensional flows of a diatomic gas . the special cases of flow through normal and oblique shocks in free air at sea level are investigated . it is found that up to a mach number of 10 the pressure ratio across a normal shock differs by less than 6 percent from its ideal gas value,. whereas at mach numbers above 4 the temperature rise is considerably below and hence the density rise is well above that predicted assuming ideal gas behavior . it is further shown that only the caloric imperfection in air has an appreciable effect on the pressures developed in the shock process considered . the effects of gaseous imperfections on oblique shock flows are studied from the standpoint of their influence on the lift and pressure drag of a flat plate operating at mach numbers of 10 and 20 . the influence is found to be small .'), (976, 'turbulent diffusion in the wake of a blunt nosed body at hypersonic speeds .', 'lees,l. and hromas,l.', 'j. ae. scs. 29, 1962, 976.', 'turbulent diffusion in the wake of a blunt nosed body at hypersonic speeds . at reynolds numbers greater than about 5 x 10 corresponding to altitudes below about 180,000 ft, the hot outer inviscid wake behind the bow shock wave produced by a blunt-nosed body at hypersonic speeds is cooled mainly by turbulent diffusion and conduction . turbulence originates in the inner wake formed by the coalescence of the free shear layers (or annulus) shed from the body surface when the boundary layer separates from the surface . as this turbulence spreads outward, it swallows enthalpy or momentum defect originally contained in the outer inviscid wake . if the turbulence is locally similar--i.e., if it behaves at each station like a slice of a low-speed /self-similar/ wake--then the turbulent diffusivity grows from a low initial value near the body to a value corresponding to the total drag of the body at about 300 body diameters downstream . at flight velocities of the order of 9,000-10,000 ft per sec. the growth of the turbulent inner wake predicted on the basis of locally similar turbulence is in good agreement with shadowgraph measurements of wake widths behind spheres obtained in ballistic ranges in the region from 200 to 4,000 body diameters downstream of the body . tentatively, one concludes that the turbulence mechanism in the wake with respect to a fixed observer is similar to the low-speed case, in spite of the large mean temperature gradients . in order to illustrate the behavior of an observable such as electron density in a turbulent wake behind a blunt body, the two limiting cases of thermodynamic equilibrium and pure diffusion (zero electron-ion recombination rate) are calculated for m = 22 at altitudes of 100,000 and 200,000 ft . even for the case of thermodynamic equilibrium, the predicted turbulent radar trail length is about 200 body diameters at l-band (1,300 mc) at 100,000-ft altitude and about 150 body diameters for uhf (400 mc) at 200,000 ft . one interesting result is that the width of the plasma cylinder corresponding to the plasma requency at l-band remains virtually constant at about 3.5 body diameters in the range 30 150 at 100,000-ft altitude . these results are sufficiently encouraging that one can consider including the effects of finite chemical and electron-ion recombination rates in the analysis in order to give a more complete picture of the wake at hypersonic speeds .'), (977, 'concerning some solutions of the boundary layer equations in hydrodynamics .', 'goldstein,s.', 'proc. cam. phil. soc. 26, 1930, 18.', 'concerning some solutions of the boundary layer equations in hydrodynamics . the boundary-layer equations for a steady two-dimensional motion are solved for any given initial velocity distribution (distribution along a normal to the boundary wall, downstream of which the motion is to be calculated) . this initial velocity distribution is assumed expressible as a polynomial in the distance from the wall . three cases are considered .. first, when in the initial distribution the velocity vanishes at the wall, but its gradient along the normal does not,. second, when the velocity in the initial distribution does not vanish at the wall,. and third, when both the velocity and its normal gradient vanish at the wall (as at a point where the forward flow separates from the boundary) . the solution is found as a power series in some fractional power of the distance along the wall, whose coefficients are functions of the distance from the wall to be found from ordinary differential equations . some progress is made in the numerical calculation of these coefficients, especially in the first case . the main object was to find means for a step-by-step calculation of the velocity field in a boundary layer, and it is thought that such a procedure may possibly be successful even if laborious . the same mathematical method is used to calculate the flow behind a flat plate along a stream . the results are shown in curves in the original .'), (978, 'temperature profiles inafinite solid with moving boundary .', 'dewey, c.f., schlesinger, s.i. and sashkin, l.', 'j. aero. sc. v. 27, 1960 .', 'temperature profiles inafinite solid with moving boundary . a numerical solution is presented to the transient heat conduction equation for a cylinder of finite thickness with one moving boundary . the implicit method of solution is developed with conductivity as an arbitrary function of temperature . application is made to a sample case of re-entry heating encountered by aerodynamic bodies, with erosion by sublimation and combustion occurring at the body surface .'), (979, 'correlation of base pressure and wake structure of sharp and blunt-nose cones with reynolds number based on boundary layer momentum thickness .', 'lehnert,r. and schermerhorn,v.l.', 'j. ae. scs. 26, 1959.', 'correlation of base pressure and wake structure of sharp and blunt-nose cones with reynolds number based on boundary layer momentum thickness . it has been established in the past that there is a certain relationship between base pressure and boundary-layer behavior . the base-pressure and wake-flow conditions were found to be dependent upon the local flow characteristics at the surface of supersonic vehicles directly upstream of the base or of the region of wake-flow separation . in order to use existing data on cones and other shapes to predict wake angle and base pressure on blunt bodies, an attempt was made recently at the naval ordnance laboratory to establish a unique relationship between given local flow conditions at the downstream end of sharp and blunt cones at supersonic speeds and the corresponding wake-flow conditions with zero heat transfer .'), (980, 'a method of computing the transient temperature of thick walls from arbitrary variation of adiabatic-wall temperature and heat-transfer coefficient .', 'hills, p.r.', 'naca report 1372, 1958 .', 'a method of computing the transient temperature of thick walls from arbitrary variation of adiabatic-wall temperature and heat-transfer coefficient . a method of calculating the temperature of thick walls has been developed in which are used relatively new concepts, such as the time series and the response to a unit triangle variation of surface temperature, together with essentially standard formulas for transient temperature and heat flow into thick walls . the method can be used without knowledge of the mathematical tools of its development . the method is particularly suitable for determining the wall temperature in one-dimensional thermal problems in aeronautics where there is a continuous variation of the heat-transfer coefficient and adiabatic-wall temperature . the method also offers a convenient means for solving the inverse problem of determining the heat-flow history when temperature history is known . a series of diversified problems were solved by exact analysis as well as by the new method . a comparison of the results shows the new method to be accurate . the labor involved is very modest in consideration of the nature of the thick-wall temperature problem . limiting solutions for the /infinitely thick/ wall and for walls so thin that thermal lag can be neglected were also obtained .'), (981, 'solutions to the heat-conduction equation with time dependent boundary conditions .', 'bergles, a.e. and kaye, j.', 'j. aero. sc. v. 28, march 1961. pp 251-252 .', 'solutions to the heat-conduction equation with time dependent boundary conditions . design charts based on the analytical solution to the problem of one-dimensional heat flow in a solid body of constant material properties with time-dependent boundary conditions were presented by kaye and yeh . this solution dealt with aerodynamic heating at hypersonic speeds where the surface coefficient of heat transfer and the temperature potential were taken to be linear functions of time of flight . in order to make these charts of more general application, general solutions are presented which, together with the charts, enable rapid and reasonable estimates to be made of the transient temperature distributions in many practical cases .'), (982, 'the temperature history in a thick skin subjected to laminar heating during entry into the atmosphere .', 'sutton, g.w.', 'jet propulsion, vol. 28, january 1959, p 40-5', 'the temperature history in a thick skin subjected to laminar heating during entry into the atmosphere . during high speed entry into the earth''s atmosphere, a vehicle can be afforded thermal protection for the short period of entry heating by a thick outer skin, sometimes called a /heat sink/ . the temperature distribution in such a heat sink has been found by integrating the product of the laminar aerodynamic heating rate and the appropriate green''s function for a finite-thickness wall over the generalized trajectory for a vehicle entering the earth''s atmosphere at high speeds dimensional heat conduction problem for laminar heating . the maximum surface temperature that occurs during the generalized entry trajectory for any combination of wall thickness and thermal properties is obtained from which the performance of any material can be found, provided that the average thermal properties may be used . as an example of the use of the solution, the performance of copper, graphite, molybdenum and tungsten are compared .'), (983, 'addendum to ''heat transfer to satellite vehicles re-entering the atmosphere .', 'detra, r.w., kemp, n.h. and riddell, f.r.', 'jet propulsion, vol. 27, december 1957 . pp 1256-57 .', 'addendum to ''heat transfer to satellite vehicles re-entering the atmosphere . the original paper gave a correlation formula for stagnation point heat transfer rate to a blunt body of revolution in hypersonic flow . this note gives a somewhat refined version, based on further calculation and shock tube data . its effect on the conclusions of the original paper is negligible except at surface temperature parameters of over 5000 r-ft1/8 . in other problems, where heat transfer rate itself is important, it can make a significant defference .'), (984, 'method of analysis for compressible flow through mixed-flow centrifugal impellers of arbitrary design .', 'hamrick, j.t. et al.', 'naca report 1082, 1952 .', 'method of analysis for compressible flow through mixed-flow centrifugal impellers of arbitrary design . a method is presented for analysis of the compressible flow between the hub and the shroud of mixed-flow impellers of arbitrary design . axial symmetry was assumed, but the forces in the meridional /hub to shroud/ plane, which are derived from tangential pressure gradients, were taken into account . the method was applied to an experimental mixed-flow impeller . the analysis of the flow in the meridional plane of the impeller showed that the rotational forces, the blade curvature, and the hub-shroud profile can introduce severe velocity gradients along the hub and the shroud surfaces . choked flow at the impeller inlet as determined by the analysis was verified by experimental results .'), (985, 'a rapid approximate method for the design of hub shroud profiles of centrifugal impellers of given blade shape .', 'smith, k.j. and hamrick, j.t.', 'naca tn 3399. 1955 .', 'a rapid approximate method for the design of hub shroud profiles of centrifugal impellers of given blade shape . a rapid approximate method for the design of centrifugal compressors of given blade shape with compressible nonviscous flow characteristics has been developed using techniques based upon stream-filament theory . axial symmetry is assumed, but meridional-plane forces derived from tangential pressure gradients are included . the method was applied to the design of an impeller in order to determine the approximate maximum meridional streamline spacing that could be used . three numerical solutions for different streamline spacings were made using the same hub profile, blade shape, and prescribed velocity distribution along the hub . the shroud profiles obtained from the three solutions, which utilized 3, 5, and 9 stream-tubes, were negligibly different . the approximate computing time required was 15 hours per streamtube .'), (986, 'design and test of mixed-flow impellers, viii - comparison of experimental results for three impellers with shroud redesigned by rapid approximate method .', 'osborn, w.m. et al.', 'naca rm e56l07, 1957 .', 'design and test of mixed-flow impellers, viii - comparison of experimental results for three impellers with shroud redesigned by rapid approximate method . three centrifugal impellers with parabolic, circular, and skewed-parabolic blading were modified by a recently developed design procedure to reduce the velocity gradients along the hub from inlet to outlet . all original dimensions except the shroud contours were retained . experimental investigation showed that the modified impellers had better performance characteristics than the original impellers at all speeds investigated, the greatest gains occurring at speeds of 1300 feet per second and higher . these large gains probably resulted primarily from more favorable velocity gradients and from designing these impellers further away from the condition necessary for eddy formation . the modified impellers were thus able to operate over a wider range of weight flows at high speeds . the modified impellers were investigated over a range of equivalent speeds of 900 to 1500 feet per second and flow rates from maximum to the point of incipient surge . at 1300 feet per second, the peak pressure ratio and maximum adiabatic temperature-rise efficiency for the parabolic-bladed impeller were 3.07 and 0.825, respectively . for the same conditions, the circular-bladed impeller and the skewed-parabolic-bladed impeller had pressure ratios of 3.13 and 3.15 and efficiencies of 0.737 and 0.805, respectively . of the three, the parabolic-bladed impeller had the highest maximum efficiencies /0.854 to 0.800/ and the best weight-flow range over the speed range tested . on the basis of the parameters investigated, it appears that parabolic blading is superior to circular blading . the experimental results indicate that the design method of naca tn 3399 is a reliable method for use in designing centrifugal impellers .'), (987, 'a general theory of three dimensional flow in subsonic and supersonic turbo-machines of axial-radial-and mixed-flow types .', 'wu, c.', 'naca tn 2604, 1952 .', 'a general theory of three dimensional flow in subsonic and supersonic turbo-machines of axial-radial-and mixed-flow types . a general theory of steady three-dimensional flow of a nonviscous fluid in subsonic and supersonic turbomachines having arbitrary hub and casing shapes and a finite number of blades is presented . the solution of the three-dimensional direct and inverse problem is obtained by investigating an appropriate combination of flows on relative stream surfaces whose intersections with a z-plane either upstream of or somewhere inside the blade row form a circular arc or a radial line . the equations obtained to describe the fluid flow on these stream surfaces show clearly the several approximations involved in ordinary two-dimensional treatments . they also lead to a solution of the three-dimensional problem in a mathematically two-dimensional manner through iteration . the equation of continuity is combined with the equation of motion in either the tangential or the radial direction through the use of a stream function defined on the surface, and the resulting equation is chosen as the principal equation for such flows . the character of this equation depends on the relative magnitude of the local velocity of sound and a certain combination of velocity components of the fluid . a general method to solve this equation by both hand and high-speed digital machine computations when the equation is elliptic or hyperbolic is described . the theory is applicable to both irrotational and rotational absolute flow at the inlet of the blade row and at both design and off-design operations .'), (988, 'nonviscous flow through a pump impeller on a blade-to-blade surface of revolution .', 'kramer, j.j. et al.', 'nasa tn d-1108, 1962,', 'nonviscous flow through a pump impeller on a blade-to-blade surface of revolution . the nonviscous incompressible flow through a typical pump impeller is analyzed on a blade-to-blade surface of revolution . solutions are obtained for a variety of inlet conditions including several with prewhirl of the assumed location of the rear stagnation point . comparison of results from two approximate methods of analysis showed good agreement for the zero-angle-of-attack case and reliable indication of the existence of an eddy on the driving face at a large positive angle of attack .'), (989, 'incompressible nonviscous blade-to-blade flow through a pump rotor with splitter vanes .', 'kramer, j.j. et al.', 'nasa tn d-1186, 1962 .', 'incompressible nonviscous blade-to-blade flow through a pump rotor with splitter vanes . the nonviscous flow through a mixed-flow pump impeller having one splitter vane between adjacent main blades has been analyzed on a blade-to-blade surface of revolution using a previously reported analysis method . solutions were obtained for a variety of flow conditions including several cases in which whirl is imparted to the flow upstream of the impeller . the velocity distributions on the main-blade surfaces and on the splitter-vane surfaces in the region of the splitter vane were strongly dependent on the assumed location of the rear stagnation points . solutions were obtained by assuming values of slip factor and of division of flow around the splitter in addition to assuming the location of the rear stagnation points . these solutions indicated that the velocity distributions in the splitter-vane region are largely determined by the division of flow around the splitter vane and that only the region in the immediate vicinity of the trailing edge is affected by the slip factor . blade surface velocities were obtained from two approximate methods by specifying flow division and slip factor, and these results are compared with the more exact solutions of the analysis .'), (990, 'a rapid approximate method for determining velocity distribution on impeller blades of centrifugal compressors .', 'stanitz, j.d. and prian, v.d.', 'naca tn 2421, 1951 .', 'a rapid approximate method for determining velocity distribution on impeller blades of centrifugal compressors . a rapid approximate method of analysis was developed for both compressible and incompressible, nonviscous flow through radial- or mixed-flow centrifugal compressors with arbitrary hub and shroud contours and with arbitrary blade shape . the method of analysis is used to determine approximately the velocities everywhere along the blade surfaces, but no information concerning the variation in velocity across the passage between blades is given . in eight numerical examples for two-dimensional flow, covering a fairly wide range of flow rate, impeller-tip speed, number of blades, and blade curvature, the velocity distribution along the blade surfaces was obtained by the approximate method of analysis and compared with the velocities obtained by relaxation methods . in all cases the agreement between the approximate solutions and the relaxation solutions was satisfactory except at the impeller tip where the velocities obtained by the approximate method did not, in general, become equal on both surfaces of the blade as required by the joukowski condition .'), (991, 'wing-flow study of pressure drag reduction at transonic speed by projecting a jet of air from the nose of a prolate spheroid of fineness ratio 6 .', 'lopatoff, m.', 'naca rm l51e09, 1951 .', 'wing-flow study of pressure drag reduction at transonic speed by projecting a jet of air from the nose of a prolate spheroid of fineness ratio 6 . a study was made at transonic speeds by the naca wing-flow method of the pressure-drag reduction obtained by projecting a high-energy jet of air from the nose of a prolate spheroid . supplementary information was obtained by taking shadowgraphs of the model mounted in a small supersonic tunnel at a constant mach number of 1.5 . the high-velocity jet was observed to alter the pressure distribution over the body in such a way that the pressure drag of the body was reduced,. thus, in a restricted sense, the nose jet produced a thrust on the body . under the conditions investigated, the thrust produced by the nose jet was never so large as that which would be expected from a conventional rearward jet . for example, under the best conditions tested /mach number of 1.07/ the reduction in body pressure drag caused by the nose jet more than compensated for the negative thrust of the jet itself . however, the magnitude of the net reduction in drag /change in body pressure drag with jet on and jet off minus the adverse thrust of the jet/ was only about one-half of the thrust which would be produced by the same jet exhausting rearward . the appearance of such an unexpectedly large effect in the first trial indicated the phenomenon to be worth further study .'), (992, 'the effects of a small jet of air exhausting from the nose of a body of revolution in supersonic flow .', 'love, e.s.', 'naca rm l52119a, 1952 .', 'the effects of a small jet of air exhausting from the nose of a body of revolution in supersonic flow . an investigation has been made at a mach number of 1.62 to determine the effects of a small jet of air exhausting from the nose of an elliptical body of revolution upon boundary-layer transition and the viscous, pressure, and total drag of the forebody at three body stations body nose were also obtained . the tests were conducted at reynolds numbers of 2.13 x 10 and 7.66 x 10, based on body length . the maximum range of thrust coefficients for the small jet was from 0 to about at the lower test reynolds number, for which the boundary layer was laminar over the entire body in the jet-off condition, a very small flow from the jet moved the point of transition forward to the vicinity of the 20-percent-body station . as the jet flow was increased, the transition point moved abruptly to the nose at a thrust coefficient of about gardless of the type of boundary layer . at the higher test reynolds number for which the boundary layer was largely turbulent in the jet-off condition the total drag, including skin friction, was reduced somewhat by the action of the jet . although the forward-exhausting small jet was found to have the above favorable effects upon the drag, these findings are not believed too important since the question arises as to the benefits of the same small jet exhausting from the rear of the body in the conventional manner . no attempt was made to establish geometric optimums in the present investigation, yet, from a general consideration of the benefits indicated by the present results and the phenomena known to occur in the vicinity of rearward-exhausting jets, the benefits of a small jet exhausting rearward would appear to exceed those of the same small jet exhausting forward, particularly so when the flow over the body is laminar in the jet-off condition .'), (993, 'the extent of the jet interference flow fields . jet effects on cylindrical afterbodies housing sonic and supersonic nozzles which exhaust against a supersonic stream at angles of attack from 90degree to 180degree .', 'hayman, l.o. and mcdearmon, r.w.', '', 'the extent of the jet interference flow fields . jet effects on cylindrical afterbodies housing sonic and supersonic nozzles which exhaust against a supersonic stream at angles of attack from 90degree to 180degree . an investigation has been made to determine jet effects on cylindrical afterbodies housing sonic and supersonic nozzles which exhaust against a supersonic stream at angles of attack from 90 to 180 . the tests were conducted at a free-stream mach number of 2.91 and at free-stream reynolds numbers, based on body diameter, of 0.15x106 and stream static pressure investigated was from jet off to about 400 . the data presented herein showed that, in general, variation of the ratio of jet total pressure to free-stream static pressure, jet-exit mach number, and ratio of jet-exit diameter to body diameter had large influences on the body pressures on the windward halves of the after-bodies and negligible influences on the leeward pressures . there was a negligible effect of reynolds number on the body pressures . the ratio of jet total pressure to free-stream static pressure also had a large influence on the base pressures at all angles of attack . schlieren studies showed details of the shock-wave structure caused by the jet and the extent of the jet interference flow fields .'), (994, 'investigation of a retrocket exhausting from the nose of a blunt body into a supersonic free stream .', 'charczenko, n. and hennessy, k.w.', 'nasa tn d-1016, 1962 .', 'investigation of a retrocket exhausting from the nose of a blunt body into a supersonic free stream . the pressure distribution and pressure drag of a blunt body with a supersonic jet issuing upstream from its center were determined at free-stream mach numbers of 1.60, 2.00, and 2.85 . the thrust of the jet issuing from the model nose was varied to study its effects on flow around the model and to determine variation of pressure distribution and pressure drag of the model with the thrust . at all mach numbers investigated, the pressure drag decreased with increasing retrorocket thrust until a minimum value was reached . further increases in retrorocket thrust resulted in increases in the pressure drag . the resultant drag /pressure drag plus retrorocket thrust but excluding base and skin-friction drag/ of the model was reduced by retrorocket operation below the drag for a jet-off condition, except at very low retrorocket thrust coefficients . the flow about the nose of the blunt body was very unstable throughout the range of mach numbers and retrorocket thrust coefficients investigated .'), (995, '', '', '', ''), (996, 'extension of boundary layer separation criteria to a m=6 .5 utilizing flat plates with forward-facing steps .', 'sterrett,j.r. and emery,j.c.', 'nasa tn.d618, 1960.', 'extension of boundary layer separation criteria to a m=6 .5 utilizing flat plates with forward-facing steps . an experimental investigation has been made of the separation phenomena on a flat plate to which forward-facing steps were attached to force separation . both laminar and turbulent flows were investigated over a mach number range of approximately distributions, shadowgraph and chemical film techniques, the pressure rise at separation, the laminar plateau pressure, and the turbulent peak pressure were determined . boundary-layer surveys were made on a smooth flat plate and on a flat plate with roughness to force transition . examinations of the separated flow showed that the predominant variable in the determination of the pressure distribution was the location of transition relative to the separation point and reattachment . pure laminar, transitional, and turbulent types of separation were found in this mach number range . the peak static-pressure-rise ratios for identical forward-facing steps at a mach number of 6.25 were approximately 1.5 and 5.0, respectively, for pure laminar and turbulent separation . the effect of reynolds number on the peak pressure rise for turbulent separation for the lower mach number range was found to be very minor provided the step height was of the order of the boundary-layer thickness . as the mach number is increased, the peak pressure coefficient for turbulent separation decreased from approximately 0.18 at a mach number of 4 to about 0.13 at a mach number of 6.25 . the pressure coefficient at the separation point for laminar separation decreases from approximately 0.014 at a mach number of value at a mach number of 6.5 . the results obtained with forward-facing steps agree with the trends predicted, based upon lower mach number studies .'), (997, 'experimental and theoretical studies of axisymmetric free jets .', 'love, e.s. et al.', 'nasa tr r-6, 1959 .', 'experimental and theoretical studies of axisymmetric free jets . some experimental and theoretical studies have been made of axisymmetric free jets exhausting from sonic and supersonic nozzles into still air and into supersonic streams with a view toward problems associated with propulsive jets and the investigation of these problems . for jets exhausting into still air, consideration is given to the effects of jet mach number, nozzle divergence angle, and jet static-pressure ratio upon jet structure, jet wavelength, and the shape and curvature of the jet boundary . studies of the effects of the ratio of specific heats of the jets are included as are observations pertaining to jet noise and jet simulation . for jets exhausting into supersonic streams, an attempt has been made to present primarily theoretical curves of the type that may be useful in evaluating certain jet interference effects and in formulating experimental studies . the primary variables considered are jet mach number, free-stream mach number, jet static-pressure ratio, ratio of specific heats of the jet, nozzle exit angle, and boattail angle . the simulation problem and the case of a hypothetical hypersonic vehicle are examined'), (998, 'equations, tables and charts for compressible flow .', 'ames research staff.', 'naca r.1135, 1953.', 'equations, tables and charts for compressible flow . this report, which is a revision and extension of naca tn useful in the analysis of high-speed flow of a compressible fluid . the equations provide relations for continuous one-dimensional flow, normal and oblique shock waves, and prandtl-meyer expansions for both perfect and imperfect gases . the tables present useful dimensionless ratios for continuous one- dimensional flow and for normal shock waves as functions of mach number for air considered as a perfect gas . one series of charts presents the characteristics of the flow of air (considered a perfect gas) for oblique shock waves and for cones in a supersonic air stream . a second series shows the effects of caloric imperfections on continuous one-dimensional flow and on the flow through normal and oblique shock waves .'), (999, 'static aerodynamic characteristics of short blunt cones with various nose and base cone angles at mach numbers of 0. 6 to 5. 5 and angles of attack to 180 .', 'treon,s.l.', 'nasa tn.d1327, 1962.', 'static aerodynamic characteristics of short blunt cones with various nose and base cone angles at mach numbers of 0. 6 to 5. 5 and angles of attack to 180 . wind-tunnel tests have been performed at mach numbers from 0.6 to 5.5 to determine coefficients of normal force, axial force, and pitching moment for short blunt cones, as affected by changes in nose and base cone angles . models with nose half-angles of 10 and 20 were investigated . the 10 nose half-angle models were tested with a flat base and with base cones of 50 and 70 half-angle . the 20 nose half-angle model had a 50 half-angle base cone . reynolds numbers for the test ranged from about maximum diameter . variations in the base cone angle resulted in significant changes in the aerodynamic characteristics, with lesser effects resulting from changes in nose cone angle . in particular, the model with the 50 half-angle conical base had only one trim angle flat base and 70 half-angle conical base had two trim angles (a = 0 and a = 180) . estimated variations of the aerodynamic characteristics with angle of attack by means of a modified newtonian theory were in good agreement with the experimental results . the theory, however, failed to predict the trim point at a = 180 for the flat-based model .'), (1000, 'free-flight measurements of the static and dynamic stability and drag of a 10 blunted cone at mach numbers 3 .5 and 8 .5 .', 'intrieri, p. f.', 'nasa tn. d1299, 1962.', 'free-flight measurements of the static and dynamic stability and drag of a 10 blunted cone at mach numbers 3. 5 and 8 .5 . tests were made of a short blunt-nosed without a 50 half-angle conical afterbody in a pressurized ballistic range at nominal mach numbers of 3.5 and of 90,000 and 220,000, respectively . it was found that the models were statically stable about the center- of-gravity location tested but exhibited neutral dynamic stability for flight at constant altitude . the static stability was not affected by the but was nonlinear with angle of attack and varied with mach number . the nonlinear variation of the pitching moment with angle of attack was accurately approximated by a cubic polynomial . the static stability was only qualitatively predicted by modified newtonian theory . the drag characteristics were in good agreement with values calculated by use of modified newtonian theory . calculations of the oscillatory behavior of the configurations flying an example entry trajectory through the martian atmosphere indicated the configurations to be dynamically satisfactory . pitching motions should converge to a small fraction of the amplitude at entry, provided the initial angle of attack and pitch rate are not large enough to cause tumbling .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1001, 'wind tunnel investigation of the static and dynamic stability characteristics of a 10degree semivertex angle blunted cone .', 'wehrend, w.r.', 'nasa tn d-1202, 1962 .', 'wind tunnel investigation of the static and dynamic stability characteristics of a 10degree semivertex angle blunted cone . the static and dynamic stability characteristics of a blunted 10degree semivertex angle cone were studied . the cone which had a modified spherical segment nose was tested with a flat base and with a truncated conical base . all tests were performed in air at mach numbers from 0.65 to 2.20 with the angle-of-attack range from -4degree to +18degree . presented are measurements of the normal force, axial force, base pressure, and pitching moment from the static tests, and the damping-in-pitch moment from the dynamic tests . both models had satisfactory stability characteristics throughout the test mach number range but the addition of the conical afterbody had a large destabilizing effect .'), (1002, 'preliminary investigations of spiked bodies at hypersonic speeds .', 'bogdonoff, s.m. and vas, i.e.', 'jnl. aero. sci. february 1959, p. 65-74 .', 'preliminary investigations of spiked bodies at hypersonic speeds . generally accepted solutions for the problems of hypersonic flight appear, at the moment, to be centered around the use of blunt bodies to minimize the heat-transfer rates . there are, however, several other solutions to the problem, and, as part of an exploratory study of these solutions, a detailed examination has been made of the flow over blunt bodies equipped with a spike . these tests, carried out at a mach number of about 14 in the princeton helium hypersonic tunnel, have investigated the effect of varying spike lengths for flat-faced and hemispherically-nosed axially symmetric bodies . detailed pressure distributions have been obtained as well as heat-transfer rates . these exploratory studies have shown that the use of a spike protruding from a hemispherical-nosed cylinder at m 14 decreased the pressure level by an order of magnitude and the heat transfer to a fraction of that measured on a hemisphere without a spike . the general technique appears to hold considerable promise for hypersonic flight .'), (1003, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . the real-gas hypersonic flow parameters for helium have been calculated for stagnation temperatures from 0 f to 600 f and stagnation pressures up to 6,000 pounds per square inch absolute . the results of these calculations are presented in the form of simple correction factors which must be applied to the tabulated ideal-gas parameters . it has been shown that the deviations from the ideal-gas law which exist at high pressures may cause a corresponding significant error in the hypersonic flow parameters when calculated as an ideal gas . for example, the ratio of the free-stream static to stagnation pressure as calculated from the thermodynamic properties of helium for a stagnation temperature of 80 f and pressure of 4,000 pounds per square inch absolute was found to be approximately 13 percent greater than that determined from the ideal-gas tabulation with a specific heat ratio of'), (1004, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . the effects of contamination of helium by air upon static-pressure, total-pressure, heat-transfer, and temperature measurements have been investigated in the 2-inch helium tunnel at the langley research center . within the scope of the tests, even a small amount of air is shown to affect these measurements . the heat-transfer and temperature measurements were made on a 26.6 half-angle cone and demonstrated the effects of contamination qualitatively . the wall static and center-line pitot pressures show that if the contaminating air is held to less than about 0.2 percent by volume, the error in indicated mach number is less than 1 percent as calculated from the rayleigh pitot equation . the corresponding errors in wall static and center-line pitot pressures are about 1.7 and 0.4 percent, respectively .'), (1005, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . equations based on newtonian impact theory have been derived and a computational procedure developed with the aid of several design-type charts which enable the determination of the aerodynamic forces and moments acting on arbitrary bodies of revolution undergoing either separate or combined angle-of-attack and pitching motions . bodies with axially increasing and decreasing cross-sectional area distributions are considered,. nose shapes may be sharp, blunt, or flat faced . the analysis considers variations in angle of attack from -90 to 90 and allows for both positive and negative pitching rates of arbitrary magnitude . the results are also directly applicable to bodies in either separate or combined sideslip and yawing maneuvers .'), (1006, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . the inviscid flow of a perfect gas over blunt-nosed axisymmetric and two-dimensional bodies at zero angle of attack has been calculated numerically on an ibm 7090 computer . the computation consisted of the fuller blunt-body solution for the subsonic and transonic regions and the method of characteristics for the supersonic region . the flow fields about a number of blunt bodies were studied, and the calculated results showed good agreement with experimental shock-wave shapes, surface-pressure distributions, and flow-field surveys .'), (1007, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . this report presents equations, tables, and figures for use in the analysis of helium flow at supersonic and hypersonic speeds . the contents of the report and presentation of the data parallel that of a similar reference work (naca rep. 1135) prepared for air flow . the perfect-gas relations for continuous one-dimensional flow, normal- and oblique-shock waves, and prandtl-meyer expansions are the same as for air but are presented here for completeness . the tables present the values of useful dimensionless ratios for continuous one-dimensional flow and for normal-shock waves as functions of mach number . the helium viscosity relation as a function of temperature, mass-flow rates as a function of mach number and temperature, and the reynolds number as a function of mach number and stagnation temperature are plotted . the oblique-shock characteristics of wedges and cones in helium at mach numbers of 12, 16, 20, and 24 are presented in a series of plots . throughout all the computations, helium is considered to be a perfect gas .'), (1008, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . representative experimental results are presented to show the current status of the panel flutter problem . results are presented for unstiffened rectangular panels and for rectangular panels stiffened by corrugated backing . flutter boundaries are established for all types of panels when considered on the basis of equivalent isotropic plates . the effects of mach number, differential pressure, and aerodynamic heating on panel flutter are discussed . a flutter analysis of orthotropic panels is presented in the appendix .'), (1009, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . charts of thermodynamic properties for equilibrium air are presented with sufficient accuracy to permit the calculation of flow parameters in hypersonic nozzles operating at stagnation temperatures up to 4,950 r and pressures up to 1,000 atm . flow parameters calculated from these charts are presented for a series of stagnation temperatures between use of these parameters, it is possible to calibrate a nozzle in the conventional way . a method is also presented from which the flow parameters for conditions other than those chosen herein may be calculated . real-gas effects on the calculation of a hypersonic nozzle contour are shown by an example calculation in which the nozzle contour for mach number 12 was determined by including real-gas effects, and this contour was compared with one calculated by ideal-gas considerations . also presented are the approximate limiting mach numbers at which equilibrium air will just condense for various combinations of stagnation temperatures and pressures .'), (1010, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . air-flow properties in nozzles were calculated and charted for equilibrium flow and two types of frozen flows . in one type of frozen flow, air was assumed to be in equilibrium from the nozzle reservoir to arbitrary points where chemical reactions and molecular vibrations became frozen . in the other type, it was assumed that molecular vibrations were in equilibrium throughout the nozzle and that chemical reactions became frozen at arbitrary points . the calculations were made for a range of stagnation pressures up to 10,000 poinds per square inch absolute and stagnation enthalpies up to 24,500 btu per pound . the flow properties charted were temperature, pressure, density, velocity, dynamic pressure, mach number, reynolds number, molecular weight fraction, and mass flow . equilibrium flow properties through normal shock waves were also included .'), (1011, 'free-flight measurements of the static and dynamic .', '', '', 'free-flight measurements of the static and dynamic . charts have been prepared relating the thermodynamic properties of air in chemical equilibrium for temperatures to 15,000 k and for pressures from 10 to 10 atmospheres . also included are charts showing the composition of air, the isentropic exponent, and the speed of sound . these charts are based on thermodynamic data calculated by the national bureau of standards .'), (1012, 'principles of creep buckling weight-strength analysis of aircraft structures .', 'shanley,f.r.', 'part iii. p 343, mcgraw-hill, new york, 1952.', 'principles of creep buckling weight-strength analysis of aircraft structures . the possibility of a gradual instability failure of a column under compressive load has been recognized for some time . marin presented an analysis of creep buckling based on a theory of creep bending, but did not take into account the average stress due to axial loading . the theory also neglected the transient (nonlinear) portion of the creep curve . in efficient column design, the average stress should be relatively high in comparison with the bending stresses,. that is, the column should be as straight as possible and the slenderness ratio should not be too great . under these conditions marin''s theory is not directly applicable, although it gives good agreement with tests of columns having large slenderness ratios or large eccentricities .'), (1013, 'principles of creep buckling weight-strength analysis .', '', '', 'principles of creep buckling weight-strength analysis . published work on creep buckling has implied that failure of columns after a critical time is caused by initial imperfections . such analyses are relatively complex and ultimately leave the choice of selecting the proper value of the initial imperfection to the designer . furthermore, recent test results on creep buckling of columns have indicated that there is a random and relatively unimportant effect of small initial imperfections on the critical time . to avoid the difficulties associated with initial imperfections, a formulation of the creep buckling phenomenon in terms of classical stability theory is presented . the theory permits the extension of known solutions for plastic buckling of certain thin plates and shells to creep buckling problems .'), (1014, 'principles of creep buckling weight-strength analysis .', '', '', 'principles of creep buckling weight-strength analysis . a problem of creep stability of columns and plates is considered . in an analysis use is made of two forms of the creep theory based on the strain hardening hypothesis . for a uniformly compressed palte a comparison is made between the results according to the flow theory and strain theory .'), (1015, 'principles of creep buckling weight-strength analysis .', '', '', 'principles of creep buckling weight-strength analysis . the determination of column deflections and column buckling loads has been considered for many years . the available theories, however, do not provide for materials which creep with time under constant loads . for the design of structural members made of these materials, a consideration of creep may be of practical importance . plastics, concrete, and some metals creep at normal temperatures while other metals creep only at high temperatures and at stress values beyond the yield point . a consideration of creep in the design of some structures appears appropriate in view of the modern developments in plastics and the presence of high stress values which are sometimes beyond the yield stress . this paper gives a rational theory for predicting creep deflections in columns . a special case using this theory is applied to the interpretation of some preliminary tests of an aluminum alloy .'), (1016, 'principles of creep buckling weight-strength analysis .', '', '', 'principles of creep buckling weight-strength analysis . the relation of the time-dependent tangent-modulus load--as conceived by shanley--to actual column capacity is clarified . it may be interpreted as a limiting case of the conservative estimate . the time-dependent tangent-modulus load is, therefore, an approximation to a conservative estimate . the approximation, however, may be either conservative or nonconservative when applied to imperfect or real columns . typical cases are discussed and experimental results for two alloys are cited .'), (1017, 'note on creep buckling of columns .', 'gerard,g.', 'j. ae. scs. 19, 1962, 714.', 'note on creep buckling of columns . it appears from librove''s interesting analysis that, for the case of creep buckling of columns, the initial imperfections contained in ordinary columns provide the mechanism by which failure due to creep occurs after a period of time . in fact, it can be concluded from this analysis that a theoretically perfect column that is initially loaded below the time-independent critical load will not buckle at all . this is an interesting contrast to the case of static buckling where small initial imperfections play an insignificant role, since the failing load of an initially imperfect column is substantially the same as that of a theoretically perfect column . it is of interest, therefore, to conjecture whether there is any possible mechanism by which a column containing no initial imperfections can fail as a result of creep when the initial load is less than the theoretical buckling load .'), (1018, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . the results of short-time elevated-temperature creep tests of objective of obtaining procedures for predicting column lifetime . semiempirical lifetime curves are obtained with the aid of a previously published column creep theory and are used for deriving column curves . the semiempirical lifetime curves are also used to study the effect of varying applied stress and out-of-straightness . in the range considered, small variations in out-of-straightness are found to be of little practical significance,. whereas, small stress variations change the column lifetime considerably . for the range of out-of-straightness encountered in the tests, the data can be presented in plots that do not explicitly include out-of-straightness, and plots of this type should be satisfactory for predicting column lifetime for design purposes .'), (1019, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . a method for estimating allowable load capacities of columns subject to creep is presented . the method, which utilizes approximate stress distributions derived from isochronous-stress-strain curves to estimate column load capacities, is shown to be conservative for the time for which the estimate is made . an application of the method is made to test data on as-received and on stabilized 24s-t4 aluminum alloy . a comparison of the computed column capacities with experimental capacities indicates that the method is satisfactory for estimating the decrease in capacity with increasing time . easily obtained, time-dependent tangent-modulus loads are discussed . they are interpreted as being approximations to allowable load-capacity estimates . a limited application is made to test data, and the results appear promising . it is concluded that if certain limitations are recognized, the method may prove to be useful because of its simplicity . a presentation of the results of an experimental investigation of the effects of column imperfection and column-material variation is made . it is found that column-capacity variations of the order of 10 per cent can result from column-imperfection differences and column-material variation . the results of an experimental study of the variation of column capacity with temperature of exposure are presented . they indicate that column efficiency, as measured by decrease in capacity, can be acceptable for very short times at the higher temperatures . the efficiency at these higher temperatures falls rapidly, however, with increasing time .'), (1020, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . the results of short-time creep-buckling and creep-bending tests of slenderness ratio 111 are presented . the tests were performed at 600 f, and strain measurements were taken with high-temperature electric-resistance strain gages . a description of the development of the gages is given in an appendix . the column tests show that the critical time decreases much more rapidly with increasing load than with increasing initial deviation from straightness . the bending tests indicate that the steady creep rate of the curvature is a simple power function of applied moment . these latter results, together with a previously derived creep-buckling theory, are used to develop a semiempirical formula suitable as a guide for the determination of the critical time for columns .'), (1021, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . this paper describes theory and tests of the creep collapse of long thin aluminum-alloy cylinders under external radial pressure . steady-state creep is assumed in the theoretical derivation . the test temperatures were between 300 and 500 f . the collapse time for each cylinder was calculated theoretically . agreement between theoretical and test results was fair .'), (1022, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . forty-three cylinders of 40-inch length and 16-inch diameter, made of 5052-0 aluminum-alloy sheets of thickness, were subjected to bending moments constant along the cylinder and in time in an oven which maintained a constant temperature of 500 f during the test . all the cylinders failed by buckling . the time that elapsed between load application and collapse was measured .'), (1023, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . a theory of creep buckling is presented in which the instantaneous elastic and plastic deformations following the application of a load, as well as the steady creep deformations, are considered in an approximate manner . equations are given from which the critical time, that is, the time elapsing between load application and the collapse of the column, can be computed .'), (1024, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . the general dynamic equation of creep bending of a beam loaded laterally and axially was derived for a linearly viscoelastic material whose mechanical properties can be characterized by four parameters . the material can exhibit instantaneous and retarded elasticity as well as pure flow . the equation derived was used to obtain the creep bending deflection of a beam in pure bending and of a column with initial sinusoidal deviation from straightness . as expected, the ratio of the creep deflections of the beam in pure bending and the deflections of a corresponding purely elastic structure is identical to the ratio of the creep strain and the corresponding elastic strain of a bar under simple tension or compression . the results of the analysis of the creep deflection of the column showed that the deflections increase continuously with time and become infinitely large only when the loading time is correspondingly large . however, large deflections are obtained in reasonably short periods of time if the applied load is near to the euler load of the column . the deflection-time curves obtained from a numerical example are of the same type as those determined by experiment with aluminum columns .'), (1025, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . the creep of a slightly crooked section column carrying a constant load is studied theoretically . the material of the column is characterized by a strain-time relationship, under constant uniaxial stress, of the form, where is the total strain, is the constant stress, is the time, and e,a,b, and k are material constants . this form was selected because it applies to at least two alloys--75s-t6 aluminum alloy at 600 f . and a low-alloy steel at 800 and 1,100 f . however, the analysis is intended for any material having creep properties of the same form and for which the material constants are known . a strain-time relationship under variable uniaxial stress, necessary for the column analysis, is formulated from the constant-stress properties with the aid of shanley''s engineering hypotheses of creep . the analysis leads to the conclusion that the lateral deflection approaches infinity--that is, the column collapses--in finite time . results are given showing the maximum length of time the column can support a given load before it collapses and the growth of stresses, strains, and deflections prior to collapse .'), (1026, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . this paper is concerned with the solution of the creep buckling of columns . instantaneous elastic and plastic deformations, as well as the transient and secondary creep, are considered . formulae for the critical time at which a column fails are presented for integral values of the exponents appearing in the creep law .'), (1027, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . a phenomenological relation between stress, strain rate, and temperature is suggested to account for the behavior of polycrystalline metals above the equicohesive temperature . the properties of the metal included in the relation are elasticity, linear thermal expansion, and viscosity . the relation may be integrated under various conditions to provide information on creep rates, creep rupture, stress-strain curves, and rapid-heating curves . it is shown that for one material - 7075-t6 aluminum-alloy sheet - the information yielded by the relation for these four applications agrees reasonably well with test data .'), (1028, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . the phenomenological theory previously proposed in naca technical note 4000 for the behavior of metals at elevated temperatures has been modified to yield transient creep curves by assuming that the metal consists of two phases, each with its own elasticity and viscosity . the extended theory satisfies the basic requirements for a theory of transient creep at elevated temperatures .. that the transient creep be closely connected with the subsequent steady creep, and that the apparent exponent of the time in the transient region be permitted wide variations between 0 and 1 . from this theory it is possible to construct nondimensional creep curves which extend continuously from the transient region into the steady-state region . the corresponding family of creep curves for any metal may be obtained from the nondimensional family by use of appropriate constants . the constants required are those obtained from steady creep measurements, together with two additional constants which represent the difference between the phases . the transient creep curves resulting from this theory are compared with the experimental curves for pure aluminum, gamma iron, lead, and agreement is found .'), (1029, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . a variational theorem is presented for a body undergoing creep . solutions to problems of the creep behavior of plates, columns, beams, and shells can be obtained by means of the direct methods of the calculus of variations in conjunction with the stated theorem . the application of the theorem is illustrated for plates and columns by the solution of two sample problems .'), (1030, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . some general topics in elastic stability are discussed . in particular, attention is given to the relationship between adjacent-equilibrium-position and energy techniques, to the effects of nonlinearity, and to the sensitivity of certain stability problems to the character of the loading .'), (1031, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . a general variational theory of elastic stability that was originated by e. trefftz (1) is applied to the problem of buckling of rings of rectangular cross section subjected to uniform external pressure . the theory is believed to be more rigorous than previous treatments of the problem, since it avoids conventional assumptions of curved-beam theory, such as the assumptions that plane sections remain plane and that radial stresses vanish . the classical result of levy (2) is confirmed for a ring of infinitesimal thickness . new results are obtained which show the effect of the finite thickness of a ring on the coefficients in the buckling formula .'), (1032, 'on the conservativeness of various distributed force systems .', 'sol r. bodner', 'assistant professor of engineering, brown university, providence, r.i.', 'on the conservativeness of various distributed force systems . the necessity of determining the conservativeness of force systems in instability problems is discussed in reference 1 . it is shown that, whereas kinetic methods are generally applicable for the determination of instability loads, the statical methods usually employed are valid only for conservative and nongyroscopic systems . small changes in the character of the loading could make an otherwise conservative system nonconservative and cause a large change in the magnitude of the buckling load . the buckling load of the cantilever column example in reference 1 is, for tangential end loading, eight times that for conservative, constant directional loading .'), (1033, 'the design of tubes under uniform external pressure on the basis of assumed inaccuracies .', '', '', 'the design of tubes under uniform external pressure on the basis of assumed inaccuracies . since the failure of tubes under uniform external pressure depends very much upon the various kinds of imperfections in them, it seems logical to derive a design formula for such tubes in which the quantities depending on imperfections will appear explicitly . the most common imperfection in tubes is an initial ellipticity, the limiting value of which in each type of tube is usually well-known from numerous inspection measurements . the deviation of the shape of the tube from a perfect circular form can be defined by the initial radial deflections w'' .'), (1034, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . a long, thin-walled cylindrical shell is loaded by a uniform external pressure . equations are developed for the time behavior of the shape of the cross section under the following conditions .. formations expressible by a power creep law,. (b) the initial and subsequent mode shape of the deviations from circularity of any cross section is two-lobed,. and (c) the shell construction is of the sandwich type, with concentric cylindrical membranes taking normal stresses and an annular core supporting shear without deformation . explicit solutions are obtained for the particular case of the cubic creep law . it is shown that the nondimensional amplitude of the cross-sectional mode shape (briefly, shape factor) will become infinite in a finite time . curves of shape factor versus time and of collapse time versus initial value of the shape factor are presented . also given are an explicit expression for and a curve of the expected variation in collapse time owing to uncontrollable deviations from a nominal initial value of the shape factor . it is shown that the expected variation is small if the nominal initial shape factor value is sufficiently large .'), (1035, 'note on creep buckling of columns .', '', '', 'note on creep buckling of columns . the stability of a compressed elastic ring has been studied by a method which can be extended to solve the problem of the stability of a flexible heavy structure spread by a system of hoops as in a crinoline skirt . the original work by levy, which was developed by timoshenko and love, cannot be generalized to problems in which the compressing forces are affected by the deformation of the ring . it is shown that the load at which a ring will buckle depends not only upon the magnitude of the load but also upon its first derivative relative to the radial distance . a positive derivative causes the ring to buckle at a higher load . when this result is applied to a cone of heavy and loosely draped fabric spread by a rigid hoop of radius and a larger and flexible hoop of radius below it, both hoops being in horizontal planes, then various modes of buckling other than oval are possible according to the relative magnitudes of and . it is found that oval buckling changes to three-wave buckling when three-wave changes to four-wave when, and as and approach nearer to equality the buckled form progressively changes to more waves . when applied to a structure spread by many horizontal hoops of which the top one is rigid and oval, it is found that all other hoops, if each is designed to the criterion, will have the same absolute deviation from circularity as the rigid hoop . if any one hoop is designed so that, then the oval shape of the rigid hoop is magnified on all flexible hoops .'), (1036, 'on transverse vibrations of thin, shallow elastic shells .', 'reissner,e.', 'q. app. math., 13, 1955, 169.', 'on transverse vibrations of thin, shallow elastic shells . according to marguerre (proc. 5th internat. congress appl. mech., cambridge, mass., 1938, wiley, new york, shells are governed by three simultaneous differential equations in the three displacements . the author has considerably simplified this theory for the case of transverse vibrations by ignoring the longitudinal inertia terms, thus reducing the problem to that of solving two simultaneous differential equations in a stress function and one displacement component . this simplification is justified by an order-of-magnitude analysis, and illustrated by considering the vibrations of a paraboloidal shell with a rectangular boundary .'), (1037, 'on transverse vibrations of thin, shallow elastic shells .', '', '', 'on transverse vibrations of thin, shallow elastic shells . it is shown that a perfectly straight bar, subjected to a state of uniform shear stress, will buckle, in a manner similar to a column under compression, if the shear stress exceeds a certain critical value . the buckling equations are obtained by the newtonian approach and also by the application of the principle of minimum potential energy . in order to provide additional insight into this buckling mechanism, a simple model is introduced and analyzed .'), (1038, 'on transverse vibrations of thin, shallow elastic shells .', '', '', 'on transverse vibrations of thin, shallow elastic shells . the deformation and complete stress distribution are determined for each of the following edge-loaded thin shells .. (1) a right circular cylinder, (2) a frustum of a right circular cone, and (3) a portion of a sphere . the locations of maximum circumferential and meridional stresses are also found . equations are developed for discontinuity shear and moment at the following junctions .. circular cylinder, (2) axial change of thickness in a cone, (3) change of thickness in a portion of a sphere, cylinder and a portion of a sphere,'), (1039, 'on transverse vibrations of thin, shallow elastic shells .', '', '', 'on transverse vibrations of thin, shallow elastic shells . an experimental investigation was made (1) to evaluate previously published theoretical procedures for the prediction of stress distribution for cases of radially symmetric abrupt change in wall thickness of thin-walled cylinders subject to internal pressure and (2) to investigate the significance of stresses attributable to the presence of thickness changes typical of design practice . one theory was adequate in itself for solution of the case of continuous middle surface,. use of the second theoretical procedure was required to determine the additional stresses arising from discontinuous middle surfaces at the change in thickness . comparisons were made between theoretical and experimental stress distributions for cases with continuous middle and continuous inner surfaces for radially symmetric changes in wall thickness of a cylinder subject to internal pressure for diameter to larger wall thickness ratios of 117 and 28 and for the case of a continuous outer surface for a ratio of 28 . in all tests the ratio of wall thicknesses at the change in wall thickness was 0.4 . there was reasonably good correlation between theoretical and experimental curves of stress distribution . on the basis of this correlation, it was concluded that the applicable theories were valid . it was shown that inclusion of the stresses arising from the condition of discontinuous middle surfaces at a change in thickness has an important effect on stress distribution . in the case of a cylinder with a continuous outer surface, the maximum mean effective stress was of sufficient magnitude to indicate that this geometry should be avoided in design if possible . the maximum mean effective stress was not increased to a significant degree by the presence of a change in wall thickness in the other cases .'), (1040, 'on transverse vibrations of thin, shallow elastic shells .', '', '', 'on transverse vibrations of thin, shallow elastic shells . experimental measurements of pressures and heat-transfer rates over three blunt afterbodies of small fineness ratio in fully separated wakes are presented . the afterbodies are generally similar in shape but have different stepdown heights from the end of the forebody . tests were made by means of a new shroud technique over a range of reynolds numbers closely corresponding to typical flight conditions at mach numbers on the order of 20, considering models on the order of 5 ft. in diameter at about 120,000 ft. altitude . stagnation temperatures on the order of 1,300 r. to strictly speaking, the test flows correspond to prototype flows which would be created by a forebody consisting of a sharp or slightly blunted 54 half angle cone which turns cylindrical for a short distance and then connects with the afterbody . judiciously interpreted, the results may be considered to have a somewhat wider applicability for approximation purposes . the results are presented and compared with each other in terms of nondimensional variables based on flow conditions at the end of the forebody . the pressure distribution along an afterbody is seen to be roughly uniform in each run . for a given point on an afterbody, the ratio of pressure to the stagnation pressure at the forebody end (or exit) decreases with increasing stagnation pressure or reynolds number . the present pressures and pressure-reynolds number variations (fig. 8) are compared with values obtained from chapman''s mach 2 or 3 base-pressure data,. qualitative and some quantitative agreement is noted . in the reynolds number range comparable to those of the present tests, chapman''s exit boundary layers were considered to be laminar . an approximate check of the heat-transfer rate at the forebody end in the present tests also indicates a laminar rate . no information was obtained concerning the possible transition of the free-mixing separated boundary layer covering the wake . an adverse pressure gradient on the cylindrical end of the forebody figs. 7(a) and 7(c) was observed . heat-transfer rates are seen to be roughly uniform over an afterbody in each run, although some increase in the streamwise direction is noted . the afterbody nusselt number (n) varies with the reynolds number (r evaluated at the forebody end) roughly in the manner n r where generally (fig. 13) . heat rates on the rear faces of the afterbodies are almost twice the values on the sides . the heat rates on the large-step body are higher than those on the body of zero stepdown height . in an addendum, it is shown that the prandtl-meyer expansion angle of the flow leaving the afterbody increases with increasing test reynolds number, and that the corresponding local mach number square increases linearly with reynolds number . the effect is to keep the local wake reynolds numbers virtually constant with increasing test reynolds number while the afterbody heat rates increase sharply . the expansion angle on the afterbody of zero stepdown height is significantly smaller than on the stepped down bodies,. this may affect the decreased heat rates on this body .'), (1041, 'analysis of stresses in the elements of shell structure .', 'c. e. taylor and e. wenk, jr.', 'structural research engineer, david taylor model basin head, structures division, david taylor model basin', 'analysis of stresses in the elements of shell structure . the love-meissner analysis for thin shells has previously been applied to cones of uniform wall thickness, and solutions for the stress resultants were given in terms of kelvin''s functions . since tabulation of these functions for large arguments is not practical, considerable computation was still required . in the present paper, the authors define special functions which eliminate the necessity of evaluating kelvin/s functions and which may be used with simple algebraic and trigonometric functions to compute the boundary forces and displacements for cones for various loading conditions . these special functions also make clear the magnitude of errors which result from geckeler/s and other approximate solutions .'), (1042, 'on transverse vibrations of thin, shallow elastic shells .', '', '', 'on transverse vibrations of thin, shallow elastic shells . the report presents information on the stress problems in the analysis of pressurized cabins of high-altitude aircraft not met with in other fields of stress analysis relating to aircraft . the material may be roughly divided into shell problems and plate problems, the former being concerned with the curved walls of the cabin or pressure vessel and the latter being concerned with small rectangular panels of its walls, framed by stiffeners, but not necessarily plane .'), (1043, 'on transverse vibrations of thin, shallow elastic shells .', '', '', 'on transverse vibrations of thin, shallow elastic shells . a numerical analysis is given for the solution of the general equations of thin shells of revolution subjected to rotationally symmetric pressure and temperature distributions . the basic differential equations are in a very general form, which permits the geometry of the shells considered, to be specified by discrete data points . the analysis determines elastic stresses, strains and displacements for multi-layer and multi-sectional shells of revolution . surface loads, temperatures, thicknesses and material properties may vary arbitrarily in the meridional direction . temperatures and material properties can also vary through the thickness . the solution is obtained by direct computation using a numerical method that employs two by two coefficient matrices,. and hence avoids the problems of slow convergence . the solution has been programmed in a semi-algebraic language which can be used on most high speed computers . comparisons of numerical solutions to known exact and approximate solutions of the thin shell equations are made to demonstrate the accuracy of this method .'), (1044, 'on the theory of thin elastic shells .', 'reissner,e.', 'reissner anniv. 1949, 231.', 'on the theory of thin elastic shells . general equations for the symmetrical finite deflection of a rotationally symmetric thin shell are first obtained . for small deflections these equations are reduced to a pair of equations for the change of slope of the shell surface and the product of the undeformed radius of the shell to the radial stress . this choice of dependent variable is shown to be advantageous . two cases of shallow shells give particularly simple solutions .. parabolic shells of nth degree and uniform thickness, and parabolic shells of mth degree with thickness varying as mth power of the distance from the apex . for the first case, the solutions can be expressed in terms of cylinder functions,. for the second, in powers of the paper concludes with a discussion of the asymptotic solutions for small thickness .'), (1045, 'the bending strength of pressurized cylinders .', 'zender,g.w.', 'j. ae. scs. 29, 1962, 362.', 'the bending strength of pressurized cylinders . discussion of previously presented experimental data for the loading of pressurized cylinders, in terms of membrane theory .'), (1046, 'the bending strength of pressurized cylinders .', '', '', 'the bending strength of pressurized cylinders . a theoretical solution is given for the critical stress of thin-walled cylinders loaded in torsion . the results are presented in terms of a few simple formulas and curves which are applicable to a wide range of cylinder dimensions from very short cylinders of large radius to long cylinders of small radius . theoretical results are found to be in somewhat better agreement with experimental results than previous theoretical work for the same range of cylinder dimensions .'), (1047, 'the bending strength of pressurized cylinders .', '', '', 'the bending strength of pressurized cylinders . be described here is attributed to the russian investigator v. g. galerkin, whose original papers are inaccessible to the present writer . his knowledge of the method is derived from a description given in a paper by e. p. grossman . grossman states that the method was given by galerkin in his treatise p. 897), and that applications to oscillation problems were first made by v. p. lyskov . it is pointed out by grossman that galerkin''s process in applications to mechanics leads to the same results as lagrange''s principle of virtual work, but employs a special co-ordinate system . the method of galerkin belongs to the same general class as those of rayleigh and ritz, for it seeks to obtain an approximate solution of a differential equation with given boundary conditions by taking a function which satisfies these conditions exactly, and proceeds to specialise the function in such a manner as to secure approximate satisfaction of the differential equation . the selected function is a linear combination of n independent functions, and the coefficients are determined by a process of integration . the galerkin process can be considered from two points of view, (a) simply as a means for the approximate solution of differential equations, and treatment of problems concerning the statics and dynamics of elastic and other deformable bodies . these two aspects are treated separately in parts 1 and 2 of the paper respectively, and will now be briefly discussed . which satisfies the boundary conditions, in the differential equation be . since the result should be zero, is the error in the differential equation . then the galerkin process consists in choosing the n coefficients in the function in such a manner that n distinct weighted means of the error, taken throughout a certain range of representation, shall all be zero . as a generalised force, and the multipliers used to weight the errors are the virtual displacements corresponding to increments of each of the generalised co-ordinates in turn . thus the vanishing of the weighted mean is here interpreted as the vanishing of the virtual work in the appropriate displacement . the degree of accuracy attaindd can be increased indefinitely by increasing the number of independent functions employed, but this entails a great increase of labour . however, when the functions are well chosen, an excellent approximation can be obtained by the use of a very small number, as is sufficiently shown by the examples included in this paper .'), (1048, 'a small deflection theory for curved sandwich plates .', 'stein,m. and mayers,j.', 'naca r.1008, 1951.', 'a small deflection theory for curved sandwich plates . a small-deflection theory that takes into account deformations due to transverse shear is presented for the elastic-behavior analysis of orthotropic plates of constant cylindrical curvature with considerations of buckling included . the theory is applicable primarily to sandwich construction .'), (1049, 'elastic constants for corrugated core sandwich plates .', 'libove,c. and hubka,r.e.', 'naca tn.2289, 1951.', 'elastic constants for corrugated core sandwich plates . the sandwich plate consisting of corrugated sheet fastened between two face sheets is considered . application of existing theories to the analysis of such a sandwich plate requires the knowledge of certain elastic constants . formulas and charts are presented for the evaluation of these constants . the formulas for three of these constants were checked experimentally and found to give values in close agreement with the experimental values .'), (1050, 'compressive buckling of simply supported curved plates and cylinders of sandwich construction .', 'stein,m. and mayers,j.', 'naca tn.2601, 1952.', 'compressive buckling of simply supported curved plates and cylinders of sandwich construction . theoretical solutions are presented for the buckling in uniform axial compression of two types of simply supported curved sandwich plates .. the corrugated-core type and the isotropic-core type . the solutions are obtained from a theory for orthotropic curved plates in which deflections due to shear are taken into account . results are given in the form of equations and curves .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1051, 'the stability of thin-walled unstiffened circular cylinders under axial compression including the effects of internal pressure .', 'harris,l.a.', 'j. ae. scs. 24, 1957, 587.', 'the stability of thin-walled unstiffened circular cylinders under axial compression including the effects of internal pressure . in the design of high-speed aircraft the importance of unpressurized and pressurized monocoque cylinders necessitates a reliable analysis procedure for the compressive buckling of cylindrical shells . analysis by the classical small-deflection theory has proved inadequate . recent large-deflection theoretical treatments of the problem have shown reasonable correlation with experiments but require a prior knowledge of the initial imperfections of the cylinder . developed in this paper is a semiempirical procedure which permits a compressive buckling analysis of cylindrical shells with a knowledge of the cylinder geometry only . this analysis is achieved by correlating experimental data statistically with theoretical parameters . in order to provide data not previously available, an extensive series of axial compression tests of pressurized cylinders has been performed . these data, together with all other known test data, are analyzed semiempirically . in the analysis best-fit curves are presented using theoretical parameters and shapes of curves where applicable . unpressurized and pressurized cylinder compressive buckling curves are then developed as 90 per cent probability curves from the test data . in general, these statistically defined design curves are significantly lower than previously available design curves .'), (1052, 'recent advances in the buckling of thin shells .', 'william a. nash', 'professor of engineering mechanics, university of florida', 'recent advances in the buckling of thin shells . the importance of the field of shell analysis is evidenced by the fact that in august, 1959, the international union of theoretical and applied mechanics conducted a symposium on the theory of thin elastic shells in delft, holland . this special meeting was attened by approximately 65 scientists in this field from 14 countries . this symposium indicated that considerable interest currently exists in such relatively new topics as the buckling of bimetallic shells, pressurized shells, creep buckling, and dynamic buckling, as well as in the more traditional problems involving isotropic shells of various geometries .'), (1053, 'spherical cap snapping .', 'keller,h.b. and reiss,e.l.', 'j. ae. scs. 26, 1959, 643.', 'spherical cap snapping . a nonlinear boundary value problem for the determination of the rotationally symmetric deformations of a clamped spherical cap under external pressure is solved by finite differences . the numerical solutions are obtained by employing a previously developed iteration procedure . a special case of the difference equations is solved explicitly and yields a justification of the iteration method as well as insight into the properties of the more accurate numerical solutions . buckled and unbuckled equilibrium states are obtained and the shape of the pressure-deflection curve which is usually assumed for these states is verified for a large class of caps . close estimates are given for the upper and lower buckling loads and an intermediate buckling load--i.e., the /dead-weight/ load . the stresses and deflections in the buckled and unbuckled states are examined and compared with an asymptotic solution valid in the interior of very thin shells . boundary layers are found to develop in the buckled states both as the loading increases and as the thickness of the shell decreases .'), (1054, 'iterative solutions for the non-linear bending of circular plates .', 'keller,h.b. and reiss,e.l.', 'comm. on pure and app. math. 11, 1958, 273.', 'iterative solutions for the non-linear bending of circular plates . the authors study non-linear von karman equations for bending of a thin circular plate under uniform normal pressure . discussion is mainly concerned with plates clamped at the edges and with zero radial displacement, but analysis is valid for other edge conditions . solution is by an iterative procedure whose convergence properties are studied by means of integral equations . method is then applied to finite difference formulation of the differential equations in order to obtain numerical solutions . numerical results are compared with previous work by other authors and the advantages of the present method are indicated .'), (1055, 'non-linear bending and buckling of circular plates .', 'keller,h.b. and reiss,e.l.', 'proc. 3rd u.s. nat. cong. app. mech. 1958, 375.', 'non-linear bending and buckling of circular plates . iterative solutions of finite difference approximations of the non-linear von karman plate equations are presented . results are obtained for circular plates under a variety of boundary conditions subjected to either uniform lateral pressure or uniform edge thrust . the solution, carried out numerically on the aec univac at new york university, yields a complete description of stresses and deflections for an apparently unlimited range of load parameters . in addition, boundary layer phenomena are discussed . for computing purposes, this iterative method proves to be superior to the previously used power series method and may be applicable to other non-linear problems .'), (1056, 'axisymmetric large deflections of circular plates subjected to thermal and mechanical load .', 'newman,m. and forray,m.', 'proc. aerospace forum i session, s.m.f. paper ff-30, 1962, 56.', 'axisymmetric large deflections of circular plates subjected to thermal and mechanical load . this paper is concerned with the nonlinear axisymmetric analysis of circular plates with in-plane edge restraint . both temperature and mechanical loads are accommodated as an extension of investigations performed for the isothermal mechanical loading problem . an exact mathematical formulation within the framework of the v. karman large strain-displacement relations is developed . the equilibrium equations and boundary conditions are then derived by utilizing the calculus of variations for arbitrary axisymmetrical temperatures and normal distributed loading . the satisfaction of equilibrium and compatibility equations requires the solution of two simultaneous nonlinear ordinary differential equations subject to the prescribed boundary conditions . analytical solutions of such equations are apparently not possible and therefore numerical procedures must be employed . a finite difference procedure utilizing /relaxed iterations,/ developed by h. keller and e. reiss, and employed by them for the solution of isothermal problems with apparently unlimited load parameter ranges, is used here for combined thermo-mechanical problems . numerical results are presented for the special case of a simply supported circular plate with radially immovable boundaries, subject to a uniform pressure and an arbitrary temperature variation through the thickness tained for a large range of temperature and load parameters . however, because of space limitations, only a limited amount of data are presented in this paper .'), (1057, 'the uniform section disk spring .', 'almen,j.o. and laszlo,a.', 'asme trans. 58, 1936, 305.', 'the uniform section disk spring . the authors point out in this paper that initially coned annular-disk springs of uniform cross section may be proportioned to give a wide variety of load-deflection curves not readily obtainable with the more conventional forms of springs, and that, although the versatility of this type spring has long been indicated, the formulas available have not been presented in a manner to disclose readily the effect of spring proportions on characteristics . therefore the authors have derived the formulas presented in this paper with the intention that the formulas will aid the designer in arriving at suitable characteristics by choice of spring geometry . these new formulas have been in use for several years at the general motors corporation research laboratories section, and their reliability has been checked by tests of springs used in a variety of special test equipment .'), (1058, 'the conical disk spring .', 'wempner,g.a.', 'proc. 3rd u.s. nat. cong. app. mech. 1958, 473.', 'the conical disk spring . this paper presents approximate formulas to describe the behavior of the conical disk spring . it is assumed that the shallow conical shell remains conical when subjected to the axially symmetric edge loads . the principle of stationary potential energy is used to derive the relations between load and deflection . formulas relating the applied loads, stresses, and deflections are given for several types of edge constraint . the analysis is essentially a refinement and extension of the previous work of almen and laszlo .'), (1059, 'the nonlinear conical spring .', 'schmidt,r. and wempner,g.a.', 'asme trans. 81e, 1959, 681.', 'the nonlinear conical spring . the large symmetric deformations of shallow conical shells are of interest in the design of nonlinear conical disk springs . in most applications a uniformly distributed axial load acts at the inner and outer edges,. these edges are otherwise free . several approximations have been proposed to describe the behavior of these springs . a first approximation (1) is based on the assumption that meridional strains are negligible . this requires that the shell remain conical after deformation and also that the extensional strain of meridional lines on the middle surface vanish . another approximation (2) retains only the assumption that the shell remains conical . the first assumption satisfies neither of the two boundary conditions at the free edges,. the latter violates the condition of vanishing moment at the free edges . recently the authors presented a series solution (3) for a special case, namely, the case of an annular plate under similar loading . numerical solutions for the shallow conical shell under these conditions of load have also been obtained (4) . an examination of these results indicates that the meridional bending stresses are of much smaller magnitude than the circumferential bending stresses . hence the present analysis is based on the neglect of the meridional bending moment .'), (1060, 'buckled states of circular plates .', 'keller,h.b., keller,j.b. and reiss,e.l.', 'q. app. math. 20, 1962, 55.', 'buckled states of circular plates . authors discuss the thin elastic circular disk of constant thickness subjected to a constant compressive thrust applied at its edge . the analysis presented is based upon the nonlinear von karman equations of plate theory and is applied to disks with completely clamped and completely, simply supported edges .'), (1061, 'turbulent mixing of a rocket exhaust jet with a supersonic stream including chemical reactions .', 'vasilu, j.', 'j. aero. sc. v. 29, january 1962, pp 19-28 .', 'turbulent mixing of a rocket exhaust jet with a supersonic stream including chemical reactions . the equations for the turbulent mixing of a two-dimensional supersonic jet issuing into an ambient supersonic stream are formulated . both streams consist of a mixture of chemically active and possibly reacting gases, therefore any heat release by chemical reaction is included,. the net mass rate of production of species is obtained on the assumption that the reaction rate constant is given by an expression reducible to the classical arrhenius law . the equations first given in terms of the x, and y coordinates, are expressed in dimensionless form and in terms of the x and coordinates, where is the stream function . the resulting expressions are all of the /heat conduction/ type,. they are put in a finite difference form by using the crank-nicolson method of substituting finite difference approximations for both the /time/ and /space/ derivatives . the mixture is assumed to consist of six species, namely h2o, h2, o2, co2, co, and n2, and the oxidation of h2 and co is assumed to take place according to a single-step chemical reaction . the solution of the problem is based on the simultaneous solution of 8n linear algebraic equations in 8n unknowns, n being the number of internal grid points at every step in the x-direction, and 8 the total number of unknowns at each grid point, namely velocity, temperature, and concentration for each of the six species . a method of obtaining initial and boundary conditions from available inviscid jet flow solutions is discussed . the equations are programed for calculation on an ibm-704 computer . finally, one typical case is considered, and plots of velocity, temperature, and concentration profiles are given for the initial stages of development of the mixing layer .'), (1062, 'an experimental and theoretical investigation of second-order wing-body interference at high mach number .', 'wilby,p.g.', 'ffa r.91, stockholm, 1962.', 'an experimental and theoretical investigation of second-order wing-body interference at high mach number . the second-order wing-body interference theory of landahl and beane is used in the theoretical calculation of the pressure distributions over the wing of a wing-body combination . results are compared with experimental values obtained from wind-tunnel tests, at a mach number of 7.35, on a cone-cylinder non-lifting body with a triangular wing of wedge section set at incidences of 0, 3, 6 and 10 . it is shown that interference effects can be very large and can be calculated theoretically with good accuracy .'), (1063, 'on obtaining solutions to the navier-stokes equations with high speed digital computers .', 'russell,d.b.', 'arc 23,797, 1962.', 'on obtaining solutions to the navier-stokes equations with high speed digital computers . the purpose of this paper is to show how to obtain steady state solutions to the navier-stokes equations on a high-speed digital computer . first the relative merits of various finite difference formulae are discussed . thereafter the main part of the paper is concerned with the methods used to solve the finite difference equations and an investigation is made of all the simpler iterative methods .'), (1064, 'propeller slipstream effects as determined from wing pressure distribution on a large-scale six-propeller vtol model at static thrust .', 'winston,m.m.', 'nasa tn.d1509, 1962.', 'propeller slipstream effects as determined from wing pressure distribution on a large-scale six-propeller vtol model at static thrust . during static-thrust tests of a large-scale general research model having a tilting wing and double-slotted flaps, static-pressure measurements were made on a wing segment behind one propeller to survey the effects of the slipstream . for the conditions of highest slipstream energy, the hovering end point of aerodynamic parameters for aircraft having vertical and short take-off and landing capability the tilt-wing configuration (zero flap deflection) was a 6 spanwise variation in effective angle of attack in a span of slightly less than 1 propeller diameter . effective changes in camber on the tilt-wing configuration as a result of slipstream rotation, the radial velocity gradient, and the resultant spanwise flow were negative and had a maximum magnitude of less than 2-percent chord . for the deflected-slipstream configuration (double-slotted flaps deflected), effects important to the hovering performance were found, including a 40-percent spanwise variation in effective thrust recovery and a 20 spanwise variation in effective thrust turning .'), (1065, 'a free-flight investigation of ablation of a blunt body to a mach number of 13 .1.', 'winters,c.w., witte,w.g., rashis,b. and hopko,h.n.', 'nasa tn.d1500, 1962.', 'a free-flight investigation of ablation of a blunt body to a mach number of 13 .1. a five-stage rocket-propelled research-vehicle system was flown to a maximum mach number of 13.1 at an altitude of approximately 78,000 feet to determine ablation characteristics of teflon in free flight . continuous in-flight measurements were made using sensors developed by the national aeronautics and space administration . the sensors were located on the blunted face of a nose cone constructed from teflon, with one at the stagnation point and two others at a surface distance of 0.62 radius on opposite sides of the stagnation point . the ablated-length measurements were in close agreement with analytical predictions . the analytical predictions, upon inclusion of the pertinent material property values, should be applicable to other materials as well as teflon .'), (1066, 'wind tunnel measurements of aerodynamic damping derivatives of a launch vehicle vibrating in free-free bending modes at mach numbers from 0. 70 to 2. 87 and comparisons with theory .', 'hanson,p.w. and doggett,r.v.', 'nasa tn.d1391, 1962.', 'wind tunnel measurements of aerodynamic damping derivatives of a launch vehicle vibrating in free-free bending modes at mach numbers from 0. 70 to 2. 87 and comparisons with theory . the aerodynamic damping of a flexibly mounted aeroelastic model with a blunted conical nose and a cylindrical afterbody was measured at mach numbers from 0.70 to 1.20 at several levels of dynamic pressure and two weight conditions and at mach numbers from 1.76 to 2.87 at one weight condition . the first two free-free flexible modes of vibration were investigated . also investigated at mach numbers from 0.9 to 1.2 was the aerodynamic damping in the first free-free modes of a model which had a /hammerhead/ nose (the base diameter of the blunted cone was greater than the diameter of the afterbody which necessitated a reflex angle downstream from the cone base) . two basically different methods, the /electrical power-input/ and the /decaying oscillations/ methods were used to determine the damping and frequencies . the experimentally determined values are compared with some applicable theories . the results of the investigation indicate that the aerodynamic damping in the elastic modes of vibration was small for all configurations tested . the maximum aerodynamic damping measured in the first mode was on the order of damping . the aerodynamic damping was found to be even less for vibration modes higher than the first . reduced-frequency effects were found to be negligible for the range investigated . agreement of calculated aerodynamic damping derivatives with the experimental results was not good . generally, the experimentally determined derivatives were larger than those predicted by the various theories used . the bond-packard theory appeared to give the best agreement for the first free-free vibration mode but gave the worst agreement for the second mode . measurements made on the configuration that had a hammerhead nose indicated small negative aerodynamic damping in the mach number range from 0.95 to 1.00 . aerodynamic stiffness effects were found to be small and within the experimental scatter . (wind-on frequency determination was accurate only to approximately 1 percent .)'), (1067, 'plastic stability theory of geometrically orthotropic plates and cylindrical shells .', 'gerard,g.', 'j. ae. scs. 29, 1962, 956.', 'plastic stability theory of geometrically orthotropic plates and cylindrical shells . a linear eighth-order equilibrium differential equation for plastic buckling of geometrically orthotropic thin cylindrical shells is derived . this equation is used to obtain explicit solutions for long flat plates and wide columns under axial compression and moderate-length cylinders under external pressure, torsion, and axial compression .'), (1068, 'instability analysis of cylindrical shells under hydrostatic pressure .', 'herrmann,g.', 'nasa tn.d1510, 1962, 239.', 'instability analysis of cylindrical shells under hydrostatic pressure . to determine the elastic buckling pressure of simply supported cylindrical shells subjected to lateral and axial hydrostatic forces, various versions of linear bending theories have been employed in the past . for certain shell dimensions, however, the expressions commonly used may yield substantially differing results . in what follows, recent work on this problem by a. e. armenakas and the writer is briefly reviewed . this work consisted primarily in employing a general bending theory of circular cylindrical shells under the influence of initial stress, developed earlier by the same authors, to re-examine the problem mentioned, and compare the results with those of previous investigations . the outcome was the establishment of a simple but accurate expression for the buckling pressure applicable to a wide range of shell dimensions .'), (1069, 'design and testing of honeycomb sandwich cylinders under axial compression .', 'cunningham,j.h. and jacobson,m.j.', 'nasa tn.d1510, 1962.', 'design and testing of honeycomb sandwich cylinders under axial compression . experimental results for 36 diameter honeycomb cylinders fabricated with thin (0.010) aluminum faces and cores prove that it is quite feasible to stabilize thin faces so they can be loaded beyond the yield point . the effect of initial imperfections and the various modes of failure are discussed .'), (1070, 'buckling of orthotropic and stiffened conical shells .', 'singer,j.', 'nasa tn.d1510, 1962,463.', 'buckling of orthotropic and stiffened conical shells . donnell type stability equations for thin circular orthotropic conical shells are presented and solved for external pressure, axial compression and combined loading . the solution is likewise applied to stiffened conical shells . correlation with equivalent cylindrical shells yields a simple approximate stability analysis for orthotropic or ring-stiffened conical shells under hydrostatic pressure . the general instability of stiffened conical shells under hydrostatic pressure is also analysed by a more accurate approach . preliminary experimental results for buckling of ring-stiffened conical shells under hydrostatic pressure are presented and discussed .'), (1071, 'stability of thin torispherical shells under uniform internal pressure .', 'mescall,j.', 'nasa tn.d1510, 1962, 671.', 'stability of thin torispherical shells under uniform internal pressure . the stability of the toroidal portion of a torispherical shell under internal pressure is considered from the point of view of the linear buckling theory . a detailed stress analysis of the prebuckled shell is made employing asymptotic integration . the change in potential energy of the shell is then minimized using a rayleigh-ritz procedure for actual computation of the critical pressure . numerical results reveal that elastic buckling may occur for very thin shells whose material has a relatively high value of the ratio of yield stress to elastic modulus .'), (1072, 'ignition and combustion in a laminar mixing zone .', 'marble,f. and adamson,t.c.', 'jet prop. 24, 1954, 85.', 'ignition and combustion in a laminar mixing zone . the analytic investigation of laminar combustion processes which are essentially two- or three-dimensional present some mathematical difficulties . there are, however, several examples of two-dimensional flame propagation which involve transverse velocities that are small in comparison with that in the principal direction of flow . such examples occur in the problem of flame quenching by a cool surface, flame stabilization on a heated flat plate, combustion in laminar mixing zones, etc . in these cases the problem may be simplified by employing what is known in fluid mechanics as the boundary-layer approximation, since it was applied first by prandtl in his treatment of the viscous flow over a flat plate . physically it consists in recognizing that if the transverse velocity is small, the variations of flow properties along the direction of main flow are small in comparison with those in a direction normal to the main flow . the analytic description of the problem simplifies accordingly . the present analysis considers the ignition and combustion in the laminar mixing zone between two parallel moving gas streams . one stream consists of a cool combustible mixture, the second is hot combustion products . the two streams come into contact at a given point and a laminar mixing process follows in which the velocity distribution is modified by viscosity, and the temperature and composition distributions by conduction, diffusion, and chemical reaction . the decomposition of the combustible stream is assumed to follow first-order reaction kinetics with temperature dependence according to the arrhenius law . for a given initial velocity, composition, and temperature distribution, the questions to be answered are .. (1) does the combustible material ignite,. and (2) how far downstream of the initial contact point does the flame appear and what is the detailed process of development . since the hot stream is of infinite extent, it is found that ignition always takes place at some point of the stream . however, when the temperature of the hot stream drops below a certain value, the distance required for ignition increases so enormously that it essentially does not occur in a physical apparatus of finite dimension . the complete development of the laminar flame front is computed using an approximation similar to the integral technique introduced by von karman into boundary layer theory .'), (1073, 'a practical method for numerical evaluation of solutions of partial differential equations of the heat-conduction type .', 'crank,j. and nicolson,p.', 'proc. cam. phil. soc., 43, 1947, 50.', 'a practical method for numerical evaluation of solutions of partial differential equations of the heat-conduction type . three approximate methods for the solution of the nonlinear equation of heat flow in a medium where heat is being generated by a chemical reaction are compared . the equations are where subscripts indicate partial differentiations and q, k, a are'), (1074, 'theoretical and experimental investigation of second-order supersonic wing-body interference .', 'landahl, m., drougge, g. and beane, b.j.', 'j. aero. sc. v. 27, september 1960. pp 694-702 .', 'theoretical and experimental investigation of second-order supersonic wing-body interference . approximate second-order solutions for the supersonic flow around wing-body combinations are calculated, using two different theoretical models small and the wing sweep small in comparison with that of the mach cone are considered . the analysis is restricted to such high mach numbers that m-2 1, and an approximate formula common to the two models is then found for the second-order interference term . this formula can also be used to correct experimental pressure distributions for the effect of nonuniformities in the wind-tunnel flow . in order to test the theory, wind-tunnel experiments on non-lifting cone-cylinder bodies in combination with wings of simple shapes were performed . pressure distributions were measured at m 3 and m 4, both around the bodies and on the wings separately, as well as in combination, and it was found that the second-order interference was predicted reasonably well by the simplified theory .'), (1075, 'an experimental and theoretical investigation of second-order supersonic wing-body interference, for a non-lifting body with wings at incidence .', 'wilby, p.g.', 'aero. res. inst. of sweden, ffa report 87, 1960 .', 'an experimental and theoretical investigation of second-order supersonic wing-body interference, for a non-lifting body with wings at incidence . pressure distributions on the wing of two wing-body combinations are measured experimentally at mach numbers 3 and 4 with the wing at various incidences in the range 0degree to 10degree . the results are compared with theoretical results which include interference effects calculated according to the second-order supersonic wing-body interference theory due to landahl and beane /1/ . this theory, having been tested previously for non-lifting wing-body combinations, is thus tested also for wings at incidence . the agreement between theory and experiment is found to vary with mach number and wing sweepback . for the higher mach number and moderate sweepback the theory gives a good prediction of pressure distribution, but for the most adverse condition of low mach number and large sweepback the theory is found to overestimate the interference effects . this is expected as the theory assumes the sweepback of the wings is small compared with that of the mach line . an empirical guide to the limit of application of the interference theory is given . within this limit the agreement between theory and experiment is found to deteriorate only a little with increase of incidence, over the range tested .'), (1076, 'an approximate method for determining the displacement effects and viscous drag of laminar boundary layers in two-dimensional hypersonic flow .', 'bertram, m.h.', 'naca tn 2773, 1952 .', 'an approximate method for determining the displacement effects and viscous drag of laminar boundary layers in two-dimensional hypersonic flow . a simplified approximate theory is presented by means of which the laminar boundary layer over an insulated two-dimensional surface may be calculated, a linear velocity profile being assumed, and an estimate made of its effect in changing the pressure distribution over the profile upon which the boundary layer is formed . skin friction is also determined . comparisons of results from this theory are made with experimental results at a mach number of 6.86 and a reynolds number of'), (1077, 'a method of solution with tabulated results for the attached oblique shock wave system for surfaces at various angles of attack, sweep, and dihedral in an equilibrium real gas including the atmosphere .', 'trimpi, r.l. and jones, r.a.', 'nasa tr r-63, 1960 .', 'a method of solution with tabulated results for the attached oblique shock wave system for surfaces at various angles of attack, sweep, and dihedral in an equilibrium real gas including the atmosphere . a new method is derived for solving the attached oblique shock-wave system for surfaces at various angles of attack, sweep, and dihedral in any real gas in equilibrium . results are tabulated for the following ranges .. angle of attack, 0degree to 65degree,. angle of sweep, 0degree to 75degree,. angle of dihedral, 0degree to 30degree,. mach number, 3 to 30,. and /effective specific-heat ratio/ parameter, 1.10 to 1.67 . both the method and tabulated solutions are easily adaptable to flight in any gas or in the atmosphere of any planet . an illustrative example is presented based on the ardc 1956 model atmosphere .'), (1078, 'the steady flow of a viscous fluid past a circular cylinder at reynolds numbers 40 and 44 .', 'apelt, c.j', 'a.r.c. r + m 3175 .', 'the steady flow of a viscous fluid past a circular cylinder at reynolds numbers 40 and 44 . this paper describes the numerical solution of the complete navier-stokes equations for the steady flow of an incompressible viscous fluid of unlimited extent past a circular cylinder at reynolds number 40 . a new device developed for the numerical solution is described . the results of the investigation are .. good agreement with experimental results . higher reynolds numbers even though they may not exist in nature . a solution has been obtained at reynolds number 44 but it has not been carried to the same accuracy as the solution at reynolds number 40 . portion of the cylinder continues to increase with reynolds number in such steady-state solutions up to a reynolds number 44 and no indication has been found that this process will not continue as the reynolds number is increased beyond 44 .'), (1079, 'finite difference formulae for the square lattices .', 'nicley,w.g.', 'q.j.mech. app. math, 1, p35.', 'finite difference formulae for the square lattices . the paper gives approximate formulae for derivatives (including combinations like and ), and integrals, of a function of two independent variables, in terms of its values at nodes of a square lattice, primarily for use in the numerical solution of partial differential equations . consideration is given to the form, as well as to the magnitude, of the leading terms in the error, and what is believed to be for most purposes optimum combinations are thus selected for the simpler compact sets of nodes .'), (1080, 'viscous flow round a sphere at low reynolds numbers . /l40/ .', 'jenson, v.g.', 'proc. roy. soc. series, a. v. 249, p. 346, 1959 .', 'viscous flow round a sphere at low reynolds numbers . /l40/ . relaxation methods are outlined, and the present problem formulated in modified spherical polar co-ordinates . the results of calculations made for r 5, 10, 20, 40 are presented in the form of stream function and vorticity distributions,. and further results of pressure distributions, velocity distributions, and drag coefficients, calculated from them . these results are shown to compare favourably with experimental work, showing a steady trend from symmetrical stokes''s flow, towards boundary layer flow . the phenomenon of separation of the forward flow and development of a circulating wake, is explained and illustrated, the first formation of a wake being at r 17 .'), (1081, 'numerical solution of the navier-stokes equations for the flow around a circular cylinder at reynolds number 40 .', 'kawaguti, m.', 'j. phys. soc. japan, v. 8, no. 6 p. 747, 1953 .', 'numerical solution of the navier-stokes equations for the flow around a circular cylinder at reynolds number 40 . the steady two-dimensional flow around a circular cylinder submerged in a viscous fluid for the case r 40 is investigated, integrating numerically the exact navier-stokes equations . the main results are as follows . /i/ the steady flow solution exists even for the reynolds number as high as 40 . moreover, it seems that the solution goes over smoothly to the solution of the kirchhoff discontinuous flow theory which seems to be the limiting flow for the case r . /ii/ the flow pattern and the coefficients of pressure and drag are in good agreement with the experimental data .'), (1082, 'the flow past pitot tube at low reynolds numbers, part 1-dash the numerical solution of the navier-stokes equations for steady viscous axisymmetric flow, part 2-dash the effects of viscosity and orifice size on a pitot tube at low reynolds numbers .', 'lester, w.g.s.', 'o.u.e.l. no. 136, a.r.c. 22, 070, f.m. 2983 .', 'the flow past pitot tube at low reynolds numbers, part 1-dash the numerical solution of the navier-stokes equations for steady viscous axisymmetric flow, part 2-dash the effects of viscosity and orifice size on a pitot tube at low reynolds numbers . in this report numerical methods used to solve the navier-stokes equations for steady viscous two-dimensional flow are extended to include the case of axial symmetry . the equations and their finite difference approximations are derived working in cylindrical polar co-ordinates with the stokes'' stream function and the vorticity as variables . a new method of dealing with the boundary conditions is given . the effects of viscosity and orlfice size on a blunt-nosed pitot tube have been theoretically investigated up to a reynolds number of ten, where the reynolds number has been based on the radius of the tube . results are expressed in terms of a pressure coefficient where p is the pressure measured in the tube, p the density of the fluid, and p and u the static pressure and velocity in an undisturbed flow at the position of the tube . the values of c for a blunt-nosed tube are found to be less than those for tubes with hemispheroidal heads, but always greater than unity in the range considered . the effect of the orifice size is to decrease c as the orifice size increases, this decrease is very small but increases with the reynolds number . at a reynolds number of ten the decrease is at most five per cent of the value of c when there is no orifice . it is suggested that the decrease of c below unity found in some experimental investigations at a higher reynolds number could be due to the effects of orifice size .'), (1083, 'an investigation of fluid flow in two dimensions .', 'thom, a.', 'a.r.c., r + m, 1194. 1928 .', 'an investigation of fluid flow in two dimensions . flow of an inviscid fluid . -dash there are in existence several methods of obtaining numerical solutions to the two-dimensional flow of a perfect fluid for given boundary conditions . part 2 of the present paper gives a method of obtaining a numerical solution for viscous steady flow solution of the simpler problem, illustrating it with examples bearing on the experimental work described in part iv .'), (1084, 'the flow past circular cylinders at low speeds .', 'thom, a.', 'proc. roy. soc. series a. vol. 141, p. 651, 1933 .', 'the flow past circular cylinders at low speeds . this paper deals chiefly with calculations and experiments on the flow past circular cylinders, but the arithmetical methods of solution of the equations of steady viscous flow proposed and used in section i, are applicable to other equations and may be of interest .'), (1085, 'note on the convergence of numerical solutions of the navier-stokes equations .', 'thom, a. and apelt, c.j.', 'a.r.c. r + m 3061. 1956 .', 'note on the convergence of numerical solutions of the navier-stokes equations . a criterion is given for the convergence of numerical solutions of the navier-stokes equations in two dimensions under steady conditions . the criterion applies to all cases of steady viscous flow in two dimensions and shows that if the local ''mesh reynolds number'', based on the size of the mesh used in the solution, exceeds a certain fixed value, the numerical solution will not converge .'), (1086, 'a note on the numerical solution of fourth order differential equations .', 'woods,l.c.', 'aero. quart. 5, 1954, 176.', 'a note on the numerical solution of fourth order differential equations . an old numerical method of solving fourth order differential equations is put in relaxation form . the higher order correction terms are included and the technique is illustrated by an example . the method has the advantage of being more rapidly convergent than the usual relaxation procedure for fourth order equations . some comments are made on the numerical solution of the viscous flow equation .'), (1087, 'convergence rates of iterative treatments of partial differential equations .', 'frankel, s.p.', 'math. tables aids comp. v. 4. p. 65, 1950 .', 'convergence rates of iterative treatments of partial differential equations . the development of high-speed digital computers has made feasible the numerical solution by iterative methods of some partial differential equations . the convergence rates of several such iterative methods are estimated here . it is found that with the familiar elementary iterative methods some quite simple problems require prohibitive computational labor . the iterative methods here considered are related to the various forms of the southwell /relaxation method/ in that they involve successively applied local corrections to improve an approximate solution . however, these iterative methods are routinized in conformity with the requirements of automatic computers while the relaxation method is flexible and depends in an essential way on the skill of its practitioners .'), (1088, 'iterative methods for solving partial difference equations of elliptic type .', 'young, d.', 'trans. amer. math. soc. vol. 76, p. 92, 1954 .', 'iterative methods for solving partial difference equations of elliptic type . this paper considers linear systems /1/ where a includes matrices of a sort frequently occurring in the solution of elliptic partial differential equations by difference methods /in particular, a o/ . rewriting superscript is number of iteration cycle/ are used to compute u when u are used . also, one may /over-relax/ .. ser. a. 210, 307-357 /1910/ who suggested changing from time to time to speed up convergence . in the present paper over-relaxation /with fixed w/ is combined with immediate introduction of newly-computed u''s, a la gauss-seidel . various theorems on convergence are proved$. in particular, it is shown that there exists an ordering of the equations and an optimum value wb such that in general /3/ converges much more rapidly than the gauss-seidel method /w 1/ . means are suggested for estimating wb,. the sensitivity of the rate of convergence to the choice of w is studied . the paper concludes with a theoretical comparison of gauss-seidel and the method proposed, /successive over-relaxation/, for solving dirichlet''s difference problem over a square using a high-speed computing machine .'), (1089, 'aerodynamic characteristics of propeller-driven vtol aircraft .', 'kirby,r.h.', 'nasa tn.d730, 1961.', 'aerodynamic characteristics of propeller-driven vtol aircraft . this paper discusses the two major configurations that are usually considered for achieving vtol while keeping the fuselage essentially horizontal - that is, the tilt-wing and the deflected-slipstream configurations . because of the high turning losses incurred by deflected-slipstream configurations in hovering and because of the wing-stalling problem of the pure tilt-wing configurations during the transition, it appears that a combination of the two principles should be used . this tilt-wing and flap configuration should make use of a programed extensible-chord slotted flap together with a leading-edge high-lift device in order to avoid the performance and handling qualities problems associated with wing stalling during the transition while keeping the wing area as low as possible for efficiency in cruising flight .'), (1090, 'pressure distribution and force measurements on a vtol tilting wing-propeller model . pt .ii, analysis of results .', 'currie,m.m. and dunsby,j.a.', 'rep. lr-284, nat. res. council of canada, 1960.', 'pressure distribution and force measurements on a vtol tilting wing-propeller model . pt .ii, analysis of results . this report presents in graphical form the results presented in reference 1 from pressure distribution and force measurements on a half-wing model of a twin-engined tilt-wing vtol configuration . the profound influence of the slipstream on these results is discussed in some detail .'), (1091, 'data from a static thrust investigation of a large scale general research vtol-stol model in ground effect .', 'huston,r.j. and winston,m.m.', 'nasa tn.d397, 1960.', 'data from a static thrust investigation of a large scale general research vtol-stol model in ground effect . the model was tested at two different elevations with the wing pivot at 1.008 and 2.425 propeller diameters above the ground . the slipstream of the propellers was deflected by tilting the wing and propellers, by deflections of large-chord trailing-edge flaps, and by combinations of flap deflection and wing tilt . tests were conducted over a range of propeller disk loadings from 7.41 to 29.70 pounds per square foot . force data for the complete model and pressure distributions for the wing and flaps behind one propeller were recorded and are presented in tabular form without analysis .'), (1092, 'wing-nacelle-propeller interference for wings of various spans . force and pressure distribution tests .', 'robinson,r.g. and herrnstein,w.h.', 'naca r.569, 1936.', 'wing-nacelle-propeller interference for wings of various spans . force and pressure distribution tests . an experimental investigation was made in the n. a. c. a. full-scale wind tunnel to determine the effect of wing span on nacelle-propeller characteristics and, reciprocally, the lateral extent of nacelle and propeller influence on a monoplane wing . the results provide a check on the validity of the previous research on nacelles and propellers with 15-foot-span wings tested in the the scale propeller and the n. a. c. a. cowling used in the former researches were tested in three typical tractor locations with respect to a thick wing of 5-foot chord and 30-foot span . the span was progressively reduced to 25,20, and 15 feet and the same characteristics were measured in each case . the efficiency factors--propulsive efficiency, nacelle drag efficiency, and net efficiency--were obtained for each wing length by means of force tests and the values are compared to determine the effect of span . pressure-distribution measurements show the lateral extent of the nacelle interference and the propeller-slipstream effect on the span loading for the various conditions . complete polar curves and curves showing the variation of nacelle drag with lift coefficient are also included . force and pressure-distribution tests concur in indicating that, for engineering purposes, the influence of a nacelle and of a propeller, in a usual combination, may be considered to extend laterally on a wing the same maximum distance, or about five nacelle diameters or two propeller diameters outboard of their common axes . all important effects of scale nacelle-propeller combinations may be measured within practical limits of accuracy by tests of a 15-foot-span wing .'), (1093, 'induced interference effects on jet and buried-fan vtol configurations in transition .', 'spreemann,k.+.', 'nasa tn.d731, 1961.', 'induced interference effects on jet and buried-fan vtol configurations in transition . recent investigations of some jet and buried-fan configurations have indicated that in the transition speed range, configurations with considerable area surrounding the jet or buried fan can encounter large losses in lift and nose-up pitching moments due to the pressures induced on the lower surfaces by the interaction of the jet and free-stream flow . the obvious way of minimizing these effects is to reduce the surface area surrounding the jets or buried fans, that is, to consider these effects in the preliminary stages of the airplane design .'), (1094, 'investigation of the effects of ground proximity and propeller position on the effectiveness of a wing with large chord slotted flaps in redirecting propeller slipstream downward for vertical take-off .', 'kuhn,r.e.', 'naca tn.3629, 1956.', 'investigation of the effects of ground proximity and propeller position on the effectiveness of a wing with large chord slotted flaps in redirecting propeller slipstream downward for vertical take-off . an investigation of the effects of ground proximity and propeller position on the effectiveness of a wing equipped with large-chord slotted flaps in redirecting the slipstreams from large-diameter propellers downward for vertical take-off has been conducted in a static-thrust facility at the langley aeronautical laboratory . the results indicate that, with the propeller thrust axis on the wing chord plane, both the angle through which the slipstream is deflected and the ratio of resultant force to thrust are reduced as the ground is approached . at positions nearest the ground some of the loss in resultant force is regained . lowering the thrust axis below the wing chord plane reduces the adverse effects of the ground and also reduces the large diving moments associated with the slotted-flap arrangement . the static-thrust efficiency of the propellers is slightly reduced by the ground effect .'), (1095, 'investigation of effectiveness of large-chord slotted flaps in deflecting propeller slipstreams downward for vertical take-off and low-speed flight .', 'kuhn,r.e. and draper,j.w.', 'naca tn.3364, 1955.', 'investigation of effectiveness of large-chord slotted flaps in deflecting propeller slipstreams downward for vertical take-off and low-speed flight . an investigation of the effectiveness of a wing equipped with large-chord slotted flaps in rotating the thrust vector of propellers through the angles required for vertical take-off and for flight at very low speeds has been conducted in the facilities of the langley 300 mph 7- by 10-foot tunnel . under conditions of static thrust and with zero incidence between the thrust axis and the wing chord plane, the slotted flaps were effective in rotating the thrust vector upward about than 10 percent of the thrust . when an auxiliary vane was added above the wing, the thrust vector was rotated upward configuration, vertical take-off could be achieved with an initial attitude of 16 and at airplane weights up to 90 percent of the total propeller thrust . the addition of 10 incidence between the thrust axis and the wing increased the upward rotation of the thrust vector about 10 . for the same turning angle, the diving moments associated with the slotted-flap configurations were approximately twice as large as the diving moments of the configurations with plain flaps and two auxiliary vanes .'), (1096, 'qualitative measurements of the effective heats of ablation of several materials in supersonic air jets at stagnation temperature up to 11,000 f.', 'rashis,b., witte,w.g. and hopko,r.n.', 'naca rm. l58e22, 1958.', 'qualitative measurements of the effective heats of ablation of several materials in supersonic air jets at stagnation temperature up to 11,000 f. the effective heats of ablation of a number of materials were derived from tests in supersonic air jets at stagnation temperatures ranging from 2,000 f to 11,000 f . the materials included the plastics teflon, nylon, lucite, and polystyrene,. the inorganic salts ammonium chloride and sodium carbonate, several phenolic resins of varied resin content and type of reinforcement,. and a melamine-fiber glass laminate .'), (1097, 'experimental ablation cooling .', 'bond,a.c., rashis,b. and levin,l.', 'naca rm.l58e15a, 1958.', 'experimental ablation cooling . this paper presents the results of an experimental investigation on the ablation of a number of promising materials for heating conditions comparable to those which may be encountered by unmanned reentry satellite vehicles, as well as for higher heating conditions comparable to those associated with reentry ballistic missiles . materials tested included the plastics teflon, nylon, and lucite,. the inorganic salts ammonium chloride and sodium carbonate,. graphite,. a phenolic resin and fiber glass composition,. and the commercial material haveg rocketon . results of these tests indicated heat-absorption capabilities which are several times greater than those of current metallic heat-sink materials . the results with teflon showed that for hemispherical noses there was no apparent effect of size or stagnation-point pressure on ablation rate for the range of variables covered in the tests . for flat-faced configurations, however, there was a definite increase in the ablation rate with increased stagnation-point pressure . the results for the several materials tested at heating rates associated with reentry ballistic missiles showed considerable increase in the effective heats of ablation over the results obtained at lower heating rates . this trend of increased effectiveness with increased heating potential is in agreement with the predictions of ablation theories . comparisons of the results for several materials tested at the higher heating rates showed graphite to have the lowest ablation rate of all materials tested .'), (1098, 'an experimental investigation of ablating material at low and high enthalpy potentials .', 'rashis,b. and walton,t.e.', 'nasa tm.x-263, 1960.', 'an experimental investigation of ablating material at low and high enthalpy potentials . the ablation performance characteristics of a number of materials were derived from tests conducted in a mach number 2.0 ethylene-heated high-temperature air jet having a maximum stagnation enthalpy potential of approximately 1,200 btu lb . the tests were conducted with 6- inch-diameter blunt nose shapes . the surface of most of the materials after testing was generally smooth and the unablated portions of the specimens were in appearance the same as before testing . in all cases, the back or inside surface of the specimens exhibited no evidence of heating . an evaluation of the enthalpy potential effect was obtained by comparison of the present data with previous tests conducted, on the in a subsonic arc-heated air jet . the stagnation enthalpy potential of this facility was approximately 7,000 btu lb . for teflon, the effective heat of ablation increased from approximately 1,250 btu lb to enthalpy potential was increased from'), (1099, 'a theoretical study of stagnation point ablation .', 'roberts, l.', 'nasa tr r-9, 1959 .', 'a theoretical study of stagnation point ablation . a simplified analysis is made of the shielding mechanism which reduces the stagnation-point heat transfer when ablation takes place at the surface . the most significant result of the analysis is that the effective heat capacity of the ablation material increases linearly with stream enthalpy . the automatic shielding mechanism is discussed and the significant thermal properties of a good ablation material are given parameters .'), (1100, 'an analytical investigation of ablation .', 'rashis, b. and hopko, r.n.', 'nasa tm x-300, 1960 .', 'an analytical investigation of ablation . an analytical procedure is described which enables the derivation of effective heat of ablation relationships for any type of boundary layer from transpiration cooling results . the procedure enables the inclusion of such effects as the ratio of wall temperature to local stream temperature, surface radiation, and surface combustion . the predicted effective heats of ablation for a three-dimensional laminar stagnation boundary layer for teflon material were in agreement with those derived from tests conducted at boundary-layer enthalpy potentials of 800 and approximately 7,000 btu/lb . the predicted equilibrium surface temperatures on nonablating surfaces behind an ablating material were in agreement with the values derived from tests conducted with inconel cylinders having teflon hemispherical nose pieces .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1101, 'a sensor for obtaining ablation rates .', 'winters, c.w. and bracalente, e.m.', 'nasa tn d-800, 1961 .', 'a sensor for obtaining ablation rates . a variable-capacitance ablation-rate sensor which allows continuous measurements of ablation rates for teflon and similar polymers has been developed and tested in an ethylene-heated high-temperature jet at stagnation temperatures ranging from 2,400degree to 3,800degree f . the data /length changes/ were measured by using the same telemeter equipment as that used in rocket-propelled flight vehicles . test results indicate measurement error to be a maximum of 4 percent between the telemetered length changes and the length changes that were obtained from photographic records of the test .'), (1102, 'a five-stage solid fuel sounding rocket system .', 'swanson, a.g.', 'nasa memo 3-6-59 l, 1959 .', 'a five-stage solid fuel sounding rocket system . a five-stage solid-fuel sounding-rocket system which can boost a payload of 25 pounds to an altitude of 525 nautical miles and that of 100 pounds to 300 nautical miles is described . data obtained from a typical flight test of the system are discussed .'), (1103, 'pressures, densities and temperatures in the upper atmosphere .', 'the rocket panel .', 'phys. rev. vol. 88, december 1, 1952. pp 1027-42 .', 'pressures, densities and temperatures in the upper atmosphere . averaged and internally consistent values of atmospheric pressure, density, and temperature from the ground to an altitude of 219 km have been determined and compiled by the united states groups active in upper-atmospheric research by rockets . additional relevant data by similar groups engaged in research on meteors and on the anomalous propagation of sound are also included, particularly in a brief discussion of variations with time and with place of these three atmospheric parameters .'), (1104, 'aerodynamic heating of blunt nose shapes at mach numbers up to 14 .', 'stoney, w.e.', 'naca rm l58 e05a, 1958 .', 'aerodynamic heating of blunt nose shapes at mach numbers up to 14 . results are presented from recent investigations of the aerodynamic heating rates of blunt nose shapes at mach numbers up to 14 . data obtained in flight and wind-tunnel tests have shown that the flat-faced cylinder has about 50 percent the stagnation-point heating rates of the hemisphere over nearly the entire mach number range . tests made at a mach number of 2 on a series of bodies made up of hemispherical segments of varying radius of curvature showed that slight amounts of curvature can decrease the local rates at the edge of the flat-faced cylinders with only a slight increase in the stagnation rate . the total heat transfer to such slightly curved bodies is also somewhat smaller than the total heat transfer to flat-faced cylinders . comparison of several tests with theoretical heating-rate distributions showed that both laminar and turbulent local rates can be predicted by available theories /given the pressure distribution about the body/ reasonably well, although the scatter of the available data still leaves open the choice between the theories at the edge of the bodies, where they usually differ . tests on a flat-faced cylinder at a mach number of 2.49 and at angles of attack up to 15degree showed the movement of the apparent stagnation point from the center of the body to the 50 percent windward station at creased about 30 percent while that near the leeward edge decreased about 20 percent at 15degree angle of attack . preliminary results on a concave nose have indicated the possibility that this type of design may be developed to give heating rates significantly lower than even the flat-faced cylinder rates . the test results have also shown, however, the existence of an unsteady flow phenomenon which can increase the heating rates to extremely high values .'), (1105, 'numerical solutions for supersonic flow of an ideal gas around blunt two-dimensional bodies .', 'fuller,f.b.', 'nasa tn.d791, 1961.', 'numerical solutions for supersonic flow of an ideal gas around blunt two-dimensional bodies . the method described is an inverse one,. the shock shape is chosen and the solution proceeds downstream to a body . bodies blunter than circular cylinders are readily accessible, and any adiabatic index can be chosen . the lower limit to the free-stream mach number available in any case is determined by the extent of the subsonic field, which in turn depends upon the body shape . some discussion of the stability of the numerical processes is given . a set of solutions for flows about circular cylinders at several mach numbers and several values of the adiabatic index is included .'), (1106, 'free-flight observation of a separated turbulent flow including heat transfer up to mach 8. 5.', 'mcconnell,d.g.', 'nasa tn.d278, 1961.', 'free-flight observation of a separated turbulent flow including heat transfer up to mach 8. 5. a turbulent boundary layer separated by a forward-facing step was observed on the cylindrical portion of a hemisphere-cone-cylinder test vehicle . tip blunting, producing a shear flow, was found to induce higher pressures on the cylindrical portion than were predicted from ballistic tunnel data of unblunted projectiles . an approximate method for predicting this blunt-body pressure distribution was hypothesized . these findings, along with the hypothesis, were substantiated by a wind tunnel test of a similar body . the peak pressure ratios of the separation were smaller in magnitude than flat plate theory predicted because of the effect of the shear flow . the decrement in heating of the separated flow, relative to the corresponding attached flow, was found to compare well with the expected results .'), (1107, 'the flow field over blunted flat plates and its effect on turbulent boundary growth and heat transfer at a mach number of 4. 7.', 'tendeland,t.', 'nasa tn.d689, 1961.', 'the flow field over blunted flat plates and its effect on turbulent boundary growth and heat transfer at a mach number of 4. 7. surface pressures, impact and static pressure distributions in the flow field over the plate, and local heating rates were measured on a flat plate with various leading-edge diameters . the tests were conducted at a mach number of 4.7 and a free-stream reynolds number of 3.8x10 per foot . it was found that the shape of the shock wave indicated the existence of an outward deflection of the flow over the plate . the flow deflection caused an outward deflection of the shock-wave asymptote of approximately the shock-wave angle calculated including boundary-layer growth . the mach number distributions in the shear layer evaluated from pitot and static pressure surveys agreed with predictions based on shock-wave shape . the predicted turbulent heat-transfer coefficients for the blunted flat plates agreed with the measured heat-transfer coefficients . a comparison between the measured heat-transfer coefficients for the blunted flat plates and the calculated coefficients for a sharp leading-edged plate indicated that the coefficients were highest near the leading edge of the most blunted plate . the measured heat-transfer coefficients dropped to approximately 80 percent of the sharp-plate values at a considerable distance from the leading edge for all of the blunted flat plates .'), (1108, 'a study of second-order supersonic flow theory .', 'vandyke,m.d.', 'naca r.1071, 1952.', 'a study of second-order supersonic flow theory . an attempt is made to develop a second approximation to the solution of problems of supersonic flow which can be solved by existing first-order theory . the method of attack adopted is an iteration process using the linearized solution as the first step . for plane flow it is found that a particular integral of the iteration equation can be written down at once in terms of the first-order solution . the second-order problem is thereby reduced to an equivalent first-order problem and can be readily solved . at the surface of an isolated body, the solution reduces to the well-known result of busemann . the plane case is considered in some detail insofar as it gives insight into the nature of the iteration process . again, for axially symmetric flow the problem is reduced to a first-order problem by the discovery of a particular integral . for smooth bodies, the second-order solution can then be calculated by the method of von karman and moore . bodies with corners are also treated by a slight modification of the method . the second-order solution for cones represents a considerable improvement over the linearized result . second-order theory also agrees well with several solutions for other bodies of revolution calculated by the numerical method of characteristics . for full three-dimensional flow, only a partial particular integral has been found . as an example of a more general problem, the solution is derived for an inclined cone . the possibility of treating other inclined bodies of revolution and three-dimensional wings is discussed briefly .'), (1109, 'unsteady laminar compressible boundary layers on an infinite plate with suction or injection .', 'yang,k-t.', 'j. ae. scs. 1959, 653.', 'unsteady laminar compressible boundary layers on an infinite plate with suction or injection . this study deals with unsteady compressible laminar boundary layers on an infinitely extended porous plate . an integral solution based on two types of assumed velocity and temperature profiles is presented for the general case where the unsteady free-stream velocity and rate of surface suction or injection are both arbitrary . also indicated is an exact solution, applicable, however, only to certain specific unsteady free-stream and surface suction or injection variations . the reliability and range of validity of the integral solutions is then established on the basis of numerical results from the exact solution . finally, several general qualitative conclusions of the unsteady effects of free-stream velocity and surface suction or injection on laminar boundary-layer behavior are made .'), (1110, 'on supersonic flow past a slightly yawing cone .', 'stone,ah.', 'j. math. phys. 27, 1948, 67.', 'on supersonic flow past a slightly yawing cone . this paper is concerned with the motion of a circular cone, of not too blunt an angle, through air at high speed . if the direction of motion of the cone coincides with its axis of symmetry, the resulting air flow is well known . here we consider the perturbation produced by a small /yaw/--i.e., the case in which the cone is moving not quite in the direction of its axis . the results are confirmed experimentally, and have applications to ballistics, though we are not concerned with the latter here,. they may also be useful as providing a check on various approximate methods of wider applicability . the square of the yaw is neglected--an approximation of which the validity is discussed . (similar methods can be applied to the second-order effects of the yaw, which are also of ballistic significance,. but the computations have not yet been completed .) it should be observed that, because of the lack of symmetry, the flow will be neither irrotational nor isentropic .'), (1111, 'some research on high speed flutter .', 'garrick, i.e.', '3rd a.a.aero. conf. september 1951. r.a.s. 1952, pp 419-446j .', 'some research on high speed flutter . paper presents brief discussions of many topics currently of interest in the flutter field . these include /a/ the sonic speed case, /b/ oscillating pressure field of propellers, /c/ wing flutter with various configurations including effects of body modes, and /d/ propeller stall flutter .'), (1112, 'unsteady aerodynamic forces on a slender body of revolution in supersonic flow .', 'bond,r. and packard,b.b.', 'nasa tn.d859, 1961.', 'unsteady aerodynamic forces on a slender body of revolution in supersonic flow . linearized slender-body theory is applied to the computation of aerodynamic forces on an oscillating, or deforming, body in supersonic flow . the undeformed body is a body of revolution and the deformed body is represented by movement of a line through the centers of the cross sections which are assumed to remain circular . the time dependence is based on sinusoidal motion . for a body of vanishing thickness the slender-body theory yields the apparent mass approximation as it is obtained for incompressible crossflow around a cylinder . both linearized slender-body theory and the apparent mass approximation are used to calculate the pitching-moment coefficients on a rigid slender body with a parabolic arc nose cone, and these coefficients are compared with some experimental results .'), (1113, 'an electronic apparatus for automatic recording of the logarithmic decrement and frequency for oscillations in the audio and subaudio frequency range .', 'olsson,c.o. and orlik-ruckeman,k.', 'aero. res. inst. sweden, r.52, 1954.', 'an electronic apparatus for automatic recording of the logarithmic decrement and frequency for oscillations in the audio and subaudio frequency range . an electronic apparatus for automatic evaluation of the damping of a harmonic oscillation has been designed and constructed . the apparatus is based on the idea of representing the harmonic damped oscillation by a rotating vector on the screen of a cathode-ray tube in such a way, that the rate of decrease of the length of the vector is a measure of the damping . the results are obtained simultaneously with the oscillation test as two numbers in decimal digits, which are inversely proportional to the logarithmic decrement and the frequency, respectively . the apparatus, which is named the /dampometer/, has been used for some time for free oscillation measurements of the dynamic stability derivatives of aeroplane models in windtunnels, and has proved to be very satisfactory . it gives results of usually higher accuracy than evaluation methods in common use, and permits a most considerable saving of time .'), (1114, 'steady and fluctuating pressures at transonic speeds on two space-vehicle payload shapes .', 'coe,c.f.', 'nasa tm.x503, 1961.', 'steady and fluctuating pressures at transonic speeds on two space-vehicle payload shapes . steady and fluctuating pressures have been measured at mach numbers which were varied from 0.6 to 1.2 on two bodies of revolution typical of two space-vehicle payload shapes, the centaur and the able v . the results of the investigation showed that significant fluctuations of pressure occurred on both bodies between mach numbers of 0.75 and 1.00 . the maximum fluctuations measured at any mach number and angle of attack occurred in the region of the normal shock wave as a result of shock-wave motion . large regions of unsteady pressure also occurred as a result of separation on the converging afterbody of the able-v model . the maximum pressure fluctuations occurring on the bodies increased with increasing angle of attack . for angles other than are indicated since pressure fluctuations were larger on the upper half of the bodies than on the lower half . no definite conclusions could be drawn regarding the form of the spectral densities of pressure fluctuations in the region of the shock wave . the spectral densities in regions of separation following the shock wave appeared flat except for some increase in energy level below due to slight model motions .'), (1115, 'measurements of aerodynamic forces for various mean angles of attack on an airfoil oscillating in pitch and on two finite-span wings oscillating in bending with emphasis on damping in the stall .', 'rainey,a.g.', 'naca r.1305, 1957.', 'measurements of aerodynamic forces for various mean angles of attack on an airfoil oscillating in pitch and on two finite-span wings oscillating in bending with emphasis on damping in the stall . the oscillating air forces on a two-dimensional wing oscillating in pitch about the midchord have been measured at various mean angles of attack and at mach numbers of 0.35 and 0.7 . the magnitudes of normal-force and pitching-moment coefficients were much higher at high angles of attack than at low angles of attack for some conditions . large regions of negative damping in pitch were found, and it was shown that the effect of increasing the mach number from 0.35 to 0.7 was to decrease the initial angle of attack at which negative damping occurred . measurements of the aerodynamic damping of a 10- percent-thick and of a 3-percent-thick finite-span wing oscillating in the first bending mode indicate no regions of negative damping for this type of motion over the range of variables covered . the damping measured at high angles of attack was generally larger than that at low angles of attack .'), (1116, 'general instability of stiffened cylinders .', 'becker,h.', 'naca tn.4237, 1958.', 'general instability of stiffened cylinders . theoretical buckling stresses are determined in explicit form for circular cylinders with circumferential and axial stiffening . the loadings are axial compression, radial pressure, hydrostatic pressure, and torsion . analyses were confined to moderate-length and long cylinders . the investigation was based upon the use of a form of donnell''s equation derived by taylor which is applicable to orthotropic cylinders . the derivation of this equation is presented in this report .'), (1117, 'stability of orthotropic cylindrical shells under combined loading .', 'hess,t.e.', 'ars j. 31, 1961.', 'stability of orthotropic cylindrical shells under combined loading . the increasing use of fiber and whisker reinforced materials makes necessary the availability of methods of analyzing cylinders and cones composed of an orthotropic material . this paper treats the buckling of such shells under a combination of axial compression and uniform external pressure . the differential equation approach of flugge is used, and the resulting buckling equation is presented in terms of axial and circumferential bending rigidities, shear rigidity, poisson''s ratio, geometry parameters and mode shapes . design curves are presented which allow quick determination of critical loads on cylinders, and, by using the equivalent cylinder concept, on conical shells of small included angle . the curves also lend themselves to /tailoring/ of materials to fit the load carrying requirements of the structure .'), (1118, 'elastic stability of orthotropic shells .', 'becker,h. and gerard,g.', 'j. ae. scs. 29, 1962.', 'elastic stability of orthotropic shells . a small-deflection theory for general instability of orthotropic circular cylindrical shells has been derived for external pressure, torsion, and axial compression . for the first two types of loading, comparison of the theory with experimental data for orthotropic cylinders reveals agreement comparable with that obtained for isotropic shells . for axial compression, experimental data have been found to agree reasonably well with theory for orthotropic cylinders, in contrast to the agreement usually obtained for isotropic cylinders .'), (1119, 'plastic stability theory of thin shells .', 'gerard,g.', 'j. ae. scs. 24, 1957.', 'plastic stability theory of thin shells . considerable interest is currently centered on the role of deformation and flow types of plasticity theories in the solution of stability problems . for thin flat plates, deformation theory combined with classical stability theory appears to yield results which are in substantially good agreement with test data . on the other hand, flow or incremental theories appear to require the introduction of initial imperfections in order to obtain a satisfactory degree of correlation with tests . thus, in view of the current state of development of plastic stability theory, it appears fruitful to exploit the mathematical simplicity inherent in deformation theory in the investigation of the plastic stability of thin shells . although there may be theoretical objections to deformation theories as a class, test data on flat plates do suggest the predictive value of the results obtained from this theory . in this paper, a set of equilibrium differential equations for the plastic buckling of thin shells of constant unequal radii is derived . this set of three equations applies to flat plates, cylinders, and spheres under any loading system leading to buckling . for particular problems such as buckling of cylinders under axial compression, torsion or lateral pressure, and spheres under external pressure, the set of equations can be reduced to a single eighth-order partial differential equation of the donnell type in terms of the radial displacement only . these donnell-type equations are used to obtain solutions for plastic buckling of spheres under external pressure and long and moderate length cylinders under lateral pressure or torsion loads . the limiting cases of a simply supported flat plate under compression or shear, represent the solutions for short cylinders under lateral pressure or torsion, respectively .'), (1120, 'a unified theory of plastic buckling of columns and plates .', 'stowell,e.z.', 'naca r.898, 1948.', 'a unified theory of plastic buckling of columns and plates . on the basis of modern plasticity considerations, a unified theory of plastic buckling applicable to both columns and plates has been developed . for uniform compression, the theory shows that long columns which bend without appreciable twisting require the tangent modulus and that long flanges which twist without appreciable bending require the secant modulus . structures that both bend and twist when they buckle require a modulus which is a combination of the secant modulus and the tangent modulus .'), (1121, 'compressive and torsional buckling of thin-wall cylinders in the yield region .', 'gerard,g.', 'naca tn.3726, 1956.', 'compressive and torsional buckling of thin-wall cylinders in the yield region . based on assumptions which have led to the best agreement between theory and test data on inelastic buckling of flat plates, a general set of equilibrium differential equations for the plastic buckling of cylinders has been derived . these equations have been used to obtain solutions for the compressive and torsional buckling of long cylinders in the yield region . test data are presented which indicate satisfactory agreement with the theoretical plasticity-reduction factors in most cases . where a difference in results exists, test data are in substantially better agreement with the results obtained by use of the maximum-shear law rather than the octahedral-shear law to transform axial stress-strain data to shear stress-strain data .'), (1122, 'on the role of initial imperfections in plastic buckling of cylinders under axial compression .', 'gerard,g.', 'j. ae. scs. 29, 1962.', 'on the role of initial imperfections in plastic buckling of cylinders under axial compression . in a recent paper lee treated the complex problem of the plastic buckling and postbuckling behavior of an axially compressed cylindrical shell containing initial imperfections, representing an important step forward in our understanding of this common, yet perplexing, structural element . lee drew two major conclusions .. (a) even with initial imperfections the incremental theory of plasticity considerably overestimates the buckling strength as compared with the deformation theory, which is in substantially good agreement with experiments, and strength of cylindrical shells subject to axial compression are significant . it is the purpose of this note to discuss the second conclusion in terms of lee''s experimental and theoretical results, other experimental data on inelastic buckling of 7075-t6 aluminum-alloy cylinders, and recent theoretical results on the inelastic buckling of cylinders in the axisymmetric and circumferential modes . in particular, this writer does not believe that lee has proved that initial imperfections are important for the group of cylinders that he has tested . on the contrary, it is believed that initial imperfections are completely insignificant for this group of cylinders although of probable significance in other cases .'), (1123, 'an extension of donnell''s equation for a circular cylindrical shell .', 'tsu-tao loo', 'associate professor of mechanics, rensselaer polytechnic institute, troy, n.y.', 'an extension of donnell''s equation for a circular cylindrical shell . in 1933, donnell succeeded in simplifying the equations of equilibrium for a circular cylindrical shell he obtained simple relations between the critical buckling shearing stress and the physical properties of a thin circular cylinder under pure torsion . his approach reduces the tedious computations involved in the classical solutions and is still in good agreement with them . furthermore, it is easy to show that the well-known classical solution for critical compressive stress of the cylinder under axial compression can readily be obtained from donnell''s equation .'), (1124, 'design of missile bodies for minimum drag at very high speeds - thickness ratio, lift, and center of pressure given .', 'strand,t.', 'j. ae. scs. 1959, 568.', 'design of missile bodies for minimum drag at very high speeds - thickness ratio, lift, and center of pressure given . newtonian flow theory has been used to develop a procedure for the design of minimum drag bodies of revolution having a given thickness ratio and center of pressure . it is shown that the optimum body shape is independent of lift . center of pressure location, however, exerts a powerful influence on both the shape of the body and on the drag coefficient at zero lift .'), (1125, 'collapse by instability of thin cylindrical shells under external pressure .', 'windenburg,d.f. and trilling,c.', 'asme trans. 56, 1934, 819.', 'collapse by instability of thin cylindrical shells under external pressure . this paper discusses the collapse by instability of thin-walled cylindrical vessels subjected to external pressure . the most important of the theoretical and empirical formulas that apply to this subject are presented in a common notation . a new and simple instability formula is developed . three classes of tubes are considered .. tubes of infinite length,. tubes of finite length with uniform radial pressure only,. and tubes of finite length with both uniform radial and axial pressure . collapsing pressures calculated by the various formulas are presented in tabular form as a means of comparing the formulas . the formulas are discussed briefly and checked against the results of tests conducted at the u. s. experimental model basin for the bureau of construction and repair, navy department . this paper is a sequel to one previously published as a part of the work of the a.s.m.e. special research committee on the strength of vessels under external pressure .'), (1126, 'an engineer''s conceptual approach to the buckling of cylindrical shell (axial loading) .', 'cunningham,j.h.', 'ias paper 62-106, 1962.', 'an engineer''s conceptual approach to the buckling of cylindrical shell (axial loading) . by using the well known analogy between the bending of a beam on an elastic foundation and the axial symmetric displacement of a cylinder, a physical insight is obtained for the buckling of cylindrical shells under axial compression . the technique is equivalent to classical small deflection theory and provides good agreement with the more elaborate solutions for the buckling strength of various sandwich, multi-layered, and orthotropic cylinders, including the effects of internal pressure or an elastic core .'), (1127, 'the buckling of sandwich type panels .', 'hoff,n.h. and mautner,s.f.', 'j. ae. scs. 1945, 285.', 'the buckling of sandwich type panels . fifty-one flat rectangular sandwich-type panels were tested in edgewise compression with the unloaded edges of the panels restrained by v-grooves . the sandwich consisted of papreg faces and a cellular cellulose acetate core . the thickness of the faces varied from 0.00675 to 0.02025 in.,. the core, from 0.066 to 0.741 in.,. the width of the panel, from 4 to 11 in . the length of the panel was always 10.5 in . the buckled shape consisted of a ripple of short wave length across the panel . it was either symmetric, the two faces bulging out symmetrically according to sine curves, or skew, the two faces deflecting in the same sense according to sine curves having a phase angle of 90 . a strain energy theory of buckling is presented for both the symmetric and the skew cases, and the buckling load in the symmetric case is also calculated by integration of the differential equation . the agreement between the theoretic and the experimental buckling stress is reasonable, that between the predicted and actual buckled shape good . a simple formula is developed which permits a choice of the most suitable core material when the mechanical properties of the face material are given .'), (1128, 'face wrinkling and core strength in sandwich construction .', 'yuseff,s.', 'j. ae. scs. 1960.', 'face wrinkling and core strength in sandwich construction . the effect of initial waviness on the wrinkling of faces in sandwich construction is studied . formulae are derived to determine the failing stress when the faces wrinkle due to failure of the core in tension, compression or shear . the importance of core strength requirements in maintaining surface smoothness is noted . a comparison of theory with experiments is made, and the agreement between the two is found to be reasonably good . the strength of the core . williams has related the strength of the core in tension and shear to an arbitrarily assumed initial irregularity which, to ensure laminar flow in a wing is assumed to have a maximum admissible value (initial wave amplitude critical wavelength=0.0005 to 0.001) .'), (1129, 'general instability of a ring stiffened circular cylindrical shell under hydrostatic pressure .', 'bodner,s.r.', 'j. app. mech. 24, 1957, 269.', 'general instability of a ring stiffened circular cylindrical shell under hydrostatic pressure . the general instability load of a ring-stiffened, circular cylindrical shell under hydrostatic pressure is determined by analyzing an equivalent orthotropic shell . a set of differential equations for the stability of an orthotropic shell is derived and solved for the case of a shell with simple end supports . the solution is presented in terms of parameters of the ring-stiffened, isotropic shell, and a relatively simple expression for the general instability load is obtained . some numerical examples and graphs of results are presented . in addition, an energy-method solution to the problem is outlined, and the energy and displacement functions that could be used in carrying out a rayleigh-ritz approximation are indicated .'), (1130, 'handbook of structural stability . pt .vi . strength of stiffened curved plates and shells .', 'becker,h.', 'naca tn.3786, 1958.', 'handbook of structural stability . pt .vi . strength of stiffened curved plates and shells . a comprehensive review of failure of stiffened curved plates and shells is presented . panel instability in stiffened curved plates and general instability of stiffened cylinders are discussed . the loadings considered for the plates are axial, shear, and the combination of the two . for the cylinders, bending, external pressure, torsion, transverse shear, and combinations of these loads are considered . general instability in stiffened cylinders was investigated . for bending and torsion loads, test data and theory were correlated . for external pressure several existing theories were compared . as a result of this investigation a unified theoretical approach to analysis of general instability in stiffened cylinders was developed .'), (1131, 'the effect of axial constraint on the instability of thin conical shells under external pressure .', 'singer,j.', 'j. app. mech. 29, 1962, 212.', 'the effect of axial constraint on the instability of thin conical shells under external pressure . author considers elastic axial restraint which may be (1) active from the beginning of loading and (2) active only at the onset of buckling . buckling loads for the two cases are related by a simple conversion factor . effect of the restraint on the axisymmetric type buckling is negligible, but the amplification of the critical load for the nonaxisymmetric type buckling may be very large for type (1) restraint . design curves are included for a range of l d.and restraint stiffness . results are of doubtful value considering the known inadequacy of the linear theory of buckling under axial compression .'), (1132, 'general instability of ring stiffened cylindrical shells subject to external hydrostatic pressure - a comparison of theory and experiment .', 'galletly,d.d.', 'j. app. mech. 25, 1958, 259.', 'general instability of ring stiffened cylindrical shells subject to external hydrostatic pressure - a comparison of theory and experiment . tests are described of a number of machined-stiffened cylinders subjected to external hydrostatic pressure, and the observed general instability strengths compared with predictions from theories of kendrick and nash . agreement with kendrick was found rather good . results also are presented from electrical strain gages which show in detail the growth of embryonic lobes and nonlinear characteristics of deformation at the threshold of buckling . weakening effects of imperfect circularity are discussed .'), (1133, 'a simple extension of southwell''s method for determining the elastic general instability pressure of ring-stiffened cylinders subject to external hydrostatic pressure .', 'g. d. galletly and t. e. reynolds', 'david taylor model basin, washington, d. c.', 'a simple extension of southwell''s method for determining the elastic general instability pressure of ring-stiffened cylinders subject to external hydrostatic pressure . a simple extension of southwell''s method is presented whereby the elastic general instability pressure of ring-stiffened cylindrical shells subject to external hydrostatic pressure may be determined . an actual application of the method is illustrated in the paper and the results of several other examples are summarized .'), (1134, 'torispherical shells - a caution to designers .', 'galletly,g.d.', 'j. eng. for industry, 1959, 51.', 'torispherical shells - a caution to designers . it has recently become apparent, through a rigorous stress analysis of a specific case that designing torispherical shells by the current edition of the asme code on unfired pressure vessels can lead to failure during proof-testing of the vessel . the purpose of the present paper is to show in what respects the code fails to give accurate results . as an illustrative example, a hypothetical pressure vessel with a torispherical head having a diameter-thickness ratio of 440 was selected . the supports of the vessel were considered to be either on the main cylinder or around the torus . the vessel was subjected to internal pressure and the elastic stresses in it were determined rigorously and by the code . a comparison of the two revealed that the code predicted stresses in the head which were less than one half of those actually occurring . furthermore, the code gave no indication of the presence of high compressive circumferential direct stresses which exceeded 30,000 psi for practically the entire torus . if the head had been fabricated using a steel with a yield point of would have failed or undergone large deformations, whereas the code would have predicted that it was safe . the code''s rules for torispherical heads are thus in need of revision for certain geometries . the implications of the foregoing results are currently being studied by the asme,. in the interim, however, designers should exercise care in applying the code to torispherical shells . it is also shown in the paper that the use of the membrane state as a particular solution of the differential equations is not a good approximation for toroidal shells of the type considered .'), (1135, 'limit analysis of symmetrically loaded thin shells of revolution .', 'drucker,d.c. and shield,r.t.', 'j. app. mech. 26, 1959, 61.', 'limit analysis of symmetrically loaded thin shells of revolution . the yield surface for a thin cylindrical shell is shown to be a very good approximation to the yield surface for any symmetrically loaded thin shell of revolution . hexagonal prism approximations to this yield surface, appropriate for pressure vessel analysis, are described and discussed in terms of limit analysis . procedures suitable for finding upper and lower bounds on the limit pressure for the complete vessel are developed and evaluated . they are applied for illustration to a portion of a toroidal zone or knuckle held rigidly at the two bounding planes . the combined end force and moment which can be carried by an unflanged cylinder also is discussed .'), (1136, 'design of thin walled torispherical and toriconical pressure - vessel heads .', 'shield,r.t. and drucker,d.c.', 'j. app. mech. 28, 1961, 292.', 'design of thin walled torispherical and toriconical pressure - vessel heads . the failure under hydrostatic test of a large storage vessel designed in accordance with current practice stimulated earlier analytical studies . this paper gives curves and a table useful for the design and analysis of the knuckle region of a thin torispherical or toriconical head of an unfired cylindrical vessel . a simple but surprisingly adequate approximate formula is presented for the limit pressure, np, at which appreciable plastic deformations occur .. where p is the design pressure, is the yield stress of the material, and n is the factor of safety . the thickness t of the knuckle region is assumed uniform . upper and lower bound calculations were made for ratios of knuckle radius r to cylinder diameter d of 0.06, 0.08, 0.10, 0.12, 0.14, and 0.16, and ratios of spherical cap radius l to d of 1.0, 0.9, 0.8, 0.7, and 0.6 . toriconic1a heads may be designed or analyzed closely enough by interpreting in table 1 as the complement of the half angle of the cone .'), (1137, 'on the theory of thin elastic toroidal shells .', 'clark,r.a.', 'j. math. phys. 29, 1950, 146.', 'on the theory of thin elastic toroidal shells . the author obtains asymptotic solutions to the problem of rotationally symmetric small deflection of thin toroidal elastic shells . he first reduces the problem to that of integrating a single linear nonhomogeneous ordinary differential equation involving two parameters . asymptotic formulae for the complementary function are obtained by applying the general method of langer (trans.amer.math.soc.33, advantage of yielding results valid near the points where the tangent plane is perpendicular to the axis of revolution, where the methods of asymptotic integration customary in shell theory fail (see the preceding review) . for two problems in which only the complementary function is required, the author''s results are compared with those obtained by wissler (dissertation, zurich, 1916) by a method of power series expansion,. the agreement is within 4 or better . the author observes that the usual method of obtaining asymptotic expressions for a particular integral, being based on using as an approximation the complementary function obtained from the membrane theory, will fail near points where the tangent plane is perpendicular to the axis of revolution . he therefore introduces a new method, which he states was developed jointly with e. reissner . he applies his results to the cases of an joint loaded symmetrically and parallel to its axis, a corrugated pipe subject to axial load, and a corrugated cylinder subject to axial pressure . many numerical calculations are involved and there are two tables of functions occuring in the solutions .'), (1138, 'asymptotic solutions of toroidal shell problems .', 'clark,r.a.', 'quart. app. math. 16, 1958, 47.', 'asymptotic solutions of toroidal shell problems . method of asymptotic integration developed by e. reissner and author is refined, and solutions previously obtained for problems of bending of curved tube and of a toroidal expansion joint subject to an axial force are generalized and extended . results are compared to those obtained by l. beskin . for large values of a certain parameter, agreement is good .'), (1139, 'the effect of entrance velocity on the flow of a rarefied gas, through a tube .', 'pond,h.l.', 'j. ae. scs. 29, 1962, 917.', 'the effect of entrance velocity on the flow of a rarefied gas, through a tube . the flow of a rarefied gas through a circular tube is considered . molecules entering the tube have a mass velocity directed down the tube, as well as a randomly directed thermal velocity . it is assumed that the conditions for free-molecule flow hold, and that molecules striking the tube wall are reflected diffusely . the mass velocity and tube dimensions are restricted only by the limitation to free-molecule flow . the theory is illustrated by an example of the effect of an entrance tube on the measurement of pressure and density by a gage placed on a satelite .'), (1140, 'shock-standoff distance for spherical bodies at high mach numbers .', 'chaudhury,p.r.', 'j. ae. scs. 29, 1962, 745.', 'shock-standoff distance for spherical bodies at high mach numbers . usaf-sponsored development of a simple expression for shock-standoff distance, by consideration of an apropriate specific-heat ratio behind the shock . the theory gives predictions which are almost as accurate as those of the van dyke and hays methods .'), (1141, 'the wake behind an oscillating vehicle .', 'zeisberg,s.l.', 'j. ae. scs. 29, 1962, 1344.', 'the wake behind an oscillating vehicle . the incompressible laminar far wake behind an oscillating vehicle is analyzed with the use of the oseen linearization, and the assumption that the wake cross section is axisymmetric . time-wise drag variations are thus represented as changes in the wake diameter and velocity defect . it is shown that the unsteadiness in this flow can be predicted by a quasi-steady theory .'), (1142, 'effect of wall divergence on sonic flows in solid wall tunnels .', 'tirumalesa,d. and satyanarayana,b.', 'j. roy. soc. 66, 1962, 125.', 'effect of wall divergence on sonic flows in solid wall tunnels . the effect of wall divergence (in excess of the normal value given for compensating the boundary layer growth on the tunnel walls), on the simulation of sonic flows in solid wall wind tunnels is presented in this note which is a condensed version of ref. 1 .'), (1143, 'a one-foot hypervelocity shock tunnel in which high-enthalpy real gas flows can be generated with flow times of about 180 milliseconds .', 'cunningham,b.e. and kraus,s.', 'nasa tn.d1428, 1962.', 'a one-foot hypervelocity shock tunnel in which high-enthalpy real gas flows can be generated with flow times of about 180 milliseconds . a shock tunnel is described in which high-enthalpy, real-gas air flows can be generated with flow times of about 180 milliseconds . this shock tunnel is operated with a combustion-heated driver gas and consists of a combustion chamber, shock tube, supersonic nozzle, test section, and vacuum tank . an essential feature of this shock tunnel is a means for achieving a constant-pressure air reservoir for the duration of the test . air streams with velocities in excess of achieved at a mach number of about 10 . the corresponding stream total enthalpy is about 4,500 btu lb and the stagnation pressure is 3.25 psia .'), (1144, 'slipstream flow around several tilt-wing vtol aircraft models operating near the ground .', 'william a. newsom, jr., and louis p. tosti', 'technical note d-1382', 'slipstream flow around several tilt-wing vtol aircraft models operating near the ground . a collection of data from a number of brief investigations made with three different models to determine the character of the slipstream flow along the ground is presented for multiple-propeller tilt-wing vtol aircraft configurations operating near the ground . in general, the tests involved tuft surveys and slipstream dynamic-pressure measurements for several tilt-wing vtol models . a more extensive series of tests, including some measurements of the erosion of gravel by the slipstream and some measurements of the unsteady rolling, yawing, and pitching moments, was also made on one of the models operating in the hovering condition near the ground . the results of the flow studies indicated the presence of a stronger and deeper slipstream flow along the center line of the aircraft, and to some extent along parallel planes between adjacent propellers (on one wing), than to the side of the aircraft . this effect is caused by an intensification of the individual slipstreams as they meet at the planes of flow symmetry . the intensified flow along the center line of the aircraft is amplified by the presence of the fuselage and causes the dynamic pressure to be greater in front of the aircraft than would be expected on the basis of the slipstream of the individual propellers . in the erosion tests it was found that gravel, if sufficiently small, was rapidly eroded by the slipstream and that this gravel could be thrown high into the air if it struck even very small fixed obstacles on the ground (obstacles with a height less than the diameter of the gravel) . results of the investigation of moment fluctuations indicated that there are large, erratic variations of rolling, yawing, and pitching moments and that the propellers, reacting to an erratic inflow from the recirculating slipstream, are the primary source of these moments .'), (1145, 'buckling of core-stabilized cylinders under axisymmetric external loads .', 'don o. brush and bo o. almroth', 'lockheed missiles and space company', 'buckling of core-stabilized cylinders under axisymmetric external loads . an equation is derived for the elastic stability of a circular cylindrical shell which is filled with a soft elastic core and is subjected to general axially-symmetric lateral pressure combined with a central axial force . numerical results are given for three lateral pressure distributions of interest in rocket motor case analysis.. uniform pressure, linearly varying pressure, and a circumferential band of pressure located at an arbitrary distance from one end of the cylinder . comparison is made with results of previous theoretical and experimental investigations, where available .'), (1146, 'thermal buckling of cylinders .', 'melvin s. anderson', 'nasa langley research center', 'thermal buckling of cylinders . several theoretical and experimental investigations on the buckling of cylinders due to both axial and circumferential thermal stresses are reviewed . differences that exist among the various results are discussed and areas of future work are indicated .'), (1147, 'heat transfer to bodies traveling at high speed in the upper atmosphere .', 'jackson r. stalder and david jukoff', 'report 944', 'heat transfer to bodies traveling at high speed in the upper atmosphere . a general method has been developed, using the methods of kinetic theory, whereby the surface temperatures of bodies can be calculated for steady flight at any speed in a rarefied gas . the particular solution was made for a flat plate., however, the calculations can be easily extended to bodies of arbitrary shape . it was found that the aerodynamic heating problem in the absence of solar radiation, that is, for the case of nocturnal flight, becomes of negligible importance at altitudes of 125 miles and higher and up to steady flight speeds of 36,000 feet per second . the effect of solar radiation, for the case of daytime flight, becomes increasingly important as the flight altitude is increased . at an altitude of 150 miles and higher, solar radiation is the predominating factor that determines skin temperature . owing to the strong effect of solar radiation on skin temperatures at high altitudes, the desirability of nocturnal flight is indicated in order to minimize skin temperatures . in order to maintain low skin temperatures, it was found that the angle of inclination of the body with respect to the flight path should be kept as small as possible . this may be accomplished in practice by designing the body to be finely tapered and by flying the body at small angles of attack . it is pointed out that skin temperatures may be reduced by insuring thermal contact between portions of the skin inclined at positive and negative angles with respect to the flight path . as much surface as possible should be inclined at negative angles . practically, this may be accomplished by boattailing the body . in the event that an internal skin-cooling system is employed, it is shown that the rate of internal cooling must be of the same order of magnitude or greater than the rate at which heat is lost naturally by emitted radiation . if the cooling rate is below the natural radiation rate, cooling has little effect upon skin temperatures . it is shown that, in the case of a missile designed to fly over a wide range of altitudes and speeds, it is desirable to make the emissivity of the skin as high as possible . this conclusion, however, is based upon a skin surface for which the emissivity is independent of the wave length of the emitted and absorbed radiant energy . a possible method of reducing surface temperatures is indicated by the decrease in skin temperature which accompanies a decrease in thermal accommodation coefficient . this phenomenon may be used to advantage if it is possible to decrease the accommodation coefficient by altering the surface characteristics of the skin .'), (1148, 'knudsen flow through a circular capillary .', 'w. c. demarcus and e. h. hopper', 'carbide and carbon chemicals company, k-25 plant, post office box p, oak ridge, tennessee', 'knudsen flow through a circular capillary . the problem of knudsen flow through a circular capillary has been often discussed, usually by the momentum transfer method . however, p. clausing gave a rigorous formulation for the problem and obtained an integral equation for which he gave an approximate solution . from time to time the accuracy of clausing''s solution has been questioned and since clausing did not give a rigorous estimate of his error we have reinvestigated the problem .'), (1149, 'similar temperature boundary layers .', 'dragutin stojanovic', 'university of beograd, jugoslavia', 'similar temperature boundary layers . conditions for the existence of similar solutions are known for (a) two-dimensional, incompressible, steady and nonsteady laminar boundary layers and (b) three-dimensional, incompressible, steady, laminar boundary layers for a body of revolution rotating in a fluid at rest or a body of revolution in a rotating fluid flow . corresponding conditions for the existence of similar temperature boundary layers in both cases are given for constant and variable wall temperatures . the general conclusion is that, in all these cases, with or without viscous heating, and with constant wall temperature, conditions for the existence of similar velocity boundary layers are at the same time the conditions for the existence of similar temperature boundary layers . if the wall temperature is variable, the conditions for the existence of similar velocity boundary layers are at the same time the conditions for the existence of similar temperature boundary layers if the wall temperature varies as a power of the local free-stream velocity or surface velocity . numberical solutions are given for the nondimensional temprature distributions function and the nondimensional temperature gradient at the wall for several prandtl numbers in the case of a rotating flow over an infinite plate at rest .'), (1150, 'preliminary results of density measurements from an air force satellite.', 'mcisaac, j., pond, h. and stergis, g.', 'air force, cambridge lab., 63-24. february, 1963 .', 'preliminary results of density measurements from an air force satellite. atmospheric density was determined from a singly mounted ionization gauge flown on an air force satellite . included is a brief description of the experiment and theory as well as a discussion of some of the problems involved in performing these measurements . density data are given for the altitude range of 370 to 400 km during early morning hours for the two days 17 and 18 june 1961 . results are compared with those of the 1961 revised u.s. standard atmosphere .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1151, 'experiments on supersonic blunt-body flows .', 'kendall,j.m.', 'j.p.l.prog. r.20-372, 1959.', 'experiments on supersonic blunt-body flows . recently, progress has been made in the theoretical calculation of the inviscid flow between the detached shock wave and the surface of a blunt body travelling at supersonic speed . detailed experimental data are needed for comparison . experiments have been made in the supersonic wind tunnels of the jet propulsion laboratory on spheres, disks, and blunted cones over the mach number range 1.8 to 5.0 in air . surface pressures, shock wave shape and detachment distance, certain velocity gradients, the sonic line location, and some streamline paths were determined . the sonic line is found as the trace of the termination point of a weak shock generated by a probe ahead of the model . for a sphere, good agreement between theory (of van dyke) and experiment is found .'), (1152, 'on periodically oscillating wakes in the oseen approximation .', 'lin,c.c.', 'studies in maths . and mechs ., presented to r.von mises, p 170, academic press, 1954.', 'on periodically oscillating wakes in the oseen approximation . studies in maths . and mechs ., the oscillating vortex wake behind an obstacle at reynolds numbers of order 10 is studied by means of the oseen approximation .'), (1153, 'a study of the simulation of flow with free stream mach number 1 in a choked wind tunnel .', 'spreiter, j.r., smith, d.w. and hyett, b.j.', 'nasa tr r-73, 1960 .', 'a study of the simulation of flow with free stream mach number 1 in a choked wind tunnel . the degree to which experimental results obtained under choking conditions in a wind tunnel with solid walls simulate those associated with an unbounded flow with free-stream mach number 1 is investigated for the cases of two-dimensional and axisymmetric flows . it is found that a close resemblance does indeed exist in the vicinity of the body, and that the results obtained in this way are generally at least as accurate as those obtained in a transonic wind tunnel with partly open test section . some of the results indicate, however, that substantial interference effects, particularly those of the wave reflection type, may be encountered under certain conditions, both in choked wind tunnels and in transonic wind tunnels, and that the reduction of these interference effects to acceptable limits may require the use of models of unusually small size .'), (1154, 'on the influence of wall boundary layers in closed transonic test sections .', 'berndt s.b.', 'ffa rep. 71, stockholm, 1957 .', 'on the influence of wall boundary layers in closed transonic test sections . the boundary layers at the test section walls of a transonic wind tunnel are known to reduce the wall interference . in the present paper this effect is studied by means of small perturbation theory, assuming viscosity to be negligible when perturbing a turbulent boundary layer . an approximation for thin boundary layers leads to a modified boundary condition at the wall of the test section, expressing the normal streamline slope induced by changes in mass flow density and crossflow within the boundary layer . this boundary condition is applied to the linearized equations of subsonic flow and to the non-linear transonic equations at choking, the cases of plane and circular test sections only being treated in detail . the results of linear theory show that all corrections except the three-dimensional angle-of-attack correction are considerably reduced by the presence of the boundary layers at mach numbers greater than 0.9, the essential part of their influence being due to the change of mass flow density with pressure . in the case of choking the analysis indicates that the presence of boundary layers will increase the maximum model size for which the flow can be interpreted as corresponding to mach number one in free flight . finally, the technique of using artificial thickening of the wall boundary layers for a reduction of wall interference is considered, though without reaching a definite conclusion as to its value as compared to other techniques .'), (1155, 'some experimental investigations on the influence of wall boundary layers upon wind tunnel measurements at high subsonic speeds .', 'petersohn, e.g.m.', 'ffa n.r. 44, stockholm, 1952 .', 'some experimental investigations on the influence of wall boundary layers upon wind tunnel measurements at high subsonic speeds . pressure distribution measurements and drag determination by means of balance measurements have been carried out for a number of models at high subsonic velocity in wind tunnels, where the boundary layer of the walls has been varied . within the investigated range it appeared that a thickening of the boundary layer reduced the disturbing influence of the walls, which also caused an increase of the choking mach number . the phenomenon described should be of a certain importance from the point of view of wind tunnel technique, since it is possible to increase the choking velocity for a given model by means of thickening the boundary layer .'), (1156, 'experimental investigation of attenuation of strong shock waves in a shock tube with hydrogen and helium as driver gases .', 'jones,j.j.', 'nasa tn.4072, 1957.', 'experimental investigation of attenuation of strong shock waves in a shock tube with hydrogen and helium as driver gases . an experimental investigation has been made of the attenuation of strong shock waves in air in a shock tube . time-history measurements were made of the static pressure at several stations in the wall of the tube . the internal diameter of the tube is 3.75 inches . shock- wave-velocity data were taken for a distance along the tube of about 120 feet . the range of the shock-wave mach number covered was from 5 to 10 and the initial pressure ahead of the shock wave varied from 5 to 100 millimeters of mercury . hydrogen and helium were used as driver gases . a helium-driven shock wave was found to decay only about one-half as rapidly as a hydrogen-driven shock wave . the pressure level had little effect on the attenuation rate of a shock wave of given strength for the pressure range investigated . the static-pressure measurements indicated that a severe pressure gradient existed in the latter portion of the air flow . this gradient limits the testing time useful for obtaining reliable aerodynamic data .'), (1157, 'hypersonic shock tunnel .', 'nagamatsu, h.t. et al.', 'ars j.v. 29, may 1959, pp 332-340 .', 'hypersonic shock tunnel . a hypersonic shock tunnel has been developed for obtaining fluid mechanic information at the high mach numbers and corresponding stagnation temperatures encountered in flight by long range ballistic vehicles and satellites . this report describes the hypersonic shock tunnel and presents some of the results obtained in the driven tube and in the nozzle helium is ignited in the driver to produce strong shock waves in air . a shock velocity in air as high as 55,000 fps with a calculated equilibrium temperature of 16,000 k has been produced in the driven tube . the effects of high stagnation temperatures upon the detached shock wave and the pressure distribution for blunt bodies have been observed in the nozzle test section . the detachment distance devreased greatly at high temperatures . the pressure distribution for the hemisphere was found to be less than that predicted by the modified newtonian theory . shock wave boundary layer interaction at the leading edge of a flat plate was observed, and the results agreed with the analytical prediction . a detached shock wave was observed for a blunt two-dimensional body at very low densities in the test section with a flow mach number of 19.6 .'), (1158, 'the tailored-interface hypersonic shock tunnel .', 'charles e. wittliff, merle r. wilson, and abraham hertzberg', 'cornell aeronautical laboratory, inc.', 'the tailored-interface hypersonic shock tunnel . the /tailored-interface/ hypersonic shock tunnel provides a means for producing the high mach number, high stagnation temperature flow conditions encountered in hypersonic flight . various gasdynamic phenomena associated with shock tunnels are discussed, and experimental evidence of the successful application of this technique is presented . as an indication of its research application, the results of heat-transfer experiments on a hemisphere-cylinder model are presented and compared with theory .'), (1159, 'experimental investigation of the effect of yaw on rates of heat transfer to transverse circular cylinders in a 6500-foot-per-second hypersonic air stream .', 'bernard e. cunningham and samuel kraus', '', 'experimental investigation of the effect of yaw on rates of heat transfer to transverse circular cylinders in a 6500-foot-per-second hypersonic air stream . a technique has been developed by which air can be shock-compressed by helium to 3660 degrees rankine to generate a 6500-foot-per-second air stream with a flow duration of 40 milliseconds . the resulting equipment is described . experiments were conducted to determine rates of heat transfer to transverse circular cylinders of 0.003-, greater than 100 . the cylinders were tested at a nominal mach number of 11 with a stagnation reynolds number (evaluated with free-stream mass flow and stagnation viscosity of 4.00 times 10 to the 4th power per foot .'), (1160, 'recent advances in gaseous detonation .', 'gross,r.a. and oppenheim,a.k.', 'ars j. 29, 1959, 173', 'recent advances in gaseous detonation . a review of recent work in gaseous detonation is presented . early work is briefly mentioned and treatises listed . theoretical calculations of chapman-jouguet detonations are reviewed, compared and the ambiguity concerning the speed of sound in a reacting gas mixture discussed . experimental chapman-jouguet measurements are reviewed . recent studies of the interior of a detonation wave are presented . standing detonation wave research, detonation limits, two-dimensional detonations, spectra, ionization and magnetohydrodynamic treatments are brought to the reader''s attention . a qualitative description of the development of a flame to a detonation is presented . experimental observations are examined and recent theoretical attempts to explain these observations are reviewed .'), (1161, 'stagnation point heat transfer measurements in dissociated air .', 'rose,p.h. and stark,w.i.', 'j. ae. scs. 26, 1958, 86.', 'stagnation point heat transfer measurements in dissociated air . the results of an experimental investigation of the laminar heat transfer at the stagnation point of a blunt body in partially dissociated air are presented and are compared to the theoretical treatment of fay and riddell . heat-transfer results are presented for air temperatures as high as 8,000 k. where more than periments were performed in a shock tube and the new experimental techniques and principles are discussed briefly . simulation of flight stagnation conditions at velocities up to satellite velocity of 26,000 ft. sec. is shown to be possible in shock tubes and data has been obtained over a large altitude range at these velocities .'), (1162, 'force-test investigation of the stability and control characteristics of a 1/8 scale model of a tilt-wing vertical take-off and landing airplane .', 'tosti,l.p.', 'nasa tn.d44, 1960.', 'force-test investigation of the stability and control characteristics of a 1/8 scale model of a tilt-wing vertical take-off and landing airplane . a force-test investigation has been made to determine the aerodynamic characteristics of a scale model of a tilt-wing vertical-take-off-and-landing airplane in the short- and long-wing configurations . the model had two 6-blade dual-rotating propellers that were not interconnected mounted on a wing that could be tilted up to an incidence angle of about 90 for vertical take-off and landing . the investigation included measurements of both the longitudinal and lateral stability and control characteristics in the normal- forward-flight, transition, and hovering ranges . tests in the forward-flight and transition conditions were made at various wing incidences and power conditions . tests in the hovering condition were made in the presence of the ground . the data are presented without analysis .'), (1163, 'force-test investigation of the stability and control characteristics of a scale model of a tilt-wing vertical take-off and landing aircraft .', 'newson,w.a. and tosti,l.p.', 'nasa memo 11-3-58l, 1959.', 'force-test investigation of the stability and control characteristics of a scale model of a tilt-wing vertical take-off and landing aircraft . a wind-tunnel investigation has been made to determine the aerodynamic characteristics of a scale model of a tilt-wing vertical- take-off-and-landing aircraft . the model had two 3-blade single-rotation propellers with hinged (flapping) blades mounted on the wing, which could be tilted from an incidence of 4 for forward flight to 86 for hovering flight . the investigation included measurements of both the longitudinal and lateral stability and control characteristics in both the normal forward flight and the transition ranges . tests in the forward-flight condition were made for several values of thrust coefficient, and tests in the transition condition were made at several values of wing incidence with the power varied to cover a range of flight conditions from forward-acceleration (or climb) conditions to deceleration (or descent) conditions the control effectiveness of the all-movable horizontal tail, the ailerons and the differential propeller pitch control was also determined . the data are presented without analysis .'), (1164, 'effect of ground proximity on the aerodynamic characteristics of a four- engined vertical take-off and landing transport airplane model with tilting wing and propellers .', 'newson,w.a.', 'naca tn.4124, 1957.', 'effect of ground proximity on the aerodynamic characteristics of a four- engined vertical take-off and landing transport airplane model with tilting wing and propellers . an investigation has been made to study the effect of ground proximity on the aerodynamic characteristics of a four-engine vertical- take-off-and-landing transport-airplane model with tilting wing and propellers . tests were made with the wing at an angle of incidence of 90, the position used for vertical take-off or landing . with the model at various heights above the ground, the lift, drag, and pitching moment were measured and tuft studies were made to determine the flow field caused by the propeller slipstream . data were obtained for the complete model, for the model with horizontal tail removed, and for the wing-propeller combination alone . the results of the investigation showed that, when the model was hovering near the ground, there was a strong upwash in the plane of symmetry and also an increase in lift of about 10 percent of the propeller thrust . about one-half of this lift resulted from an increase in propeller thrust and one-half resulted from an up load on the fuselage induced by the upwash . as the model approached the ground, it also experienced an increasing nose-down pitching moment that evidently resulted from the up load on the fuselage, the rear part of which was longer than the front part . the addition of the horizontal tail which was located about halfway up the vertical tail did not increase the nose-down pitching moment because the fuselage decreased the energy of the upwash before it reached the tail .'), (1165, 'an investigation of the effect of downwash from a vtol aircraft and a helicopter in the ground environment .', 'o''bryan,t.c.', 'nasa tn.d977, 1961.', 'an investigation of the effect of downwash from a vtol aircraft and a helicopter in the ground environment . dynamic-pressure measurement, in ground effect, have been obtained about a single-rotor helicopter and a dual-propeller vtol aircraft . the results indicate that the slipstream dynamic pressure along the ground, some distance from the center of rotation, is not a function of disk loading but merely a function of the gross weight or thrust of the aircraft . furthermore, for a given gross weight the thickness of this outward flowing sheet of air is less for a small-diameter propeller (higher disk loading propeller) . the variation of the dynamic-pressure flow field for single and dual propellers or rotors is significantly different in the plane of symmetry between the two rotors than in a direction normal to this plane . the interaction of the two flows produces a region of upflow in this plane where the fuselage is located, and the decay of the maximum dynamic pressure with distance ahead of the fuselage is slower .'), (1166, 'an investigation to determine conditions under which downwash from vtol aircraft will start surface erosion from various types of terrain .', 'kuhn,r.e.', 'nasa tn.d56, 1959.', 'an investigation to determine conditions under which downwash from vtol aircraft will start surface erosion from various types of terrain . results of an investigation with small-scale equipment of the conditions under which the downwash from a hovering vertical-take-off- and-landing (vtol) aircraft will start surface erosion indicate that the onset of erosion depends only on the dynamic pressure of the outward flow of air near the surface . for a rotor or propeller at a height of about 1 slipstream diameter above the surface, this surface dynamic pressure was found to be equal to the disk loading . for the vtol aircraft supported by a ducted fan, the surface dynamic pressure with the ducted fan exit at a height of about one-half the exit-area loading . the surface dynamic pressure decreases rapidly with increasing height of the vtol device . erosion of sand and loose dirt started at surface dynamic pressures of 1 to 3 lb sq ft, which is in general agreement with helicopter experience . thoroughly soaking the sand and loose-dirt surfaces increased the resistance to erosion to surface dynamic pressures of 30 to 50 lb sq ft . spray from water started at surface dynamic pressures of 1.5 to pressures up to about 1,000 lb sq ft .'), (1167, 'an experimental study of the effect of downwash from a twin-propeller vtol aircraft on several types of ground surfaces .', 'o''bryan,t.c.', 'nasa tn.d1239, 1962.', 'an experimental study of the effect of downwash from a twin-propeller vtol aircraft on several types of ground surfaces . a full-scale, twin-propeller vtol aircraft with a maximum gross weight of 3,400 pounds has been operated on the ground to study the effect of downwash on several types of ground surfaces . static operation over loose snow indicated a zone of obliterated vision ahead of the pilot in an arc of approximately 10 on each side of the plane of symmetry . an arc 10 to 45 each side of the center line was found to be an area of fair visibility while the arc from 45 to 90 was an area of poor visibility . static operation in the presence of loose surface material indicated that the downwash cleared the area near the aircraft of these particles without recirculation or damage to any components . short-time operation at moderate forward speed over loose gravel, with the thrust axis at an angle of in propeller-blade erosion and numerous small dents and fabric punctures in the sides of the fuselage . the propeller-blade erosion was superficial except for the leading edges where several layers of glass fiber were eroded .'), (1168, 'damage incurred on a tilt-wing multipropeller vtol/stol aircraft operating over a level, gravel-covered surface .', 'pegg,r.j.', 'nasa tn.d535, 1960.', 'damage incurred on a tilt-wing multipropeller vtol/stol aircraft operating over a level, gravel-covered surface . a summary is presented of the damage experienced by a tilt-wing vtol stol aircraft as a result of operating from a level surface covered with loose gravel . the damage was inadvertently incurred as the aircraft was performing a taxiing-turn maneuver over an area of level macadam surface containing loose and embedded crushed stone . observers from a chase aircraft commented that the wing was tilted at approximately 76 with respect to the ground when the damage occurred . deposits of stone in the open fuselage, punctures in the rotor blade skin, and damage to the compressor blades of the engine occurred due to the circulation of the crushed gravel .'), (1169, 'hangling qualities experience with several vtol research aircraft .', 'reeder,j.p.', 'nasa tn.d735, 1961.', 'hangling qualities experience with several vtol research aircraft . all of the vtol research aircraft discussed in this paper have successfully demonstrated conversion from hovering to airplane flight and vice versa . however, control about one or more axes of these aircraft has been inadequate in hovering flight . furthermore, ground interference effects have been severe in some cases and have accentuated the inadequacy of control in hovering and very low speed flight . stalling of wing surfaces has resulted in limitations in level-flight deceleration and in descent, particularly for the tilt-wing aircraft, which in this case is a very rudimentary type . minor modifications to the wing leading edge have, however, produced surprisingly large and encouraging reductions in adverse stall effects . height control in hovering and in low-speed flight has proved to be a problem for the aircraft not having direct control of the pitch of the rotors . the other systems have shown undesirable time lags in development of a thrust change .'), (1170, 'structural loads surveys on two tilt-wing vtol configurations .', 'ward,j.f.', 'nasa tn.d729, 1961.', 'structural loads surveys on two tilt-wing vtol configurations . the results of two structural-loads surveys are summarized . the first loads program discussed concerns the airframe vibratory loads encountered during flight tests of the vz-2 tilt-wing vtol aircraft throughout the operational range from hover to cruise flight . the primary sources of airframe vibration were wing-stall buffeting and tail buffeting in descents . the second loads program discussed concerns the initial results of a structural-loads survey conducted as part of the wind-tunnel test of a large-scale tilt-wing research model . this loads program deals with the steady wing loads measured throughout simulated transition from hover to cruise .'), (1171, 'the stability under axial compression and lateral pressure of circular cylindrical shells with a soft elastic core .', 'seide,p.', 'j. ae. scs. 1962.', 'the stability under axial compression and lateral pressure of circular cylindrical shells with a soft elastic core . the stability under axial compression and lateral pressure of a finite circular-cylindrical shell with an elastic core is treated by means of donnell''s equations . the stability criterion is investigated in detail for the general cylinder under axial compression or lateral pressure and for a particular cylinder under combined loading . comparisons are made with available experimental data .'), (1172, 'elastic stability of circular cylindrical shells stabilized by a soft elastic core .', 'goree,w.s. and nash,w.a.', 'exp. mech. 2, 1962.', 'elastic stability of circular cylindrical shells stabilized by a soft elastic core . the effect of a soft elastic core upon the buckling strength of a thin, circular, cylindrical shell is investigated experimentally . two types of loading are considered .. (a) axial compression, and (b) uniform radial-band loading, where the width of the band is small compared to the length of the shell . for each type of loading it is shown that the strengthening effect of the elastic core becomes more significant with the increasing values of the radius-thickness ratio . for example, it is shown that for the geometric and elastic constants considered it is possible, with the presence of the core, to increase the axial buckling stress by as much as 65 percent over the values found for those without an elastic core . the elastic core is even more effective in stabilizing the shell against buckling due to band loading, the peak pressure required to buckle the filled specimen being 7.30 times that required to buckle the unfilled shell .'), (1173, 'the buckling of cylindrical shells under longitudinally varying loads .', 'v. i. weingarten', 'space technology laboratories, inc., los angeles, calif. now with aerospace corp., los angeles, calif.', 'the buckling of cylindrical shells under longitudinally varying loads . two problems illustrating the effect of nonuniformity of loading on the buckling characteristics of circular cylinders are investigated . the first problem deals with the effect of linearly varying axial compressive stress, such as would be produced by the weight of the propellant in a solid-propellant engine case . the results indicate that the ratio of the maximum critical compressive stress induced by the shear load to the critical uniform compressive stress varies from 1.9 for the curvature parameter z equal to 1.6 as z becomes infinite . in particular, the increase in stress is less than 20 per sq. ft. for z greater than 100 . the stability of thin cylinders loaded by lateral external pressure varying linearly in the longitudinal direction is also investigated . the results indicate that for z greater than 100, the buckling coefficients are proportional to square root z .'), (1174, 'general theory of buckling .', 'langhaar,h.l.', 'app. mech. rev. 11, 1958.', 'general theory of buckling . various aspects of the theory of buckling are expounded in many treatises (1 to 15) . the books of bleich (2) and salmon (3) contain large bibliographies . goodier developments in buckling theory . numerous references are appended to hoff''s article . nearly all publications on buckling of shells, available in the u.s.a. (to 1956), are listed in the bibliographies on shells by nash (18) . the section ready guide to recent literature .'), (1175, 'stresses from local loadings in cylindrical pressure vessels .', 'bijlaard,p.p.', 'asme trans. 77, 1953.', 'stresses from local loadings in cylindrical pressure vessels . a short discussion is given of the possible methods for computing the stresses caused in cylindrical shells by local loadings . it is concluded that the method of developing the loads and displacements into double fourier series leads to formulas which are best suited for numerical evaluation . with this method the pertinent expressions for the displacements caused by radial loads are found by reducing the three partial differential equations of the shell theory to an eighth-order differential equation in the radial displacements, which is similar to, but not identical with, those derived by donnell and yuan . insertion of the fourier series for the radial displacements and the external loading in this equation leads directly to a double series expression of the radial displacement w in terms of the load factors of the radial load . this results in the pertinent expressions for the other displacements and for the bending moments and membrane forces . the cases of radial loading considered here and those which can be reduced to it are (a) a load uniformly distributed within a rectangle, tion, uniformly distributed over a short distance in the circumferential direction, (d) a moment in the circumferential direction, uniformly distributed over a short distance in the longitudinal direction . for all these loadings the load factors, which have to be used in the pertinent formulas for the displacements, bending moments, and membrane forces, are computed . for the case of tangential loading an eighth-order differential equation is derived in terms of the radial displacement and the tangential load . using this equation, formulas for the displacements, bending moments, and membrane forces for tangential loading within a rectangle are found .'), (1176, 'bending tests of ring-stiffened circular cylinders .', 'peterson,j.p.', 'naca tn.3735, 1956.', 'bending tests of ring-stiffened circular cylinders . twenty-five ring-stiffened circular cylinders were loaded to failure in bending . the results are presented in the form of design curves which are applicable to cylinders with heavy rings that fail as a result of local buckling .'), (1177, 'effects of rapid heating on strength of airframe components .', 'pride,r.a.', 'naca tn.4051, 1957.', 'effects of rapid heating on strength of airframe components . results of several experimental investigations are presented which indicate the effects of rapid heating on the bending strength of multiweb beams and ring-stiffened cylinders . it is shown that thermal stresses reduce the bending load carried at buckling by both beams and cylinders . the influence of thermal stress on maximum load is found to depend largely on the mode of buckling . for beams that buckle locally, no apparent effect of thermal stress on the maximum load has been found . a reduction in maximum load has been observed for beams that buckle in the wrinkling mode and for cylinders .'), (1178, 'buckling of ring-stiffened cylinders under a pure bending moment and a nonuniform temperature distribution .', 'anderson,m.s. and card,m.f.', 'nasa tn.d1513, 1962.', 'buckling of ring-stiffened cylinders under a pure bending moment and a nonuniform temperature distribution . thirteen stainless-steel ring-stiffened cylinders were subjected to a pure bending load and heated rapidly until buckling occurred . for most of the cylinders the heating was not uniform around the circumference so that appreciable axial thermal stresses were present . elementary thermal stress theory was found to be inadequate for the prediction of these thermal stresses, but a method was developed that would give satisfactory thermal stress results . by properly accounting for the thermal stress, the buckling load could be correlated with a theory for the buckling of an axially compressed ring-stiffened cylinder that is uniformly heated .'), (1179, 'a theory of asymmetric hypersonic blunt-body flows .', 'swigart,r.j.', 'aiaa jnl. 1963, 1034.', 'a theory of asymmetric hypersonic blunt-body flows . two-dimensional asymmetric and three-dimensional inviscid blunt-body flows are analyzed using a new method . the method is inverse, that is, the shock-wave shape and freestream conditions are taken as known, and the body shape and flow field are to be determined . results at zero angle of attack are obtained as a special case of the general problem . solutions at zero angle are calculated for a variety of body shapes at freestream mach numbers ranging from infinity to 1.85 . the ratio of specific heats, is taken as 1.4 . comparison with results obtained using van dyke''s and garabedian''s numerical solutions indicates that the method under consideration is more accurate than the van dyke method for determining stand-off distance . solutions are obtained for parabolic and paraboloidal shock waves at small angle of attack and infinite freestream mach number,. assumes the values 1.4, 1.2, 1.1, and 1.05 . for all cases, the streamline that wets the body passes through the shock wave slightly above the point where the shock is normal and thus does not possess maximum entropy . these results provide counter examples to the conjecture that any isolated convex body in a supersonic stream is wetted by the streamline of maximum entropy .'), (1180, 'approximate analysis of the slot injection of a gas in laminar flow .', 'libby,p.a. and schetz,j.a.', 'aiaa jnl. 1963, 1056.', 'approximate analysis of the slot injection of a gas in laminar flow . the laminar diffusion and combustion of a gas injected into a high-speed uniform stream by means of a wall slot are considered . the dorodnitzin-howarth transformation is employed to reduce the boundary layer equations to incompressible form,. the nonsimilar flow field is treated by a modified oseen approximation in conjunction with the integral method . thermal boundary conditions corresponding to an adiabatic wall and to constant wall enthalpy are discussed . the injection of homogeneous, heterogeneous, nonreactive, and reactive gases is treated . for the latter case, the models usually employed for chemical behavior, namely, frozen and equilibrium flow, are considered . the analysis is applicable to a wide variety of laminar flows, e.g., those involving cooling, thermal protection, skin-friction reduction, and supersonic deflagration . a numerical example of practical interest in connection with the venting of gaseous hydrogen boiloff from a rocket booster is presented .'), (1181, 'steady magnetohydrodynamic flow past a non-conducting wedge .', 'chu,c.k. and lynn,y.m.', 'aiaa jnl.', 'steady magnetohydrodynamic flow past a non-conducting wedge . this paper presents a study of the steady two-dimensional magnetohydrodynamic flow of an infinitely conducting fluid past a nonconducting wedge with nonaligned flow and magnetic field . the flows considered are in the /superfast/ or fully hyperbolic regime . the flows consist of several regions of uniformity connected by shocks and expansion waves . because of the boundary condition on the magnetic field, the magnetic field must be the same in the regions above and below the wedge,. thus the flows in these regions are coupled, unlike in the case of ordinary supersonic gasdynamics . only small wedge angles and weak waves (characteristics) are considered . the problem thus is linearized, and explicit solutions are obtained which are qualitatively similar to the nonlinear solutions . some interesting and unexpected features arise, and they are discussed in detail .'), (1182, 'an approximate solution for laminar boundary layer flow .', 'robert l. kosson', 'grumman aircraft engineering corporation, bethpage, n.y.', 'an approximate solution for laminar boundary layer flow . this paper presents an approximate solution for two-dimensional, incompressible, laminar boundary layer flow with arbitrary pressure gradient . von mises'' form of the boundary layer equation is linearized by making a change in the coefficient of one of the terms . the linearized equation yields a solution that is accurate for the outer portion of the flow but inaccurate near the surface . a separate inner solution then is developed which is accurate at the surface and which joins with the outer solution at some point within the boundary layer . the method may be considered a major modification of one developed earlier by von karman and millikan, with changes in both outer and inner solutions, and the point at which the two solutions are joined . the changes improve the accuracy of the method and in some respects simplify the calculations . as examples, results are presented for flow with a linear variation of velocity (including flat plate and stagnation point flow as special cases), flow with sinusoidal variation of velocity, flow past a circular cylinder (heimenz'' velocity distribution), and flow past an ellipse (schubauer''s data) . agreement with theoretically exact solutions is good, and better than results obtained using the pohlhausen method .'), (1183, 'laminar hypersonic trail in the expansion-conduction region .', 'lykoudis,p.s.', 'aiaa jnl. 1963, 772.', 'laminar hypersonic trail in the expansion-conduction region . the usual procedure in calculating the cooling process in a wake behind a blunt object is to assume a region of pure expansion up to a distance where the pressure has reached its ambient value, followed by a region where the mechanism of pure heat conduction is operative . in the present paper both mechanisms are assumed to be valid simultaneously, and the result is compared with previous calculations . the following criterion is established .. the minimum radius of a hemisphere-cylinder configuration, above which a simultaneous conduction-expansion calculation is not needed, is given by the approximation where is the nondimensional value of the enthalpy at the axis of the wake below which the two methods of computation give the same result, and m is the flight mach number .'), (1184, 'three dimensional effects in viscous wakes .', 'steiger,m.h. and bloom,m.h.', 'aiaa jnl. 1963,776', 'three dimensional effects in viscous wakes . three-dimensionality in wakelike or jetlike free mixing may stem from initial geometric configurations, nonuniformities in flow variables over a cross section, or boundary conditions along the flow . these may be generated by bodies at angle of attack, nonaxisymmetric bodies, mixing of nonaxisymmetric jets with an outer flow, finite wings, or more artificial means . this paper is devoted to studies bearing on such configurations . the first section deals with the general mathematical model, in which the boundary layer approximations are used, and with methods of solution . laminar and turbulent flow, compressibility, unsteadiness, and streamwise pressure gradients are admitted initially . the flux forms of the equations are given . algebraic integrals of the energy equations and the diffusion ( frozen-flow) equations are obtained . a simplification of the convective terms, roughly corresponding to the oseen approximation, is used in the asymptotic downstream region . the second section contains explicit solutions for specific configurations, in particular for flows whose initial isovels are of elliptic shape . these flows may be wakelike or jetlike . compressibility is admitted,. however, the flows must have uniform pressure and must be steady . the final section deals with interpretation and evaluation of the results .'), (1185, 'an integral method for calculating heat and mass transfer in laminar boundary layers .', 'culick,f.e.c.', 'aiaa jnl. 1963, 783.', 'an integral method for calculating heat and mass transfer in laminar boundary layers . an integral method previously used successfully in several kinds of boundary layer problems has been extended to treat simultaneous heat and mass transfer in a binary-mixture laminar boundary layer when the pressure is uniform . the principal results are two pairs of dual integral relations arising from solutions to the integral concentration and energy equations . one pair connects the surface mass transfer rate and surface concentration of injected gas,. the other relates surface temperature and heat transfer rate in the presence of mass transfer . only the cases of helium and air injection into an undissociated air stream are discussed in detail, but the method can be applied to problems involving other gases . the approximate results agree quite well with some numerical solutions and with recent experimental results for which no numerical solutions are available .'), (1186, 'lift of slender delta wings according to newtonian theory .', 'messiter,a.f.', 'aiaa jnl. 1963,795', 'lift of slender delta wings according to newtonian theory . an approximate system of equations is derived to describe the inviscid flow past a flat slender wing at angle of attack, in the limit and . the aspect ratio is required to approach zero at the same rate as the mach angle in the flow behind the shock wave . only a single parameter appears in the resulting equations, and a similarity law therefore can be written expressing a correction to the newtonian normal-force coefficient . for the delta wing, a correlation of experimental data according to the similarity law is shown, and the first terms of the solution are derived under the assumption that the similarity parameter is small ( vertex angle much smaller than mach angle) .'), (1187, 'shock-induced boundary layer separation in overexpanded conical exhaust nozzles .', 'arens, m. and spiegler, e', 'aiaa journal, march 1963. p.578 .', 'shock-induced boundary layer separation in overexpanded conical exhaust nozzles . the flow in overexpanded supersonic nozzles is reviewed . although five essentially different flow regimes can be discerned, depending on the nozzle pressure ratio, the regime of most interest to the engine designer is the one characterized by oblique shock patterns in the nozzle and flow separation from the nozzle wall . it is shown that the pressure rise associated with the separation correlates well with the mach number at the separation point . a simple analytical formulation for the pressure rise required to separate the flow provides excellent agreement with experimental data over a wide range of nozzle operating conditions and allows prediction of overexpanded nozzle performance .'), (1188, 'factors affecting lift-drag ratios at mach numbers from 5 to 20 .', 'goebel,t.p. martin,j.j. and boyd,j.a.', 'aiaa jnl. 1963, 640.', 'factors affecting lift-drag ratios at mach numbers from 5 to 20 . yawed-cone working charts and an engineering method are presented and used to calculate lift-drag ratios of flat-top conical wing-body arrangements at mach numbers from 5 to 20 . viscous interaction effects are considered, but bluntness effects are neglected . correlations of wind-tunnel data in the range show that boundary layer displacement corrections to surface pressure and skin friction are required to calculate lift-drag ratios by this method whenever is greater than 0.2 . is the freestream mach number and is the freestream reynolds number based on body length . double- and single-type shock patterns, transition from one pattern to the other, and the variation of inner-shock position with angle of attack are described . lift-drag ratios are calculated at selected flight design points for flat-top, conical body arrangements with triangular and hyperbolic wing planforms . the hyperbolic wing arrangement offers a potential l d benefit at mach 5 but not at mach 10 or above .'), (1189, 'nonequilibrium flow past a wedge .', 'capiaux,r. and washington,m.', 'aiaa jnl. 1963, 1, 650.', 'nonequilibrium flow past a wedge . an exact numerical solution is obtained for the chemically reacting flow past a wedge . the freestream is either in equilibrium or out of equilibrium but nonreacting . the attached shock wave is shown to be either concave, convex, or straight, depending on the values of the amount of dissociation in the freestream and a parameter describing the amount of energy contained in the freestream relative to the gas dissociation energy . numerical examples are presented illustrating these regimes . the flow field is characterized by the presence of an entropy layer and a relaxation layer, both easily identifiable in the presentation of the numerical results .'), (1190, 'flow of a gas near a solid surface .', 'ziering,s.', 'aiaa jnl. 1, 1963, 661.', 'flow of a gas near a solid surface . the state of a gas near a solid surface is investigated . it is assumed that at a sufficiently large distance from the surface the particle distribution function is of the chapman-enskog form . the half-range analysis previously employed for parallel plate geometrics (symmetric problems) can be adapted to the single-plate problem . although the mathematical analysis differs, the slip coefficients are essentially identical with those obtained from the parallel plate problem (e.g., couette flow) . detailed calculations are presented for both hard sphere and maxwellian molecules . the recent work of bakanov and deryagin for hard sphere molecules, which is based on incorrect approximations, is discussed .'), (1191, 'heat transfer to a hemisphere-cylinder at low reynolds numbers .', 'hickman,r.s. and giedt,w.h.', 'aiaa jnl. 1, 1963, 665.', 'heat transfer to a hemisphere-cylinder at low reynolds numbers . measurements of the local heat flux to hemisphere-cylinder models in a supersonic rarefied air stream are presented . two different steady-state methods were developed, and five individual models were used . data were obtained throughout the mach number range of 2 to 6, with reynolds numbers (based on conditions behind the bow shock and model diameter) varying from 38 to 1730 . the stagnation point data indicated a gradual increase from continuum boundary layer theory at the higher reynolds numbers to about 10 above at the lower end of the range investigated . pressure distribution measurements on cooled and uncooled models were found to agree well with modified newtonian theory . local recovery factor measurements showed a small rarefaction effect at the lowest reynolds numbers .'), (1192, 'an integral method for calculation of supersonic laminar boundary layer with heat transfer on yawed cone .', 'yen,shee-mang and thyson,n.a.', 'aiaa jnl. 1, 1963, 672.', 'an integral method for calculation of supersonic laminar boundary layer with heat transfer on yawed cone . an integral method for calculating the three-dimensional boundary layer over the surface of a cone at angle of attack is investigated . the numerical procedure of integration for that method on the basis of a simplifying assumption concerning the boundary layer development along the cone generator is developed and illustrated by applying the method to find the solutions of integral equations for a specific example . the results obtained for the example for the range of circumferential angle of 40 investigated are summarized and given as heat transfer coefficients, coefficients of friction, and other friction parameters . the distribution of heat transfer coefficients checked with available experimental data fairly well .'), (1193, 'some exact solutions for cavitating curvilinear bodies .', 'ehrich,f.f.', 'aiaa jnl. 1, 1963, 675.', 'some exact solutions for cavitating curvilinear bodies . a special case of cavitating flow solutions is postulated and transformed to a semi-infinite plane . the complete, exact solution then is synthesized by superposition of singularities . the solution is relevant to a general, two-parameter family of curvilinear bodies . the parameters are the flow angles at the two points of flow separation . the body reduces, in the special case, to the rayleigh solution for a flat plate . the equations of the cavity boundaries are given in explicit form . the body form and the stagnation streamline are given as the locus of the roots of a cubic equation . local static pressures and, hence, lift and drag, also may be calculated . the generated solutions constitute a technique involving simple computation for exact solutions of a special family of cavitating curvilinear bodies at finite angles of attack .'), (1194, 'magnetohydrodynamic flow past a thin airfoil .', 'cumberbatch,e., sarason,l. and weitzner,h.', 'aiaa jnl. 1, 1963, 679.', 'magnetohydrodynamic flow past a thin airfoil . the steady flow of a perfectly conducting magnetohydrodynamic fluid past a thin nonconducting airfoil is studied with the usual model in which the fluid variables obey the lundquist equations linearized about a constant unperturbed flow . /hyperliptic/ flows, in which hyperbolic and elliptic fields are superimposed, are considered . results of grad, mccune and resler, and sears and resler are extended and considered in detail for the case of an arbitrarily inclined unperturbed field . the general solution contains four line singularities along the characteristics through the ends of the body and has two arbitrary constants . by a / generalized kutta-joukowski condition,/ these constants are fixed so that two of the line singularities disappear . specifically, it is required that the solution be locally square integrable . behavior of the exponents of the singularities is investigated by numerical computation and, in limiting cases, analytically . the singular parts of some flows are investigated numerically .'), (1195, 'experiments with two-dimensional, transversely impinging jets .', 'dosanjh,d.s. and sheerlan,w.j.', 'aiaa jnl. 1963, 1, 329.', 'experiments with two-dimensional, transversely impinging jets . experiments on the interaction of transversely impinging two-dimensional jet flows were performed in which a low pressure control jet flow interacted with a relatively high pressure power jet flow . the ratio of the control jet to the power jet supply chamber gauge stagnation pressure was adjusted at 0, 10, and 15 . shadowgraphs of the power jet alone, as well as the corresponding interacting jet flows, were recorded to establish the nature of and changes in the shock structure . the jet flows were traversed by a pitot tube to record the pitot pressure distributions at various locations downstream of the power jet exit . it was discovered that with the addition of only a small percent control jet flow, the normal shock front of the highly underexpanded power jet flow changed to an oblique shock structure and, downstream of the previous location of the normal shock which appeared in the power jet flow alone, the maximum recovery stagnation pressures were proportionally much higher . the mechanism for this behavior of the normal shock is proposed . possible practical importance of this behavior of interacting jet flows with reference to aerodynamic noise, supersonic diffuser losses, etc., is also pointed out . for the power jet flow alone it was found that by considering the actual jet boundaries as simply an extension of the actual nozzle, the average axial flow quantities, computed from the area-mach number relation using the observed cross-sectional area of the jet flow, agreed quite favorably with the experimental results .'), (1196, 'growth of the turbulent wake behind a supersonic sphere .', 'murphy,c.h. and dickinson,e.r.', 'aiaa jnl. 1, 1963, 339.', 'growth of the turbulent wake behind a supersonic sphere . experimental data are presented on the growth of turbulent wakes up to 8000 calibers behind and spheres traveling at supersonic velocities . experimental determination of the exponential coefficient in the growth law is very difficult, if not impossible . data are presented in the form of both . in the representation, two regions of different wake growths are observed . by means of a quasi-steady state assumption, the effect of drag deceleration is eliminated and growth of the far wake compared with theoretical predictions . the agreement with the lees-hromas theory in this region was found to be quite good .'), (1197, 'unsteady aerodynamic forces on slender supersonic aircraft with flexible wings and bodies .', 'yates,j.e. and zeydel,e.f.e.', 'aiaa jnl. 1, 1963, 350.', 'unsteady aerodynamic forces on slender supersonic aircraft with flexible wings and bodies . the present paper derives generalized aerodynamic forces for slender supersonic aircraft on the basis of slender-body theory . particular consideration is given to configurations which are spanwise flexible . to treat configurations with flexible wings and bodies, the slender wing-body problem is first reduced to a simple body problem whose solution is well known, and a solution of the latter is obtained, utilizing the circle theorem or method of images and a known solution of the airfoil equation for a double interval . with this approach, it is not necessary to apply conformal mapping techniques, and the solution so obtained is valid for arbitrary spanwise downwash distributions . on the basis of slender-body theory, the velocity potential and, subsequently, the generalized aerodynamic forces are derived for a general class of spanwise flexible wing-body configurations .'), (1198, 'the blunt-leading-edge problem in hypersonic flow .', 'oguchi,h.', 'aiaa jnl. u, 1963, 361.', 'the blunt-leading-edge problem in hypersonic flow . the present paper is mainly concerned with the hypersonic flow over a flat plate with a blunt nose . the analysis is based on the flow model in which the flow field behind the shock wave may be divided into two regions .. the inviscid-hypersonic-flow region and the entropy layer, across which the pressure has no appreciable change . the equations for the entropy layer can be reduced to those of the usual boundary-layer problem with the exception that the outer edge of the entropy layer, as well as the pressure remain unknown . these unknowns are determined so as to approximately match the entropy-layer solution with the inviscid hypersonic solution in which the shock wave has the shape of the power law of the distance from the leading edge . the assumed flow model is shown to be valid over a restricted range depending on the wall-to-stagnation temperature ratio and (where is the reynolds number based on half the thickness of nose t, m the freestream mach number, and c the chapman-rubesin constant . actual calculations have been carried out for the case with typical values of and the wall-to-stagnation temperature ratio . the calculated values for both the surface pressure and heat-transfer rate are compared with the experimental data . as regards surface pressure in particular, a satisfactory agreement with the data is obtained . the validity of the assumptions upon which the present analysis is based has been examined from the numerical results, and the region of the validity has been found to extend over a certain large range of the nondimensional distance from the leading edge .'), (1199, 'theoretical investigations of a supersonic laminar boundary layer with foreign-gas injection .', 'freedman,s.i., radbill,j.r. and kaye,j.', 'aiaa jnl. 1, 1963.', 'theoretical investigations of a supersonic laminar boundary layer with foreign-gas injection . the phenomena arising from the uniform injection of helium, air, argon, and iodine into the laminar boundary layer of a supersonic stream of air in a tube were investigated theoretically . the partial differential equations describing the energy, mass, and momentum transfers through the boundary layer were obtained, and a series solution was found for the case of uniform injection through the tube wall . the results of the analysis are in the form of axial distributions of wall temperature and recovery factor and of radial distribution of concentration, velocity, static, and stagnation temperatures . the gas mixture was assumed to be a perfect gas . properties of the mixture were calculated in accordance with the gibbs-dalton rule and the mixing rules based on the kinetic theory of dilute gases . transport properties for pure air were taken from the n.b.s. tabulations . transport properties for the other gases were calculated by kinetic-theory methods, employing a lennard-jones 6-12 model for the interaction potential . the theoretical predictions for the recovery factor along the tube with air or argon injection agree with experimental data to within one percent . the theoretical predictions for helium injection indicate an 8-percent rise in the recovery factor along the tube, while experiments have shown only a 1-percent rise . these differences between theory and experiment are attributed to inaccuracies in the approximations to the transport properties of the binary mixtures .'), (1200, 'hypersonic viscous flow over a sweat-cooled flat plate .', 'tien,c.l. and gee,c.', 'aiaa jnl. 1963, 159.', 'hypersonic viscous flow over a sweat-cooled flat plate . this paper presents a theoretical analysis of the hypersonic viscous flow over a sweat-cooled flat plate . the physical system under consideration is the hypersonic laminar boundary layer over a porous flat plate with homogeneous, normal injection of a coolant into the external stream . a heat balance at the porous surface is made between the heat transferred to the surface and the heat absorbed by the coolant . the existence of similar solutions requires a nonuniform distribution of coolant injection . the method of solution consists of the integration of three simultaneous first-order equations, the momentum and the energy integral equations in the boundary layer, and the tangent-wedge approximation in the inviscid layer . first-order asymptotic formulas are given in both the strong and the weak pressure interaction regions for the induced surface pressure, the skin-friction coefficient, and the nusselt number . numerical results for three specific cases are presented and discussed .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1201, 'a study of slender shapes of minimum drag using the newton-busemann pressure coefficient law .', 'miele,a.a.', 'aiaa jnl. 1, 1963, 168.', 'a study of slender shapes of minimum drag using the newton-busemann pressure coefficient law . the problem of minimizing the drag of a slender, two- dimensional or axisymmetric body in hypersonic flow at zero angle of attack is considered under the assumption that the pressure coefficient law is newton''s impact law as modified by busemann in order to include centripetal acceleration effects . after the condition that the pressure coefficient be nonnegative is accounted for and after arbitrary conditions are imposed on, in addition to the thickness and the length, the enclosed area and the moment of inertia of the contour in the two-dimensional case and the wetted area and the volume in the axisymmetric case, the minimal problem is formulated as a problem of the mayer type and solved by the combined use of the euler-lagrange equations, the transversality condition, the erdmann-weierstrass corner condition, and the properties of the switching function . particular attention is devoted to the class of problems such that, among the four quantities being considered, two are prescribed while the remaining are free . for these problems, the extremal arc is composed of two subarcs .. one is characterized by a positive pressure coefficient and is called the regular shape,. the other is characterized by a zero pressure coefficient and is called the free layer . in this connection, the analysis shows the existence of two different types of solutions depending on whether the thickness is given or free . if the thickness is given, the expression for the regular shape is a power law, and the transition from the regular shape to the free layer occurs in the second half of the body . in the two- dimensional case, the exponent of the power law is 1 if the length is given if the enclosed area is given, and 3 if the moment of inertia of the contour is given,. the transition point from the power body to the free layer is located at 50 percent of the length if the length is given, at 66 percent if the enclosed area is given, and at the axisymmetric case, the exponent of the power law is if the length is given, 1 if the wetted area is given, and if the volume is given,. the transition point from the power body to the free layer is located at 60 percent of the length if the length is given, at 70 percent if the wetted area is given, and at 80 percent if the volume is given . on the other hand, for problems where the thickness is free, the equation governing the regular shape is not that of a power body, and the point of transition to the free layer is located in the first half of the body . in the two-dimensional case, the transition point is at 28 percent of the length if the length and the enclosed area are given, at 32 percent if the length and the moment of inertia of the contour are given, and at 45 percent if the enclosed area and the moment of inertia of the contour are given . in the axisymmetric case, the transition point is located at 35 percent of the length if the length and the wetted area are given, at 39 percent if the length and the volume are given, and at 46 percent if the wetted area and the volume are given . for all of the cases considered, analytical expressions are obtained for the optimum shapes, the thickness ratios, and the drag coefficients .'), (1202, 'uniformly valid second-order solution for supersonic flow over cruciform surfaces .', 'wallace,j. and clarke,j.h.', 'aiaa jnl. 1, 1963, 179.', 'uniformly valid second-order solution for supersonic flow over cruciform surfaces . considered is the second-order supersonic flow over a cruciform configuration consisting of two intersecting rectangular wings of high aspect ratio . the practical interest is in application to supersonic inlets, wing-body junctions and vehicle fins . the fundamental interest centers about identification and adjustment of the severe local failures of the ordinary second-order theory . for wings with discontinuous slopes, discontinuous potentials occur across the planar shock and square-root singularities in the velocities occur at the intersection of these shocks with the cruciform surfaces . the problem is simple enough so that these interesting features stand out clearly . a second-order solution uniformly valid to first order is constructed by adjustment of the ordinary second-order solution obtained first . the uniformly valid solution has two different series representations in the thickness parameter . one is the ordinary second-order series in ascending integral powers of the thickness parameter which is valid in the interior of the vertex-centered undisturbed mach cone, and the other is a series containing fractional powers which is valid adjacent to and upstream of this mach cone . the uniformly valid solution gives the detailed wave structure and shows a flow regime upstream of the vertex-centered undisturbed mach cone not predicted by the ordinary theory . the two solutions are otherwise identical . the wave structure consists of a pyramidal arrangement of planar shocks adjacent to and upstream of the above cone, followed by weaker oblique expansion fans and finally by two extremely weak shocks coincident with the vertex-centered undisturbed mach cone . as an example of the above, detailed results are presented for the case of two intersecting wedges . application of the techniques to other quasi-cylindrical problems is discussed .'), (1203, 'the propagation of a nonuniform magnetohydrodynamic shock wave into a moving monatomic fluid .', 'gundersen,r.m.', 'j. ae. scs. 1962, 1421.', 'the propagation of a nonuniform magnetohydrodynamic shock wave into a moving monatomic fluid . an initially uniform magnetohydrodynamic shock wave of arbitrary strength propagates through a channel which consists of two portions of which one has uniform cross-sectional area while the other is of varying cross-sectional area . it is assumed that the flow in the nonuniform section in front of the shock is initially a uniform state and no perturbations (due to the area variations) of this flow reach the shock until the area variation is encountered . when the shock enters the nonuniform section, it is perturbed, the shock strength altered and the subsequent flow is nonisentropic . in addition to the perturbation due to the effect of the area variations on the initially uniform upstream flow, there are two further contributions--viz., a permanent perturbation caused directly by the area changes and a transient disturbance--which propagates with true sonic speed with respect to the flow behind the shock, due to reflections of the permanent perturbation at the shock . expressions for these various contributions are obtained . the results presented include as special cases propagation of a nonuniform conventional gas dynamic shock into a moving nonconduction fluid and propagation of a nonuniform hydromagnetic shock wave into a stationary fluid .'), (1204, 'experimental effect of bluntness and gas rarefaction on drag coefficients and stagnation heat transfer on axisymmetric shapes in hypersonic flow .', 'bloxson,d.e. and rhodes,b.v.', 'j. ae. scs.1962.', 'experimental effect of bluntness and gas rarefaction on drag coefficients and stagnation heat transfer on axisymmetric shapes in hypersonic flow . inverted hemispheres, circular discs (normal to stream), spheres, 26 total angle 0.368 blunt hemisphere cones, 18 total-angle sharp cones, and other axisymmetric shapes were run in a hypervelocity wind tunnel . hypersonic drag coefficients at zero angle of attack were measured in the air velocity range, 7,000- efficient is defined as drag force . knudsen number is defined as mean free path behind shock sphere shock detachment distance . in the case of nonsphere shapes, the knudsen number is defined as the knudsen number of a sphere with the same base diameter . these drag coefficients cover the range of gasdynamics to free molecule flow and are given in graphical form . the drag coefficients were measured by means of a ballistic balance in millisecond intervals, and referenced to the drag coefficient of a sphere in the gasdynamics region, for a gamma of 1.4, of 0.92 . tunnel stagnation conditions of pressure, temperature, density, and pressure drop with time were measured directly . in the tunnel test section, velocity, q density, total pressure, and static pressure were measured directly . these experimental curves have been found useful in the analysis of complex shapes if the complex shapes can be easily broken down into simple components with small interactions between components . heat-transfer distributions have also been obtained on these and other complex shapes in the hypervelocity wind tunnel, by means of a special paint which changes through several visible spectral orders within a heat transfer range of x10 for a single application . heat transfer rates, so obtained, have been performed in the hypersonic gasdynamic and slip flow regions and are presented for spheres . these data, in the vorticity interaction region, agree with the data of ferri and zakkay .'), (1205, 'effects of cooling on boundary layer transition on a hemi- sphere in simulated hypersonic flow .', 'dunlap,r. and kuethe,a.m.', 'j. ae. scs. 1962, 1454.', 'effects of cooling on boundary layer transition on a hemi- sphere in simulated hypersonic flow . an experimental investigation of the effects of cooling on boundary-layer transition on a 9-in. diameter hemisphere in simulated hypersonic flow is reported . the newtonian pressure distribution was obtained by use of a shroud and boundary layer cooling was achieved by internally cooling the model . transition was detected with hot wires and with a pitot tube at the surface . attained . transition was observed in the subsonic and near-sonic flow region at and upstream of n = 45 . in this region the stagnation reynolds number at which transition occurred when the surface was highly polished was only slightly affected by cooling within the temperature range . thus, transition reversal does not occur on a polished spherical surface within the range of these tests, and we therefore conclude that the cooling did not cause the linear stability of boundary layer to decrease significantly . an essential feature of transition studies with boundary-layer cooling is the close control of surface roughness . in the present experiments this control required, in addition to a highly polished surface, the necessity for low water vapor dewpoint, the avoidance of carbon dioxide condensation and the utilzation of every available means for removing the dust from the airstream .'), (1206, 'magnetohydrodynamic mach cones .', 'cumberbatch,e.', 'j. ae. scs. 1962, 1476.', 'magnetohydrodynamic mach cones . features of the surfaces of main disturbance created by a small object in steady motion through a conducting fluid are examined . these surfaces are found by drawing tangent cones from the object to the relevant wave-front diagrams . the outer wave cone (when present) is smooth, but the two inner cones have cross sections similar to the cusped figures of the inner wave-front diagram . it is conjectured that the disturbance may be concentrated along such line cusps . this has particular relevance in the application of known two-dimensional results to three-dimensional problems, say in the well-known techniques of aerodynamics . in mhd the omission of the large disturbance characteristics implicit in a two-dimensional solution may invalidate its use in any practical three-dimensional problem .'), (1207, 'supersonic airfoil performance with small heat addition .', 'mager,a.', 'j.ae.scs. 1959, 99.', 'supersonic airfoil performance with small heat addition . an analytical method is presented which permits a very rapid evaluation of the acrodynamic effects arising from the addition of small amounts of heat near supersonic two-dimensional airfoils . this method applies to shockless inviscid flow without heat conduction . also, the mechanism by which the sesired heat addition is achieved is not considered . it is shown that even small amounts of heat generate a substantial pressure rise and thus cause appreciable changes in the acrodynamic coefficients . the results of this analysis compare favorably with those obtained by a more accurate, but also more tedious, graphical method of characteristics . two possible modes of application to an airplane design are considered from the energy requirements standpoint . in this connection, it is shown that the decrease of the required wing area resulting from heat addition may, in some cases, lead to savings in the rate of the fuel consumption . in general, however, one should not expect any substantial reduction in energy requirements resulting from the application of the wing heat addition .'), (1208, 'a linearized analysis of the forces exerted on a rigid wing by a shock wave .', 'ehlers,f.e. and shoemaker,e.m.', 'j. ae. scs. 1959, 75.', 'a linearized analysis of the forces exerted on a rigid wing by a shock wave . solutions are obtained in closed form for the pressures exerted on a rigid half plane by an incident, plane acoustic shock wave . the angle of incidence of the wave front is arbitrary and the half plane is considered to be traveling at constant velocity, subsonic of supersonic with respect to the acoustic medium . a closed-form solution is obtained also for a rigid wedge which is motionless with respect to the acoustic medium . the analysis is carried out by transforming the wave equation to laplace''s equation by the busemann conical transformation and then applying conformal mapping .'), (1209, 'aerodynamic processes in the downwash-impingement problem .', 'vidal,r.j.', 'j. ae. scs. 1962.', 'aerodynamic processes in the downwash-impingement problem . theoretical and experimental data relating to the downwash impingement problem are examined in order to arrive at a coherent understanding of the process of entrainment of ground particles in the flow . it is demonstrated that a key mechanism in the process is the interaction of nonuniform flow in the ground boundary layer with bluff ground particles . this interaction produces a lift force which, under typical conditions, equals or exceeds the particle weight . in the interest of quantitative prediction of the conditions necessary for particle entrainment, four subsidiary problem areas in the impinging jet are examined . these are the viscous decay, the inviscid flow field, the ground boundary layer, and the forces on a bluff body in nonuniform flow . applicable theories are used in conjunction with experimental data to assess the accuracy and range of validity of the theories, and to define the stream conditions which will cause particle entrainment . available data are applied to the establishment of criteria for particle entrainment in the vicinity of the impinging-jet stagnation point . these criteria show that entrainment occurs in a finite annular region on the ground plane, and that the particles most readily entrained are those with a diameter equal to about two-thirds the thickness of the ground boundary layer . the configuration size is shown to influence the process in that the onset of entrainment is fixed by the jet diameter and velocity, and the size of the ground particles . the criteria established provide a quantitative estimate of the conditions causing entrainment and provide a basis for scaling experimental results to a variety of full-scale situations .'), (1210, 'on slender airfoil theory for nonequilibrium flow .', 'ryhming,i.l.', 'j.ae.scs. 1962, 1076.', 'on slender airfoil theory for nonequilibrium flow . an exact linear theory for nonequilibrium flow past a thin airfoil is given . green''s function technique is used to solve the boundary value problem for the governing third order equation . upon satisfying the boundary condition on the airfoil surface an integral equation is obtained which has an exact solution . the final expression for the velocity potential, given as an integral over the source strength times the green''s function, shows that the solution is dependent not only on the slope variation of the airfoil but also on its curvature variation . this turns out to be the case for all free-stream mach numbers . as an example, the supersonic flow past a wedge is considered .'), (1211, 'boundary layer transition at supersonic speeds-three-dimensional roughness effects (spheres).', 'van driest,e.r. and blumer,c.b.', '', 'boundary layer transition at supersonic speeds-three-dimensional roughness effects (spheres). further experiments carried out in the 12-in. supersonic wind tunnel of the jet propulsion laboratory of the california institute of technology to investigate the effect of three-dimensional roughness elements (spheres) on boundary-layer transition on a the local mach number for these tests was 2.71 . the data show clearly that the minimum (effective) size of trip required to bring transition to its lowest reynolds number varies as the one-fourth power of the distance from the apex of the cone to the trip . use of available data for other mach numbers indicates that the mach-number influence for effective tripping is taken into account by the simple expression . some remarks concerning the roughness variation for transition on a blunt body are made .'), (1212, 'effect of uniformly distributed roughness on turbulent skin-friction drag at supersonic speeds .', 'goddard,f.e.', 'j. ae. scs. 1959, 1.', 'effect of uniformly distributed roughness on turbulent skin-friction drag at supersonic speeds . an experimental program was carried out in the 18-in. by 20-in. supersonic wind tunnel of the jet propulsion laboratory to determine the effect of uniformly distributed sand-grain roughness on the skin-friction drag of a body of revolution for the case of a turbulent boundary layer . the mach number range covered was 1.98 to 4.54, and the reynolds number varied from about 3 x 10 to 8 x 10 . some data were also obtained at a mach number of 0.70 . at speeds up to a mach number of 5 and for roughness sizes such that the quadratic resistance law holds, the compressibility effect is indirect, and the skin-friction drag is a function of only the roughness reynolds number, exactly as in the incompressible case . it is shown that the entire compressibility effect is a reduction of the fluid density at the surface as the mach number increases . the critical roughness, below which the surface is hydraulically smooth, is,. this is equal to the thickness of the laminar sublayer for a smooth surface for both compressible and incompressible flow . over the range of roughness sizes considered here, there appears to be no wave drag associated with the drag due to roughness . the shift in the turbulent veocity profile for a rough surface at supersonic speeds is a function of only the roughness reynolds number and quantitatively follows exactly the same law as that for the incompressible case .'), (1213, 'heat transfer to slender cones in hypersonic flow, including effects of yaw and nose bluntness .', 'wittcliff,c.e. and wilson,m.r.', 'j. ae. scs. 1962, 761.', 'heat transfer to slender cones in hypersonic flow, including effects of yaw and nose bluntness . as part of a general study of the aerothermodynamic characteristics of flight of hypersonic vehicles, an investigation of laminar heat transfer to slender yawed cones has been conducted . experiments have been made in the cal 11- by 15-in. shock tunnel at mach numbers from 11 to 13 and at yaw angles up to were tested . the heat-transfer rates are compared with theoretical predictions . the effects on the local heat-transfer rates of the boundary-layer displacement thickness, transverse curvature, yaw, nose bluntness, and the entropy sublayer are discussed . it is shown that, at zero yaw, the experimental data for the sharp cone are in good agreement with theory when boundary-layer displacement and transverse-curvature effects are included . for the yawed sharp cone, the heat-transfer rates along the most windward streamline are in good agreement with reshotko''s theory for yaw angles up to 3 . at larger yaw angles, the experimental heat transfer was found to be greater than that predicted theoretically . however, at these yaw angles the heat-transfer distribution on the windward side was in good agreement with laminar-boundary-layer calculations based on an assumption of local similarity . the zero-yaw tests of the blunted cones showed qualitative agreement with cheng''s shock-layer theory for slender blunt-nose bodies .'), (1214, 'the drag of elongated bodies over a wide reynolds number range .', 'robertson,j.m. and clark,m.e.', 'j. ae. scs. 1962, 842.', 'the drag of elongated bodies over a wide reynolds number range . the resistance of bodies in motion through an incompressible viscous fluid is predictable from stokes- or oseen-type solutions in the creeping-motion range, while some test information is available in the boundary-layer range . with the exception of experimental results for spheres or circular cylinders and analytical and experimental results for flat plates, almost no information is available on other bodies, particularly in the intermediate range of reynolds numbers extending from unity to a million . experimental results as obtained from hydroballistic studies in water and glycerin-water solutions are presented for finned ellipsoids of fineness-ratio .4 over a 20,000-fold range and are correlated with available information on other bodies . although results do not extend down to the creeping-motion region where analytical predictions are available, comparison with the drag coefficient trends for spheres and flat plates indicates that an appropriate curve for the ellipsoid could be extended so as to cover the entire laminar-viscous range . less extensive results are presented on the drag of fineness-ratio 8 ellipsoids and on laminar-turbulent transition occurrences .'), (1215, 'the effect of slip particularly for highly cooled walls .', 'rott,n. and lenard,m.', 'j. ae. scs. 1962, 591.', 'the effect of slip particularly for highly cooled walls . it is found that for boundary conditions on the velocity slip and on the temperature jump which are not oversimplified in an unrealistic way, the effect of these phenomena on the heat transfer and the shear at a stagnation point is of the order of the ratio of the mean free path outside the boundary layer to the boundary-layer thickness, even for highly cooled walls . a simplified theory of this effect is given, which puts the physical reasons for the results in evidence and agrees closely with the more accurate calculations . it is concluded that the effects of slip and jump are not negligible in comparison with other low-reynolds-number corrections, even for very cold walls .'), (1216, 'pressure distribution in regions of step-induced turbulent separation .', 'vasilu,j.', 'j. ae. scs. 1962, 596.', 'pressure distribution in regions of step-induced turbulent separation . an analysis is made of the pressure distribution in the separated-flow region ahead of a step, using the concept of the turbulent mixing coefficient of crocco and lees and the /jet-flow/ model of chapman with some modification . on the basis of a variable mixing coefficient, a differential equation for the pressure distribution is derived, which gives the pressure rise as a function of the distance from the separation point . this equation contains the separation length as an unknown . a second equation is obtained by making a mass balance of the air entering and, leaving the /dead-air/ region ahead of the step . the pressure rise and the separation distance for a given mach number are determined by solving the two equations simultaneously . the analysis yields results which are in close agreement with the experimental data on steps, obtained at princeton, particularly for m = 3.85 . for lower mach numbers, a maximum variation of 5 percent is found between theory and experiment . use of the velocity profiles of jets, as required by the jet-flow model, necessarily restricts the applicability of the present study to flows with thin boundary layers at the separation point .'), (1217, 'application of inequality constraints to variational problems of lifting re-entry .', 'levinsky,e.s.', 'j. ae. scs. 1962, 400.', 'application of inequality constraints to variational problems of lifting re-entry . inequality constraints are introduced into the variational formulation of the optimum re-entry problem for a lifting vehicle to prevent human and or structural tolerances from being exceeded . these constraints consist of minimum and maximum angle of attack, maximum load factor, and maximum convective heat transfer (equilibrium temperature) . the equations have been programed for the ibm 704 computer, and sample trajectories are presented for which the total heat transferred to certain critical areas on the windward surface of the vehicle is minimized . these trajectories indicate the dominant effect of the constraints on the optimum flight path, which is shown to consist of both unconstrained and constrained arcs .'), (1218, 'experimental lift and drag of a series of glide configurations at mach numbers 12 .6 and 17 .5.', 'geiger,r.e.', 'j. ae. scs. 1962, 410.', 'experimental lift and drag of a series of glide configurations at mach numbers 12 .6 and 17 .5. a series of semiballistic-type bodies consisting of three half sphere cones of 0.3 bluntness ratio with half-cone angles of 8.6, laboratory hypersonic shock tunnel at m = 17.5 and 12.4 . in addition, a representative winged glide configuration consisting of a sharp-edged, 60 swept delta wing with cone-segment the range of angle of attack for the half sphere-cone tests was the technique for force coefficient determination consists of analyzing high-speed motion pictures of the motion of very light balsa and isofoam plastic models which are literally free-flown for several milliseconds in the test section of the shock tunnel . because of viscous effects the newtonian prediction of half sphere cone drag is consistently less than, but generally parallel these bodies is generally well predicted by the newtonian theory except at small and moderate positive angles of attack where it is generally less than newtonian . this lift deficiency appears to increase with cone half angle . maximum lift-drag ratios fall considerably short of the newtonian predictions . several exploratory tests at mach 11.7 and low reynolds number ( approximately reduction in) on the 13 model produced an approximate doubling of minimum drag and a 35 percent decrease in (l d) max,. this demonstrates the importance of viscous effects for blunt bodies in the reynolds number range of these tests . the sharp leading-edge, 60 sweep delta wing-body configuration exhibited the same (l d) max, as the wing alone, about 2.80 at both positive and negative angles of attack .'), (1219, 'determination of lift or drag programs to minimize re-entry heating .', 'bryson,a.e.', 'j. ae. scs. 1962, 420.', 'determination of lift or drag programs to minimize re-entry heating . a study of single-pass re-entry from escape speed and from circular satellite speed is made to determine the lift program for a hypersonic glider and the drag-modulation program for a non-lifting vehicle that minimize the heating of the vehicles within acceleration or range constraints . a new method of numerical solution is used, similar to kelley''s /method of gradients,/ that permits rapid convergence to the optimum lift program starting with an original good estimate . this method avoids the two-point boundary-value problem of the calculus-of-variations formulation, and is applicable to any optimum-programing problem . an acceleration-tolerance limit is introduced which describes the human pilot''s capability to withstand acceleration more accurately than a simple acceleration limit .'), (1220, 'boundary layer transition in the presence of streamwise vortices .', 'tani,i. and komoda,h.', 'j. ae. scs.1962, 440.', 'boundary layer transition in the presence of streamwise vortices . results of an experimental investigation of instability leading to transition in the subsonic boundary layer flow along a flat plate are presented . a series of wings was placed outside the boundary layer to produce streamwise vortices, which in turn made the boundary layer three-dimensional--i.e., periodic in thickness in the spanwise direction . hot-wire measurements were made to trace the downstream development of the disturbance or wave created by the vibrating ribbon . as the wave travels downstream, it is deformed into a three-dimensional configuration by the three-dimensionality of the boundary-layer flow, but it is eventually damped out so long as it remains small in intensity . it is only after the wave intensity exceeds a certain amount (which depends on the degree of boundary-layer three-dimensionality) that the nonlinear effect manifests itself by the rapid amplification of wave intensity, the rapid increase in wave three-dimensionality, and the distortion in mean velocity profile . the appearance of nonlinear development inevitably leads to the breakdown of laminar flow, and hence the onset of turbulence . there is present a mechanism by which the energy is transferred from one spanwise position to another so that the breakdown of laminar flow occurs as a consequence of three-dimensional development of the wave as a whole .'), (1221, 'steady flow of conducting fluids in channels under transverse magnetic fields, with consideration of hall effect .', 'tani,i.', 'j. ae. scs.1962, 297.', 'steady flow of conducting fluids in channels under transverse magnetic fields, with consideration of hall effect . an approximate method of solution based on a minimum principle is presented for the steady laminar incompressible flow of an electrically conducting fluid through a straight channel of arbitrary cross section with conducting or nonconducting walls in the presence of a uniform transverse magnetic field . the hall effect is taken into account by making simplifying assumptions that the gas is fully ionized and that both reynolds number and magnetic reynolds number are small . numerical calculations are carried out for the case of a rectangular channel .'), (1222, 'axisymmetric magnetohydrodynamic channel flow .', 'hains,f.d. and holer,y.a.', 'j. ae. scs. 1962, 143.', 'axisymmetric magnetohydrodynamic channel flow . the axisymmetric subsonic and supersonic flow fields, and the skin friction and heat transfer of an electrically conducting compressible fluid flowing in a channel of constant circular area through a magnetic field are investigated when the magnetic reynolds number is small . the inviscid-flow field for flow through a dipole field is solved by the method of characteristics in the supersonic case . for the subsonic case, linearized equations are derived for small values of the magnetic interaction parameter . numerical results are obtained by the relaxation method . the inviscid-flow-field solutions are used as boundary conditions for the laminar boundary layer along the wall, in which axial pressure gradients form an important feature . the exact continuum-flow equations are reduced by an order-of-magnitude analysis to the boundary-layer equations, which are solved numerically by an integral method using a fourth-degree velocity profile and a fifth-degree stagnation-enthalpy profile . pressure, temperature, and heat-transfer measurements are made with a shock tube under supersonic-flow conditions closely approaching those used in the numerical computations . general agreement is found between the theoretical and the experimental results .'), (1223, 'inviscid-incompressible-flow theory of static two-dimensional solid jets, in proximity to the ground .', 'strand,t.', 'j. ae. scs. 1962, 170.', 'inviscid-incompressible-flow theory of static two-dimensional solid jets, in proximity to the ground . the inviscid-incompressible-flow theory of static two- dimensional solid jets impinging orthogonally on the ground is presented using conformal mapping methods . it is shown that the thrust of a solid jet at constant power initially decreases as the ground is approached . the magnitude of the thrust out of ground effect is regained only at a very low height-to-jet width ratio (approximately 0.55) . the maximuin decrease is about 6 percent . the ground effect on solid jets is thus largely unfavorable .'), (1224, 'on the plk method and the supersonic blunt-body problem .', 'vaglio-laurin,r.', 'j. ae. scs. 1962, 185.', 'on the plk method and the supersonic blunt-body problem . detailed analysis of the subsonic and transonic portious of the flow field about either very blunt or asymmetric configurations requires successive approximations,. these can be carried out in a systematic fashion only when an appropriate convergent perturbation procedure is available . the problem of producing successively refined sets of initial conditions for either /direct/ or /inverse/ analysis of the flow is formulated in the following terms .. given reasonable estimates for shock shape and pressure distribution on the body, can one determine the flow field of interest to any desired degree of approximation by a perturbation approach .qm a procedure to this effect is developed which involves stretching of coordinates in the spirit of the poincare-lighthill-kuo are transformed along body, shock, and intermediate lines so as to annul perturbations of the local resultant velocity,. (b) for the integral method the coordinate along the boundary of each strip is shifted so as to control perturbations of the velocity component that determines the critical point . the approach is justified by a study of the equations governing the direct method, and by consideration of model transonic flow problems for which closed form solutions are available . the range of validity of the proposed procedure is assessed by practical application and comparison with experiment,. results are presented for a disk set normal to a low-termperature air stream at m = 4.76, and for a highly asymmetric two-dimensional configuration at m = 8 .'), (1225, 'the effect of adverse pressure gradients on the characteristics of turbulent boundary layers in supersonic streams .', 'george h. mclafferty and robert e. barber', 'united aircraft corporation research laboratories', 'the effect of adverse pressure gradients on the characteristics of turbulent boundary layers in supersonic streams . tests were conducted at mach numbers from 2.0 to 3.5 to determine the thickness and profile shape characteristics of turbulent boundary layers on two-dimensional and axisymmetric curved-surface models having adverse pressure gradients . the magnitude of the gradients relative to the boundary-layer thickness at the beginning of the gradient was varied by employing models having different radii of curvature and by changing the boundary-layer thickness at the beginning of the gradient . the overall pressure rise in most cases was greater than the value which would cause a turbulent boundary layer to separate if the pressure rise were created by an oblique shock wave . an analytical investigation was also conducted so that the results of the experimental investigation could be applied to the prediction of cases outside the range of the experiments . it is shown that boundary-layer momentum thickness can be predicted from the von karman boundary-layer momentum equation, but that measured values of boundary-layer profile shape are in poor agreement with values computed from procedures derived by extending conventional methods for predicting profile shape in subsonic flow . a new procedure for calculating boundary-layer profile shapes, developed in this paper, is shown to provide a good correlation between experimental and calculated values of boundary-layer profile shapes in adverse pressure gradients created by curved surfaces . this procedure is based on the experimental observation that the station at which high-energy free-stream flow actually mixes into a turbulent boundary layer in an adverse pressure gradient is well downstream of the station at which flow would have to mix in order to maintain a flat-plate profile .'), (1226, 'heat transfer in the laminar boundary layer with ablation of vapor of arbitrary molecular weight .', 'faulders,c.r.', 'j. ae. scs. 1962, 76.', 'heat transfer in the laminar boundary layer with ablation of vapor of arbitrary molecular weight . the reduction of heat transfer in the laminar boundary layer under the condition of vaporizing ablation is analyzed for arbitrary molecular weight of the vapor . primary assumptions are that the pressure gradient is zero, the individual components of the binary system are perfect gases, the prandtl number is constant, and the viscosity is proportional to temperature . variations through the boundary layer of the schmidt number for binary diffusion and the density-viscosity product, are included in the analysis . the wall temperature is held constant . numerical results are obtained for prandtl numbers of 0.75 and varying from 0.25 to 4.00, wall concentration of the foreign gas as high as 0.9 (corresponding to the high heat rates encountered during re-entry), and ratio of specific heats of foreign gas equal to that of air . kinetic theory is used to obtain schmidt number as a function of molecular weight and concentration . the departure of schmidt number and prandtl number from unity and the variation of reynolds analogy factor with prandtl number, blowing parameter, wall concentration, and molecular weight ratio are found to have relatively minor influence on the heat block ratio at high rates of ablation . the primary factor governing the influence of molecular weight ratio on the heat block ratio is the variation of across the boundary layer . little loss of accuracy is incurred, in the range of molecular weight ratios considered here, by assuming schmidt and prandtl numbers of unity as long as the variation is properly taken into account .'), (1227, 'pressure-gradient effects on the preston tube in supersonic flow .', 'naleid,j.f. and thompson,m.j.', 'j. ae. scs. 1961, 940.', 'pressure-gradient effects on the preston tube in supersonic flow . this paper is concerned with an experimental investigation of the effects of a longitudinal pressure gradient in a supersonic stream of air over a bounding surface on the performance of a preston or impact-pressure tube at the surface . evidence is presented which indicates that for the mach number considered and for the range of pressure gradients covered, the preston tube functions in a completely satisfactory manner for the determination of local shear stress .'), (1228, 'leading-edge separation of laminar boundary layers in supersonic flow .', 'brower,w.b.', 'j. ae. scs. 1961, 957.', 'leading-edge separation of laminar boundary layers in supersonic flow . a brief description of the flow field is given for the interaction of shock wave and laminar boundary layer on a compression corner in supersonic flow . a special sub-case--that of leading-edge laminar separation--is analyzed by extension of chapman''s laminar mixing-layer theory . results are tabulated for ranges of mach number, and compression-corner angle . a limited region of possible leading-edge laminar separation with an attached leading-edge shock (or in certain cases an expansion) followed by a second shock due to the reattachment flow is found to exist . comparison with existing experimental data is found to be satisfactory in several cases .'), (1229, 'the effect of sweep angle on hypersonic flow over blunt wings .', 'bogdonoff,s.m. and vas,r.e.', 'j. ae. scs. 1961.', 'the effect of sweep angle on hypersonic flow over blunt wings . a series of tests were carried out in the princeton university helium hypersoule wind tunnel on blunt two-dimensional wings at zero angle of attack with sweep angles up to 70 at mach numbers from 7 to 15 . the leading edge reynolds number varied from 3,000 to 25,000 . the measured pressure distributions were compared with the simple summation of the theoretical inviscid effect (based on blast wave theory using the normal mach number) added to the viscous effect (calculated as if no sweep were present) . for the unswept wing, the slope of the pressure decay was reasonably well predicted by the theoretical calculations . the viscous theory reasonably predicted the variation in the pressure distribution due to changes in leading-edge reynolds number . by subtracting the theoretical viscous effects, an inviscid mach number dependence of the 2.2 power was found as compared to the value of 2.0 predicted by the inviscid theory . the same approach for the swept wing did not give consistently satisfactory results . deviations avove and below the calculated value by as much as 40-50 percent were measured and there seemed to be no systematic variation with either mach number or reynolds number . at a constant high reynolds number, it was found that the pressure distribution varied with the distance along the wing with an exponent between about--0.53 and--0.58 except for a rather sharp decrease which occurred for the 70 sweep case . the pressure at a given station for a fixed mach number and given leading edge thickness varied as the cosine of the sweep angle to the 1.1 power as compared to the 1.3 power predicted from general geometrical considerations .'), (1230, 'hypersonic nozzle expansion of air with atom recombination present .', 'nagamatsu,h.t., workman,j.b. and sheer,r.e.', 'j. ae. scs. 1961, 833.', 'hypersonic nozzle expansion of air with atom recombination present . an experimental investigation on the expansion of high- temperature, high-pressure air to hypersonic flow mach numbers in a conical nozzle of a hypersonic shock tunnel has been carried out . the equilibrium temperature and pressure ranges after the reflected shock wave were 1400 to 6000 k and 100 to 1000 psia . static-pressure measurements, which are sensitive to the state of the gas, were made along the axis of the nozzle for different reservoir conditions . these results are compared with the calculated equilibrium and /frozen/ data for the same geometry and initial reservoir conditions . for reservoir pressures greater than 500 psia, the expansion of the air in the nozzle is essentially in equilibrium up to reservoir temperatures of about 4,500 k . for temperatures greater than almost frozen . at a given area ratio for the nozzle and reservoir pressure, the expansion process remains in equilibrium up to a certain reservoir temperature, and beyond this temperature the flow expansion deviates rapidly from the equilibrium process and approaches the frozen case .'), (1231, 'hypersonic flow over an elliptic cone: theory and experiment .', 'chapkis,r.l.', 'j. ae. scs. 1961, 844.', 'hypersonic flow over an elliptic cone: theory and experiment . by applying hypersonic approximations to ferri''s linearized characteristics method, simple results were obtained for the shock shape and surface pressure distribution for an unyawed conical body of arbitrary cross section . calculations were carried out for an elliptic cone having a ratio of major to minor axes of, and a semivertex angle of about 12 in the meridian plane containing the major axis . an experimental investigation of the flow over this body conducted at a mach number of 5.8 in the galcit hypersonic wind tunnel showed that the surface pressure distribution at zero angle of attack agreed quite closely with the theoretical prediction . on the other hand, the simple newtonian approximation predicts pressures that are too low . surface pressure distributions and schlieren photographs of the shock shape were obtained at angles of attack up to 14 at zero yaw, and at angles of yaw up to 10 at zero pitch . at the higher angles of attack the newtonian approximation for the surface pressures is quite accurate .'), (1232, 'the curtain jet .', 'ehrich,f.f.', 'j. ae. scs. 1961, 855.', 'the curtain jet . a detailed analytic study is made of the curtain jet, the two-dimensional fluid wall used to contain support pressure on the underside of ground effect machines . two variations of the jet are studied in detail--the bifurcated jet, in which a portion of the flow streams into the support pressure region, and the deflected jet, in which none of the flow penetrates into the support pressure region . kirchhoff-helmholtz free steamline analysis is used to construct the flow field, and quantitative results are presented for the effect of nozzle inclination and detailed geometry on flow requirements and support pressure differential at varying altitudes .'), (1233, 'supersonic shear flow past an airfoil between two parallel walls .', 'rhyming,i.l.', 'j. ae. scs. 1961, 861.', 'supersonic shear flow past an airfoil between two parallel walls . the supersonic flow with assigned mach number gradient in the span direction past a straight wing between two parallel walls is studied using the small-disturbance theory . the governing equation for the disturbance pressure on the airfoil, together with the boundary conditions on the airfoil and at the walls, is solved by the method of separation of variables . upon separation the problem is reduced to a sturm-liouville eigenvalue problem and to the solution of the telegraph equation . as an application, a certain mach number profile is selected and the resulting pressure distribution on a parabolic arc airfoil is computed .'), (1234, 'direct calculation of pressure distribution on blunt hypersonic nose shapes with sharp corners .', 'holt,m.', 'j. ae. scs. 1961, 872.', 'direct calculation of pressure distribution on blunt hypersonic nose shapes with sharp corners . the method of belotserkovskii for calculating hypersonic flow fields past a circular cylinder is extended to deal with axially-symmetric flow past sharp-cornered nose shapes, in particular, spherical segments and flat-headed cylinders . results on spheres are also included . in the present paper belotserkovskii''s first approximation is considered, and comparison of calculated pressure distribution and shock shape with experimental results shows very good agreement .'), (1235, 'a theory of the two dimensional laminar bounary layer over a curved surface .', 'yen,k.t. and toba,k.', 'j. ae. scs. 1961, 877.', 'a theory of the two dimensional laminar bounary layer over a curved surface . the purpose of this paper is to present a theory to account for surface curvature effects on the two-dimensional boundary-layer flow which approaches a potential flow at free stream . the problem of two-dimensional viscous flow is first formulated by using the streamlines and their orthogonal trajectories as the generalized coordinates . a boundary-layer approximation is applied to the navier-stokes equations and the gauss equation in the generalized coordinates to yield the boundary-layer equations . the conditions under which similar solutions of the boundary-layer equations exist are determined . by a simple transformation, the governing differential equation can be expressed in a form which reduces to the falkner-skan equation for zero surface curvature . numerical results for a similar solution which corresponds to a flow over a curved surface with zero surface pressure gradient have been obtained . the velocity profiles in the boundary layer and the wall skin-friction distribution for concave and convex surfaces are presented . the wall skin friction for a convex wall is found to be higher than the blasius value for a flat plate . on the other hand, for a concave wall, the skin friction will drop below the blasius value as the curvature increases, but it appears to reach a minimum, and beyond this minimum point it will increase again . the same flow problem was treated by murphy by a different method of analysis . comparison of murphy''s results with those obtained by the present method reveals some basic differences in the boundary-layer characteristics . in particular, murphy''s results indicate that the wall skin friction for a convex surface is smaller than the blasius value, while for a concave wall it is higher .'), (1236, 'the stagnation point boundary layer in the presence of an applied magnetic field .', 'bush,w.b.', 'j. ae. scs. 1961, 610.', 'the stagnation point boundary layer in the presence of an applied magnetic field . similarity equations for axisymmetric compressible flow are obtained, assuming that the magnetic field is uniform, normal to the surface, and unaffected by the flow, and that the conductivity varies as the nth power of the enthalpy . numerical solutions are given for a number of values of n and of the field strength, and are used to modify the estimates of heat-transfer made by the author using inviscid theory (title source 26, 536-537, 1959) .'), (1237, 'foreign-gas injection into a compressible turbulent boundary layer on a flat plate .', 'ness,n.', 'j. ae. scs. 1961, 645.', 'foreign-gas injection into a compressible turbulent boundary layer on a flat plate . the distributed injection of a foreign gas into a compressible, turbulent boundary layer in the absence of a pressure gradient is considered . the analysis is performed within the framework of the binary-mixture concept, that is, the primary fluid flowing over the surface represents one component while the injected species represents the second . calculations have been performed for the injection of helium into undissociated air . the results indicate an effect of mach number on surface shear and energy transfer when distributed light-gas injection normal to the surface exists . a comparison with experimental data indicates reasonable agreement over a range of mach numbers .'), (1238, 'the newtonian approximation in magnetic hypersonic stagnation-point flow .', 'lykoudis,p.s.', 'j. ae. scs. 1961.', 'the newtonian approximation in magnetic hypersonic stagnation-point flow . the hypersonic flow of an electrically conducting fluid around the stagnation region of a sphere carrying a radial magnetic field is examined . by assuming a newtonian pressure distribution and constant density, the differential equation of the inviscid flow is integrated and a simple closed-form solution is obtained . it is found that the ratio of the stand-off distances of the shock wave for the magnetic and nonmagnetic cases does not depend explicitly on the magnetic parameter s (ratio of the ponderomotive force to the free-stream inertia force) nor on the density ratio (the value at the free stream divided by the value behind the shock wave) but on the product s at least for values of between and . the velocity gradient on the body is also calculated and the ratio of the magnetic to the nonmagnetic case is shown to depend on the parameter . the case of cylindrical shocks is also examined,. the same general conclusions are drawn .'), (1239, 'body under lifting wing .', 'chen,c.f. and clarke,j.h.', 'j. ae. scs. 1961, 547.', 'body under lifting wing . an investigation is made of supersonic-aircraft configurations composed of a cambered body positioned a certain distance beneath an arbitrary lifting wing . the geometry of the wing is regarded as given and the geometry of the body may be given or optimum . expressions for the drag and lift are obtained from reverse-flow considerations,. these greatly implement such a study when interference cross flows must be cancelled . the drag advantage to be gained when a given body and wing assume a given orientation is studied . treated more extensively is the variational problem of determining the optimum wing incidence and optimum body shape, for the given volume and length, to yield the minimum drag for prescribed lift . numerical results are provided to indicate the significance of the large number of parameters appearing in the problem . of these, the gap between the wing and the body is found to be particularly important . it is found that at low gap moderate body distortions have a significant influence on the drag . drag reductions of up to 44 relative to the case of no interference have been found at a mach number of 2.24 in a configuration having a gap approximately equal to the maximum diameter of the body, and a wing chord of about three eighths of the length of the body . comparison is made with the conventional wing-body combination including the effects of skin friction, and it is concluded that the advantage suggested by the preceding considerations is not appreciably diminished . finally, it is shown that the configurations studied lead to bodies of fineness ratios much lower than are appropriate to conventional wing-body combinations . tests were made on an arrangement consisting of a scars-haack body located under a lifting rectangular diamond-profile wing . the mach number was 1.6 and the reynolds number was 9.17 x 10 based on the body length . it was found that the measured lift developed on the wing due to the flow field of the body agrees very well with the theoretical value . downstream of the impinging shock from the wing, flow separation was observed on the exterior of the body but not in the interior . the separation is attributed not to the pressure rise across the shock but to the pressure field arising from the reflection from the body of the shock-induced cross flow . further observations suggest that the separation can be avoided by pitching the body or by kinking the body at the shock wave to accommodate the shock-induced cross flow .'), (1240, 'nonsimilar solutions of the compressible laminar boundary layer equations with applications to the upstream-transpiration cooling problem .', 'pallone,a.', 'j. ae. scs. 1961, 449.', 'nonsimilar solutions of the compressible laminar boundary layer equations with applications to the upstream-transpiration cooling problem . a new method is presented for predicting the boundary-layer characteristics downstream of the porous region of an injection-cooled surface . the method consists of a general scheme for obtaining nonsimilar solutions of the compressible- laminar-boundary-layer equations and is formulated along the following lines . the viscous domain is divided into n curvilinear strips . the governing equations are then integrated along the coordinate normal to the body from the surface to the boundary of each strip . as a result, one obtains a set of independent integro-differential relations . the integration is carried out by expressing the integrands as polynomials, the coefficients of which are functions of the unknown values of the velocity and temperature on the strip boundaries as well as of the imposed boundary condition at the wall and at the outer edge . after the integration is performed, a set of ordinary first-order differential equations is obtained . the set of equations may be solved for given initial conditions by a numerical integration scheme such as the runge-kutta method . several numerical examples of interest are presented .'), (1241, 'the turbulent boundary layer on chemically active ablating surfaces .', 'denison,m.r.', 'j. ae. scs. 1961.', 'the turbulent boundary layer on chemically active ablating surfaces . incompressible turbulent-boundary-layer analysis is extrapolated analytically to the case of a compressible turbulent boundary layer with ablation or mass injection at the surface . the effects of chemical reactions such as dissociation and recombination as well as combustion are included . the analysis applies to blunt as well as sharp bodies which are either axisymmetric or two-dimensional . when the turbulent lewis and prandtl numbers are unity, it is found that, as in the laminar case, little detailed knowledge of the chemistry inside the boundary layer is required in most instances . the conditions at the surface and the outer edge of the boundary layer are often sufficient for prediction of heat and mass transfer . comparison is made with experiments on the combustion of graphite under turbulent flow conditions . prediction of ablation rates within about 30 percent accuracy is obtained when empirical constants obtained from incompressible velocity profiles with no mass injection are used .'), (1242, 'some considerations on the laminar stability of time-dependent basic flows .', 's. f. shen', 'u.s. naval ordnance laboratory', 'some considerations on the laminar stability of time-dependent basic flows . as a stability criterion for infinitesimal disturbances in an incompressible, parallel but time-dependent basic flow, it is proposed to introduce the concept of /momentary stability,/ which is said to prevail at the instant if the kinetic energy of the disturbances, as a fraction of the kinetic energy of the basic flow, tends to decrease . the significance of such a criterion is briefly discussed . for special time-dependent basic flows which are described by similar velocity profiles at all times (except for changes in amplitude), in the inviscid limit only a change of the time scale is needed to reduce the solution essentially to that for the steady case . the disturbances may be of either the transverse-wave or the longitudinal-vortices type . the result indicates a very strong destabilizing influence of deceleration, which is likely to overshadow that of the velocity profile under normal circumstances . the observations of fales rotating cylinders) are believed to be largely due to the deceleration . at finite reynolds numbers, the usual procedure of calculating the stability solution on the basis of the instantaneous profile is further shown to be valid only for extremely slow acceleration or deceleration . even when the solution is acceptable, the condition for neutral stability may not be used without reservation . to calculate momentary stability properly, a procedure for a slowly varying but more general profile is also described .'), (1243, 'supersonic boom of wing-body configurations .', 'ryhming,i.l. and yoler,y.a.', 'j. ae. scs. 1961, 313.', 'supersonic boom of wing-body configurations . the supersonic boom in steady, level flight of a wing-body configuration is due to the effects of body volume, wing volume, wing incidence or lift, and wing-body interference . the contribution in the far field of each of these factors can, in any given azimuthal plane, be represented as that of an equivalent body of revolution . this concept is developed to investigate the possibilities of using interference among the components of a wing-body configuration to reduce or suppress the boom due to lift . results of wind tunnel experiments are also presented and discussed in light of the theoretical indications .'), (1244, 'on the aerodynamic noise of a turbulent jet .', 'cheng,sin-i.', 'j. ae. scs. 1961, 321.', 'on the aerodynamic noise of a turbulent jet . a new model is advanced for analyzing the broad-spectrum noise of a turbulent jet . the shear layer bounding the turbulent jet is assumed to play an important role in modifying the / quadrupole sound radiation/ from the interior . to the sound- emitting small-scale turbulent eddies (with frequencies much higher than those of large-scale eddies), the laminar shear layer has an irregular contour, as if the large-scale turbulent motions were frozen . the linearized analysis is then applied to the laminar shear layer to relate the acoustic oscillations across it . the concept of geometrical acoustics is generalized to represent the passage of an acoustic ray through a laminar shear layer . acoustic rays may be traced across the shear layer as transmission and refraction, but they may also be apparently /absorbed/ or /generated/ by the laminar layer . this /generation/ is visualized as the schematic representation, within the framework of geometrical acoustics, of the action of the reynolds stress in transferring energy from the shearing mean flow to the acoustic waves . such action of the reynolds stress can be neglected in ordinary acoustics when the acoustic medium is not moving at speeds comparable to the speed of sound in the medium . however, this action is of crucial importance in the aerodynamic noise of high-speed turbulent jets where the reynolds stress is the fundamental element of the radiating quadrupoles, according to lighthill . those acoustic waves that become /stationary/ with respect to the local mean flow somewhere in the interior of the shear layer are significantly modified by the viscous action through the critical layer . the shear layer therefore serves as a selective amplifier of the acoustic waves passing through it . kinematically, the shear layer brings about the preferred downstream emission .. dynamically, the shear-layer augmentation significantly increases the polar peak noise level . the acoustic power output per unit solid angle for such downstream emissions augmented by the shear layer (including the polar peak) varies as, predicted by lighthill, but without lighthill''s convective corrections . on the other hand, the acoustic power output per unit solid angle nearly normal to the jet, due to the transmitted downstream-propagating waves, varies roughly as . heating the jet gas increases the shear-layer augmentation and may increase the polar peak noise level by several db . the silencing action of the edge notches and edge teeth may also be interpreted as due apparently to the result of possible distortion of the shear-layer profiles .'), (1245, 'some aspects of nonequilibrium flows .', 'sedney,r.', 'j. ae. scs. 1961, 189.', 'some aspects of nonequilibrium flows . in this paper are discussed some of the general features of nonequilibrium flow . in particular, vibrational relaxation is discussed in detail . this case is somewhat simpler than dissociation and ionization but it illustrates some of the main new features of nonequilibrium flow . those aspects of two-dimensional and axisymmetric flow behind shock waves are examined analytically which yield significant information without requiring numerical solution of the governing equations . the thermodynamics of a vibrational relaxing gas are discussed . the conditions for simulating flows are noted . crocco''s theorem and the characteristic equations are derived . then a simple method of obtaining the initial gradients of the flow variables behind a shock is shown . these gradients are used in discussing two particular flows . an exact solution for flow over a cusped body is obtained . flow over a wedge near the tip and far from the tip is considered . it is found that far from the tip a boundary-layer type phenomenon occurs .'), (1246, 'solution of subsonic nonplanar lifting surface problems by means of high-speed digital computers .', 'kilakowski,l.j. and haskell,r.n.', 'j. ae. scs. 1961.', 'solution of subsonic nonplanar lifting surface problems by means of high-speed digital computers . the method proposed in this paper is based on an approximate solution of the integral equation which represents the potential flow about a finite wing, with no restrictions beyond those necessary for linearization . after assuming the usual series representation of the wing surface vorticity distribution, the solution is achieved by approximating portions of the kernels of the transformed integral equation by single and double fourier series and performing termwise integrations analytically . this is followed by the routine inversion of the aerodynamic influence coefficient matrix, after satisfying appropriate boundary conditions at selected control points . in this procedure the number of control point used is limited only by the storage capacity of the computer . control points may be located so as to cover the entire wing surface, with due regard to certain physical requirements, permitting the accurate representation of complicated mean surface shapes . an evaluation of the proposed method is included . comparisons with other theoretical methods and electrical analogy tank results are used to substantiate the accuracy of the proposed method when applied to plane wings . a final evaluation involves a comparison of calculated surface pressure distribution with wind-tunnel measurements on a swept, tapered wing with a cambered and twisted mean surface . the agreement evidenced in the latter comparison has the same order of overall accuracy as similar comparisons on plane wing planforms . in either case, the results given by the proposed method are within the accuracy requirements for most aircraft design studies .'), (1247, 'the supersonic boom of a projectile related to drag and volume .', 'ryhming,i.l.', 'j. ae. scs. 1961.', 'the supersonic boom of a projectile related to drag and volume . the whitham theory predicting the far-flow field around a projectile is used to derive body shapes which produce extreme bow shock-wave pressure jump or /boom,/ subject to constraining conditions regarding the drag due to the bow shock and fineness ratio of the bodies . it is found that the minimum drag body is also the minimum boom body . the body-volume effect and the effect of discontinuities in slope of the body meridian section on the boom intensity is investigated . as a general result of the investigation, it can be said that the boom of a projectile for given mach number and flight altitude is primarily determined by its length and fineness ratio . the maximum variation in the boom intensity for pointed bodies with given length and fineness ratio is of the order of 10 per cent . the geometry of the bodies is thus found to play a minor role .'), (1248, 'an analytic extension of the shock-expansion method .', 'waldman,g.d. and probstein,r.f.', 'j. ae. scs. 1961, 119.', 'an analytic extension of the shock-expansion method . the problem is considered of calculating approximately the inviscid rotational flow field and pressure distribution about a smooth two-dimensional airfoil with sharp leading and trailing edges in a uniform supersonic or hypersonic stream . the assumption of a perfect gas is made, and the basic flow pattern for the analysis is taken to be given by the simple isentropic shock-expansion method with straight characteristics . an elementary characteristics treatment is discussed to show when the simple shock-expansion method should be satisfactory for computing the surface pressure distribution, and under what circumstances it may be expected to break down . by utilizing characteristic variables the isentropic shock-expansion method is then formulated analytically, and an analytic result is obtained for the shock shape corresponding to this zero-order approximation . in the special case where hypersonic similitude is applicable, that is, for slender bodies and high mach numbers, the shock-shape expression for large distances is found to reduce to the result previously given by mahony, which for weak shocks and slender bodies in turn reduces to the simple-wave result first given by friedrichs . employing the analytic form of the isentropic shock-expansion method as a zero-order approximation, an analytically consistent perturbation method is developed by expanding the dependent flow variables in the exact partial differential equations in powers of the reflection coefficient for simple waves interacting with an oblique shock . the scheme by its nature helps to define those regions in which shock expansion can be used, in addition to taking into account in a perturbation sense the factors neglected in simple shock-expansion theory, namely, the curvature and reflection of the mach waves and the correct boundary conditions at the shock wave . analytic solutions are obtained for the first-order corrections, including the surface pressure distribution . the necessary numerical computation of the integrals involved is considerably simpler than a direct application of the method of characteristics . to illustrate the method and its accuracy, the zero-order shock shape and first-order pressure distribution are calculated for a family of parabolic arc airfoils at an infinite free-stream mach number . these results are compared with rotational characteristic solutions where available, and the present method is found to be in excellent agreement .'), (1249, 'plasma flow over a thin charged conductor .', 'yoshihara,h.', 'j. ae. scs. 1961, 141.', 'plasma flow over a thin charged conductor . the flow of a dense plasma over a wavy conducting wall of small amplitude is investigated where magnetic effects are negligible . these results are then used to analyze the flow over a thin conductor with cusped edges . it is found that the coulomb drag vanishes identically, while the fluid-pressure drag corresponds to the ackeret value for a neutral particle gas at the reduced-plasma mach number .'), (1250, 'high-speed viscous corner flow .', 'bloom,m.h. and rubin,s.', 'j. ae. scs. 1961, 145.', 'high-speed viscous corner flow . a boundary-layer integral method analysis is set up for compressible laminar flow in a symmetric corner with varying angle and streamwise pressure gradient . it represents an extension and modification of the constant density analysis of loitsianskii and bolshakov . the analysis is applied to the case of constant pressure, constant corner angle, and isothermal surfaces, for which the crocco velocity-enthalpy relation holds . although simplifying assumptions limit the quantitative accuracy outside the 60 to 120 angle range, some qualitative trends are probably correct outside this range . the limiting cases near 0 and 180 are not considered . favorable agreement between some results obtained by the integral method and by other methods is demonstrated for the isothermal, constant-density case . results show an increasingly sharp merger of the outermost isovels of streamwise velocity as the mach number increases . this sharp merging of the outer isovels is increased by increasing corner angle and by insulation of heating of the surfaces . within the interior of the viscous layer the spreading or contraction of the disturbed region of merging is influenced by surface heat-transfer conditions . surface shear and heat flux are decreased in the disturbed region, and are zero at the apex . for cases corresponding roughly to the higher mach numbers of wider corner angles, the /specific momentum-area/ exhibits the same decrease with mach number as its two-dimensional counterpart, whereas the /specific displacement-area,/ a measure of stream-tube dilation, increases more rapidly with mach number than the comparable two-dimensional parameter .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1251, 'viscous flow past a quarter infinite plate .', 'stewartson,k.', 'j. ae. scs. 1961, 1.', 'viscous flow past a quarter infinite plate . a study is made of the motion of an incompressible viscous fluid past a quarter-infinite plate, whose leading edge is perpendicular to and whose side edge is parallel to the undisturbed direction of the stream . it is assumed that the kinematic viscosity is small . the first approximation is taken to be the undisturbed motion, and successive approximations are obtained by iferation . the second approximation is the blasius shear layer necessary to satisfy the boundary conditions on the plate . in turn, this layer leads to a velocity component normal to the plate which needs a potential solution, in which the velocities are 0, to match with the conditions at infinity . further, the match at the edge of the blasius shear layer must be completed to 0 by introducing a secondary shear layer . the regions near the leading and side edges are considered separately,. in particular, the neighborhood of the side edge needs special care, because the determination of the chief terms is complicated by the presence of powers of log . in particular it is shown that the effect of the edge is to change the skin-friction coefficient by a factor'), (1252, 'on the approach to chemical and vibrational equilibrium behind a strong normal shock wave .', 'dorrance,w.h.', 'j. ae. scs. 161, 43.', 'on the approach to chemical and vibrational equilibrium behind a strong normal shock wave . the concurrent approach to chemical and vibrational equilibrium of a pure diatomic gas passing through a strong normal shock wave is investigated . it is demonstrated that the equilibrium degree of dissociation behind the shock front, and hence the density, for the case where the vibrational degrees of freedom are frozen out can exceed the degree of dissociation, and hence the density, for the case where all degrees of freedom are in equilibrium . thus the necessary condition for a maximum of the density between the shock front and the position of full equilibrium flow downstream of the shock front is established . the sufficient condition that such a maximum be observable is shown to be that the approach to equilibrium of the vibrational degrees of freedom (or any other internal degrees of freedom) must lag the approach to dissociation equilibrium by a significant amount,. that is, there must be at least an order of magnitude difference in the respective relaxation times before such a maximum might be observed . an example calculation for a mach 13 strong shock wave in oxygen illustrates the appearance of such a maximum of the density and its dependency upon the relative values of the vibration and dissociation relaxation times .'), (1253, 'hypersonic viscous flow near the stagnation point in the presence of magnetic field .', 'wu,ching-sheng.', 'j. ae. scs. 1960, 882.', 'hypersonic viscous flow near the stagnation point in the presence of magnetic field . the present study investigates the hypersonic viscous flow past blunt-nosed bodies with hydromagnetic interaction . local-similarity solutions of flow field and temperature distribution are near the stagnation-point region . the discussions may be grouped into two parts .. the two-dimensional problem (circular cylinder) and axisymmetric problem (sphere) . numerical computations have been carried out for the sphere problem for the /viscous-layer regime,/ with various magnetic field strengths and electrical conductivities .'), (1254, 'combustion in the boundary layer on a porous surface .', 'eschenroeder,a.q.', 'j. ae. scs. 1960, 901.', 'combustion in the boundary layer on a porous surface . the position of the diffusion flame in a boundary layer with uniform mixture injection from a porous wall parallel to a uniform air stream is determined under the conditions of laminar, steady flow with zero streamwise pressure gradient . under the assumption of fast forward reaction rate, solutions of the boundary layer forms of the conservation laws of acrothermochemistry are obtained leading to a formula for the downstream velocity at the flame in terms of composition and flow variables . the rates of change of conditions at the wall in the streamwise direction are assumed to be relatively small . methods of treating complex reaction systems are described, and generalized form of the reynolds analogy is developed .'), (1255, 'the flow about a charged body moving in the lower atmosphere .', 'hunziker,r.r.', 'j. ae. scs. 1960.', 'the flow about a charged body moving in the lower atmosphere . the flow about an electrically charged body traveling at high speeds through the lower ionosphere is analyzed . a simple gas model composed of electrons, ions, and neutral particles is used and the hydrodynamic description given is based on maxwell''s transfer equations for a mixture . the conditions under which local statistical equilibrium can be assumed are discussed, and different approaches to determine the gasdynamic force in the subsonic, supersonic, and hypersonic cases are indicated . the reciprocal action of the electric field of the flow on the body is also analyzed and a formula for the resultant electric force is given . the total force on the body is equal to the sum of the gasdynamic force and the electric force . the negative potential acquired by a plane body is also calculated . finally, the lack of validity of debye''s linearization in this case and the solution of the exterior nonlinear problem which characterize the electric potential and the electron distribution are discussed .'), (1256, 'fluctuating lift and drag acting on a cylinder in a flow at supercritical reynolds numbers .', 'fung,y.c.', 'j. ae. scs.1960, 801.', 'fluctuating lift and drag acting on a cylinder in a flow at supercritical reynolds numbers . the fluctuating lift and drag acting on a circular cylinder in a flow of an incompressible fluid at large reynolds numbers were measured . data on the root-mean-square values of the lift and drag coefficients, the extreme values of these coefficients, and their power spectra at various reynolds numbers are presented .'), (1257, 'an optical boundary-layer probe .', 'daiber,j.w.', 'j. ae. scs. 1960.', 'an optical boundary-layer probe . the applicability of the schlieren-photomultiplier technique to obtain quantitative density measurements in the laminar boundary layer induced by a traveling shock wave in a shock tube is investigated . tests were conducted at a mach number of 1.58 so that the data could be compared with the exact theoretical solution tabulated by mirels . the data obtained are in good agreement with the theory if the distance of the light beam above the floor of the shock tube is adjusted to fit the theoretical curve,. this would not be necessary if a larger shock tube were used . values of the transition reynolds number were also determined which are slightly less than those found by martin using an interferometer . it is shown that this technique is sensitive enough to detect changes in density that are only 0.000 per cent of atmospheric density .'), (1258, 'heat transfer, recovery factor and pressure distributions around a circular cylinder normal to a supersonic rarefied air stream .', 'tewfik,o.k. and giedt,w.h.', 'j. ae. scs. 1960, 721.', 'heat transfer, recovery factor and pressure distributions around a circular cylinder normal to a supersonic rarefied air stream . measurements of the heat transfer, recovery factor, and pressure distributions around a circular cylinder normal to a supersonic rarefied-air stream (total temperature 300 k.) are described for the mach number range of 1.3 to 5.7, the reynolds number range of 37 to 4,100 and at cylinder wall average temperature levels of 90 k. and 210 k . study of the results yielded .. (1) a correlation equation for the stagnation-point nusselt number as a function of the reynolds number just after the normal part of the detached bow shock wave,. and (2) fourier series expressions for the heat-transfer coefficient and pressure coefficient distributions in terms of the stagnation point values . in comparing these measurements with predictions based on recent analytical studies, exceptionally good agreement for the heat-transfer coefficient distribution was obtained with lees'' theory . in the mach number range of 3.55 to 5.73 the pressure decreased less rapidly with distance from the stagnation point than predicted by the modified newtonian theory .'), (1259, 'second-order theory for unsteady supersonic flow past slender pointed bodies of revolution .', 'revell,j.d.', 'j. ae. scs.1960, 730.', 'second-order theory for unsteady supersonic flow past slender pointed bodies of revolution . an analysis is made of the second-order effects of thickness on the unsteady aerodynamic forces on a slender pointed body of revolution in supersonic flow . the theory is restricted to harmonic oscillations for small angles of attack . the solution is obtained by approximating the nonlinear terms in the second-order potential equation by their first-order values and solving the resulting inhomogeneous partial differential equation, subject to more refined boundary conditions . the pressure equation is likewise refined and integrated to give the second-order corrections to lift and pitching moment coefficients . the analysis can be considered as an extension of the second-order, slender body theory of lighthill to the case of unsteady flow . the results indicate appreciable reductions in unsteady lift and damping moment coefficients when applied to slender cones . the present theory is estimated to be reliable provided that is less than 0.7 .'), (1260, 'on the response of the laminar boundary layer to small fluctuations of the free-stream velocity .', 'rott,n. and rosenzweig,m.l.', 'j. ae. scs. 1960, 741.', 'on the response of the laminar boundary layer to small fluctuations of the free-stream velocity . the linearized treatment of small time-dependent disturbances of a laminar boundary layer, initiated by lighthill, is extended in several ways . in particular, the high-frequency expansion is continued beyond the leading (stokes) term . several interesting questions of /joining/ occur, which are discussed but left unresolved . in addition, a practical method for obtaining the response to the laminar boundary layer to an impulsive change in velocity is presented . the methods are applied to the case in which the basic steady flow belongs to the falkner and skan family of similarity solutions .'), (1261, 'a method of calculating velocity distribution for turbulent boundary layers in adverse pressure distributions .', 'uram,e.m.', 'j. ae. scs. 1960, 659.', 'a method of calculating velocity distribution for turbulent boundary layers in adverse pressure distributions . a new method of calculating the behavior of turbulent boundary layers in adverse pressure distributions is developed which permits direct determination of the velocity profile rather than the gross integral parameters normally used to infer the general character of the boundary layer . the method offers the simplicity of algebraic equations coupled with the use of charts rather than the laborious simultaneous solution of coupled differential equations required by existing methods . the method also affords, for the first time, a means of determining the total boundary-layer thickness, thus allowing calculation of the absolute as well as the nondimensional velocity distribution . the velocity profile is considered to be composed of two regions--an inner region which is described by the law of the wall and an outer region which is described by a function depicting the deviation from that law . the deviation function involves two parameters which are uniquely dependent upon the skin-friction coefficient and a third parameter which, for practical purposes, can be considered a constant . since the entire velocity distribution was found to be almost uniquely dependent upon the local skin friction, serious doubt is cast upon the generally accepted /history concept/ which considers the outer region of the boundary layer to be dependent on integrated upstream conditions . agreement between experimental velocity distributions and those calculated by the method presented here is generally very good . the analysis and calculation procedures which are presented are applicable to two-dimensional, pseudo-two- dimensional, and axisymmetric conical flows .'), (1262, 'an extension of the linearized characteristics method for calculating the supersonic flow around elliptic cones .', 'martellucci,a.', 'j. ae. scs. 1960, 667.', 'an extension of the linearized characteristics method for calculating the supersonic flow around elliptic cones . the method of linearized characteristics as applied by ferri to the flow about elliptic cones can be used to determine the surface pressure distribution, even when only linear terms are kept in the boundary conditions, provided an area rule requirement is satisfied . in addition, the method can be applied for angles of attack provided the elliptic body geometry is specified in a manner that does not distort the cross section . the surface pressure distribution obtained by this modified method is in reasonable agreement with experiment over the range of mach numbers and semidiameter ratios considered . experimental results for several conical bodies are presented .'), (1263, 'turbulent heat transfer through a highly cooled, partially dissociated boundary layer .', 'rose,p.h., probstein,r.f. and adams,m.c.', 'j. ae. scs. 1958, 751.', 'turbulent heat transfer through a highly cooled, partially dissociated boundary layer . the problem of heat transfer from high-temperature air through a turbulent boundary layer to a cold surface is considered both analytically and experimentally . heat-transfer data obtained in shock tubes are presented and correlated by a semiempirical theory which includes the effect of atomic diffusion . the distinguishing characteristics of turbulent boundary layers with dissociation and large cooling are considered . it is shown that the equations governing such flow, after certain approximations, can be represented in a form similar to the classical equations for a turbulent boundary layer . an approximate theory is proposed for turbulent heat transfer for a highly cooled boundary layer on portions of the body where the pressure gradient is negligible in the case of blunted bodies of revolution in high-speed flight . experimental results obtained on the cylindrical portion of a hemisphere-cylinder model are presented for conditions simulating flight speeds to 21,350 ft. sec., where up to 30 per cent of the molecules are dissociated . reynolds numbers of 2.5 x 10, based on local fluid properties external to the boundary layer, were achieved . the larger values of reynolds number and flight speed were not obtained simultaneously, due to structural limitations of the shock tubes,. however, the experiments were conducted in such a way that the important effects of each could be determined . in the experiments the mach number external to the boundary layer varied between 1.7 and 2.2 . the corresponding mach number for blunted nonslender bodies in flight would have a maximum value between 2.5 and 4,. however, it is shown that these differences in mach number are not important for such bodies .'), (1264, 'boundary layer transition and heat transfer in shock tubes .', 'hartunian,r.a., russo,a.l. and marrone,p.v.', 'j. ae. scs. 1960, 587.', 'boundary layer transition and heat transfer in shock tubes . an experimental study is made of the wall boundary layer in a shock tube operated over a wide range of shock mach numbers and pressure levels in air, including those for which real-gas effects exist . transition distances are determined and correlated in terms of the transition reynolds number based on a characteristic length for this boundary layer . data from independent shock-tube studies are also included in this correlation . the results indicate a weak dependence of transition reynolds number on shock strength up to moderate values of shock mach number, followed by a larger stabilizing tendency . comparison of these data with transition data obtained in the same manner in argon indicate that the increased cooling rates are largely responsible for the stabilization . a dependence of transition reynolds number on the unit reynolds number is found at the lower shock strengths . specifically, higher transition reynolds numbers are achieved at larger unit reynolds numbers . the phenomenon of transition reversal does not appear within the range of the experiments reported . laminar- and turbulent-flow heat-transfer rates to the walls of the shock tube are determined experimentally . the results of the heat-transfer measurements substantiate existing theories in both the laminar- and turbulent-flow regimes .'), (1265, 'on the thrust hypothesis for the jet flap including jet-mixing effects .', 'yen,k.t.', 'j. ae. scs. 1960, 607.', 'on the thrust hypothesis for the jet flap including jet-mixing effects . this paper is concerned with the thrust generated by a jet flap . it is shown that a /linear/ thrust hypothesis can be obtained, provided linearized potential flow is assumed . in fact, the linearized problem of a jet-flap system is found to be the linear combination of a lift problem and a thrust problem . the lift problem gives all the lift generated, but it is of interest to note that the thrust problem would yield all the thrust developed by the jet flap within the limitation of the linearized theory . the mixing of the jet flap with the surrounding fluid is analyzed by the momentum-integral method . the analysis substantiates stratford''s suggestion for obtaining an increase of thrust by causing the jet to mix with the main stream in a region of high suction . finally, some approximate formulas, relating the thrust and the jet angle, are derived . the drag of the airfoil section and other viscous effects are, however, not considered .'), (1266, 'minimum wing wave drag with volume constraint .', 'strand,t.', 'j. ae. scs. 1960, 615.', 'minimum wing wave drag with volume constraint . a numerical method is developed for calculating the minimum thickness drag for a given wing planform and volume using linearized supersonic flow theory . the corresponding optimum volume distribution is also determined . the results show that considerable drag reduction is possible by improved volume distribution .'), (1267, 'on supersonic flow past thick airfoils .', 'kogan,a.', 'j. ae. scs. 1960, 504.', 'on supersonic flow past thick airfoils . the inviscid rotational supersonic flow behind the shock wave attached to the sharp leading edge of an airfoil is studied by a transformation of coordinates which introduces the crocco stream function as an independent variable . using expansions in the power series of, an iterative process is developed for the determination of pressure distribution along the airfoil surface .'), (1268, 'stable combustion of a high-velocity gas in a heated boundary layer .', 'turcotte,d.l.', 'j. ae. scs.1960.', 'stable combustion of a high-velocity gas in a heated boundary layer . it is generally recognized that stable combustion processes in heated boundary layers may be achieved by either of two conceptual mechanisms . in one mechanism it is pictured that the heat transfer to the wall quenches the propagating flame at a certain distance from the surface . the equality between the flow velocity and the normal burning velocity at this quenching distance determines the position of the propagating flame . in the second mechanism it is conceived that the hot surface provides a continuous source of ignition in much the same manner that the hot recirculation zone of a bluff body flame holder provides continuous ignition to the gas flowing around it . in this case it is the characteristic time during which the gas must be heated that determines the position of the flame . all experimental work reported to date has been concerned with conditions where the first picture has apparently been applicable . in the present paper, experiment and analysis are given that show under what conditions the continuous ignition mechanism provides the appropriate model and also how the two models are related . to differentiate the two mechanisms an experiment was set up to study flame stabilization in high-velocity boundary layers over a wall heated in the form of a step function . with a turbulent boundary layer and a wall temperature above 1,700f., the characteristic time was found to be a systematic and reproducible variable . these observations led to the conclusion that a continuous ignition mechanism governs stabilization in heated turbulent boundary layers . a rational explanation is made for the transition from the low-speed mechanism known to be applicable in unheated turbulent boundary layers and heated laminar boundary layers to the ignition mechanism applicable in heated turbulent boundary layers . as a further verification of the continuous ignition mechanism an apparent ignition energy was found . the logarithm of the heat added at the lower stability limit was found to be a linear function of the reciprocal of the limiting wall temperature . the activation energy derived from this arrhenius type of relation agreed reasonably well with the estimated value for the fuel used .'), (1269, 'a study of supersonic combustion .', 'gross,r.a. and chinitz,w.', 'j. ae. scs. 1960, 517.', 'a study of supersonic combustion . steady, stable, plain, and oblique detonation waves were created in a high-temperature, steady flow supersonic tunnel . ignition conditions and properties across the wave were measured . the local-wave fluid-dynamic properties agree well with detonation theory . experimental data are presented in detail and compared with other studies and theory . experimental behavior of these detonations and their possible utility are discussed .'), (1270, 'supersonic inlet dynamics .', 'fraiser,h.r.', 'j. ae. scs. 1960, 429.', 'supersonic inlet dynamics . an approximation of the differential equation for compressible duct flow is presented . the equation is linear and of the second order . the duct transfer function and response characteristics are obtained by applying small-perturbation theory to the differential equation . the resulting equations describe duct natural frequency as a function of duct areas and volumes, and damping ratio as a function of the slope of the steady-state mass flow, pressure-recovery curve . the calcualted response agrees, to a first approximation, with measured response as obtained from tests of a fixed-geometry, sugar-scoop inlet model with hypass for matching airflows . testing was done in the 10 x 10 and 8 x 6 ft. supersonic tunnels at nasa lewis flight propulsion laboratory . further agreement was obtained during flight tests of the f8u-3 airplane .'), (1271, 'theory of supersonic propeller aerodynamics .', 'ordway,d.e. and hale,r.w.', 'j. ae. scs. 1960, 437.', 'theory of supersonic propeller aerodynamics . a supersonic propeller with blades attached to an infinite cylinder as a hub is studied . the forward speed may be subsonic, but the relative speed at each section is supersonic . the lightly loaded blades are represented by a surface distribution of appropriate /modified/ sources in a fashion similar to ordinary supersonic thin-wing theory . these sources are found by approximating the exact potential for a constant-strength compressible source traveling along a helical path . the usual relationship between the source strength and boundary condition is found,. and subsequently the source distribution is given, to the appropriate order, in terms of the blade geometry . tip effects are considered by extending the theory of evvard and krasilshchikova . the present investigation, however, is restricted to those planforms for which no vortex sheet appears off the tip . for points in the tip region, the potential is obtained through the appropriate distribution of /modified/ sources in the upwash region off the tip . by transforming to a curvilinear, nonorthogonal coordinate system coincident with the modified mach lines described by the infinities of the potential, an integral equation for the required source distribution in the upwash region is derived . without having to solve this equation, it is shown that the potential for a point in the tip region can be obtained in terms of an integration of known source distributions over the blade surface only . the case of a twisted flat plate of particular planform is treated, and a sample calculation is made of the pressure distribution at selected radial positions within the noncommunicating portion of the blade, as well as over the entire tip region . though this analysis is carried out explicitly for the supersonic propeller, it could also be extended to calculate various rotary derivatives for highspeed flight vehicles .'), (1272, 'oscillatory aerodynamic coefficients for a unified supersonic hypersonic strip theory .', 'rodden,w.p. and revell,j.d.', 'j. ae. scs. 1960, 451.', 'oscillatory aerodynamic coefficients for a unified supersonic hypersonic strip theory . this investigation presents a derivation of the oscillatory aerodynamic coefficients for wings with supersonic leading edges from the second-order, nonlinear, unsteady, supersonic flow theory of van dyke . the theory is considered applicable throughout the supersonic-hypersonic regime at mach numbers normal to the leading edge and reduced frequencies for which . the coefficients are modified for sweep, and a finite-span correction is suggested to increase the accuracy of strip-theory flutter analyses . the limiting values of the coefficients in steady flow are also discussed .'), (1273, 'magnetohydrodynamic effects on the formation of couette, flow .', 'tao,l.n.', 'j. ae. scs. 1960, 334.', 'magnetohydrodynamic effects on the formation of couette, flow . this paper is concerned with the problem of the formation of couette flow--i.e., the problem of how the velocity profile varies with the time tending asymptotically to that of the steady flow of an electrically conducting viscous fluid in the presence of a magnetic field . the governing equations and boundary conditions are established and discussed . the cases of both vanishing and nonvanishing mean induced electric field strengths are solved in terms of complimentary error functions as well as some elementary functions . it is shown that the solutions are reducible to that of the steady case as the time approaches infinity, and to that of the nonmagnetic field as the hartmann number becomes zero . some numerical calculations are given . the results indicate that in the presence of a magnetic field the flow rate is reduced depending on the magnitude of the hartmann number, and that the magnetic field /assists/ the flow to reach its steady condition .'), (1274, 'real gas effects in flow over blunt bodies at hypersonic speeds .', 'nagamatsu, h.t., geiger, r.e. and sheer, r.e.', 'j. aero. sc. april, 1960. p241 .', 'real gas effects in flow over blunt bodies at hypersonic speeds . a hypersonic shock tunnel has been developed to investigate the aerodynamic characteristics of flow over bodies at conditions comparable to those encountered by ballistic missiles and satellites re-entering the atmosphere . some results for a shock velocity of over 50,000 ft. per sec. in the shock tube portion of the facility are presented . static pressure investigations were made in the nozzle to determine the flow condition and the expansion process . the results of the investigation of representative blunt bodies at hypersonic mach numbers and nozzle stagnation temperatures up to approximately 6,000degreek. are presented . these include body pressure distributions, shock-wave shapes, detachment distances, and photographs of the luminous gas region in the shock layer . it is seen that the shock detachment distance is smaller at higher stagnation temperatures due to the real gas effects . for the hemisphere the pressure distribution was less than that predicted by the modified newtonian theory for all stagnation temperatures . for a 50degree cone-hemisphere the pressure distribution and the shock detachment distance were appreciably affected by the real gas effects . the observed shock-wave shape and the approximate boundary layer on a flat plate are compared with the analytical prediction . some preliminary results for the detached shock wave produced by a blunt two- dimensional body in a low density flow at a mach number of 19.6 are presented'), (1275, 'flow about an unsteadily rotating disc .', 'sparrow,e.m. and gregg,j.l.', 'j. ae. scs.1960,252.', 'flow about an unsteadily rotating disc . an analysis is made of the unsteady laminar flow about a rotating disc whose angular velocity may vary with time . the deviation of the actual instantaneous state of the flow from the quasi-steady state (instantaneous steady state) is determined . from this, a simplified criterion, is derived to define the conditions under which the flow can be considered as quasi-steady for the purposes of shear stress and torque computations . since a turbulent flow responds more rapidly than a laminar flow, the quasi-steady criterion found here should also serve for the turbulent situation .'), (1276, 'a three-dimensional linearized analysis of the forces exerted on a rigid wing by a shock wave .', 'ehlers,f.e. and shoemaker,e.m.', 'j. ae. scs. 1960, 257.', 'a three-dimensional linearized analysis of the forces exerted on a rigid wing by a shock wave . the pressure distribution on a moving flat plate induced by an acoustic shock front striking the edge of the plate obliquely has been found in terms of the two-dimensional solution of the authors'), (1277, 'a study of vortex cancellation .', 'schaffer,a.', 'j. ae. scs. 1960, 193.', 'a study of vortex cancellation . the cancellation of a vortex by means of another concentric vortex of equal strength but opposite spin is investigated . when such a cancellation occurs, there is a recovery of static pressure . the vortices are generated by means of two three-dimensional airfoils cantilevered from the duct wall, one being situated in the wake of the other . the airfoils have opposite effective angles of attack and therefore have trailing vortices of opposite spin, as required . it is demonstrated experimentally that there exists an optimum angle of attack for the second airfoil which cancels the vortex from the first airfoil and restores uniform flow downstream of the two airfoils . a theoretical solution of this optimum angle of attack is presented, and it is found to depend upon the angle of attack of the first airfoil and upon the geometrical properties of the wings . the pressure recovery accompanying the vortex cancellation is also studied . theoretical considerations, based on the model of a vortex filament in the center of a circular tube show that a maximum of 62 per cent of the static pressure drop across the first airfoil can be recovered . this maximum is imposed, irrespective of skin friction and separation losses, by the irreversibility associated with establishing a vortex field . experimental pressure recoveries of 50 per cent are realized . perhaps the primary value of the present study is the opportunity it provides to verify certain of the fundamental concepts of fluid mechanics which are brought into play when the trailing vortex system of a lifting wing is cancelled by a second wing .'), (1278, 'transition in a separated laminar boundary layer .', 'lochtenberg,b.h.', 'j.ae.scs.1960, 92.', 'transition in a separated laminar boundary layer . transition to turbulence was studied in a separated laminar boundary layer on a flat plate 24 in. long and thick . steps with a height of to were provided at a distance of 4 to transition was observed through a hot-wire anemometer . the author concludes that transition was always initiated by tollmien-schlichting waves . two types of transition were observed . in one type, bursts suddenly appeared in the wavy flow . the other type consists of amplification, distortion, and breaking up of the waves . which type of transition occurs depends on the value of the following parameter .. boundary-layer displacement thickness times step height times free-stream velocity squared divided by kinematic velocity squared . the burst type has been observed for values of this parameter larger than 4.2 x 10 . the separated laminar boundary layer becomes unstable and develops waves when the critical reynolds number based on boundary-layer displacement thickness at the step location exceeds a value of 350 . some conclusions on the development of separation bubbles on air foils are drawn from the present studies .'), (1279, 'sublimation in a hypersonic environment .', 'scala,s.m.', 'j. ae. scs. 1960, 1.', 'sublimation in a hypersonic environment . a priori knowledge of the response of materials subjected to a severe aerothermal environment is essential in the space age . the successful design of space and re-entry vehicles demands that the fundamental problem of the interaction between a material and dissociated air be properly formulated and solved . in this paper, the problem of sublimation in a hypersonic environment is considered . in this study of hypersonic ablation, the pertinent conservation equations are derived and the simultaneous processes of diffusion, convection, and thermal exchange are analyzed for the vaporization of a refractory material which is subjected to the environmental conditions encountered during hypersonic reentry . for simplicity, only the forward stagnation point of an axially symmetric body is treated . it is shown that the quantity, called the effective heat of vaporization, which includes all heat absorbing or heat blocking effects, is an increasing function of flight speed, independent of body size, except where nonequilibrium vaporization effects or radiative effects appear .'), (1280, 'wings with minimum drag due to lift in supersonic flow .', 'ginzel,i. and multhopp,h.', 'j. ae. scs. 1960, 13.', 'wings with minimum drag due to lift in supersonic flow . it has been shown by r. t. jones that, in order to produce minimum drag, the given lift must be distributed over the wing surface in such a way that the sum of the downwash induced by this distribution and the downwash induced in reversed flow is constant over the wing surface . this combined downwash can be expressed by an integral which contains the load as a function of the spanwise and chordwise coordinate . the problem of finding the appropriate load distribution is thus reduced to the problem of finding the solution of a rather cumbersome integral equation . the severe spanwise singularity of the kernel function is handled most easily, as in corresponding subsonic problems, by an approximate integration over interpolation polynomials . the chordwise load distribution is represented by a limited series development in legendre polynomials . the sigularity of the kernel function along the mach lines through any pivotal point can be avoided by a similar legendre development of the combined induced downwash which is constant . the integral equation is thus converted into a system of linear equations for the unknown coefficients of the legendre functions of the load distribution at a limited number of spanwise stations . practical calculations are carried out on an electronic computer . the solutions yield the optimum load distribution and the local incidence (twist, camber, etc.) necessary to realize this distribution . for many wing plan forms, considerable gains over a plane wing appear possible .'), (1281, 'turbulent heat transfer on blunt-nosed bodies in two-dimensional and general three-dimensional hypersonic flow .', 'vaglio-laurin,r.', 'j. ae. scs. 1960, 27.', 'turbulent heat transfer on blunt-nosed bodies in two-dimensional and general three-dimensional hypersonic flow . recent results obtained for three-dimensional laminar boundary layers are extended to the turbulent case . it is shown that in the presence of highly cooled surfaces and of moderate mach numbers of the outer stream, the crossflow and the pertaining reynolds stresses in a general three-dimensional turbulent boundary layer are negligible even for large transverse pressure gradients . a correlation due to mager between two- dimensional compressible and incompressible turbulent boundary layers is extended to the problem in question . from a study of the transformation and of its implications, a rapid method for the analysis of the boundary-layer flow under the subject conditions is established . in the absence of general three- dimensional data, a comparison with experiments and with the predictions of other known analyses is carried out for several axisymmetric configurations,. the results of the method presented here exhibit good agreement with the data . the range of validity of the cold wall approximation for general three-dimensional problems is estimated qualitatively on the basis of recent measurements in laminar flow, the argument being that, for either zero or favorable streamwise pressure gradients, smaller three-dimensional effects are to be expected in a turbulent boundary layer, as compared to a laminar layer .'), (1282, 'compressible flat-plate boundary-layer flow with an applied magnetic field .', 'bush,w.b.', 'j. ae. scs. 1960, 49.', 'compressible flat-plate boundary-layer flow with an applied magnetic field . the laminar boundary-layer equations are formulated and solved for a flat plate in high-speed compressible air flow where equilibrium dissociation and ionization are assumed and where there is an applied magnetic field having its component normal to the plate proportional to . the skin-friction and heat-transfer characteristics are determined for free-stream velocities of up to 17,500 meters sec. and magnetic fields of up to about the results show that the skin friction and heat transfer at a given free-stream velocity decrease with increasing magnetic field strength, and the percentage reduction is constant along the length of the plate . they also exhibit the same hysteresis behavior as was first found in the case of magnetoaerodynamic couette flow,. however, for the flat plate the hysteresis effect disappears at a higher mach number . furthermore, it was found that the reduction in heat transfer with increasing field strength is opposite in behavior from that for couette flow .'), (1283, 'on shearing flow between porous coaxial cylinders .', 'dunwoody,n.t.', 'j. ae. scs. 29, 1962, 494.', 'on shearing flow between porous coaxial cylinders . the flow between concentric porous cylinders in relative axial motion with a pressure gradient is considered . the analysis is restricted by the assumption that the velocity distribution is a function of the radial coordinate only so that there is no net injection or withdrawal of fluid at any station . this assumption reduces the problem to a soluble system of ordinary differential equations . an associated heat-transfer problem is also discussed briefly .'), (1284, 'the transition to tubulence in a boundary layer on a blunt cone in supersonic flow .', 'rogers,r.h.', 'j. ae. scs. 29, 1962, 501.', 'the transition to tubulence in a boundary layer on a blunt cone in supersonic flow . experiments were made with a series of cones, each having an included angle of 15 degrees, and having different tip radii (sharp to blunt nose) . the cones were tested in streams, undisturbed mach numbers of 3.12 and 3.81, and the position of transition to turbulence was observed with a shadowgraph technique . for each mach number the distance to transition (distance downstream from the tip of the cone) increased with increase in tip radius, reached a maximum at a certain tip radius, and then decreased with increase in tip radius . a study indicates that a reynolds number based on the momentum thickness, instead of the length from tip to transition, is a more appropriate parameter for correlating the data . the research scientist active in this field probably would derive the most benefit from the original paper .'), (1285, 'experiments at hypersonic speeds on circular cones at incidence .', 'peckham,d.h.', 'rae tn.aero.2863, 1963.', 'experiments at hypersonic speeds on circular cones at incidence . pressure distribution measurements on five circular cones with total apex-angles ranging from 25 to 45 degrees are described . the tests covered a range of angles of incidence from 0 to and 8.60 . the extent to which various analytical and empirical theories predict the measured pressures is assessed .'), (1286, 'equilibrium real-gas performance charts for a shypersonic shock-tube wind-tunnel employing nitrogen .', 'bernstein,l.', 'arc cp.633, 1961.', 'equilibrium real-gas performance charts for a shypersonic shock-tube wind-tunnel employing nitrogen . charts are presented covering a wide range of reflected-shock wind-tunnel operating conditions, using nitrogen as the working gas . a statistical-mechanical model of the gas is assumed which takes account of molecular vibration, electronic excitation and dissociation . the gas is assumed to be constantly in equilibrium that is, the reaction rates are taken to be infinitely fast . the equations of motion are solved with the aid of a digital computer, previously reported results for the state of the shock-processed gas in the shock-tube being used .'), (1287, 'progress report on an experiment on the effect of surface flexibility on the stability of laminar flow .', 'gregory,n. and love,e.m.', 'arc cp.602, 1961.', 'progress report on an experiment on the effect of surface flexibility on the stability of laminar flow . this paper describes the flexible surfaces whose properties have been examined and which have been tested on an aerofoil in a wind tunnel . the experiment has been rather inconclusive as no drag reductions have been found in turbulent flow, whilst the only rearward movements of transition occurred in conditions where the alteration has been inhibited by the onset of laminar separation . the limitations of the experiment are discussed carefully in order to clarify the next steps which are to be taken with more flexible surfaces with less damping .'), (1288, 'analysis of the fluid mechanics of secondary injection for thrust vector control .', 'broadwell,j.e.', 'aiaa jnl. 1963, 1067.', 'analysis of the fluid mechanics of secondary injection for thrust vector control . an analysis is made of the interaction of an injected gas or liquid with a supersonic stream, and the force induced on an adjacent wall is predicted . the study deals only with the freestream-injectant interaction,. the modifications to the flow introduced by the boundary layer are not considered . in the case of liquids, it is shown that the momentum deficit of the injectant relative to the freestream may play a larger part in producing the side force than the volume generation by vaporization and reaction . the analytical results are compared with those obtained from experiments in a wind tunnel and in nozzles .'), (1289, 'numerical technique to lifting surface theory for calculation of unsteady aerodynamic forces due to continuous sinusoidal gusts on several wing planforms at sobsonic speeds .', 'murrow,h.n., pratt,k.g. and drischler,j.a.', 'nasa tn.d1501, 1963.', 'numerical technique to lifting surface theory for calculation of unsteady aerodynamic forces due to continuous sinusoidal gusts on several wing planforms at sobsonic speeds . a numerical lifting-surface method has been used to calculate direct gust forces and moments on wings of several planforms . the gust velocities are continuous and vary sinusoidally in the stream direction and are also uniform across the wing span . the procedure has the advantage of rapid machine calculation and includes the effects of wing planform, nonsteady subsonic flow, and induced flow effects . the method provides for calculation of gust forces on a basis consistent with that for the calculation of forces due to motion and deformation . the results include the in-phase and quadrature components of the following quantities .. (a) spanwise distribution of section lift coefficient, (b) total lift coefficient, and (c) total pitching-moment coefficient . in addition, generalized gust forces on approximate fundamental cantilever bending modes ( parabolic) are also included . results have been obtained for 60 and 75 delta wings, ratio 11.60, and an unswept wing of aspect ratio 6.00 . conditions for which calculations were made include two mach numbers reduced-frequency range of 0 to 1.0 . the direct gust forces and moments are in forms suitable to be inserted in equations of motion used in the calculation of the dynamic responses of flexible lifting vehicles to random turbulence and to be compared with results from other methods .'), (1290, 'measured and calculated subsonic and transonic flutter characteristics of a 45 sweptback wing planform in air and in freon-12 in the langley transonic dynamics tunnel .', 'yates,e.c., land,n.s. and foughner,j.t.', 'nasa tn.d1616, 1963.', 'measured and calculated subsonic and transonic flutter characteristics of a 45 sweptback wing planform in air and in freon-12 in the langley transonic dynamics tunnel . in order to investigate the reliability of flutter data measured in the langley transonic dynamics tunnel, an experimental and theoretical subsonic and transonic flutter study has been conducted in air and in freon-12 in this facility . the wing planform employed had an aspect ratio of 4.0, a taper ratio of 0.6, and 45 of quarter-chord sweepback . a sting-mounted full-span model was tested in addition to three sizes of wall-mounted semispan models . a wide range of mass ratio was covered by the tests in air and by flutter calculations made by the modified strip-analysis method of naca research memorandum l57l10 . a limited amount of data was obtained in freon-12 . results of the tests in air and in freon-12 are in good agreement with the flutter calculations at all mach numbers . the test data compare favorably with previously published transonic flutter data for the same wing planform . the results indicate that flutter characteristics obtained in freon-12 may be interpreted directly as equivalent flutter data in air at the same mass ratio and mach number .'), (1291, 'atmosphere entries with spacecraft lift-drag ratios modulated to limit decelerations .', 'levy,l.l.', 'nasa tn.d1427, 1962.', 'atmosphere entries with spacecraft lift-drag ratios modulated to limit decelerations . an analysis has been made of atmosphere entries for which the spacecraft lift-drag ratios were modulated to limit the maximum deceleration . the parts of the drag polars used during modulation were from maximum lift coefficient to minimum drag coefficient . five drag polars of different shapes were assumed for the spacecraft . the entries covered wide ranges of initial velocity, initial flight-path angle, initial and maximum lift-drag ratio . two-dimensional trajectory calculations were made for a nonrotating, spherical earth with an exponential atmosphere . the results of the analysis indicate for four of the five drag polars that, relative to the maximum deceleration of an unmodulated entry at maximum lift-drag ratio, the greatest reduction in maximum deceleration obtainable by modulation depends upon a single parameter . this parameter is the ratio of the value of the aerodynamic resultant-force coefficient at minimum drag coefficient to the value at maximum lift coefficient . thus, the reduction in maximum deceleration is independent of initial velocity, initial flight-path angle, initial maximum lift-drag ratio, and the shape of the drag polar . for the fifth drag polar, the reduction in maximum deceleration was found to depend upon the maximum lift-drag ratio . also, relative to the depth of a given deceleration-limited corridor, the greatest increase in corridor depth obtainable by modulation (for four of the five drag polars) depends upon the same ratio of aerodynamic resultant-force coefficients . the fractional increase in corridor depth can be expressed as an empirically determined analytical function of this ratio .'), (1292, 'effect of jet pluming on the static stability of cone-cylinder-flare configurations at a mach number of 9. 65 .', 'hinson,w.f. and falanga,r.a.', 'nasa tn.d1352, 1962.', 'effect of jet pluming on the static stability of cone-cylinder-flare configurations at a mach number of 9. 65 . the effects of jet pluming on normal force and pitching moment of have been measured at a free-stream mach number of 9.65 with reynolds numbers based on model length of 500,000 to 600,000 . geometric variables included nose bluntness, flare half-angle, and nozzle geometry and exit displacement . two test nozzles with design mach numbers of 3.74 and 4.60 were operated with compressed air to simulate the initial jet-boundary shape of a particular solid-propellant rocket motor operating between altitudes of 165,000 and 215,000 feet . the ratio of the jet pressure to free-stream static pressure varied from a jet-off condition to approximately 1,300 for the nozzle with design mach number of 3.74, and from a jet-off condition to approximately 280 for the nozzle with design mach number of 4.60 . the angle-of-attack range was from 0 to approximately 6 . the results indicate that as the jet-pressure ratio was increased the size of the jet plume increased, and as a result the model static stability was decreased . increasing the angle of attack resulted in a reduction in static instability during the jet-on condition . increasing nose bluntness resulted in a more forward movement of the center of pressure when jet-plume interference was not present and a rearward movement in the center of pressure when jet interference was present . increasing the nozzle-area expansion ratio and displacing the nozzle exit downstream of the flare base resulted in a more rearward location of the center of pressure .'), (1293, 'design of stiffened cylinders in axial compression .', 'hedgepeth,j.m.', 'nasa tn.d1510, 1962.', 'design of stiffened cylinders in axial compression . the problem of optimum design of axially compressed cylinders stiffened by rings and stringers is discussed . particular attention is devoted to configurations suitable for large launch vehicles . consideration is given to the analytical techniques for determining strength as well as the procedures for optimization .'), (1294, 'non-linear shallow shell analysis by the matrix force method .', 'lansing,w., jones,i.w. and ratner,p.', 'nasa tn.d1510, 1962, 753.', 'non-linear shallow shell analysis by the matrix force method . the matrix force method of redundant structure analysis is currently being extended by various users to cover a number of non-linear problems . one of these is the non-linear analysis of heated cambered wings, such as might be used in advanced flight vehicles . in this case the approach used by the present authors is equally applicable to shallow shells, the formulation of the strain-displacement and equilibrium relations being a finite element equivalent to that used by marguerre . the solution is obtained by a combined iteration and step by step procedure utilizing a tangent flexibility matrix . divergence in the calculations indicates that the range of stable configurations has been exceeded . cambered plates subjected to several loadings are given as examples,. for one, an exact solution is available for comparison . it is believed that the basic concepts involved in this shallow shell analysis can be extended to apply to other, more general shell instability problems, and that useful solutions to the latter are probably within the capability of present day digital computers .'), (1295, 'recent advances in nonequilibrium dissociating gasdynamics .', 'li,t.y.', 'ars jnl. 1961.', 'recent advances in nonequilibrium dissociating gasdynamics . the purpose of this paper is to review some recent advances in the study of gasdynamic problems including effects of chemical reactions . to provide a background for the study the general concepts shall be outlined briefly . the discussions of the recent developments are restricted to inviscid flow problems only, neglecting viscosity, heat conduction and diffusion . particular attention is directed to recent advances in analyses of nonequilibrium dissociating gas flows . in the hypersonic flight regime, high stagnation enthalpies sufficient to cause dissociation are realized . when the time to reach equilibrium is comparable with the time it takes for a fluid particle to pass through the flow, then there exist regions of the flow field where nonequilibrium states are encountered . a brief survey of both the linear and the nonlinear methods of treatment of these nonequilibrium flows, including some new developments that have not appeared elsewhere, will be presented .'), (1296, 'non-equilibrium expansions of air with coupled chemical reactions .', 'eschenroeder,a.q., boyer,d. and hall,j.g.', 'phys. fluids, 1962.', 'non-equilibrium expansions of air with coupled chemical reactions . analysis and solutions of the streamtube gas dynamics involving coupled chemical rate equations are carried out . results are presented for airflows along the surface of blunt bodies and through hypersonic nozzles . speeds and altitudes corresponding to re-entry were selected to obtain initial conditions for the external flow calculations . conditions appropriate to hypersonic tunnel testing were chosen for the nozzle flow calculations . composition histories are shown for a kinetic mechanism including 6 species and 14 reactions . gas-dynamic effects of nonequilibrium processes qualitatively resemble those reported earlier . however, the freezing process is complicated by the coupling of the nitric oxide shuffle reactions with the dissociation-recombination reactions . in many cases of hypersonic nozzle flows where the energy in nitrogen dissociation is significant, the fast shuffle reactions prevent nitrogen-atom freezing which would otherwise occur if three-body recombination were the only process operating . nitric oxide concentrations undershoot the equilibrium values if the ratio of nitric oxide to oxygen molecule concentrations exceeds unity in the freezing region . this depletion of nitric oxide leads to nitrogen-atom freezing .'), (1297, 'ionization nonequilibrium in expanding flows .', 'eschenroeder,a.q.', 'ars jnl. 1962, 32.', 'ionization nonequilibrium in expanding flows . approximate methods are developed for solving the electron-ion kinetic equations in expanding quasi one-dimensional airflows . results are obtained for inviscid nozzle flows at conditions appropriate to shock tunnel testing and are compared with exact numerical solutions . effects on rf transmission and d-c conductivity are examined . since two-body deionization never fully freezes in the flows considered here, the assumption of sudden freezing gives an upper bound on the residual ionization at large area ratios . the use of an asymptotic form of solution with the freezing criterion provides an improved estimate for such cases . ionization nonequilibrium is also considered for the plasma sheath associated with blunt hypersonic bodies flying at high altitudes . the influence of atomic ions is examined for typical re-entry conditions .'), (1298, 'theory of radiation from luminous shock waves in nitrogen .', 'hammerling,p., teare,j.d. and kivel,b.', 'phys. fluids, 1959, 2, 422.', 'theory of radiation from luminous shock waves in nitrogen . the physical properties behind a normal shock in nitrogen are calculated as a function of time . these include the variation of temperature, composition, ionization, and the intensity of radiation from the n first negative band system . this calculation incorporates a rate equation for the dissociation of nitrogen, the conservation laws, an equation describing vibrational relaxation, and a method of coupling the vibrational relaxation with the dissociation rate . the n radiation is computed assuming excitation of the radiating state by collision with vibrationally excited nitrogen molecules . a particular case is considered for which experimental data are available, and regions sensitive to particular rates are indicated .'), (1299, 'hypersonic viscous shock layer .', 'scala,s.m.', 'ars jnl. 1959, 29, 520.', 'hypersonic viscous shock layer . a decade ago tsien (1) (as well as others) and, more recently, adams and probstein (2) have attempted to define the different regimes of gaseous interactions during high altitude flight . in this note some results are presented which are pertinent to the flight of hypersonic lifting vehicles composed of axially symmetric and two-dimensional elements,. sec. fig. 1 .'), (1300, 'some effects of bluntness on boundary layer transition and heat transfer at supersonic speeds .', 'moeckel,w.e.', 'naca r.1312, 1957.', 'some effects of bluntness on boundary layer transition and heat transfer at supersonic speeds . large downstream movements of transition observed when the leading edge of a hollow cylinder or a flat plate is slightly blunted are explained in terms of the reduction in reynolds number at the outer edge of the boundary layer due to the detached shock wave . the magnitude of this reduction is computed for cones and wedges for mach numbers to 20 . concurrent changes in outer-edge mach number and temperature occur in the direction that would increase the stability of the laminar boundary layer . the hypothesis is made that transition reynolds number is substantially unchanged when a sharp leading edge or tip is blunted . this hypothesis leads to the conclusion that the downstream movement of transition is inversely proportional to the ratio of surface reynolds number with blunted tip or leading edge to surface reynolds number with sharp tip or leading edge . this conclusion is in good agreement with the hollow-cylinder result at mach 3.1 . application of this hypothesis to other mach numbers yields the result that blunting the tip of a slender cone or the leading edge of a thin wedge should produce downstream movements of transition by factors ranging from 2 at mach 3.0 to 30 at mach the possible reduction in over-all heat-transfer rate and friction drag for aircraft flying at high supersonic speeds . mach number profiles near the surfaces of blunted cones and wedges are computed for an assumed shape of the detached shock wave at flight mach numbers to 20 . the dissipation and stability of these profiles are discussed, and a method is described for estimating the amount of blunting required to produce the maximum possible downstream movement of transition .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1301, 'compressible boundary layers on bodies of revolution .', 'mangler,k.w.', 'arc r.9740, 1946.', 'compressible boundary layers on bodies of revolution . in a former paper (1) it has been shown that the behaviour of the laminar boundary layer on a body of revolution can be described mathematically by the same equations which are also applied to the processes in the laminar boundary layer in the two-dimensional flow along a body contour, the form of which is determined by the shape of the body of revolution . a simple relation exists between the two-dimensional boundary layers and the axially symmetrical ones . the flow had been assumed to be incompressible . in this report it shall be shown that this relation is still valid when the compressibility is taken into consideration . the distribution of velocity as well as that of temperature in the laminar boundary layer of a body of revolution can be calculated by solving the corresponding problem for the two-dimensional flow around a suitable contour . the method is made clear by the example of the supersonic flow towards a cone tip,. this example has already been treated by another method by hantzsche and wendt (2) .'), (1302, 'the development of the boundary layer in supersonic shear flow .', 'rogers,r.h.', 'rae tn.aero.2738, 1961.', 'the development of the boundary layer in supersonic shear flow . the development of the boundary layer in a velocity shear layer is discussed for two-dimensional flow and for axisymmetric flow of both compressible and incompressible fluids . it is shown that the solutions obtained by li and glauert for the two-dimensional flow of an incompressible fluid are applicable in the more general case after suitable transformations of coordinates have been made . new definitions are shown to be necessary, and are given, for the displacement and momentum thicknesses of such a boundary layer . reynolds numbers based on these thicknesses are given, and it is shown that any phenomenon which occurs at a constant value of such a reynolds number will occur at a point which, as the length scale of the flow increases, first moves down-stream and then moves slightly upstream . this is shown to be in qualitative agreement with experimental results on a blunt cone in a supersonic flow . a quantitative comparison of the theoretical and experimental values of displacement and momentum thicknesses is attempted, and no disagreement is obvious,. unfortunately the accuracy of the experiments so far available is insufficient to give positive confirmation of the theory of this note .'), (1303, 'air pressure on a cone moving at high speeds .', 'taylor,g.i. and maccoll,j.w.', 'proc. roy. soc. a, 139, 1933, 278.', 'air pressure on a cone moving at high speeds . the cone is considered to be moving at a velocity higher than that of sound, so that there is in front of it a shock wave, moving with the same speed as the cone itself . in the first part of the paper, the case is investigated mathematically where the flow is irrotational, and the pressure, velocity and density of the air stream are each constant over the surfaces of cones coaxial with the moving solid cone . the complete solution is obtained in numerical form, for cones of semi-vertical angle of the paper, the results are compared with experiment, both in respect of pressure distribution as measured in a wind tunnel, and also (for the 30 cone) by comparison with photographs of bullets in flight . in the latter case the theory should only be applicable if the speed is 1.46 or more times the velocity of sound, and it is in fact found in the photographs, that the nature of the wave alters at about this velocity . the exact solution found, is compared with an approximation given recently by v. karman and moore . this should be valid for thin spindle-shaped bodies, and does in fact agree well in the case of the cone of 10 semi-vertical angle, but diverges increasingly from the truth as the angle is increased .'), (1304, 'newtonian flow over a surface .', 'giraud,j.p.', 'colston symposium on hypersonic flow, bristol, 1959.', 'newtonian flow over a surface . a general method is presented for the study of a three-dimensional hypersonic flow about a body of arbitrary shape when . the manner of constructing a double asymptotic development in and is shown . formulae are given which enable the first three terms of this development to be obtained while neglecting . the theory is then applied to the case of a body of circular-cone shape . the pressure is given as a triple development in accordance with the preceding parameters and the angle of attack,. this development neglects . a. ferri''s vortical layer is brought into evidence . a second application is devoted to calculation of the total forces acting upon bodies of revolution at angles of incidence, while neglecting . general formulae are established for the coefficients of axial force, normal force and moments . the formulae are developed according to the powers of incidence, the first terms of each formula being of very simple form .'), (1305, 'a proposed programme of wind tunnel tests at hypersonic speeds to investigate the lifting properties of geometrically slender shapes .', 'peckham,d.h.', 'rae tn.aero.2730.', 'a proposed programme of wind tunnel tests at hypersonic speeds to investigate the lifting properties of geometrically slender shapes . a programme of tests at hypersonic speeds on slender bodies is described, which has the aim of investigating how lift is generated, and the compromises that may be enforced by aerodynamic heating . the programme is based on models of simple geometric shape, from which lifting configurations will later be built up .'), (1306, 'experiments on circular cones at yaw in supersonic flow .', 'holt,m. and blackie,j.', 'j. ae. scs. 23, 1956, 931.', 'experiments on circular cones at yaw in supersonic flow . pressure measurements made in the fort halstead supersonic tunnel on two circular cones, of semiapex angles 15 and coefficients are compared with corresponding values calculated by theoretical methods, and the relative merits of these methods are then discussed .'), (1307, 'laminar heat-transfer and pressure measurements at a mach number of 6 on sharp and blunt 15 half-angle cones at angles of attack up to 90 .', 'conti,r.j.', 'nasa tn.d962, 1961.', 'laminar heat-transfer and pressure measurements at a mach number of 6 on sharp and blunt 15 half-angle cones at angles of attack up to 90 . two circular conical configurations having 15 half-angles were tested in laminar boundary layer at a mach number of 6 and angles of attack up to 90 . one cone had a sharp nose and a fineness ratio of blunted nose with a bluntness ratio of 0.1428 and a fineness ratio of 1.66 . pressure measurements and schlieren pictures of the flow showed that near-conical flow existed up to an angle of attack of approximately near the base and the bow shock wave was considerably curved . comparison of the results with simply applied theories showed that on the stagnation line pressures may be predicted by newtonian theory, and heat transfer by local yawed-cylinder theory based on the yaw angle of the windward generator and the local radius of the cone . base effects increased the heat transfer in a region extending forward approximately circumferential pressure distributions were higher than the corresponding newtonian distribution and a better prediction was obtained by modifying the theory to match the pressure at 90 from the windward generator to that on the surface of the cone at an angle of attack of 0 . circumferential heat-transfer distributions were predicted satisfactorily up to about 60 from the stagnation line by using lees'' heat-flux distribution based on the newtonian pressure . the effects of nose bluntness at large angles of attack were very small in the region beyond two nose radii from the point of tangency .'), (1308, 'a guide to the use of the m. i. t. cone tables .', 'roberts,r.c. and riley,j.d.', 'j. ae. scs. 21, 1954, 336.', 'a guide to the use of the m. i. t. cone tables . the second and third volumes of the m.i.t. cone tables have been found to be unsatisfactory in two respects . they have been criticized because of their inconvenient tabulation and because the theory on which they are based is inadequate near the cone surface . the former is climinated by means of a coordinate transformation . empirical evidence is presented to show that the latter may be ignored in practice . the exact nature of certain numerical errors in the table is also pointed out .'), (1309, 'hypersonic flows past a yawed circular cone and other pointed bodies .', 'cheng,h.k.', 'j. fluid mech. 12, 1962, 169.', 'hypersonic flows past a yawed circular cone and other pointed bodies . a detailed treatment of inviscid hypersonic flow past a circular cone is given, for small and moderate yaw angles, within the framework of shock-layer theory . the basic problem of non-uniform validity associated with the singularity of the entropy field is examined and a valid first-order solution is obtained which provides an explicit description of a thin vortical layer at the inner edge of the shock layer . analytic formulas for pressure and circumferential velocity are given consistent to the second-order approximation including the non-linear yaw effect . the study of the entropy field (which is not restricted to the hypersonic case) also provides corrections to previous work on the yawed cone and confirms the validity of the linear yaw effect on pressure field in the stone theory . a related investigation of three-dimensional flow fields is presented with special reference to the flow structure near the surface of a pointed, but otherwise arbitrary body . the inviscid streamline pattern on the surface is given by the geodesics originting from the pointed nose as a leading approximation of shock-layer theory . associated with this streamline pattern is a vortical sublayer which exists generally at small as well as at large angle of attack . at the base of the sublayer, enthalpy and flow speed remain essentially uniform .'), (1310, 'survey of inviscid hypersonic flow theory for geometrically slender shapes .', 'crabtree,l.f.', 'rae tn. aero.2695, 1960.', 'survey of inviscid hypersonic flow theory for geometrically slender shapes . a survey is made of existing theories for the calculation of pressure distributions on slender bodies at hypersonic speeds . no account is taken of boundary layer displacement effects which are expected to become important above a mach number of about 10 for a slender body . first the breakdown of linearised supersonic theory is demonstrated as mach number increases above about 5, and this is followed by a derivation of the hypersonic similarity rule . this section includes a description of the piston-analogy . next a physical interpretation of hypersonic flow is outlined and a simple derivation of the modified newtonian pressure formula is given . the equations of flow through an oblique shock wave are simplified by assuming a strong shock, and various results are thereby derived . these include the tangent-wedge and tangent-cone formulae . this is followed by a description of the newtonian approximation for slender bodies, including the effect of centrifugal forces, and the connection with newtonian flow theory is emphasized for . the shock-expansion method is described in some detail for both two-and three-dimensional bodies, and finally some remarks are made about the available data sheets and tables for estimating pressures on cones and ogive-cylinders in yaw . the note does not claim to be original, even in presentation . the aim has been to prepare a reasonably complete survey of available theory for hypersonic flow over slender bodies, excluding viscous and explicit real gas effects . this will provide the background for further work in which experimental data will be analysed and in conjunction with which it is hoped to produce accurate design methods for estimating pressures and forces on shapes intended for sustained flight at hypersonic speeds .'), (1311, 'some simple solutions to the problem of predicting boundary layer self-induced pressures .', 'bertram,m.h. and blackstock,t.a.', 'nasa tn.d798, 1961.', 'some simple solutions to the problem of predicting boundary layer self-induced pressures . simplified theoretical approaches are shown, based on hypersonic similarity boundary-layer theory, which allow reasonably accurate estimates to be made of the surface pressures on plates on which viscous effects are important . the consideration of viscous effects includes the cases where curved surfaces, stream pressure gradients, and leading-edge bluntness are important factors .'), (1312, 'tabulated solutions of the equilibrium gas properties behind the incidents and reflected normal shock-wave in a shock-tube .', 'bernstein,l.', 'i. nitrogen, ii. oxygen. arc cp.626, 1961.', 'tabulated solutions of the equilibrium gas properties behind the incidents and reflected normal shock-wave in a shock-tube . tabulated solutions are presented for the equilibrium gas properties behind the incident and reflected normal shock-waves in the shock-tube, for nitrogen and oxygen . they cover the range of shock-wave mach numbers up to 12 at intervals of undisturbed gas pressure between 1 and the thermodynamic model of the gas used in the calculations is described in some detail, as is the method of solving the equations . the limitations of the assumption of thermodynamic equilibrium are discussed with regard to shock-tube applications, and the estimated accuracy of the tables is indicated .'), (1313, 'on the flow in a reflected shock tunnel .', 'holder, d.w. and schultz, d.l.', 'arc r + m 3265, august 1960 .', 'on the flow in a reflected shock tunnel . the performance of a shock tunnel operated by the reflected-shock technique is examined theoretically neglecting viscous effects and high-temperature real-gas effects . particular attention is given to disturbances to the flow at the nozzle entry caused by waves reflected from the contact surface when the operating conditions depart from those for that the first disturbance reflected from the contact surface is weak enough to be tolerated only within a small range of primary-shock mach number, m /e.g., 5 7 m 6 3 if the pressure at entry to the nozzle is to remain constant to 10 per cent/ . within this range, running times much longer than those obtained in ''straight-through'' shock tunnels are predicted, the limitation usually being imposed by the arrival of the expansion wave originating at the diaphragm . outside this range of mach number, the uniform-flow duration between the arrival at the nozzle entry of the primary shock and the first disturbance reflected from the contact surface is shown to be approximately equal to the time between the arrival of the primary shock and the contact surface in a ''straight-through'' shock tunnel . at first sight it appears, therefore, that the advantages of reflected-shock operation are confined to a very narrow range of shock mach number, unless a heated driver gas is used in order to vary the mach number for further analysis suggests, however, that subsequent disturbances in the multiple wave reflection process between the contact surface and the end of the tube are relatively weak over a useful range of shock mach number . thus, if the flow after the arrival of the early reflected disturbances is used for test purposes, long running times seem possible in theory without severe restrictions to the shock mach number . experiments have been made in a shock tube and a shock tunnel to provide data for comparisons with the results of the simple theory . if allowance is made for viscous effects on the motion of the contact surface, fair agreement is found for the disturbances reflected and transmitted by the contact surface, and for the arrival of the expansion wave reflection process increases when the shock mach number is raised substantially above the ''tailored'' value, and a limit to the usable flow duration may result . a striking feature of the results is a fall of pressure at the end of the tube immediately after reflection of the primary shock . this is attributed to attenuation of the reflected shock resulting from its interaction with the boundary layer on the wall of the tube . further research is required to check this explanation, and to investigate the effects of reynolds number and of the cross-sectional shape and size of the tube . the effects of the tail and reflected head of the expansion wave originating at the main diaphragm are discussed . it is shown that the arrival of the reflected head at the nozzle entry may impose a severe limitation to the duration of uniform conditions at low shock mach number, and that the arrival of the tail may limit the flow duration at high shock mach number . unless means can be devised to suppress the expansion wave, it is demonstrated that it is desirable to have alternative diaphragm positions in a tube required to operate over a range of shock mach number . it is concluded that running times of order 10 milliseconds at a shock mach number of 4, falling to, perhaps, 1 millisecond at a shock mach number of 8 seem possible in a shock tunnel of reasonable size by using reflected-shock operation with unheated hydrogen driving air . because of the simplifying assumptions of the theoretical investigations, and the deficiencies of the apparatus used for the experiments, the present investigation must, however, be regarded as preliminary in character . further research is required to check and extend the findings, and topics particularly requiring investigation are listed in the paper .'), (1314, 'production of high temperature gases in shock tubes .', 'resler,e.l., lin,s.c. and kantrowitz,a.', 'j. app. phys., 23, 1852, 1390.', 'production of high temperature gases in shock tubes . this paper is intended to set forth aerodynamic and thermodynamic calculations which are useful in the production of strong shock waves . the experimental production of strong shock waves is discussed . comparison of the experimental shock strengths with the theoretical calcualtions is made, and finally, some preliminary results of shock tube studies in high temperature gases (up to 18,000k) are briefly surveyed .'), (1315, 'performance estimates for the rae 6in . high-pressure shock tube .', 'woods,n.a.', 'arc 20,440, 1958.', 'performance estimates for the rae 6in . high-pressure shock tube . estimates are made of the performance of the rae 6'''' high pressure shock tube, with various driver gases, over a range of pressure ratios giving shock mach numbers from 6 to 22 . the calculations are based on a simplified model of shock tube flow, in which the working fluid (argon-free air) is assumed to be always in chemical equilibrium, and the driver gas (either hydrogen, or the products of combustion of a hydrogen-oxygen mixture) is assumed to behave as an ideal gas with constant specifiic heats . the results are presented in graphical form and comprise charts normal shock waves in argon-free air shock wave mach number and diaphragm pressure ratio under various initial conditions, and of the shock-induced flows, both in the uniform-sectioned shock tube, and when expanded in a divergent nozzle'), (1316, 'temperature measurements of shock-waves by spectrum-line reversal, ii a double beam method .', 'clouston, j.g., gaydon, a.g. and hurle, i.r', 'proc. roy. soc. a252, p143, september 1959 .', 'temperature measurements of shock-waves by spectrum-line reversal, ii a double beam method . the sodium-line reversal method, as previously described, using a photomultiplier and oscillograph, has been modified . two light beams are now employed, and interference filters are used in front of the photomultipliers instead of a spectrograph . in one beam the background source is viewed directly, through the shock tube, and in the other beam the background source is viewed through the shock tube by a mirror system with a neutral filter interposed to reduce its effective brightness temperature . with a suitably chosen temperature for the background, one oscillograph trace indicates absorption and the other indicates emission of the sodium lines . it is thus possible, from the records of a single shock, to determine the temperature history behind the shock wave to about 20degreec . nitrogen and oxygen again show relaxation effects near the front . temperatures in argon tend to come low, owing to radiative disequilibrium,. excitation processes in argon are discussed . with this system it is possible to determine temperatures rather higher than that of the background source . some work has also been done, with a single-beam method, using a carbon arc as background and following reversal of the indium blue line . temperatures up to 3600degreek have been measured in shocks through nitrogen, but the time resolution is not so good .'), (1317, 'shock-tube testing time .', 'anderson, g.', 'j. aero. sc. vol. 26, march 1959, p184 .', 'shock-tube testing time . in a theoretical investigation of attenuation effects of the shock wave the conservation of mass equation led to an explanation of the difference between the ideal theoretical test time and the experimentally obtained time . a numerical example is given .'), (1318, 'stagnation temperature measurements in a hypersonic gun tunnel using the sodium-line reversal method .', 'stollery, j.l.', 'arc 22854, may 1961 .', 'stagnation temperature measurements in a hypersonic gun tunnel using the sodium-line reversal method . the sodium line reversal /s.l.r./ method has recently been used to measure transient temperatures in the 1400degreek to 3000degreek range, for example ref.6 reports measurements of gun flash temperatures . in the gun tunnel, stagnation temperatures in the above range can be generated flow between a blunt body and its bow shock wave . the gas temperature in this zone is close to the stagnation value .'), (1319, 'real gas effects in flow over blunt bodies at hypersonic speeds .', 'nagamatsu, h.t., geiger r.e. and sheet, r.e.', 'arc 21, 083, june 1959 .', 'real gas effects in flow over blunt bodies at hypersonic speeds . a hypersonic shock tunnel has been developed to investigate the aerodynamic characteristics of flow over bodies at conditions comparable to those encountered by ballistic missiles and satellites re-entering the atmosphere . some results for a shock velocity of over 50,000 ft/sec in the shock tube portion of the facility are presented . static pressure investigations were made in the nozzle for different stagnation conditions in order to determine the flow condition and the expansion process . the results of the investigation on representative blunt bodies at hypersonic mach numbers and nozzle stagnation temperatures up to approximately 6000degreek are presented . these include body pressure distributions, shock wave shapes, detachment distances, and photographs of the luminous gas region in the shock layer . it is seen that the shock detachment distance is smaller at higher stagnation temperatures owing to the real gas effects . for the hemisphere the pressure distribution was less than that predicted by the modified newtonian theory for all stagnation temperatures . for a 50degree cone-hemisphere the pressure distribution and the shock wave detachment distance were appreciably affected by the real gas effects . the experimentally obtained shock wave shape and the approximate boundary layer on a flat plate are correlated with the analytical prediction . some preliminary results for the detached shock wave produced by a blunt two-dimensional body in a low density flow at a mach number of 19.6 are presented .'), (1320, 'divergence of plate airfoils of low aspect ratio at supersonic speeds .', 'hancock,g.', 'j. ae. scs. 1959, 495.', 'divergence of plate airfoils of low aspect ratio at supersonic speeds . in part (1), as a first approach to a theoretical investigation of low aspect ratio rectangular plate wings of constant thickness, the two assumptions are made that .. (a) the spanwise form of the structural distortion is known, leaving the chordwise distortion arbitrary,. and (b) the aerodynamic forces are approximations of the supersonic linearized theory . the form of the chordwise distortion is then deduced from the differential equation representing the state of neutral equilibrium for small displacements at the critical divergence speed . secondly, this problem is investigated using measured structural flexibility coefficients together with theoretical aerodynamic coefficients . thirdly, the usual series solution based on the rayleigh-ritz approach is discussed, using the same assumptions as in the first method . all the results of these methods are consistent and indicate that the transonic regime at m = 1 is the most critical for divergence . in part (2), it is established that sweeping the leading edge of a plate airfoil of constant thickness increases its stability . for angles of sweep less than 30, the critical conditions occur when the leading edge is sonic, but for angles greater than 30 the critical conditions occur when m = 1 .'), (1321, 'effects of a flexible boundary on hydrodynamic stability .', 'benjamin, t.b.', 'j. fluid mech. vol. 9, 4, 1960. p513 .', 'effects of a flexible boundary on hydrodynamic stability . purpose of paper is to examine theoretically the use of coatings of elastic materials to prevent transition from laminar to turbulent flow . theory is extension to flexible boundary of the small-disturbance tollmien-schlichting stability theory and makes use of /tietjens function/ and other functions that occur in solution of orr-sommerfeld equation . it is shown how solutions for flexible wall can be obtained from solutions for rigid boundary . outline and discussion is given first for tollmien-schlichting stability theory for rigid wall, then for theory for flexible boundary . theory is given both for a nondissipative and a dissipative flexible boundary . behavior of flexible medium itself is also examined . practical requirements are discussed . for example, a conclusion is that to avoid tollmien-schlichting instability, the wave velocity of surface waves in absence of flow should coincide with tollmien-schlichting wave velocity at wavelength of /most dangerous/ tollmien-schlichting waves . moreover, damping should be large enough to prevent surface waves from developing but not so large that tollmien-schlichting waves are permissable . author states that a boundary that is both soft and light, one whose elastic constants are of same order as the dynamic pressure of the flow, may be practical for use at high speeds . this surface should have a small damping to avoid tollmien-schlichting type of instability and a large enough wave speed without flow to avoid surface wave instability . although paper is somewhat sketchy in places, it gives comprehensive coverage of stability of laminar flow over a flexible wall .'), (1322, 'qualitiative solutions of the stability equation for a boundary layer in contact with various forms of flexible surface .', 'nonweiler, t.r.f.', 'arc, 22670, march 1961 .', 'qualitiative solutions of the stability equation for a boundary layer in contact with various forms of flexible surface . an appropriate form of the boundary layer stability equation is developed for the condition where the fluid is in contact with an isotropic and homogeneous elastic medium, and various approximate analytical solutions obtained for certain types of surface, so as to reveal at least qualitatively the origin and characteristics of neutral oscillations . in the worked solutions the elastic medium is treated as nondissipative, and the interior boundary is supposed either fixed, or free of stress, or exposed to fluid .. the boundary layer, also, is treated as that over a flat-plate in an incompressible fluid . the results obtained show that the presence of such a resiliant surface introduces the possibility of a number of other modes of oscillation schlichting waves . most of these modes have speeds of propagation determined largely by the properties of the elastic material, and their presence may well be effectively a matter of ''non-viscous'' flow stability -dash a subject not treated here . the tollmien-schlichting mode has its minimum reynolds number increased by the presence of the surface, but if the interior boundary is free there may be an upper limit as well . indeed, a sufficiently thin free surface, or one of low rigidity, apparently eliminates neutral oscillations of this mode altogether, only at the expense, however, of the introduction of a mode of flexural waves .'), (1323, 'an investigation of the use of an auxiliary slot to re-establish laminar flow on low drag aerofoils .', 'cumming,r.w., gregory,b. and walker,w.s.', 'arc r + m.2742, 1950.', 'an investigation of the use of an auxiliary slot to re-establish laminar flow on low drag aerofoils . the use of an auxiliary slot on a laminar-flow aerofoil has been investigated to check whether laminar flow can be re-established by suction at the rear of the region of deposited dirt, flies, etc . results indicate that in the absence of unfavourable pressure gradients, it is possible to re-establish a laminar boundary layer by removing a little more than the whole turbulent layer reaching the slot, and preliminary estimates suggest that with efficient ducting it should be possible to achieve a reduction in overall effective drag coefficient by this means .'), (1324, 'the effect on transition of isolated surface excrescences in the boundary layer .', 'gregory,n. and walker,w.s.', 'pt. 1, arc r + m 2779, 1950.', 'the effect on transition of isolated surface excrescences in the boundary layer . the effect of isolated surface excrescences in a laminar boundary layer in producing disturbances which may lead to turbulent flow has been examined experimentally by several methods . photographs of some of the flow patterns visualised by smoke and china-clay techniques are given . the critical heights of pimple which just give rise to spreading wedges of turbulent flow have been measured on a flat plate and on two aerofoils at several angles of incidence . the results are analysed and are presented in a form which enables approximate estimates to be made of the protuberances permissible on laminar-flow surfaces at full-scale flight reynolds numbers . the estimates suggest that at an altitude of 30,000 ft the critical pimple height is 0.004 in. for a speed of 350 m.p.h., whilst 0.002 in. may be permissible at all subsonic speeds . at sea-level, however, the tolerances are approximately halved .'), (1325, 'experiments on the use of suction through perforated strips for maintaining laminar flow . transition and drag measurements .', 'gregory,n. and walker,w.s.', 'arc r + m 3083, 1955.', 'experiments on the use of suction through perforated strips for maintaining laminar flow . transition and drag measurements . wind-tunnel tests are described in which suction is applied at perforated strips, as an alternative to porous strips or slots, in order to maintain a laminar boundary layer . a test was first carried out on a single row of perforations on a cambered plate, as a preliminary to the main tests which were performed on strips of multiple rows of perforations drilled through the surface of a low-drag-type aerofoil 13 per cent thick and of 5-ft chord . up to a wind speed of 180 ft sec it has been ascertained that suction may be safely applied to extend laminar flow provided the ratio of hole diameter to boundary-layer displacement thickness is less than 2, the ratio of hole pitch to diameter is less than 3 and there are at least three rows of holes in the strip . with less than three rows, the criteria are much more restrictive . it is possible to extend laminar flow by suction through perforations whose diameters and pitches exceed these values slightly, but only with the risk that excessive suction quantities will produce wedges of turbulent boundary layer originating at the holes . a uniform distribution of suction through the holes was necessary . this was successfully obtained by two methods, the use of cells and throttle holes, and with tapered holes . in particular, tests were carried out on some panels supplied by handley page, ltd., in which the cells and tapered holes had been constructed by commercial methods, and the suction distribution proved satisfactory . the resistance of some of the cellular arrangements was measured . it was found that when the suction quantities were the minimum required to maintain laminar flow, the additional losses in total head of the sucked air due to the resistance of the throttle holes could be made small compared with the loss in total head of the sucked boundary layer .'), (1326, 'interaction of secondary injectants and rocket exhaust for thrust vector control .', 'newton,j.f. and spaid,f.w.', 'ars jnl. 32, 1962, 1203.', 'interaction of secondary injectants and rocket exhaust for thrust vector control . tests were conducted with 1300- to 1500-lb thrust solid rocket motors in order to investigate the side-force generation mechanisms associated with the injection of a secondary fluid into the expansion cone of a solid propellant rocket nozzle for thrust-vector control . the nozzles were 15 conicals with a nominal expansion ratio of . all firings were conducted in zero-flow ejectors . freon-12, water, and gascous nitrogen were used as the injectant . nozzle-wall pressure profiles, side thrust, and the nozzle-wall shock interface were recorded . the general character of the pressure disturbance was defined . the major portion of the side force was generated by the pressure disturbance downstream of the injector . the axial-thrust augmentation generated by the injectant was calculated . the effects of nozzle-expansion ratio and injector location on the side force were clearly illustrated .'), (1327, 'on the propagation and structure of the blast wave .', 'sakurai,a.', 'j. phys. soc. japan, 8, 1953, 662.', 'on the propagation and structure of the blast wave . concerning blast waves with front surfaces of plane, cylindrical and spherical shape, the propagation velocity u and the distribution of hydrodynamical quantities are discussed . the solutions are constructed in the form of power series in (c u), where c is the sound velocity of undisturbed fluid . especially r, the distance of shock front from the charge, is represented as, where r is the characteristic length related to the energy of explosion, j and are constants, and a=0,1,2 correspond to plane, cylindrical and spherical case, respectively . in this paper the first approximations for a=0,1 are discussed (the case a=2 has been discussed by g. i. taylor) . the solution is obtained numerically for the case of the adiabatic index . the approximate solution is also considered . using these solutions, is found to be . the second approximation will appear in part 2 to be published subsequently .'), (1328, 'the production of aerodynamic forces by heat addition on external surfaces of aircraft .', 'willmarth,w.w.', 'rand corp. rm.2078, 1957.', 'the production of aerodynamic forces by heat addition on external surfaces of aircraft . within the framework of linearized flow theory an equivalence between a fluid mass source, a heat source, and streamwise body forces is developed . the equivalence between the fluid mass source and heat source was first noticed by hicks(2) and later by chu . (3) using the equivalence the flow field produced by heat addition and by magnetohydrodynamical body forces can be computed . examples for a two-dimensional flat plate, a delta wing, an axially symmetric slender body, and a wedge-shaped afterbody are computed at subsonic and supersonic speeds . the efficiency of lift or thrust production by surface heat addition is very low at subsonic speeds . at supersonic speeds the efficiency is compared with the efficiency of a conventional turbojet-powered aircraft configuration . it is found that the efficiency of lift or thrust production by heat addition on two-dimensional bodies is approximately the same as that for a turbojet-powered two-dimensional body . the efficiency is somewhat higher at low supersonic mach numbers and behaves as, decreasing to a constant value as increases . on the other hand, the efficiency of thrust production by heat addition increases linearly with mach number when heat is added on the rear surface of an axially symmetric afterbody of parabolic shape .'), (1329, 'some aspects of non-stationary airfoil theory and its practical application .', 'sears,w.r.', 'j. ae. scs. 8, 1951, 104.', 'some aspects of non-stationary airfoil theory and its practical application . this paper consists of three notes on the theory of two- dimensional thin airfoils in non-uniform motion .. oscillating airfoil are collected from an earlier paper and are presented in convenient forms for practical application . rigid airfoil passing through a vertical-gust pattern having a sinusoidal distribution of intensity . the lift is determined as a function of the reduced frequency (which in this case is proportional to the ratio of the airfoil chord and the wave length of the gust pattern) and is presented in the form of a vector diagram . it is shown that the lift acts at the quarter-chord point of the airfoil at all times . calculation of the amplitude of torsional oscillation of a fan blade operating in the wake of a set of pre-rotation vanes . in a numerical example the amplitude is found to be small even when the vanes are spaced so that the exciting frequency coincides with the natural frequency of the fan blade .'), (1330, 'on some fourier transforms in the theory of non-stationary flows .', 'garrick,i.e.', 'proc. 5th int. cong. app. mech. 1938, j. wiley, 590.', 'on some fourier transforms in the theory of non-stationary flows . the growth of lift on a airfoil starting impulsively from rest to a uniform velocity has been given by wagner (1925) . the steady-state lift due to circulation on an airfoil oscillating sinusoidally and moving with uniform velocity has been given by theodorsen the present paper based essentially on the material of n. a. c. a. report no. 629 by the author, discusses some reciprocal relations of the nature of fourier transforms existing between the functions of wagner and theodorsen . kussner (1936) has already shown that wagner''s function may be derived from theodorsen''s function . by means of a superposition principle it is possible to utilize these fundamental functions to treat general problems in transient expression which is accurate to within 2 percent is given for wagner''s function . this expression leads to a good approximate expression for theodorsen''s function in terms of the exponential integral, instead of hankel functions . an analogy is drawn between transient hydrodynamic flows and transient electrical flows . kussner (1936) has introduced a function describing the growth of lift on an airfoil entering a sharp edged vertical gust region . this function bears a certain relation to wagner''s function which is briefly discussed .'), (1331, 'calculated responses of a large sweptwing airplane to continuous turbulence with flight-test comparisons .', 'bennett,f.v. and pratt,k.g.', 'nasa tr r.69, 1960.', 'calculated responses of a large sweptwing airplane to continuous turbulence with flight-test comparisons . calculated responses of symmetrical airplane motions, wing deformations, and wing loads due to gusts are shown to compare favorably with available flight-test results . these calculated responses are based on random-process theory, five degrees of freedom, lifting-surface aerodynamics, and one-dimensional vertical turbulence . the extent to which various degrees of freedom contribute to the responses is examined and in this connection the relative effects of static and dynamic aeroelasticity are determined .'), (1332, 'calculated spanwise lift distributions, influence functions and influence coefficients for unswept wings in subsonic flow .', 'diederich,f.w. and zlotnick,m.', 'naca r.1228, 1955.', 'calculated spanwise lift distributions, influence functions and influence coefficients for unswept wings in subsonic flow . spanwise lift distributions have been calculated for nineteen unswept wings with various aspect ratios and taper ratios and with a variety of angle-of-attack or twist distributions, including flap and aileron deflections, by means of the weissinger method with eight control points on the semispan . also calculated were aerodynamic influence coefficients which pertain to a certain definite set of stations along the span, and several methods are presented for calculating aerodynamic influence functions and coefficients for stations other than those stipulated . the information presented herein can be used in the analysis of untwisted wings or wings with known twist distributions, as well as in aeroelastic calculations involving initially unknown twist distributions .'), (1333, 'aerodynamic forces on wings in non-uniform motion .', 'jones,w.p.', 'arc r + m.2117, 1945.', 'aerodynamic forces on wings in non-uniform motion . the problem of determining the aerodynamic forces acting on wings of finite span in non-uniform motion in an incompressible, inviscid fluid is investigated . the underlying theory is outlined in 2, and some known results for the case of an aerofoil of infinite span are included in 3 . it is shown in 4, by the use of operational methods, that the growth of lift function k (s) corresponding to a sudden unit change of incidence can be derived from the lift function corresponding to simple harmonic translational motion . from results given by the writer for rectangular wings (1943) and tapered wings (1945) in simple harmonic motion the corresponding values of k (s) are determined . the growth of lift function k (s) for a wing penetrating a uniform vertical gust can then be estimated as shown in 4 and 5 . by the use of approximate formulae for the growth of lift curves given in fig. 2, the aerodynamic forces corresponding to damped and growing translational oscillations are derived . certain integrals involved in the theory are evaluated in appendix 1, and in appendix 2 the method of determining k (s), when k (s) is known, is discussed in detail . it is suggested that the aerodynamic forces acting on wings of finite span for any type of motion can best be derived from a knowledge of the forces corresponding to purely divergent motion, which can be calculated by the methods outlined in this report .'), (1334, 'calculated spanwise lift distributions and aerodynamic influence coefficients for swept wings in subsonic flow .', 'diederich,f.w. and zlotnick,m.', 'naca tn.3476, 1955.', 'calculated spanwise lift distributions and aerodynamic influence coefficients for swept wings in subsonic flow . spanwise lift distributions have been calculated for 61 swept wings with various aspect ratios and taper ratios and with a variety of angle-of-attack or twist distributions, including flap and aileron deflections, by means of the weissinger method with eight control points on the semispan . also calculated for these plan forms were aerodynamic influence coefficients which pertain to a certain definite set of stations along the span . the information presented herein can thus be used both in the analysis of untwisted wings or wings with known twist distributions and in aeroelastic calculations involving initially unknown twist distributions . this paper supplements and is intended to be used in conjunction with naca tn 3014, where the same type of information, calculated in the same way, is presented for 19 unswept wings .'), (1335, 'use of freon-12 as a fluid for aerodynamic testing .', 'huber,p.w.', 'naca tn.1024, 1946.', 'use of freon-12 as a fluid for aerodynamic testing . the thermodynamic properties of freon-12 have been investigated to determine the possibilities of the use of this gas as a fluid for aerodynamic testing . the values of velocity of sound in freon-12, which are less than one-half those in air, are presented as functions of temperatures and pressure, including measurements at room temperature . the density of freon-12 is about four times that of air . changes in state of freon-12 may be predicted by means of the ideal gas law with an accuracy of better than 1 percent at pressures below freon-12 is shown not to condense during an adiabatic expansion from normal conditions up to a mach number of 3 . the values of the ratio of specific heats for freon-12 are lower than that for air, and therefore an additional parameter is introduced, which must be considered when comparisons are made of aerodynamic tests using freon-12 with those using air . the time lag of the vibrational heat capacity of freon-12 to a change in temperature has been measured and found to be of the order of 2 x 10 second at atmospheric temperature and pressure . this time is so short that no important energy dissipations should result in most engineering applications .'), (1336, 'studies of the use of freon-12 as a wind tunnel testing medium .', 'vondoenhoff.', 'naca tn.3000, 1953.', 'studies of the use of freon-12 as a wind tunnel testing medium . a number of studies relating to the use of freon-12 as a substitute medium for air in aerodynamic testing have been made . the use of freon-12 instead of air makes possible large savings in wind-tunnel drive power . because of the fact that the ratio of specific heats is approximately 1.13 for freon-12 as compared with 1.4 for air, some differences exist between data obtained in freon-12 and in air . methods for predicting aerodynamic characteristics of bodies in air from data obtained in freon-12, however, have been developed from the concept of similarity of the streamline pattern . these methods, derived from consideration of two-dimensional flows, provide substantial agreement in all cases for which comparative data are available . these data consist of measurements throughout a range of mach number from approximately 0.4 to 1.2 of pressure distributions and hinge moments on swept and unswept wings having aspect ratios ranging from 4.0 to 9.0, including cases where a substantial part of the wing was stalled . the freon charging and recovery system used for the langley low-turbulence pressure tunnel is described .'), (1337, 'study of effects of sweep on the flutter of cantilever wings .', 'barmby,j.g.', 'naca r.1014, 1951.', 'study of effects of sweep on the flutter of cantilever wings . an experimental and analytical investigation of the flutter of sweptback cantilever wings is reported . the experiments employed groups of wings swept back by rotating and by shearing . the angle of sweep ranged from 0 to 60 and mach numbers extended to approximately 0.85 . a theoretical analysis of the air forces on an oscillating swept wing of high length-chord ratio is developed, and the approximations inherent in the assumptions are discussed . comparison with experiment indicates that the analysis developed in the present report is satisfactory for giving the main effects of sweep, at least for nearly uniform cantilever wings of high and moderate length-chord ratios . a separation of the effects of finite span and compressibility in their relation to sweep has not been made experimentally but some combined effects are given . a discussion of some of the experimental and theoretical trends is given with the aid of several tables and figures .'), (1338, 'investigation to determine effects of center of gravity location on the transonic flutter characteristics of a 45degree sweptback wing .', 'jones, g.w. and unangst, j.r.', 'naca rm l55k30, 1956 .', 'investigation to determine effects of center of gravity location on the transonic flutter characteristics of a 45degree sweptback wing . an experimental investigation has been conducted in the 26-inch langley transonic blowdown tunnel to determine effects of center-of-gravity location on the transonic flutter characteristics of a 45degree swept-back-wing plan form of aspect ratio 4.0 and taper ratio 0.6 . solid-construction models of the plan form with streamwise naca 65a004 airfoil sections and center-of-gravity locations at approximately 34 percent chord, 46 percent chord, and 58 percent chord, respectively, were fluttered at several mach numbers between 0.8 and 1.35 . it was found that, for streamwise mach numbers from 0.8 to 1.0, the variation with mach number of the ratio of experimental flutter speed to a calculated incompressible flutter speed was not affected by center-of-gravity location . however, for mach numbers from 1.0 to 1.35, there was an increase in flutter-speed ratio with mach number which was different for each center-of-gravity position . data from wings with successively more forward center-of-gravity locations showed successively larger values of flutter-speed ratio at mach numbers from 1.0 to'), (1339, 'calculation of flutter characteristics for finite-span swept or unswept wings at subsonic and supersonic speeds by a modified strip analysis .', 'yates, e.c.', 'naca rm l57l10, 1958 .', 'calculation of flutter characteristics for finite-span swept or unswept wings at subsonic and supersonic speeds by a modified strip analysis . a method has been developed for calculating flutter characteristics of finite-span swept or unswept wings at subsonic and supersonic speeds . the method is basically a rayleigh type analysis and is illustrated with uncoupled vibration modes although coupled modes can be used . the aerodynamic loadings are based on distributions of section lift-curve slope and local aerodynamic center calculated from three-dimensional steady-flow theory . these distributions are used in conjunction with the /effective/ angle-of-attack distribution resulting from each of the assumed vibration modes in order to obtain values of section lift and pitching moment . circulation functions modified on the basis of loadings for two-dimensional airfoils oscillating in a compressible flow are employed to account for the effects of oscillatory motion on the magnitudes and phase angles of the lift and moment vectors . flutter characteristics have been calculated by this method for 12 wings of varying sweep angle, aspect ratio, taper ratio, and center- of-gravity position at mach numbers from 0 to as high as 1.75 . comparisons of the results with experimental flutter data indicate that this method gives generally good flutter results for a broad range of wings .'), (1340, 'method of controlling stiffness properties of a solid-construction model wing .', 'land, n.s. and abbott, f.t.', 'naca tn 3423, 1955 .', 'method of controlling stiffness properties of a solid-construction model wing . a simple method is presented for controlling the bending and torsional stiffnesses of a solid-construction model wing . the method consists of weakening the wing by drilling holes through the wing normal to the chord plane . aerodynamic continuity is maintained by filling the holes with a relatively soft material . the important parameters controlling the stiffnesses are the amount of material removed by drilling, the ratio of hole diameter to wing thickness, and the plan-form pattern of the holes . data are given which may be used for predicting the stiffness of a model wing weakened in this manner .'), (1341, 'investigation of wing flutter at transonic speeds for six systematically varied wing plan forms .', 'jones, g.w. and dubose, h.c.', 'naca rml53g10a, 1953 .', 'investigation of wing flutter at transonic speeds for six systematically varied wing plan forms . an investigation of the effects of systematic variations in wing plan form on the flutter speed at mach numbers between 0.73 and 1.43 has been conducted in the 26-inch langley transonic blowdown tunnel . the angle of sweepback was varied from 0degree to 60degree on wings of aspect ratio 4, and the aspect ratio was varied from 2 to 6 on wings with experimental flutter speed and the reference flutter speed calculated on the basis of incompressible two-dimensional flow . this ratio, designated as the flutter-speed ratio, is plotted as a function of mach number for the various wings . it is found that the flutter-speed ratio increased rapidly past sonic speed for sweep angles of 45degree and less, indicating a favorable effect of mach number . for sweepback of mach number range of the tests . reducing the aspect ratio had a favorable effect on the flutter-speed ratio which was of the order of 100 percent higher for the aspect-ratio-2 wing than for the aspect-ratio-6 wing . this percentage difference was nearly constant throughout the mach number range, indicating that the effect of mach number was about the same for all aspect ratios tested .'), (1342, 'the calculation of aerodynamic loading on surfaces of any shape .', 'falkner, v.m.', 'r + m 1910, august 1943 .', 'the calculation of aerodynamic loading on surfaces of any shape . the object of the report is to establish a routine method for the calculation of aerodynamic loads on wings of arbitrary shape . the method developed is based on potential theory and uses a general mathematical formula for continuous loading on a wing which is equivalent to a double fourier series with unknown coefficients . in order to evaluate the unknown coefficients the continuous loading is split up into a regular pattern of horseshoe vortices, the strengths of which are proportional to the unknown coefficients and to standard factors which are given in a table . the total downwash at chosen pivotal points is obtained by summing the downwashes due to the individual vortices, a process which is simplified by the use of specially prepared tables of the properties of the horseshoe vortex . by equating the downwash to the slope of the wing at each pivotal point, simultaneous equations are obtained, the solution of which defines the unknown coefficients . the first layout involves a total of 76 vortices over the wing, and a second layout, involving a total of 84, is shown to be of superior accuracy . the effect on the solution of the number of pivotal points is investigated and it is concluded that by a suitable choice, it is unnecessary to use a large number . results for a rectangular wing at with those obtained by other workers and it appears that there may be errors in published results in at least one of these cases . immediate development includes the application to the calculation of the characteristics of actual sweptback wings, including rotary derivatives, and future development includes also applications in wind tunnel design and technique .'), (1343, 'formulas for the supersonic loading, lift and drag of flat swept back wings with leading edges behind the mach lines .', 'cohen,d.', 'naca r.1050, 1951.', 'formulas for the supersonic loading, lift and drag of flat swept back wings with leading edges behind the mach lines . the method of superposition of linearized conical flows has been applied to the calculation of the aerodynamic properties, in supersonic flight, of thin flat, swept-back wings at an angle of attack . the wings are assumed to have rectilinear plan forms, with tips parallel to the stream, and to taper in the conventional sense . the investigation covers the moderately supersonic speed range where the mach lines from the leading-edge apex lie ahead of the wing . the trailing edge may lie ahead of or behind the mach lines from its apex . the case in which the mach cone from one tip intersects the other tip is not treated . formulas are obtained for the load distribution, the total lift, and the drag due to lift . for the cases in which the trailing edge is outside the mach cone from its apex (supersonic trailing edge), the formulas are complete . for the wing with both leading and trailing edges behind their respective mach lines, a degree of approximation is necessary . it has been found possible to give practical formulas which permit the total lift and drag to be calculated to within 2 or 3 percent of the accurate linearized-theory value . the local lift can be determined accurately over most of the wing, but the trailing-edge-tip region is treated only approximately . charts of some of the functions derived are included to facilitate computing, and several examples are worked out in outline .'), (1344, 'atmospheric entries with vehicle lift-drag ratio modulated to limit deceleration and rate of deceleration vehicles with maximum lift-drag ratio of 0. 5.', 'katzen,e.d. and levy,l.l.', 'nasa tn.d1145, 1961.', 'atmospheric entries with vehicle lift-drag ratio modulated to limit deceleration and rate of deceleration vehicles with maximum lift-drag ratio of 0. 5. an analysis has been made of atmosphere entries for which the vehicle lift-drag ratio was modulated to maintain specified maximum decelerations and or maximum deceleration rates . the part of the vehicle drag polar used during modulation was from maximum lift coefficient to minimum drag coefficient . the entries were at parabolic velocity and the vehicle maximum lift-drag ratio was 0.5 . two-dimensional trajectory calculations were made for a nonrotating, spherical earth with an exponential atmosphere . the results of the analysis indicate that for a given initial flight-path angle, modulation generally resulted in a reduction of the maximum deceleration to 60 percent of the unmodulated value or a reduction of maximum deceleration rate to less than 50 percent of the unmodulated rate . these results were equivalent, for a maximum deceleration of 10g, to lowering the undershoot boundary 24 miles with a resulting decrease in total convective heating to the stagnation point of 22 percent . however, the maximum convective heating rate was increased 18 percent,. the maximum radiative heating rate and total radiative heating were each increased about 10 percent .'), (1345, 'the use of aerodynamic lift during entry into the earth''s atmosphere .', 'lees,l.', 'space tech. lab. r. gm-tr-0165-00519, 1958.', 'the use of aerodynamic lift during entry into the earth''s atmosphere . by employing aerodynamic lift during entry into the earth''s atmosphere at either orbital or /escape/ velocity, the range of allowable entry angles for a prescribed peak deceleration is greatly increased, while the total heat energy transferred to the vehicle can be held to about the same value as for a nonlifting vehicle . only modest lift-drag ratios are required beyond peak g to prevent the deceleration from exceeding the peak value, or to prevent the vehicle from skipping out of the earth''s atmosphere . thus, the difficult guidance and control problem is greatly alleviated,. in particular, for return from the moon or other planets the necessity for multiple-pass drag braking is eliminated .'), (1346, 'modulated entry .', 'grant,f.c.', 'nasa tn.d452, 1960.', 'modulated entry . the technique of modulation, or variable coefficients, is discussed and the analytical formulation is reviewed . representative numerical results of the use of modulation are shown for the lifting and nonlifting cases . these results include the effects of modulation on peak acceleration, entry corridor, and heat absorption . results are given for entry at satellite speed and escape speed . the indications are that coefficient modulation on a vehicle with good lifting capability offers the possibility of sizable loading reductions or, alternatively, wider corridors,. thus, steep entries become practical from the loading standpoint . the amount of steepness depends on the acceptable heating penalty . the price of sizable fractions of the possible gains does not appear to be excessive .'), (1347, 'approximate analysis of atmospheric entry corridors and angles .', 'luidens,r.w.', 'nasa tn.d590, 1961.', 'approximate analysis of atmospheric entry corridors and angles . a simple closed-form solution for the achievable corridor depths and entry angles as a function of g-load limit, entry velocity, and vehicle aerodynamics and thermodynamics is developed for two modes of vehicle operation, constant angle of attack and modulated angle of attack . for constant angle of attack, operation at maximum negative lift coefficient on the overshoot bound, and at an angle of attack between zero and that for maximum lift-drag ratio on the undershoot bound, gives the deepest corridor . for modulated angle of attack, operating at maximum negative lift coefficient on the overshoot bound and modulating the angle of attack from maximum positive lift coefficient to zero on the undershoot bound give the deepest corridor . the modulated angle of attack gives corridor depths two to four times larger than the fixed angle of attack . for both cases the corridor depth is increased by increasing maximum lift-drag ratio, increasing g limit, and decreasing entry velocity . consideration of hot-gas radiation places a limit on the maximum angle of attack for either mode of operation . if a maximum free-stream reynolds number limit must be placed on the vehicle to ensure a laminar boundary layer, the deep atmospheric penetrations associated with configurations with high lift-drag ratio may be ruled out . both of these thermodynamic considerations reduce the acceptable corridor depth below the value calculated from aerodynamic considerations alone .'), (1348, 'radiative heat transfer during atmosphere entry at parabolic velocity .', 'yoshikawa,k.k. and wick,b.h.', 'nasa tn.d1074, 1961.', 'radiative heat transfer during atmosphere entry at parabolic velocity . stagnation point radiative heating rates for manned vehicles entering the earth''s atmosphere at parabolic velocity are presented and compared with corresponding laminar convective heating rates . the calculations were made for both nonlifting and lifting entry trajectories for vehicles of varying nose radius, weight-to-area ratio, and drag . it is concluded from the results presented that radiative heating will be important for the entry conditions considered .'), (1349, 'effects of simulated rocket jet exhaust on stability and control of a research type airplane configuration at a mach number of 6. 86 .', 'fetterman,d.e.', 'nasa tm.x127, 1959.', 'effects of simulated rocket jet exhaust on stability and control of a research type airplane configuration at a mach number of 6. 86 . an investigation has been undertaken in the langley 11-inch hypersonic tunnel at a free-stream mach number of 6.86 to determine the jet-interference effects at high jet-static-pressure ratios on the stability and control of a research-type airplane configuration . compressed-air tests with a jet exhausting from the base of the fuselage were conducted over a reynolds number range of 0.57 x 10 to and over a jet-static- pressure-ratio range of 0 to 1460 . the results of these tests indicated that the operation of the jet induced a sizable separated-flow region over the vertical- and horizontal-tail surfaces which could be approximately duplicated at low angles of attack by use of metal jet-boundary simulators . the results of force tests, during which these metal jet-boundary simulators were used, indicated that this separated-flow region caused a large reduction in the longitudinal stability and control and a smaller reduction in the lateral and directional stability and control . by extending the divergent section of the nozzle and thus reducing the jet-static-pressure ratio, these losses were diminished .'), (1350, 'effects of jet billowing on stability of missile-type bodies at mach 3. 85 .', 'salmi,r.j.', 'nasa tn.d284, 1960.', 'effects of jet billowing on stability of missile-type bodies at mach 3. 85 . the interference effects of a billowing jet on the forces and moments of two missile-type bodies were investigated in the nasa lewis 2-by 2-foot mach 3.85 wind tunnel . to simulate a rocket jet, pressurized nitrogen was exhausted from an annular sonic nozzle . the results indicate that for both models the stability parameter moment coefficient with angle of attack) in the region of zero angle of attack was favorably influenced by the interference resulting from separation due to jet billowing . schlieren photographs are presented that show the separation due to the jet billowing at various pressure ratios and angles of attack .'); INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) VALUES (1351, 'exploratory tests of the effects of jet plumes on the flow over cone- cylinder flare bodies .', 'falanga,r.a.', 'nasa tn.d1000, 1962.', 'exploratory tests of the effects of jet plumes on the flow over cone- cylinder flare bodies . schlieren photographs have been taken of the flow over cone-cylinder-flare bodies to study the extent of boundary-layer separation due to the presence of rocket jet plumes . tests were made of three cone-cylinder-flare configurations in the langley 11-inch hypersonic tunnel at a mach number of 9.65 and in the langley unitary plan wind tunnel at a mach number of 4.65 with two additional configurations . the stream reynolds number varied from approximately 317,000 to 582,000 based on model length . the conical flares had half-angles of 7 or 13 and contained one of two test nozzles with a design mach number of 3.72 or 4.53 . the test nozzles were operated with compressed air and were designed to simulate a solid-propellant rocket motor operating at altitudes between to free-stream static-pressure ratio varied from jet off to 1,150 for the test nozzle with a design mach number of 3.72 and from jet off to mach number of 4.53 . for most of the tests the angle-of-attack range was 0 to -4,. some additional tests were made at 2 and 4 . measurements taken from flow pictures indicated that at zero angle of attack on all configurations tested with jet on the boundary layer separates ahead of the flare-cylinder juncture and the separation point moves toward the cone-cylinder juncture with an increase in pressure ratio . increasing angle of attack reduced the extent of boundary-layer separation on the windward side as did increasing the stream mach number from 4.65 to 9.65 . other parameters which tended to reduce the extent of boundary-layer separation were .. number, (b) decreasing stream reynolds number, and (c) displacing nozzle exit rearward .'), (1352, 'aerodynamic investigation of a parabolic body of revolution at mach number of 1. 92 and some effects of an annular supersonic jet exhausting from the base .', 'love,e.s.', 'naca tn.3709, 1956.', 'aerodynamic investigation of a parabolic body of revolution at mach number of 1. 92 and some effects of an annular supersonic jet exhausting from the base . an aerodynamic investigation of a parabolic body of revolution was conducted at a mach number of 1.92 with and without an annular supersonic jet exhausting from the base . measurements with the jet inoperative were made of lift, drag, pitching moment, radial and longitudinal pressure distributions, and base pressures . with the jet in operation, measurements were made of the pressures over the rear of the body with the primary variables being angle of attack, ratio of jet velocity to freestream velocity, and ratio of jet pressure to stream pressure . the results with the jet inoperative showed that the radial pressures over the body varied appreciably from the distribution generally employed in most approximate theories . the linearized solutions for lift, pitching moment, and center of pressure gave relatively poor predictions of the experimental results . an analysis of several theoretical methods for calculating pressure distribution and wave drag showed that some methods gave results in considerable disagreement with experimental values . maximum effects of the jet were obtained at the lower ratio of jet velocity to stream velocity and the highest ratio of jet pressure to stream pressure . these effects amounted to a slight decrease in fore-drag, a reduction in lift, and a shift of center of pressure in a destabilizing direction .'), (1353, 'investigation of a two-step nozzle in the langley 11in .', 'hypersonic tunnel .', 'mclellan,c.h. naca tn.2171, 1950.', 'investigation of a two-step nozzle in the langley 11in . hypersonic tunnel . flow surveys have been made in the first of several nozzles to be investigated in the langley 11-inch hypersonic tunnel . the nozzle was designed by the method of characteristics for a mach number of 6.98 . two step expanded the air in the horizontal plane to a mach number of 4.36 and the second in the vertical plane to a mach number of 6.98 . the test results showed that, although a maximum mach number of about 6.5 was obtained, the flow in the test section was not sufficiently uniform for quantitative wind-tunnel test purposes . deviations from the design flow were traced to the presence of a thick boundary layer which developed in the first step along the parallel walls .'), (1354, 'investigation of the flow through a single stage two dimensional nozzle in the langley 11in . hypersonic tunnel .', 'mclellan,c.h.', 'naca tn.2223, 1950.', 'investigation of the flow through a single stage two dimensional nozzle in the langley 11in . hypersonic tunnel . flow surveys have been made in the second of several nozzles to be investigated in the langley 11-inch hypersonic tunnel . the single-stage, two-dimensional nozzle was designed by the method of characteristics for a mach number of 7.08 without boundary-layer corrections . the test results show that reasonably uniform flow at an average mach number of about 6.86 was obtained in a central region of the stream at the test section . this region had a cross section nearly 5 inches square and had a deviation from uniform flow of less than 1 percent in mach number and 0.3 in flow angle . an increase in mach number of about 3 percent occurred during test runs of about 60 seconds duration because of distortions of the boundaries at the first minimum due to nonuniform heating of the nozzle blocks during the tests .'), (1355, 'boundary layer displacement effects in air at mach numbers of 6. 8 and 9. 6.', 'bertram,m.h.', 'nasa tr.r22, 1959.', 'boundary layer displacement effects in air at mach numbers of 6. 8 and 9. 6. measurements are presented for pressure gradients induced by a laminar boundary layer on a flat plate in air at a mach number of 9.6 and for the drag of thin wings at a mach number of about 6.8 and zero angle of attack . the pressure measurements at a mach number of 9.6 were made in the presence of substantial heat transfer from the boundary layer to the plate surface . the measured pressure distribution on the surface of the plate was predicted with good accuracy by a modification to insulated-plate displacement theory which allows for the effect of the heat transfer and temperature gradient along the surface on the boundary-layer displacement thickness . the total drag of thin wings with square and delta plan forms was measured at a nominal mach number of 6.8 over a reasonably wide range of reynolds numbers . the total drag was found to be greater than can be explained by adding a classical value of laminar skin friction to the estimated pressure drag . the difference is, in general, explained by the increase in skin friction (20 to 40 percent) caused by the boundary-layer-induced pressures .'), (1356, 'secondary flow fields embedded in hypersonic shock layers .', 'seiff,a.', 'nasa tn.d1304, 1962.', 'secondary flow fields embedded in hypersonic shock layers . when a ramp or other compression surface is located in a locally supersonic region behind a hypersonic bow shock wave, it generates a secondary shock wave . the ramp flow disturbance may be viewed as an embedded newtonian impact flow if the embedded shock layer is thin . examination of the applicability of newtonian flow theory to cones and wedges in uniform streams suggests that this theory can be expected to give a useful approximation to the surface pressures . a pressure equation based on this concept predicts a number of interesting things .. first, pressures can differ from simple newtonian theory by factors of 1 5 to 3,. for example, on flare stabilizers on blunt-nosed bodies of revolution, pressures are lower than newtonian and diminish with increasing flight speed in the hypersonic speed range . the calculated pressures vary over the flare surface as a result of the nonuniformity of its incident stream, and depend on the axial location of the flare . in the case of a flap mounted on a large-angled blunt-nosed cone, the pressure coefficients vary from 1 to 5 through the variable entropy layer . a pressure coefficient of 5 greater than the maximum possible in newtonian flow can occur because the compression process is more efficient than a single shock wave process . on areas of the flap that protrude through the main bow wave, the pressure coefficient should revert to the simple newtonian value . equations are developed for the initial slopes of the normal-force and pitching-moment curves of a flare stabilizer . in the simplest case these differ from conventional newtonian theory by the ratio of local dynamic pressure to free-stream dynamic pressure . this ratio takes values as low as 0.1 in some of the examples considered .'), (1357, 'compressive buckling of simply supplorted plates with longitudinal stiffeners .', 'seide,p. and stein,m.', 'naca tn.1825, 1949.', 'compressive buckling of simply supplorted plates with longitudinal stiffeners . charts are presented for the analysis of the stability under compression of simply supported rectangular plates with one, two, three, and an infinite number of identical equally spaced longitudinal stiffeners that have zero torsional stiffness .'), (1358, 'compressive buckling of simply supported plates with transverse stiffeners .', 'budiansky,b. and seide,p.', 'naca tn.1557, 1948.', 'compressive buckling of simply supported plates with transverse stiffeners . charts are presented for the analysis of the stability under longitudinal compression of simply supported rectangular plates with several equally spaced transverse stiffeners that have both torsional and flexural rigidity .'), (1359, 'compression tests on circular cylinders stiffened longitudinally by closely spaced z-section stringers .', 'peterson,j.p. and dow,m.b.', 'nasa memo. 2-12-59l, 1959.', 'compression tests on circular cylinders stiffened longitudinally by closely spaced z-section stringers . six circular cylinders stiffened longitudinally by closely spaced z-section stringers were loaded to failure in compression . the results obtained are presented and compared with available theoretical results for the buckling of orthotropic cylinders . the results indicate that the large disparity that exists between theory and experiment for unstiffened compression cylinders may be significantly smaller for stiffened cylinders .'), (1360, 'simplified analysis of general instability of stiffened shells in pure bending .', 'shanley,f.r.', 'j. ae. scs.16, 1949, 590.', 'simplified analysis of general instability of stiffened shells in pure bending . although much work has been done to develop a theory for the failure of shells by general instability, there is at present no simple method by which the size of the frames may be determined for any given diameter, bending moment,.and frame spacing . such a method is needed in determining the optimum design for stiffened shells, to be used as a basis for weight analysis of fuselages, and other shell structures . in an extension of the work done for the rand corporation, a simple coefficient has been determined for this purpose . since it appears that this method may also be useful in design calculations, a brief description is presented below .'), (1361, 'large deflections of structures subjected to heating and external loads .', 'turner,j.m.', 'j. ae. scs. 1960.', 'large deflections of structures subjected to heating and external loads . the method of direct formulation of the stiffness matrix is extended to include the effects of nonuniform heating and large deflections . the purpose is to develop an analytical tool for the treatment of actual structures . in the solution of aeroelastic problems the relations between forces and deflections must be determined . the usual stiffness matrix formulation of this relationship is limited to small temperature changes and small deflections . for large temperature changes additional terms are required . also the problem becomes geometrically nonlinear when large deflections are involved . to overcome the inherent difficulties of the nonlinear problem for practical structures either an iterative or a step- by-step procedure must be used . the force-deformation relations necessary for this step-by-step or iterative approach are derived for an axially loaded member and for a plate element including the effects of thermal strains .'), (1362, 'non-linear analysis of heated, cambered wings by the matrix force method .', 'warner lansing, irving w. jones, paul ratner', 'grumman aircraft engineering corporation', 'non-linear analysis of heated, cambered wings by the matrix force method . various extensions of the matrix force method for complex structure analysis are presented and illustrated with the objective of expanding its range to handle the problems likely to be encountered in advanced vehicle wing design . methods are covered in detail for (1) determining the change in flexibility that occurs when thermal stresses are present, and also how large these stresses must be to cause buckling, (2) including the non-linear effect of large deflections by an iterative procedure, and (3) analyzing a wing that is initially slightly cambered and warped with either or both of the aforementioned effects present . formulas are given for calculating the input matrix terms as are the matrix equations and supporting theoretical discussion . an example illustrates the nature and magnitude of the effects being examined .'), (1363, 'a characteristic type of instability in the large deflections of elastic plates .', 'ashwell,d.g.', 'proc. roy. soc. a, 214, 1952, 98.', 'a characteristic type of instability in the large deflections of elastic plates . part 1. from a general equation governing the bending of thin elastic plates into certain types of surfaces of revolution are derived expressions for the behaviour of rectangular plates with initial curvatures, subjected to pure bending about one axis . it is found that such plates exhibit the type of instability characteristic of thin-walled structures which depend for their stiffness on curvature . curves are drawn showing the deformation suffered by such plates, and an expression for the critical bending moment at which instability occurs is obtained . experimental results show satisfactory agreement . part 2. the analysis of part 1 is extended to deal with the case of flat square or rectangular plates loaded by distributed bending moments applied to all four edges . curves are drawn to describe their behaviour, and they are found to exhibit the characteristic instability displayed by thin-walled curved structures . experimental verification is satisfactory .'), (1364, 'an experimental investigation of the interaction between shock waves and boundary layers .', 'gadd, g.e., holder, d.w. and regan, j.d.', 'proc. of the roy. soc. of london, ser. a., vol. 226, 1954, pp. 227-253', 'an experimental investigation of the interaction between shock waves and boundary layers . an account is given of an investigation into the interaction between the boundary layer on a flat plate and a shock wave produced either externally, by a wedge in the supersonic mainstream, or from within the boundary layer, by a wedge held in contact with the plate . a wide range of free-stream mach numbers, boundary-layer reynolds numbers, and shock strengths has been covered, shock strength being defined as the ratio of the static pressure downstream of the shock to the static pressure upstream of it . variations in these parameters can have large effects on the interaction, and there are also large differences between cases with externally generated shocks and cases where the shock is generated from within the boundary layer . the investigation has thrown light on the physical mechanisms involved . it is found that many of the major features of the interaction arise because the boundary layer separates from the surface ahead of the shock wave . the conditions under which separation occurs and the behaviour of the separated boundary layer thus have important effects, in terms of which, for example, the differences between the interactions observed with laminar and with turbulent boundary layers may be explained .'), (1365, 'approximate calculation of the laminar boundary layer .', 'thwaites,b.', 'aero. quart. 1, 1949-1950.', 'approximate calculation of the laminar boundary layer . after analyzing a large class of boundary-layer velocity-profiles, the author discovered that the functions l(m) and h(m) for all such cases differ only slightly from each other over the whole range of positive and negative pressure gradients . here l, m and h are defined by being the velocity-component in the x direction and u the value of u at the edge of the boundary-layer and and the displacement and momentum thickness, respectively . based on this discovery, an approximate method is proposed by constructing two universal curves l(m) and h(m) for all conceivable boundary-layer flows found in practice . once these are chosen, karman''s momentum-integral can be written in the form, v being the kinematic viscosity coefficient, and can be integrated numerically . as examples, both howarth''s and hartree''s tained is considered good for practical purposes .'), (1366, 'the compressible laminar boundary layer with heat transfer and arbitrary pressure gradient .', 'cohen,c.b. and reshotko,e.', 'naca r.1294, 1956.', 'the compressible laminar boundary layer with heat transfer and arbitrary pressure gradient . an approximate method for the calculation of the compressible laminar boundary layer with heat transfer and arbitrary pressure gradient, based on thwaites'' correlation concept, is presented . the method results from the application of stewartson''s transformation to prandtl''s equations, which yeilds a nonlinear set of two first-order differential equations . these equations are then expressed in terms of dimensionless parameters related to the wall shear, the surface heat transfer, and the transformed free-stream velocity . thwaites'' concept of the unique interdependence of these parameters is assumed . the evaluation of these quantities is then carried out by utilizing exact solutions recently obtained . with the resulting relations, methods are derived for the calculation of the two-dimensional and axially symmetric laminar boundary layer with arbitrary free-stream velocity distribution . mach number, and surface temperature level . the combined effect of heat transfer and pressure gradient is demonstrated by applying the method to calculate the characteristics of the boundary layer on thin supersonic surfaces and in a highly cooled, convergent-divergent, axially symmetric rocket nozzle .'), (1367, 'a theoretical investigation of the effects of mach number, reynolds number, wall temperature and surface curvature on laminar separation in supersonic flow .', 'gadd, g.e.', 'a.r.c. 18,494, june 1956 .', 'a theoretical investigation of the effects of mach number, reynolds number, wall temperature and surface curvature on laminar separation in supersonic flow . laminar separation in supersonic flow is investigated by an extension of stratford''s method . it is assumed that separation is of the usual practical type, taking place upstream of the shock wave or other agency provoking it . the results of the analysis agree well in most respects with experiment .'), (1368, 'three dimensional viscous wakes .', 'steiger,m.h. and bloom,m.h.', 'j.fluid mech. 14, 1962,233.', 'three dimensional viscous wakes . the velocity fields of three-dimensional viscous wakes are examined with the use of the boundary-layer approximations, osoen''s linearization of the convective terms, and the assumption of constant fluid properties . transform methods yield solutions for general types of initial conditions . as an illustration, the axial velocity distribution of a wake whose initial isovels (lines of constant velocity) are of elliptic shape and their decay to axial symmetry are demonstrated . both laminar and turbulent flows are considered .'), (1369, 'steady motion of a sphere., oseen''s criticism and solution .', '', '', 'steady motion of a sphere., oseens''s criticism and solution . the formula of stokes for the resistance experienced slowly moving sphere has been employed in physical researches of fundamental importance, as a means of estimating the size of minute globules of water, and thence the number of globules contained in a cloud of given mass . consequently the conditions of its validity has been much discussed both from the experimental and from the theoretical side .'), (1370, 'some remarks on the flat plate boundary layer .', 'lewis,j.a. and carrier,g.f.', 'q. app. math. 7, 1949, 228.', 'some remarks on the flat plate boundary layer . the authors discuss the solutions for the flow of a viscous incompressible fluid near the leading edge of a semi-infinite flat plate without pressure gradient . the oseen linearization is employed which approximates the equations of motion and continuity by where are the coordinate directions, the corresponding velocity components and the uniform free stream velocity which is parallel to the plate . defining a perturbation stream function by the differential equation to be solved is with boundary conditions far from the plate and when y=0 and . the authors discuss the problem by applying the two-dimensional fourier transform and obtain an explicit solution for the velocity gradient at the plate which is in disagreement with the result of the blasius solution . from this the authors conclude that it would be more appropriate to use a velocity other than in the linearization of the equations of motion and suggest replacing by where . this choice does not affect the solution far from the plate but gives on the plate and in comparison with blasius solution indicates that c=0.35 . the solution of the modified oseen equation with this value of c then seems acceptable as the approximate solution in the region intermediate between the stokes flow and the free stream . on the basis of these considerations, the authors suggest an iteration procedure for obtaining the exact solution for the above problem as well as a solution for the plate of finite length .'), (1371, 'axisymmetric free mixing with swirl .', 'steiger,m.h. and bloom,m.h.', 'pibal r.628.', 'axisymmetric free mixing with swirl . viscous laminar axially-symmetric free-mixing with small, moderate, and large swirl is investigated by a boundary layer type of analysis with integral methods . moderate and small swirls are formally the same, differing only in the order of their associated radial pressure gradients . neither induces significant axial pressure gradients,. consequently their effect on the axial flow is negligible . for moderate and small swirl an interesting feature is the swirl decay . in both compressible and incompressible flow, it is shown that jet swirl decays more rapidly than wake swirl whereas both swirls decay more rapidly than the non-uniformity in axial velocity . large swirl generates axial pressure gradients as well as large radial pressure gradients, and therefore alters the streamwise flow . examples calculated for incompressible flow, show that the wake is lengthened by large swirl . it is expected that this effect will be diminished in the presence of higher free-stream mach numbers which lead to decreased densities, due to decreased centrifugal effects, decreased radial pressure gradients, and decreased axial pressure gradients .'), (1372, 'on axially symmetric, turbulent, compressible mixing in the presence of initial boundary layer .', 'gdalia kleinstein', 'polytechnic institute of brooklyn, farmingdale, n.y.', 'on axially symmetric, turbulent, compressible mixing in the presence of initial boundary layer . recent experimental results have shown that the mixing of heterogeneous gases having an initial velocity ratio close to unity occurs faster than is predicted by classical eddy-viscosity theory . the theoretical analysis of two uniform streams of different gases but of nearly equal velocity, performed with the usual assumptions for eddy viscosity and prandtl number equal to a constant, shows that mixing will take place very slowly, i.e., at the rate corresponding to laminar diffusion . it has been suggested that the difference between analysis and experiment could be attributed to the presence of a boundary layer in the experiments . it is the purpose of this note to show that the use of the classical eddy-viscosity law, admitting the existence of a boundary layer, is not sufficient to explain the rapid mixing that is observed physically . instead, it is shown that rapid mixing can be explained on the basis of a different eddy-viscosity law, as was suggested in ref. 1 . these conclusions are obtained through application of the analysis presented briefly below .'), (1373, 'nose drag in free-molecule flow and its minimization .', 'tan,h.s.', 'j.ae.scs. 1959, 360.', 'nose drag in free-molecule flow and its minimization . the superaerodynamic nose drag of a body in a free-molecule flow involves two parameters .. the speed ratio s between ordered and random molecular motions (modified mach number), and the temperature ratio between the solid surface and undisturbed gas . simplifications of the drag formula are obtained at hypersonic as well as low-subsonic extremes . to minimize the drag on a nose of specified length and base radius, the ordinary method of calculus of variation was found inadequate . a generalized approach has, accordingly, been developed, and the specification of end conditions is discussed at length . results of the present investigation indicate that in all cases an optimum nose requires a flat tip . the optimum nose curve for the hypersonic extreme does not depend on the temperature ratio, but that for the low-subsonic extreme varies in the following manner .. for a hot body the curve is convex,. for a cold body, concave . an optimum solution exists in a restricted range of specification only . with prescribed tip and base radii the admissible nose length is bounded below for the cases of hypersonic and low-subsonic hot body and bounded above for the case of low-subsonic cold body . a vanishing tip radius leads to an infinitely long nose in the former and a vanishing nose in the latter case . optimum nose curves for several temperature ratios at the low-subsonic extreme, as well as the one for hypersonic extreme, are presented . it is observed that at the low-subsonic extreme, with, the hot-body solution asymptotically approaches the hypersonic solution--i.e., a slender conventional warhead with a flat tip,. whereas with, the cold-body solution asymptotically approaches the minimal-surface solution--i.e., tip radius, a flat disc .'), (1374, 'theoretical analysis of turbulent mixing of reactive gases with application to supersonic combustion of hydrogen .', 'libby,p.a.', 'ars jnl. 32, 1962, 388.', 'theoretical analysis of turbulent mixing of reactive gases with application to supersonic combustion of hydrogen . the turbulent mixing of an axisymmetric jet of a reactive gas is considered . by assuming a convenient model for the compressible eddy viscosity, the momentum equation is reduced to a form amenable to approximate solution . the resulting velocity distribution in both incompressible and compressible flows is in reasonable agreement with experiment . the usual assumptions with respect to chemical behavior, namely either frozen or equilibrium flow, and to unity lewis numbers and prandtl number are employed . the theoretical results for chemical equilibrium are shown to be in reasonable agreement with experimental data from low speed hydrogen flames . a numerical example of interest in connection with a hypersonic, air breathing vehicle is carried out in detail .'), (1375, 'an approximate solution for the axisymmetric jet of a laminar compressible fluid .', 'gdalia kleinstein', 'polytechnic institute of brooklyn', 'an approximate solution for the axisymmetric jet of a laminar compressible fluid . an extension of the modified-oseen method of carrier, based on the linearization of the viscous term of the von mises transformation, is presented . the method is employed to determine the velocity field associated with the laminar axisymmetric jet flow of a compressible gas with an arbitrary but constant external flow . the approximate solution is shown to be in good agreement with the exact numerical calculation of pai . in many boundary layer problems it is not possible to make the assumption of flow similarity . the solution in these cases can be obtained either by laborious finite difference techniques or by resort to approximate solutions . carrier and lewis (1), and more recently carrier (2), have suggested a method of obtaining approximate solutions to problems involving convection and diffusion . this method, termed by carrier /the modified-oseen method/, overcomes an essential difficulty of integral methods, namely, the generation of reasonable profiles . it is well known that the integral method gives accurate results only if the analytical profiles represent closely the true profiles . according to the modified-oseen method the convective operator in the original partial differential equation is replaced by a linear one . the resulting equation for the boundary layer problem is the heat conduction equation which can be treated by well-known techniques . it is the purpose of this paper to indicate a modification of this procedure and to demonstrate its simplicity and accuracy by treating the axisymmetric laminar flow of a compressible gas with arbitrary but constant external flow . the modification is based on the use of the von mises transformation with a subsequent linearization of the viscous term, rather than the linearization of the convective term . pai''s problem (3), originally treated by a finite difference technique, is considered to illustrate the effectiveness of this method .'), (1376, 'some applications in physics of the p-function .', 'masters,j.i.', 'j.chem.phys. 23, 1955, 1865.', 'some applications in physics of the p-function . the mathematical background and typical applications in physics are presented for a recently tabulated function . because of its properties, the p function should prove to be a useful aid in the solution of certain problems in applied mathematics involving surface integrations in cylindrical coordinates . a tabulation of the function in its normalized form is appended . particular attention is paid to the application of the p function to multiple scattering problems involving circular symmetry .'), (1377, 'theoretical investigation of the flow field about blunt nosed bodies in supersonic flight .', 'vaglio-laurin,r. and ferri,a.', 'j. ae. scs. 25, 1958, 761.', 'theoretical investigation of the flow field about blunt nosed bodies in supersonic flight . a numerical method ofr obtaining the solution to the inverse problem of the flow behind a given detached shock to any desired accuracy is presented . the cases of zero and small incidence are considered . the combination of sets of such solutions satisfying prescribed boundary conditions (body shapes) is described . particular attention is devoted to the analysis of the sonic and subsonic region of the flow field . convergence and stability of the stepwise integration from the shock in the elliptic region are discussed . numerical examples are also included .'), (1378, 'blunt-cone pressure distributions at hypersonic mach numbers .', 'andrew f. burke and james t. curtis', 'head, research section, and research aerodynamicist, respectively, applied hypersonic research dept., cornell aeronautical laboratory, inc., buffalo, n.y.', 'blunt-cone pressure distributions at hypersonic mach numbers . the static pressure distributions on the surface of a blunted 7.5 degree-half-angle cone have recently been experimentally determined in the cal 48 inch hypersonic shock tunnel . this facility and the associated instrumentation are described in detail in ref. 1 . these tests covered a mach number range of 8 to 18 at a reynolds number per foot of approximately 1 times 10 to the 5th power., the models included one flat-faced cone and two hemispherically blunted cones .'), (1379, 'hypersonic flight and the re-entry problem .', 'allen, h.j.', 'j. aerospace sce. 25, 217-227, 1958 .', 'hypersonic flight and the re-entry problem . paper reviews the possibilities and some of the main problems of three types of long-range vehicle, the ballistic, the glide, and the skip rocket . performance assessments are made on the basis of an ingenious, if debatable, conversion of the vehicle characteristics to the breguet formula . the problems of aerodynamic heating, aerodynamic loads and stability are briefly discussed, and other aspects of hypersonic flight free molecule flow-dash are touched upon . the results on the whole favor the glide vehicle for manned flight . the subsequent discussion on the paper includes references to multistaging and the use of high-energy fuels .'), (1380, 'the problem of obtaining high lift-drag ratios at supersonic speeds .', 'clinton e. brown and francis e. mclean', 'langley research center, nasa', 'the problem of obtaining high lift-drag ratios at supersonic speeds . the importance of the lift to drag ratio is well known to all aircraft designers since it gives, to a great extent, the aerodynamic efficiency of the airplane . aerodynamic efficiency, however, is only one component of the grand compromise that a completed airplane represents . at subsonic speeds, lift-drag ratios of well over 200 have been measured in wind tunnels on airfoil sections., but few powered aircraft have attained (lift to drag ratio) value of 20 . it is invariably true that the requirements of stability and control, structure, and flight operation all contribute to reducing the design (lift to drag ratio) considerably below those exotic values which can be predicted from unrestricted aerodynamic theory . if, however, a certain range or operating efficiency is required, there is most certainly a minimum if we examine the range equation we see that range is proportional to the lift-drag ratio, the thermopropulsive efficiency, and the logarithm of the initial to final weight ratio . the appearance of the lift-drag ratio as a linear factor in the range equation indicates that every attempt should be made to increase (lift to drag ratio)., however, the search for higher (lift to drag ratio) may lead to strange and unorthodox configurations . most frequently, such configurations are ruled out by the adverse effects of their geometry on the weight ratios . in the present paper, we will deal with the maximum lift-drag ratio problem for conventional configurations having a wing and a body in close proximity to each other . no attempt will be made to select a particular configuration as being the best . however, the promising direction to go from the aerodynamic view will be stressed with the understanding that the other factors may outweight the aerodynamics .'), (1381, 'effect of mach number on boundary layer transition in the presence of pressure rise and surface roughness on an ogive-cylinder body with cold wall conditions .', 'carros,r.j.', 'naca rm a56b15, 1956.', 'effect of mach number on boundary layer transition in the presence of pressure rise and surface roughness on an ogive-cylinder body with cold wall conditions . the effect of mach number variation from 1.8 to 7.4 on boundary-layer transition was investigated on a slender fin-stabilized ogive-cylinder body in free flight at a constant length reynolds number of 13.8 million . the wall to free-stream temperature ratio was constant at a value of 1.0 below mach number 4.5 and at a value of of the test showed that increasing mach number had a very favorable effect of increasing the extent of the laminar boundary layer for a given surface roughness . the transition data, when plotted as a function of a factor indicative of heat transfer, showed that heat transfer was possibly responsible for a good deal of the increase in transition reynolds number with mach number . transition was found to occur farther forward on the sheltered side of the body than on the windward side for angles of attack as low as 0.4 and for all mach numbers . the pressure rise along sheltered-side stream-lines was examined and it was found that the pressure-rise coefficient at the transition point, showed no variation with mach number . data from other sources for different test conditions, when reduced to values of pressure-rise coefficient, were also found to correlate well with that of the present investigation with the exception of data at low subsonic mach numbers . these present results also show that mach number, surface roughness, pressure rise, and length reynolds number all affected boundary-layer transition in the region of theoretical infinite laminar stability to small two-dimensional disturbances as calculated for a flat plate with zero pressure gradient .'), (1382, 'the solution of the equations of the laminar boundary layer for schubauer''s observed pressure distribution for an elliptic cylinder .', 'hartree,d.r.', 'arc r + m.2427, 1949.', 'the solution of the equations of the laminar boundary layer for schubauer''s observed pressure distribution for an elliptic cylinder . the solution of the equations of the laminar boundary layer has been carried out for the pressure distribution for an elliptic cylinder of axial ratio 2.96 .. 1 with its major axis in the direction of the incident stream . the solution has been obtained by the method of hartree and womersley . in applying this method the derivatives parallel to the boundary are replaced by finite differences, so that the partial differential equation of the boundary layer is replaced by an ordinary equation relating the velocity distribution through the boundary layer at one section to that at another, at an interval upstream . by two independent integrations covering the same range by finite intervals of different sizes, it is possible to estimate the errors involved in replacing the derivatives by finite differences, and so to correct for these errors . the process of solution requires the values of the pressure gradient along the solid boundary, and there is a certain tolerance in the derivation of the pressure gradient distribution from a limited number of observed values of pressure . an analysis of schubauer''s pressure distribution is outlined, and the results were used for the main solution calculated . it is found that the solution, for the distribution of pressure gradient so derived, does not give separation of the boundary layer from the solid boundary, whereas the actual flow does separate . it is found that the calculated solution is very sensitive to the pressure distribution, and a comparatively small modification of the pressure distribution gives a solution which does indicate separation close to the point at which separation is observed to occur . the solution with this pressure distribution also gives very good agreement with the observed velocity distribution through the boundary layer at points upstream from separation .'), (1383, 'on the theory of laminar boundary layer involving separation .', 'vonkarman,t. and millikan,c.b.', 'naca r.504, 1934.', 'on the theory of laminar boundary layer involving separation . the paper presents a mathematical discussion of the laminar boundary layer, which was developed with a view of facilitating the investigation of those boundary layers in particular for which the phenomenon of separation occurs . the treatment starts with a slight modification of the form of the boundary layer equation first published by von mises . two approximate solutions of this equation are found, one of which is exact at the outer edge of the boundary layer while the other is exact at the wall . the final solution is obtained by joining these two solutions at the inflection points of the velocity profiles . the final solution is given in terms of a series of universal functions for a fairly broad class of potential velocity distributions outside of the boundary layer . detailed calculations of the boundary layer characteristics are worked out for the case in which the potential velocity is a linear function of the distance from the upstream stagnation point . finally the complete separation point characteristics are determined for the boundary layer associated with a potential velocity distribution made up of two linear functions of the distance from the stagnation point . it appears that extensions of the detailed calculations to more complex potential flows can be fairly easily carried out by using the explicit formulae given in the paper .'), (1384, 'a theoretical calculation of the laminar boundary layer around an elliptic cylinder and its comparison with experiment .', 'millikan,c.b.', 'j. ae. scs. 3, 1936, 91.', 'a theoretical calculation of the laminar boundary layer around an elliptic cylinder and its comparison with experiment . the author, in conjunction with th. von karman, has recently given a new method of approximate integration of the prandtl boundary layer equations, which was developed in order to treat cases in which separation of a laminar boundary layer might be expected . the method was developed because some doubt was felt as to the accuracy with which the well-known pohlhausen analysis would describe conditions in the neighborhood of such a separation point . numerical calculations were carried out for certain cases involving theoretical simplifications, and very considerable discrepancies were found between the results of the new and pohlhausen methods . the method was also used in developing a theory for the maximum lift coefficient of certain classes of airfoils . this theory gave satisfactory agreement with experiment but no direct experimental check on the boundary layer analysis itself has been given up to the present .'), (1385, 'air flow in a separating laminar boundary layer .', 'schubauer,g.b.', 'naca r.527, 1935.', 'air flow in a separating laminar boundary layer . the speed distribution in a laminar boundary layer on the surface of an elliptic cylinder, of major and minor axes 11.78 and 3.98 inches, respectively, has been determined by means of a hot-wire anemometer . the direction of the impinging air stream was parallel to the major axis . special attention was given to the speed distribution in the region of separation and to the exact location of the point of separation . an approximate method, developed by k. pohlhausen for computing the speed distribution, the thickness of the layer, and the point of separation, is described in detail,. and speed-distribution curves calculated by this method are presented for comparison with experiment . good agreement is obtained along the forward part of the cylinder, but pohlhausen''s method fails shortly before the separation point is reached and consequently cannot be used to locate this point . the work was carried out at the national bureau of standards with the cooperation and financial assistance of the national advisory committee for aeronautics .'), (1386, 'analysis and calculation by integral methods of laminar compressible boundary layer with heat transfer and with and without pressure gradient .', 'morduchow,m.', 'naca r.1245, 1955.', 'analysis and calculation by integral methods of laminar compressible boundary layer with heat transfer and with and without pressure gradient . a survey of integral methods in laminar-boundary-layer analysis is first given . a simple and sufficiently accurate method for practical purposes of calculating the properties layer in an axial pressure gradient with heat transfer at the wall is then presented . for flow over a flat plate, the method is applicable for an arbitrarily prescribed distribution of temperature along the surface and for any given constant prandtl number close to unity . for flow in a pressure gradient, the method is based on a prandtl number of unity and a uniform wall temperature . a simple and accurate method of determining the separation point in a compressible flow with an adverse pressure gradient over a surface at a given uniform wall temperature is developed . the analysis is based on an extension of the karman-pohlhausen method to the momentum and thermal energy equations in conjunction with fourth- and especially higher degree velocity and stagnation-enthalpy profiles . from the equations derived here, conclusions regarding the effect of pressure gradient, mach number, and wall temperature on the boundary-layer characteristics are derived and illustrated . in particular the effects on skin-friction, heat-transfer coefficient, separation point in an adverse pressure gradient, and stability of the laminar boundary layer are analyzed .'), (1387, 'the buckling of a square panel under shear, when one pair of opposite edges is clamped and the other pair is simply supported .', 'leggett,d.m.a.', 'arc r + m.1991, 1941.', 'the buckling of a square panel under shear, when one pair of opposite edges is clamped and the other pair is simply supported . reasons for investigation.--for an efficient design of spar with thin sheet web it is important to know the load which will just cause the web to buckle . as stiffeners divide the web into panels, it is required to find the buckling stress of rectangular panels bounded on two sides by spar flanges and on the other two sides by stiffeners . boundary conditions which represent closely this type of edge fixing are clamping (along the flanges) and simple support critical shear stress for a square panel held in this way . conclusions and further development.--it is found that the value of the critical shear stress is almost midway between its values when all four edges are clamped and all four edges are simply supported . the method of solution developed in this report is of very general application, and can be used to investigate the stability of rectangular panels when the loading is any combination of shear and compression or tension, and the edges are clamped or simply supported, and not necessarily all clamped or all simply supported . by an easy extension the method of solution can also be used to find the periods of transverse vibration of rectangular panels for the same types of loading and edge fixing .'), (1388, 'a process for the step-by-step integration of differential equations in an automatic digital computing machine .', 'gills,s.', 'proc. cam. phil. soc. 47, 1951, 96.', 'a process for the step-by-step integration of differential equations in an automatic digital computing machine . it is advantageous in automatic computers to employ methods of integration which do not require preceding function values to be known . from a general theory given by kutta, one such process is chosen giving fourth-order accuracy and requiring the minimum number of storage registers . it is developed into a form which gives the highest attainable accuracy and can be carried out by comparatively few instructions . the errors are studied and a simple example is given .'), (1389, 'numerical construction of detached shock waves .', 'garabedian,p.r.', 'j.math.phys. 36, 1957, 192.', 'numerical construction of detached shock waves . this article proposes a new method for solving the problem of the detached shock wave . if the shock wave is assumed known, a cauchy problem for a system of partial differential equations arises . this has been solved by several authors in the region where the system is elliptic (near the peak of the shock wave) . considering the plane stationary case, the author seeks an analytic continuation of the propagation function (x,y) in the complex plane y = y1 + y2, x real . in the plane (x, y2) the propagation function satisfies a hyperbolic equation (near the peak of the shock wave) . a new cauchy problem is solved and the solution of the original problem obtained by analytic continuation . a numerical example is treated with many details .'), (1390, 'on the numerical calculation of detached bow shock waves in hypersonic flow .', 'garabedian,p.r. and liebersten,h.m.', 'j. ae. scs. 25, 1958, 109.', 'on the numerical calculation of detached bow shock waves in hypersonic flow . a method is described for calculating examples of hypersonic flow with a detached bow shock wave past a bluff axially symmetric body . the form of the shock wave is assumed, and the analysis is based on a cauchy problem for the stream function in the subsonic region, where the motion is governed by a partial differential equation of elliptic type . through analytic continuation into the complex domain, the cauchy problem is reformulated in such a manner that it becomes properly set in the subsonic region . this leads to a stable scheme for computation of the flow by finite differences . numerical examples at freestream mach number 5.8 are presented in which the flow is determined throughout the subsonic region, and, in particular, the detachment distance, the location of the sonic line, and the pressure distribution along the body are calculated . these results are in excellent agreement with experimental data obtained at the california institute of technology .'), (1391, 'shock wave and flow field development in hypersonic re-entry .', 'probstein, r.f.', 'ars. j. 31, 185-194, 1961 .', 'shock wave and flow field development in hypersonic re-entry . a study is made of when and how a shock wave and continuum-type flow field develop in the nose region of a highly cooled blunt body re- entering the atmosphere at hypersonic speed and in a free molecular flow regime . the various types of flow regimes encountered down to low altitude conditions are delineated, and the nature of the flow field and behavior of some of the aerodynamic characteristics are discussed . it is shown that for a highly cooled body, free molecule flow conditions occur at a higher altitude than previously indicated . based on available evidence, it is suggested that kinetic theory solutions, which are essentially modified free molecule results, along with the navier-stokes equations with no surface slip, serve to define all of the flow regimes except for a narrow transitional layer regime which has a height of less than one factor of 10 in free stream density change . it is also suggested that the appearance of a definable shock wave occurs very rapidly in terms of density change near the beginning of the transitional layer regime, and that its location, as in continuum flow, is governed principally by the body geometry, whereas its thickness is determined by a local mean free path .'), (1392, 'the solution of small displacement, stability or vibration problems concerning a flat rectangular panel when the edges are either clamped or simply supported .', 'hopkins,h.g.', 'arc r + m.2234, 1945.', 'the solution of small displacement, stability or vibration problems concerning a flat rectangular panel when the edges are either clamped or simply supported . this report describes an energy method for the exact solution of problems concerning the small displacements, stability or vibration of a flat rectangular panel when the edges are either clamped or simply supported . the influence of stiffeners which are parallel to one pair of edges, and situated in pairs on opposite sides of the panel so that the neutral axis of each stiffener pair lies in the middle surface of the panel, is taken into account . the method is not only applicable to isotropic panels but also to aeolotropic panels when the material of the panel has two directions of elastic symmetry parallel to the edges . the final solution of the problems depends on an infinite set of linear equations for small displacement problems or on an infinite determinantal equation for stability and vibration problems . the important feature of the analysis given is that it enables a direct approach to be made to these equations in any particular problem . it is not in general possible to obtain a direct solution of the final equations and it is necessary to approximate and consider a finite set of linear equations or a finite determinantal equation derived from the more important terms in the analytical expression for the transverse displacement of the panel . here, physical intuition and, if available, experimental data serve as a guide and the accuracy of the final results so obtained is gauged by the rate of convergence with the increase in the number of terms considered . the general method of solution is applied first to the free vibration of a square panel when all the edges are clamped, and second to the buckling of a square panel under shear when three edges are clamped and one edge is simply supported .'), (1393, 'heat transfer near the forward stagnation point of a body of revolution .', 'sibulkin,m.j.', 'j. ae. scs. 19, 1952, 570.', 'heat transfer near the forward stagnation point of a body of revolution . in order to determine the temperature distribution over a body moving through the atmosphere, a knowledge of the local heat-transfer coefficients is required . for slender sharp-nosed bodies, the heat-transfer coefficients are frequently approximated by using the comparable flat-plate values . however, for blunt-nosed bodies, flat-plate solutions are not applicable near the forward stagnation point . since the greatest rate of heat transfer may occur at the forward stagnation point, its value should be investigated . in this note a theoretical solution is given for the heat transfer near the forward stagnation point of a body of revolution assuming laminar, incompressible, low-speed flow . the comparable solution for two-dimensional flow has been given by squire . in the case of a blunt-nosed body moving with supersonic velocity, the flow behind the central portion of the bow wave is subsonic, and it is possible that a low-speed solution, using as /free-stream/ conditions those behind the center of the bow wave, will apply near the stagnation point .'), (1394, 'stagnation point heat transfer measurements in hypersonic low density flow .', 'neice,s.e., rutkowski,r.w. and chann,k.k.', 'j. ae. scs.27, 1960, 387.', 'stagnation point heat transfer measurements in hypersonic low density flow . in hypersonic, low reynolds number flow around a blunt body, the boundary-layer thickness approaches the shock-layer thickness (shock standoff distance) within the region of continuum flow . in this instance, the customary boundary-layer approximations no longer apply . hoshizaki and probstein have obtained solutions to the incompressible navier-stokes equations in the stagnation region of a blunt body in this hypersonic low reynolds number flow . the results indicate that heat-transfer rates are substantially higher than those predicted by incompressible boundary-layer theory . probstein indicated that the actual heat-transfer rates would be correspondingly higher than the predictions of fay and riddell . these findings are of particular importance in the atmospheric entry phase of recoverable satellites .'), (1395, 'low density stagnation point heat transfer measurements in the hypersonic shock tunnel .', 'wilson,m.r. and wittliff,c.e.', 'ars jnl. 32, 1962, 275.', 'low density stagnation point heat transfer measurements in the hypersonic shock tunnel . presents absolute heat-transfer measurements, using resistance thermometer, for hemisphere-cylinder at mach numbers 9.2 to 11.2 . results show vorticity-interaction and viscous-layer effects increase heat transfer above values predicted by boundary-layer theory . data are correlated using cheng''s reynolds-number- dependent parameter . investigation covers vorticity-interaction to incipient-merged-layer regimes (free-stream unit reynolds numbers'), (1396, 'shear buckling of clamped and simply-supported infinitely long plates reinforced by transverse stiffeners .', 'cook,i.t. and rockey,k.c.', 'aero. quart. 13, 1962, 41.', 'shear buckling of clamped and simply-supported infinitely long plates reinforced by transverse stiffeners . the paper presents a solution to the buckling of infinitely long plates clamped along the edges, together with an extension of the solution obtained by stein and fralich for the case when the edges are simply-supported . it is shown that, as a consequence of increasing the edge support from that of a simple support to one in which both deflection and rotation are prevented, the rigidity which an intermediate transverse stiffener must possess in order to support the plate effectively is much reduced . agreement between the theoretical relationships and existing experimental data is good .'), (1397, 'critical shear stress of an infinitely long simply supported plate with transverse stiffeners .', 'stein,m. and fralich,r.w.', 'naca tn.1851, 1949.', 'critical shear stress of an infinitely long simply supported plate with transverse stiffeners . a theoretical solution is given for the critical shear stress of an infinitely long, simply supported, flat plate with identical, equally spaced, transverse stiffeners of zero torsional stiffness . results are obtained by means of the lagrangian multiplier method and are presented in the form of design charts . experimental results are included and are found to be in good agreement with the theoretical results .'), (1398, 'stability of rectangular plates under shear and bending forces .', 'way,s.', 'j. app. mech. 3, 1936, a131.', 'stability of rectangular plates under shear and bending forces . the author first discusses the problem of a plane, simply supported rectangular plate loaded by shearing forces in the plane of the plate on all four edges . there are two stiffeners attached one third and two thirds of the way along the plate . the critical load is calculated for various stiffener rigidities . also, the rigidity necessary to keep the stiffeners straight when the plate buckles is found . this stiffener rigidity is found to be slightly larger than that necessary for a plate with one stiffener and the same panel dimensions as the plate with two stiffeners . the second problem discussed by the author is that of a plane, simply supported rectangular plate loaded by uniformly distributed edge shearing forces in the plane of the plate and linearly distributed tension and compression in the plane of the plate at the ends . the end forces vary from tension, at one corner to, at the other corner, so that their resultant is a bending moment . the presence of the edge shearing forces is found to diminish the critical bending stress in this case . calculations are made for various magnitudes of bending and shearing forces for plates of various proportions .'), (1399, 'buckling of transverse stiffened plates under shear .', 'wang,t.k.', 'j.app.mech. 3,1947, a269.', 'buckling of transverse stiffened plates under shear . this paper presents an analysis of buckling of simply supported rectangular plates reinforced by any number of transverse stiffeners and subjected to shearing forces uniformly distributed along the edges . two cases are considered .. (a) the case of a plate with a finite length,. ing stresses in both cases are expressed in similar forms, that is, in equation (13), and k in equation (24), respectively . design curves are drawn as shown in figs. 2,3, and 5 .'), (1400, 'the buckling shear stress of simply-supported infinitely long plates with transverse stiffeners .', 'kleeman,p.w.', 'arc r + m.2971, 1953.', 'the buckling shear stress of simply-supported infinitely long plates with transverse stiffeners . this report is an extension of previous theoretical investigations of the elastic buckling in shear of flat plates reinforced by transverse stiffeners . the plates are treated as infinitely long and simply-supported along the long sides . stiffeners are spaced at regular intervals, dividing the plate into a number of panels of uniform size . the effect ob bending and torsional stiffnesses of the stiffener upon the buckling shear stress is calculated for the complete range of stiffnesses, for panels with ratios of width to stiffener spacing of graphical forms .'); -- Insert all Cranfield queries INSERT INTO cranfield_full_queries (query_id, query_text) VALUES (1, 'what similarity laws must be obeyed when constructing aeroelastic models of heated high speed aircraft .'), (2, 'what are the structural and aeroelastic problems associated with flight of high speed aircraft .'), (4, 'what problems of heat conduction in composite slabs have been solved so far .'), (8, 'can a criterion be developed to show empirically the validity of flow solutions for chemically reacting gas mixtures based on the simplifying assumption of instantaneous local chemical equilibrium .'), (9, 'what chemical kinetic system is applicable to hypersonic aerodynamic problems .'), (10, 'what theoretical and experimental guides do we have as to turbulent couette flow behaviour .'), (12, 'is it possible to relate the available pressure distributions for an ogive forebody at zero angle of attack to the lower surface pressures of an equivalent ogive forebody at angle of attack .'), (13, 'what methods -dash exact or approximate -dash are presently available for predicting body pressures at angle of attack.'), (15, 'papers on internal /slip flow/ heat transfer studies .'), (18, 'are real-gas transport properties for air available over a wide range of enthalpies and densities .'), (22, 'is it possible to find an analytical, similar solution of the strong blast wave problem in the newtonian approximation .'), (23, 'how can the aerodynamic performance of channel flow ground effect machines be calculated .'), (26, 'what is the basic mechanism of the transonic aileron buzz .'), (27, 'papers on shock-sound wave interaction .'), (29, 'material properties of photoelastic materials .'), (31, 'can the transverse potential flow about a body of revolution be calculated efficiently by an electronic computer .'), (32, 'can the three-dimensional problem of a transverse potential flow about a body of revolution be reduced to a two-dimensional problem .'), (33, 'are experimental pressure distributions on bodies of revolution at angle of attack available .'), (34, 'does there exist a good basic treatment of the dynamics of re-entry combining consideration of realistic effects with relative simplicity of results .'), (35, 'has anyone formally determined the influence of joule heating, produced by the induced current, in magnetohydrodynamic free convection flows under general conditions .'), (39, 'why does the compressibility transformation fail to correlate the high speed data for helium and air .'), (40, 'did anyone else discover that the turbulent skin friction is not over sensitive to the nature of the variation of the viscosity with temperature .'), (41, 'what progress has been made in research on unsteady aerodynamics .'), (49, 'what are the factors which influence the time required to invert large structural matrices .'), (50, 'does a practical flow follow the theoretical concepts for the interaction between adjacent blade rows of a supersonic cascade .'), (51, 'what is a single approximate formula for the displacement thickness of a laminar boundary layer in compressible flow on a flat plate .'), (52, 'how is the design of ring or part ring wings by linear theory affected by thickness .'), (53, 'what application has the linear theory design of curved wings .'), (54, 'what is the effect of cross sectional shape on the flow over simple delta wings with sharp leading edges .'), (55, 'papers on flow visualization on slender conical wings .'), (56, 'what size of end plate can be safely used to simulate two-dimensional flow conditions over a bluff cylindrical body of finite aspect ratio .'), (57, 'to find an approximate correction for thickness in slender thin-wing theory .'), (58, 'how do interference-free longitudinal stability measurements (made using free-flight models) compare with similar measurements made in a low-blockage wind tunnel .'), (59, 'have wind tunnel interference effects been investigated on a systematic basis .'), (61, 'are there any papers dealing with acoustic wave propagation in reacting gases .'), (62, 'has anyone investigated relaxation effects on gaseous heat transfer to a suddenly heated wall .'), (66, 'are there any theoretical methods for predicting base pressure .'), (67, 'does transition in the hypersonic wake depend on body geometry and size'), (68, 'how can one detect transition phenomena in boundary layers .'), (69, 'how can one detect transition phenomena in hypersonic wakes .'), (71, 'has anyone investigated and developed a simple model for the vortex wake behind a cruciform wing .'), (72, 'what is a criterion that the transonic flow around an airfoil with a round leading edge be validly analyzed by the linearized transonic flow theory .'), (74, 'can the transonic flow around an arbitrary smooth thin airfoil be analysed in a simple approximate way .'), (79, 'what are the details of the rigorous kinetic theory of gases . (chapman-enskog theory) .'), (80, 'has anyone investigated the effect of surface mass transfer on hypersonic viscous interactions .'), (81, 'what is the combined effect of surface heat and mass transfer on hypersonic flow .'), (82, 'what are the existing solutions for hypersonic viscous interactions over an insulated flat plate .'), (83, 'what controls leading-edge attachment at transonic speeds .'), (84, 'can the three-point boundary-value problem for the blasius equation be integrated numerically, using suitable transformations, without iteration on the boundary conditions .'), (85, 'what are the effects of small amounts of gas rarefaction on the characteristics of the boundary layers on slender bodies of revolution .'), (86, 'what is the available information pertaining to boundary layers on very slender bodies of revolution in continuum flow (the ?transverse curvature effect) .'), (87, 'what is the available information pertaining to the effect of slight rarefaction on boundary layer flows (the ?slip? effect) .'), (93, 'what investigations have been made of the flow field about a body moving through a rarefied, partially ionized gas in the presence of a magnetic field .'), (94, 'how is the heat transfer downstream of the mass transfer region effected by mass transfer at the nose of a blunted cone .'), (95, 'to what extent can the available information for incompressible boundary layers be applied to problems involving compressible boundary layers .'), (97, 'to what extent can readily available steady-state aerodynamic data be utilized to predict lifting-surface flutter characteristics .'), (98, 'what are the significant steady and non-steady flow characteristics which affect the flutter mechanism .'), (99, 'is it possible to determine rates of forced convective heat transfer from heated cylinders of non-circular cross-section, (the fluid flow being along the generators) .'), (100, 'how much is known about boundary layer flows along non-circular cylinders .'), (101, 'is there any simple, but practical, method for numerical integration of the mixing problem (i.e. the blasius problem with three-point boundary conditions) .'), (102, 'does there exist a closed-form expression for the local heat transfer around a yawed cylinder .'), (103, 'how far around a cylinder and under what conditions of flow, if any, is the velocity just outside of the boundary layer a linear function of the distance around the cylinder .'), (104, 'where can i find pressure data on surfaces of swept cylinders .'), (105, 'can''t the static deflection shapes be used in predicting flutter in place of vibrational shapes . if so, can we provide a justification by means of an example .'), (106, 'does the boundary layer on a flat plate in a shear flow induce a pressure gradient .'), (107, 'can the procedure of matching inner and outer solutions for a viscous flow problem be applied when the main stream is a shear flow .'), (108, 'can series expansions be found for the boundary layer on a flat plate in a shear flow .'), (109, 'what possible techniques are available for computing the injection distribution corresponding to an isothermal transpiration cooled hemisphere .'), (110, 'what is known regarding asymptotic solutions to the exact boundary layer equations .'), (111, 'previous solutions to the boundary layer similarity equations .'), (112, 'experimental results on hypersonic viscous interaction .'), (113, 'what has been done about viscous interactions in relatively low reynolds number flows, particularly at high mach numbers .'), (114, 'what role does the effect of chemical reaction (particularly when out of equilibrium) play in the similitude laws governing hypersonic flows over slender aerodynamic bodies .'), (116, 'how significant is the possible pressure of a dissociated free stream with respect to the realization of hypersonic simulation in high enthalpy wind tunnels .'), (118, 'do the discrepancies among current analyses of the vorticity effect on stagnation-point heat transfer result primarily from the differences in the viscosity-temperature law assumed .'), (119, 'how far can one trust the linear viscosity-temperature solution assumed in some of the analyses of hypersonic shock layer at low reynolds number .'), (120, 'how close is the comparison of the shock layer theory with existing experiments in the low reynolds number (merged-layer) regime .'), (121, 'has anyone explained the kink in the surge line of a multi-stage axial compressor .'), (122, 'have any aerodynamic derivatives been measured at hypersonic mach numbers and comparison been made with theoretical work .'), (123, 'are methods of measuring aerodynamic derivatives available which could be adopted for use in short running time facilities .'), (126, 'what are wind-tunnel corrections for a two-dimensional aerofoil mounted off-centre in a tunnel .'), (128, 'how do kuchemann''s and multhopp''s methods for calculating lift distributions on swept wings in subsonic flow compare with each other and with experiment .'), (130, 'what is the present state of the theory of quasi-conical flows .'), (131, 'references on the methods available for accurately estimating aerodynamic heat transfer to conical bodies for both laminar and turbulent flow .'), (132, 'what parameters can seriously influence natural transition from laminar to turbulent flow on a model in a wind tunnel .'), (133, 'can a satisfactory experimental technique be developed for measuring oscillatory derivatives on slender sting-mounted models in supersonic wind tunnels .'), (135, 'what effect has the boundary layer in modifying the basic inviscid flow behind the shock, neglecting effects of leading edge and corner .'), (136, 'how does a satellite orbit contract under the action of air drag in an atmosphere in which the scale height varies with altitude .'), (137, 'how is the flow at transonic speeds about a delta wing different from that on a closely-related tapered sweptback wing .'), (138, 'recent data on shock-induced boundary-layer separation .'), (139, 'what interference effects are likely at transonic speeds .'), (140, 'given complete freedom in the design of an airplane, what procedure would be used in order to minimize sonic boom intensity, and is there a limit to the degree of minimizing that can be accomplished .'), (141, 'can methane-air combustion product be used as a hypersonic test medium and predict, within experimental accuracies, the results obtained in air .'), (142, 'what is the theoretical heat transfer rate at the stagnation point of a blunt body .'), (143, 'what is the theoretical heat transfer distribution around a hemisphere .'), (145, 'has anyone investigated the unsteady lift distributions on finite wings in subsonic flow .'), (146, 'what information is available for dynamic response of airplanes to gusts or blasts in the subsonic regime .'), (147, 'will forward or apex located controls be effective at low subsonic speeds and how do they compare with conventional trailing-edge flaps .'), (148, 'given that an uncontrolled vehicle will tumble as it enters an atmosphere, is it possible to predict when and how it will stop tumbling and its subsequent motion .'), (149, 'what are the effects of initial imperfections on the elastic buckling of cylindrical shells under axial compression .'), (150, 'why does the incremental theory and the deformation theory of plastic stress-strain relationship differ greatly when applied to stability problems .'), (152, 'basic dynamic characteristics of structures continuous over many spans .'), (153, 'is the information on the buckling of sandwich sphere available .'), (154, 'can the load deformation characteristics of a beam be obtained with the material being inelastic and a non uniform temperature being present .'), (155, 'what is the effect of an internal liquid column on the breathing vibrations of a cylindrical shell .'), (156, 'experimental techniques in shell vibration .'), (157, 'in summarizing theoretical and experimental work on the behaviour of a typical aircraft structure in a noise environment is it possible to develop a design procedure .'), (158, 'what data is there on the fatigue of structures under acoustic loading .'), (160, 'panels subjected to aerodynamic heating .'), (161, 'can increasing the edge loading of a plate beyond the critical value for buckling change the buckling mode .'), (163, 'have the effects of an elastic edge restraint been considered in previous papers on panel flutter .'), (164, 'has the solution of the clamped plate problem, in the classical theory of bending, been reduced to two successive membrane boundary value problems .'), (165, 'what data exists on oscillatory aerodynamic forces on control surfaces at transonic mach numbers .'), (167, 'it is not likely that the airforces on a wing of general planform oscillating in transonic flow can be determined by purely analytical methods . is it possible to determine the airforces on a single particular planform, such as the rectangular one by such method .'), (168, 'is the problem of similarity for representative investigations of aeroelastic effects in heated flow as intractable as previous investigations imply .'), (169, 'what is the magnitude and distribution of lift over the cone and the cylindrical portion of a cone-cylinder configuration .'), (170, 'is there any information on how the addition of a /boat-tail/ affects the normal force on the body of various angles of incidence .'), (171, 'what are the aerodynamic interference effects on the fin lift and body lift of a fin-body combination .'), (173, 'what is the effect of initial axisymmetric deviations from circularity on the non linear (large-deflection) load-deflection response of cylinders under hydrostatic pressure .'), (175, 'are previous analyses of circumferential thermal buckling of circular cylindrical shells unnecessarily involved or even inaccurate due to the assumed forms of buckling mode .'), (176, 'what papers are there dealing with circumferential buckling either thermal buckling or due to mechanical loading .'), (177, 'what analytical investigations have been made of the stability of conical shells . how do the results compare with experiment .'), (181, 'has any work been done on determining the nature of compressible viscous flow in a straight channel .'), (182, 'in what areas, other than low density wind tunnel flows, is viscous compressible flow in slender channels a problem . what analytical investigations have been made of the stability of conical shells . how do the results compare with experiment .'), (183, 'jet interference with supersonic flow -dash experimental papers .'), (184, 'thrust vector control by fluid injection -dash papers .'), (187, 'is it possible to obtain a reasonably simple analytical solution to the heat equation for an exponential (in time) heat input .'), (189, 'has anyone programmed a pump design method for a high-speed digital computer .'), (190, 'has anyone derived simplified pump design equation from the fundamental three-dimensional equations for incompressible nonviscous flow .'), (196, 'what are the flutter characteristics of the exposed skin panels of the x-15 vertical stabilizer when subjected to aerodynamic heating .'), (200, 'what agreement is found between theoretically predicted instability times and experimentally measured collapse times for compressed columns in creep .'), (201, 'theoretical studies of creep buckling .'), (202, 'experimental studies of creep buckling .'), (203, 'is it possible to correlate the results on the creep buckling of widely different structures within the framework of a single theory .'), (204, 'what are the experimental results for the creep buckling of columns .'), (205, 'what are the results for the creep buckling of round tubes under external pressure .'), (206, 'have any analytical studies been conducted on the time-to-failure mechanism associated with creep collapse for a long circular cylindrical shell which exhibits both primary and secondary creep as well as elastic deformations under various distributed force systems .'), (208, 'has the effect of initial stresses, on the frequencies of vibration of circular cylindrical shells, been investigated .'), (209, 'has the effect of the change of initial pressure due to deformation, on the frequencies of vibration of circular cylindrical shells been investigated .'), (210, 'what are the discontinuity stresses at junctions in pressurized structures .'), (211, 'what analytical solutions are available for stresses in edge-loaded shells of revolution .'), (212, 'what dome contours minimize discontinuity stresses when used as closures on cylindrical pressure vessels .'), (213, 'what general solutions for the stresses in pressurized shells of revolution are available .'), (214, 'can studies of pure membrane cylinders having no wall bending stiffness but maintaining their shape by virtue of internal pressure provide any insight into the behaviour of pressurized cylinders with finite wall stiffness .'), (215, 'what are the best experimental data and classical small deflection theory analyses available for pressurized cylinders in bending .'), (216, 'does a membrane theory exist by which the behaviour of pressurized membrane cylinders in bending can be predicted .'), (217, 'what are the equations which define the stability of simply supported corrugated core sandwich cylinders .'), (218, 'papers on small deflection theory for buckling of sandwich cylinders .'), (219, 'has anyone developed an analysis which accurately establishes the large deflection behaviour of conical shells .'), (223, 'what is the magnitude of second-order wing-body interference at high supersonic mach number .'), (224, 'what is the best theoretical method for calculating pressure on the surface of a wing alone .'), (225, 'how can the effect of the boundary-layer on wing pressure be calculated, and what is its magnitude .'), (226, 'how should the navier-stokes difference equations be solved .'), (227, 'which iterative method for solving linear elliptic difference equations is most rapidly convergent .'), (230, 'technical report on measurement of ablation during flight .'), (231, 'what qualitative and quantitative material is available on ablation materials research .'), (232, 'have flow fields been calculated for blunt-nosed bodies and compared with experiment for a wide range of free stream conditions and body shapes .'), (233, 'what are the available properties of high-temperature air .'), (234, 'what is the magnitude of aerodynamic damping in flexible vibration modes of a slender body of revolution characteristic of launch vehicles .'), (241, 'compressive circumferential stresses in a torispherical shell reveal the possibility of buckling under internal pressure . has anyone investigated for which ranges of shell parameters these stresses are sufficiently large to cause elastic buckling .'), (245, 'is there an integral method to give a single and sufficiently accurate method of calculating the laminar separate point for various incompressible and compressible boundary layers with zero heat transfer .'), (246, 'what accurate or exact solutions of the laminar separation point for various incompressible and compressible boundary layers with zero heat transfer are available .'), (247, 'can the hypersonic similarity results be applied to the technique of predicting surface pressures of an ogive forebody at angle of attack .'), (250, 'what determines the onset of shock-induced boundary-layer separation .'), (251, 'are the stable profiles of a compressible boundary layer induced by a moving wave known .'), (252, 'are there experimental results on the stability of a compressible boundary layer induced by a moving wave .'), (253, 'exact solution methods for calculating the ablative mass loss of a material ablating at high temperatures in a hypersonic flight environment .'), (254, 'what approximate solutions are known to the direct problem of transonic flow in the throat of a nozzle, i.e. finding the flow in a given nozzle .'), (255, 'what approximate solutions are known to the indirect problem of transonic flow in the throat of a nozzle, i.e. finding a nozzle which has a given axial velocity distribution .'), (257, 'why do users of orthodox pitot-static tubes often find that the calibrations appear to be,. - (a) significantly different from those formerly specified, (b) wildly variable at low reynolds numbers .'), (259, 'has a comparison been made between interference-free drag measurements using free-flight models and similar measurements made in a low-blockage wind tunnel .'), (261, 'solution of the blasius problem with three-point boundary conditions .'), (264, 'references on lyapunov''s method on the stability of linear differential equations with periodic coefficients .'), (265, 'obtain all papers and reports that contain shock detachment distance data .'), (266, 'work on flow in channels at low reynolds numbers .'), (267, 'some approximate analytical heat conduction solutions using methods other than biot''s principle .'), (268, 'what mode of stalling can be expected for each stage of an axial compressor .'), (269, 'has a criterion been established for determining the axial compressor choking line .'), (272, 'has a theory of quasi-conical flows been developed, in supersonic linearised theory, for which the upwash distribution on the lifting surface, apart from being a homogeneous function in the co-ordinate, is permitted to have a quite general functional form .'), (273, 'how does scale height vary with altitude in an atmosphere .'), (274, 'jet interference with supersonic flows theoretical papers .'), (275, 'effects of leading-edge bluntness on the flutter characteristics of some square-planform double-wedge airfoils at mach numbers less than 15.4.'), (277, 'what factors have been shown to have a primary influence on sonic boom strength .'), (283, 'work on small-oscillation re-entry motions .'), (284, 'experimental studies on panel flutter .'), (285, 'how can wing-body, flow field interference effects be approximated rationally .'), (288, 'has anyone analytically or experimentally investigated the effects of internal pressure on the buckling of circular-cylindrical shells under bending .'), (291, 'what theoretical and experimental work has been done on the excitation and response of typical structures in a noise environment .'), (292, 'is there a design method for calculating thermal fatigue endurances of components of various types and sizes in a variety of circumstances .'), (293, 'will an analysis of panel flutter based on arbitrarily assumed modes of deformation prove satisfactory, and if so, what is the minimum number of modes that need be considered .'), (294, 'what is the criterion for true panel flutter, as opposed to small amplitude vibration arising from acoustic disturbances .'), (295, 'papers dealing with uniformly loaded sectors .'), (296, 'general methods of solving clamped plate problems .'), (297, 'how can the analytical solution of the buckling strength of a uniform circular cylinder loaded in axial compression be refined so as to lower the buckling load .'), (298, 'in the problem of the buckling strength of uniform circular cylinders loaded in axial compression, does the linear solution help with improving the non-linear one .'), (299, 'the problem of similarity for representative investigation of aeroelastic effects in a flow with the absence of heating effects .'), (300, 'how is fatigue damage estimated using the normal long-hand method .'), (301, 'is there any information available on the difference in the effects of various edge conditions on the buckling of cylindrical shells .'), (303, 'have non-linear large deflection analyses been conducted for shell shapes other than conical .'), (304, 'are asymptotic methods sufficiently accurate in the determination of pre-buckling stresses in torispherical shells, or must we resort to numerical methods .'), (306, 'what are the nonequilibrium chemical constituents in the viscous shock layer ahead of a blunt re-entry vehicle .'), (314, 'how accurate are existing analytical theories in estimating pressure distributions on cones at incidence, at hypersonic speeds .'), (315, 'are simple empirical methods of any use for estimating pressure distribution in cones .'), (316, 'do viscous effects seriously modify pressure distributions .'), (317, 'has anyone investigated theoretically whether surface flexibility can stabilize a laminar boundary layer .'), (321, 'how do subsonic and transonic flutter data measured in the new langley transonic dynamics tunnel compare with similar data obtained in other facilities .'), (323, 'how do large changes in new mass ratio quantitatively affect wing-flutter boundaries .'), (327, 'what is the effect of the shape of the drag polar of a lifting spacecraft on the amount of reduction in maximum deceleration obtainable by continuously varying the aerodynamic coefficients during re-entry .'), (331, 'what are the physical significance and characteristics of separated laminar and turbulent boundary layer flows .'), (332, 'has anyone analytically investigated the stabilizing influence of soft elastic cores on the buckling strength of cylindrical shells subjected to non-uniform external pressure .'), (333, 'what papers are available on the buckling of empty cylindrical shells under non-uniform pressure .'), (335, 'what effect do thermal stresses have on the compressive buckling strength of ring-stiffened cylinders .'), (336, 'what is the effect on cylinder buckling of a circumferential stress system that varies in the axial direction .'), (338, 'can non-linear shallow shell analysis be reduced to an engineering technique by use of the matrix .'), (339, 'is it possible to predict the shape of a shroud which will allow simulation of the nose region flow field for a sphere in hypersonic flow .'), (340, 'what investigations have been made of the wave system created by a static pressure distribution over a liquid surface .'), (347, 'has anyone investigated the effect of shock generated vorticity on heat transfer to a blunt body .'), (348, 'what is the heat transfer to a blunt body in the absence of vorticity .'), (349, 'what are the general effects on flow fields when the reynolds number is small .'), (352, 'find a calculation procedure applicable to all incompressible laminar boundary layer flow problems having good accuracy and reasonable computation time .'), (353, 'papers applicable to this problem (calculation procedures for laminar incompressible flow with arbitrary pressure gradient) .'), (355, 'has anyone investigated the shear buckling of stiffened plates .'), (356, 'papers on shear buckling of unstiffened rectangular plates under shear .'), (360, 'in practice, how close to reality are the assumptions that the flow in a hypersonic shock tube using nitrogen is non-viscous and in thermodynamic equilibrium .'), (365, 'what design factors can be used to control lift-drag ratios at mach numbers above 5 .'); -- Insert expected BM25 rankings (from reference implementation with rank_bm25) INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (1, 184, 1, 26.9306), (1, 486, 2, 25.0290), (1, 13, 3, 24.5083), (1, 12, 4, 21.6931), (1, 1268, 5, 20.7507), (1, 51, 6, 18.5473), (1, 878, 7, 16.9867), (1, 746, 8, 16.2631), (1, 875, 9, 16.2098), (1, 792, 10, 15.7770), (2, 12, 1, 46.7355), (2, 746, 2, 34.9082), (2, 792, 3, 31.2848), (2, 14, 4, 28.5485), (2, 1089, 5, 27.8692), (2, 141, 6, 27.5183), (2, 51, 7, 26.8913), (2, 172, 8, 26.7776), (2, 724, 9, 25.4725), (2, 810, 10, 25.3916), (4, 399, 1, 33.8574), (4, 5, 2, 27.7760), (4, 181, 3, 27.4079), (4, 144, 4, 27.2114), (4, 485, 5, 24.1227), (4, 542, 6, 22.4812), (4, 826, 7, 18.2206), (4, 828, 8, 17.9881), (4, 584, 9, 17.7884), (4, 980, 10, 17.4085), (8, 166, 1, 60.4063), (8, 488, 2, 46.4740), (8, 1189, 3, 44.0338), (8, 1061, 4, 42.5243), (8, 185, 5, 41.8160), (8, 1085, 6, 39.6523), (8, 1252, 7, 39.1814), (8, 1255, 8, 39.0805), (8, 1275, 9, 38.9365), (8, 236, 10, 38.2772), (9, 103, 1, 22.1655), (9, 1296, 2, 17.6810), (9, 1032, 3, 17.1076), (9, 943, 4, 17.0732), (9, 746, 5, 15.6854), (9, 1272, 6, 15.5515), (9, 540, 7, 15.2913), (9, 28, 8, 15.2751), (9, 625, 9, 14.9580), (9, 552, 10, 14.8529); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (10, 491, 1, 24.1835), (10, 257, 2, 20.0736), (10, 315, 3, 18.7022), (10, 121, 4, 17.5892), (10, 386, 5, 17.1517), (10, 1273, 6, 16.8873), (10, 651, 7, 16.6105), (10, 294, 8, 16.2522), (10, 767, 9, 16.0524), (10, 406, 10, 16.0242), (12, 492, 1, 101.8693), (12, 56, 2, 71.2319), (12, 57, 3, 68.7536), (12, 973, 4, 66.6767), (12, 434, 5, 64.3064), (12, 1231, 6, 63.0180), (12, 122, 7, 62.2814), (12, 1040, 8, 62.0730), (12, 248, 9, 57.8157), (12, 232, 10, 54.7030), (13, 122, 1, 33.2403), (13, 711, 2, 29.0532), (13, 232, 3, 28.6580), (13, 492, 4, 28.4718), (13, 907, 5, 28.3554), (13, 443, 6, 27.0710), (13, 1082, 7, 26.2319), (13, 556, 8, 24.9331), (13, 569, 9, 24.6405), (13, 237, 10, 24.5076), (15, 21, 1, 24.8641), (15, 45, 2, 19.9060), (15, 22, 3, 19.1787), (15, 550, 4, 19.0796), (15, 306, 5, 17.8790), (15, 270, 6, 17.4464), (15, 571, 7, 16.4618), (15, 303, 8, 14.8020), (15, 1215, 9, 14.4836), (15, 326, 10, 13.8854), (18, 493, 1, 40.0424), (18, 302, 2, 31.7219), (18, 949, 3, 30.7637), (18, 1199, 4, 30.6034), (18, 691, 5, 30.6011), (18, 1286, 6, 29.3041), (18, 1009, 7, 28.4414), (18, 1214, 8, 28.4215), (18, 1264, 9, 26.9164), (18, 781, 10, 26.5190); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (22, 495, 1, 49.2152), (22, 110, 2, 34.3760), (22, 654, 3, 34.1961), (22, 262, 4, 33.4014), (22, 1327, 5, 33.2473), (22, 667, 6, 33.0857), (22, 304, 7, 33.0586), (22, 1238, 8, 32.7276), (22, 341, 9, 32.4722), (22, 1356, 10, 32.0198), (23, 624, 1, 39.8507), (23, 1232, 2, 22.7958), (23, 543, 3, 22.7617), (23, 650, 4, 21.4138), (23, 649, 5, 21.0395), (23, 1164, 6, 19.5386), (23, 1223, 7, 19.3796), (23, 1221, 8, 18.8033), (23, 1209, 9, 18.6898), (23, 939, 10, 18.5389), (26, 496, 1, 40.4183), (26, 903, 2, 28.8059), (26, 520, 3, 25.5876), (26, 313, 4, 23.5004), (26, 38, 5, 22.3034), (26, 199, 6, 22.2685), (26, 1268, 7, 22.1814), (26, 440, 8, 22.1566), (26, 880, 9, 20.4607), (26, 797, 10, 20.1107), (27, 64, 1, 21.3411), (27, 256, 2, 14.9068), (27, 132, 3, 14.2345), (27, 291, 4, 13.9282), (27, 170, 5, 13.6349), (27, 1364, 6, 13.1356), (27, 568, 7, 12.7388), (27, 439, 8, 11.9410), (27, 65, 9, 11.9251), (27, 798, 10, 11.7567), (29, 462, 1, 20.0617), (29, 463, 2, 16.9749), (29, 553, 3, 14.4014), (29, 761, 4, 14.1666), (29, 1097, 5, 13.8261), (29, 1098, 6, 13.7226), (29, 1117, 7, 13.2740), (29, 82, 8, 12.3981), (29, 1025, 9, 12.3360), (29, 1340, 10, 12.3290); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (31, 498, 1, 43.0305), (31, 106, 2, 36.4870), (31, 1255, 3, 29.0844), (31, 869, 4, 27.9502), (31, 1301, 5, 27.2543), (31, 93, 6, 27.0614), (31, 1108, 7, 26.9388), (31, 927, 8, 26.8149), (31, 801, 9, 26.4162), (31, 1259, 10, 26.4062), (32, 1108, 1, 46.4396), (32, 106, 2, 44.0691), (32, 1301, 3, 43.1403), (32, 1281, 4, 41.0052), (32, 410, 5, 40.8019), (32, 987, 6, 39.0473), (32, 498, 7, 38.6014), (32, 1304, 8, 38.3695), (32, 2, 9, 38.2959), (32, 266, 10, 38.1644), (33, 248, 1, 34.7439), (33, 56, 2, 31.2119), (33, 498, 3, 30.6222), (33, 234, 4, 30.5263), (33, 197, 5, 30.1820), (33, 57, 6, 29.7047), (33, 232, 7, 29.4457), (33, 1005, 8, 27.3242), (33, 122, 9, 26.8622), (33, 124, 10, 26.6144), (34, 82, 1, 39.3744), (34, 1219, 2, 34.8502), (34, 706, 3, 34.7481), (34, 1346, 4, 34.7087), (34, 274, 5, 34.3128), (34, 713, 6, 33.8339), (34, 1279, 7, 33.2312), (34, 1296, 8, 32.3989), (34, 716, 9, 31.7575), (34, 1295, 10, 31.5736), (35, 500, 1, 58.3499), (35, 268, 2, 40.0470), (35, 88, 3, 40.0459), (35, 270, 4, 34.9734), (35, 450, 5, 33.5598), (35, 87, 6, 32.5316), (35, 966, 7, 30.1014), (35, 123, 8, 28.7336), (35, 190, 9, 28.1102), (35, 1194, 10, 27.6136); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (39, 502, 1, 44.4621), (39, 302, 2, 30.2758), (39, 68, 3, 26.9025), (39, 271, 4, 26.1492), (39, 343, 5, 26.1112), (39, 377, 6, 25.9742), (39, 628, 7, 25.4159), (39, 96, 8, 25.1009), (39, 686, 9, 25.0172), (39, 538, 10, 24.9915), (40, 125, 1, 46.7278), (40, 560, 2, 46.2935), (40, 16, 3, 44.1920), (40, 307, 4, 43.3768), (40, 413, 5, 43.0974), (40, 406, 6, 42.7489), (40, 962, 7, 42.6921), (40, 1212, 8, 42.4949), (40, 565, 9, 41.9345), (40, 165, 10, 40.8868), (41, 892, 1, 19.5125), (41, 902, 2, 18.9811), (41, 28, 3, 18.9734), (41, 1287, 4, 17.6099), (41, 718, 5, 17.2943), (41, 1151, 6, 15.0672), (41, 11, 7, 14.0829), (41, 237, 8, 14.0240), (41, 251, 9, 13.8743), (41, 360, 10, 13.8035), (49, 756, 1, 29.0238), (49, 46, 2, 27.1496), (49, 12, 3, 23.8703), (49, 51, 4, 21.3618), (49, 724, 5, 21.3125), (49, 883, 6, 21.0704), (49, 47, 7, 21.0535), (49, 75, 8, 19.9990), (49, 1170, 9, 19.6605), (49, 893, 10, 19.5952), (50, 215, 1, 39.4017), (50, 121, 2, 38.7981), (50, 277, 3, 37.8089), (50, 511, 4, 36.1558), (50, 214, 5, 34.0460), (50, 212, 6, 33.5854), (50, 216, 7, 33.4979), (50, 945, 8, 32.8475), (50, 989, 9, 32.7325), (50, 540, 10, 31.9066); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (51, 145, 1, 49.4284), (51, 611, 2, 47.4564), (51, 382, 3, 47.2412), (51, 96, 4, 45.3755), (51, 4, 5, 44.0264), (51, 346, 6, 42.2882), (51, 306, 7, 42.1771), (51, 629, 8, 41.9202), (51, 309, 9, 41.8789), (51, 1355, 10, 41.4750), (52, 1035, 1, 32.3178), (52, 1031, 2, 31.5745), (52, 888, 3, 31.3141), (52, 1129, 4, 29.2571), (52, 1178, 5, 29.0649), (52, 1362, 6, 28.3072), (52, 428, 7, 27.8698), (52, 1070, 8, 27.3647), (52, 1176, 9, 26.9974), (52, 838, 10, 26.4895), (53, 251, 1, 22.1205), (53, 28, 2, 17.2495), (53, 1362, 3, 16.6285), (53, 893, 4, 16.6167), (53, 677, 5, 16.2386), (53, 367, 6, 16.0948), (53, 36, 7, 15.9747), (53, 503, 8, 15.8517), (53, 675, 9, 15.7289), (53, 674, 10, 15.6468), (54, 465, 1, 44.9665), (54, 420, 2, 44.9555), (54, 612, 3, 41.0582), (54, 146, 4, 40.0419), (54, 514, 5, 39.3818), (54, 211, 6, 39.0623), (54, 901, 7, 38.3814), (54, 222, 8, 38.2130), (54, 464, 9, 37.8267), (54, 601, 10, 37.6540), (55, 513, 1, 18.8314), (55, 420, 2, 17.2344), (55, 147, 3, 16.3139), (55, 464, 4, 16.2567), (55, 633, 5, 16.1220), (55, 600, 6, 15.6765), (55, 466, 7, 15.2920), (55, 545, 8, 15.2590), (55, 602, 9, 15.2178), (55, 388, 10, 15.0636); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (56, 751, 1, 36.4095), (56, 1209, 2, 32.2681), (56, 228, 3, 28.9263), (56, 1082, 4, 28.2802), (56, 1153, 5, 28.0078), (56, 1339, 6, 27.9771), (56, 895, 7, 27.8755), (56, 433, 8, 27.8417), (56, 247, 9, 27.7204), (56, 916, 10, 27.1891), (57, 752, 1, 34.3195), (57, 1186, 2, 24.6308), (57, 467, 3, 21.1603), (57, 921, 4, 20.8727), (57, 683, 5, 19.8514), (57, 230, 6, 19.7167), (57, 468, 7, 19.1754), (57, 530, 8, 19.0804), (57, 919, 9, 18.9996), (57, 1259, 10, 18.5981), (58, 516, 1, 60.6433), (58, 431, 2, 44.2438), (58, 141, 3, 39.6677), (58, 252, 4, 36.8344), (58, 904, 5, 32.5209), (58, 808, 6, 32.4456), (58, 1153, 7, 30.3660), (58, 755, 8, 29.9795), (58, 1000, 9, 29.5960), (58, 1163, 10, 29.5586), (59, 516, 1, 24.2481), (59, 799, 2, 21.2949), (59, 431, 3, 20.8459), (59, 1153, 4, 19.8454), (59, 1350, 5, 19.5590), (59, 175, 6, 18.3959), (59, 795, 7, 18.3561), (59, 1155, 8, 18.1983), (59, 1062, 9, 17.4495), (59, 252, 10, 17.2400), (61, 1203, 1, 19.8600), (61, 488, 2, 18.5979), (61, 319, 3, 18.1047), (61, 1208, 4, 18.0238), (61, 1327, 5, 17.6749), (61, 875, 6, 17.2234), (61, 75, 7, 16.2284), (61, 654, 8, 15.4943), (61, 1061, 9, 15.3106), (61, 739, 10, 15.2897); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (62, 123, 1, 20.6862), (62, 1268, 2, 20.1871), (62, 550, 3, 19.0009), (62, 518, 4, 18.9367), (62, 546, 5, 18.9297), (62, 789, 6, 18.8761), (62, 274, 7, 18.8413), (62, 646, 8, 18.7591), (62, 135, 9, 18.6649), (62, 44, 10, 18.4986), (66, 186, 1, 19.3582), (66, 179, 2, 16.2425), (66, 461, 3, 16.2063), (66, 188, 4, 14.8083), (66, 232, 5, 14.6090), (66, 923, 6, 13.3555), (66, 121, 7, 13.2227), (66, 283, 8, 12.6195), (66, 1088, 9, 12.4125), (66, 1311, 10, 12.3068), (67, 536, 1, 25.6675), (67, 1238, 2, 24.8055), (67, 1205, 3, 23.4541), (67, 1373, 4, 22.7758), (67, 976, 5, 22.5850), (67, 338, 6, 21.8147), (67, 85, 7, 21.8001), (67, 186, 8, 21.4942), (67, 25, 9, 21.0855), (67, 154, 10, 20.9344), (68, 315, 1, 18.6716), (68, 1268, 2, 14.6012), (68, 207, 3, 14.2124), (68, 1257, 4, 14.0786), (68, 610, 5, 13.8807), (68, 959, 6, 13.7810), (68, 37, 7, 12.8621), (68, 281, 8, 12.4051), (68, 526, 9, 12.3734), (68, 272, 10, 12.3510), (69, 536, 1, 18.2062), (69, 37, 2, 15.4682), (69, 17, 3, 13.3910), (69, 315, 4, 13.1891), (69, 207, 5, 12.9645), (69, 1257, 6, 12.8380), (69, 281, 7, 12.4051), (69, 171, 8, 12.1826), (69, 1152, 9, 11.6812), (69, 1205, 10, 11.4936); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (71, 289, 1, 48.3356), (71, 433, 2, 39.7627), (71, 927, 3, 32.5240), (71, 288, 4, 31.3499), (71, 520, 5, 31.0336), (71, 432, 6, 30.6921), (71, 229, 7, 29.7014), (71, 1277, 8, 29.6812), (71, 1091, 9, 25.2864), (71, 1202, 10, 25.1805), (72, 521, 1, 66.2764), (72, 526, 2, 56.2245), (72, 496, 3, 50.0587), (72, 439, 4, 48.3051), (72, 468, 5, 46.8662), (72, 903, 6, 46.2476), (72, 797, 7, 45.7415), (72, 503, 8, 44.7371), (72, 469, 9, 44.0679), (72, 467, 10, 44.0449), (74, 467, 1, 31.7135), (74, 903, 2, 27.6506), (74, 469, 3, 26.9950), (74, 404, 4, 26.7374), (74, 496, 5, 26.6542), (74, 521, 6, 26.4030), (74, 39, 7, 26.1194), (74, 503, 8, 25.7203), (74, 916, 9, 25.5693), (74, 1194, 10, 25.1171), (79, 103, 1, 27.8565), (79, 1190, 2, 27.3609), (79, 1199, 3, 26.7530), (79, 357, 4, 24.7932), (79, 236, 5, 24.2993), (79, 108, 6, 24.1993), (79, 367, 7, 23.8901), (79, 1031, 8, 23.6357), (79, 760, 9, 23.4993), (79, 1226, 10, 23.2067), (80, 305, 1, 28.7307), (80, 525, 2, 27.6343), (80, 123, 3, 24.2581), (80, 84, 4, 22.3343), (80, 540, 5, 22.1414), (80, 1159, 6, 20.8576), (80, 1299, 7, 20.7158), (80, 353, 8, 20.6458), (80, 274, 9, 20.3742), (80, 1204, 10, 20.0768); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (81, 305, 1, 33.9842), (81, 525, 2, 31.7491), (81, 123, 3, 30.2247), (81, 84, 4, 28.8089), (81, 623, 5, 28.5123), (81, 655, 6, 28.3738), (81, 353, 7, 28.2665), (81, 1159, 8, 28.2115), (81, 37, 9, 27.7148), (81, 36, 10, 27.5357), (82, 525, 1, 35.4386), (82, 568, 2, 29.0470), (82, 307, 3, 28.6644), (82, 305, 4, 28.1313), (82, 304, 5, 26.9810), (82, 310, 6, 26.5859), (82, 1200, 7, 26.3221), (82, 663, 8, 25.2068), (82, 25, 9, 24.9269), (82, 570, 10, 24.8138), (83, 526, 1, 29.6740), (83, 440, 2, 28.7110), (83, 797, 3, 21.9061), (83, 683, 4, 21.8192), (83, 222, 5, 19.3320), (83, 334, 6, 15.3970), (83, 521, 7, 15.2888), (83, 796, 8, 14.5302), (83, 655, 9, 14.2846), (83, 1320, 10, 14.0461), (84, 527, 1, 44.5303), (84, 321, 2, 41.2233), (84, 322, 3, 38.5108), (84, 320, 4, 36.5707), (84, 1370, 5, 36.5085), (84, 349, 6, 35.8845), (84, 1235, 7, 35.7669), (84, 476, 8, 35.0239), (84, 366, 9, 32.6700), (84, 356, 10, 31.5159), (85, 192, 1, 45.0682), (85, 435, 2, 43.9948), (85, 1301, 3, 43.4412), (85, 1259, 4, 42.7335), (85, 922, 5, 42.5940), (85, 332, 6, 41.8964), (85, 248, 7, 41.6275), (85, 541, 8, 41.4348), (85, 233, 9, 41.4137), (85, 494, 10, 41.1148); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (86, 494, 1, 51.0369), (86, 528, 2, 39.4657), (86, 326, 3, 38.8224), (86, 1281, 4, 37.8564), (86, 261, 5, 37.8335), (86, 1301, 6, 36.6657), (86, 25, 7, 36.4714), (86, 1259, 8, 36.2173), (86, 36, 9, 36.0471), (86, 23, 10, 35.9693), (87, 1215, 1, 36.4908), (87, 306, 2, 35.6589), (87, 251, 3, 34.6715), (87, 96, 4, 34.4914), (87, 36, 5, 34.1998), (87, 550, 6, 33.5475), (87, 534, 7, 33.1969), (87, 8, 8, 31.9414), (87, 1204, 9, 31.6585), (87, 1214, 10, 31.5511), (93, 208, 1, 63.4332), (93, 1221, 2, 52.7209), (93, 531, 3, 52.3467), (93, 269, 4, 48.6698), (93, 298, 5, 47.6393), (93, 61, 6, 47.2880), (93, 407, 7, 47.2830), (93, 1255, 8, 46.5871), (93, 267, 9, 46.4850), (93, 625, 10, 46.4445), (94, 123, 1, 62.1796), (94, 84, 2, 51.2413), (94, 44, 3, 51.1082), (94, 1307, 4, 50.0594), (94, 274, 5, 49.8960), (94, 305, 6, 49.1999), (94, 1300, 7, 48.2982), (94, 651, 8, 47.8223), (94, 365, 9, 47.3142), (94, 1213, 10, 47.0286), (95, 460, 1, 36.7468), (95, 17, 2, 35.3120), (95, 1185, 3, 35.1431), (95, 377, 4, 31.2327), (95, 1301, 5, 30.7300), (95, 344, 6, 29.6908), (95, 251, 7, 29.6429), (95, 406, 8, 29.4531), (95, 1154, 9, 28.5170), (95, 798, 10, 28.1117); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (97, 753, 1, 41.3128), (97, 14, 2, 24.3384), (97, 783, 3, 23.7657), (97, 1339, 4, 21.4635), (97, 251, 5, 21.0945), (97, 441, 6, 20.4202), (97, 1331, 7, 20.3386), (97, 685, 8, 20.0704), (97, 711, 9, 19.7875), (97, 627, 10, 19.2513), (98, 753, 1, 34.6229), (98, 1181, 2, 31.1558), (98, 380, 3, 28.2358), (98, 916, 4, 27.0819), (98, 1339, 5, 27.0185), (98, 52, 6, 26.5661), (98, 117, 7, 26.5022), (98, 444, 8, 25.7866), (98, 1296, 9, 24.7233), (98, 894, 10, 24.4779), (99, 754, 1, 52.6397), (99, 1159, 2, 41.6935), (99, 270, 3, 41.4845), (99, 873, 4, 40.5032), (99, 788, 5, 40.1549), (99, 120, 6, 39.8976), (99, 785, 7, 39.5524), (99, 789, 8, 36.1910), (99, 872, 9, 34.4032), (99, 269, 10, 33.9240), (100, 292, 1, 17.1554), (100, 801, 2, 16.5340), (100, 842, 3, 16.4973), (100, 786, 4, 15.7888), (100, 1105, 5, 15.4827), (100, 785, 6, 14.7136), (100, 1309, 7, 14.5802), (100, 541, 8, 14.2759), (100, 610, 9, 13.9093), (100, 458, 10, 13.6729), (101, 321, 1, 52.5502), (101, 527, 2, 52.4665), (101, 322, 3, 51.4617), (101, 320, 4, 44.2134), (101, 476, 5, 36.0303), (101, 1235, 6, 35.4904), (101, 150, 7, 34.2922), (101, 1240, 8, 33.8150), (101, 107, 9, 33.7947), (101, 1219, 10, 33.5060); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (102, 539, 1, 34.7435), (102, 566, 2, 31.8147), (102, 564, 3, 30.9283), (102, 435, 4, 27.9866), (102, 333, 5, 27.4374), (102, 1258, 6, 26.9642), (102, 352, 7, 26.5195), (102, 122, 8, 26.5133), (102, 565, 9, 26.0288), (102, 944, 10, 25.6239), (103, 1268, 1, 58.3686), (103, 1258, 2, 57.8294), (103, 105, 3, 57.5194), (103, 459, 4, 57.1462), (103, 452, 5, 56.7767), (103, 787, 6, 55.9128), (103, 381, 7, 55.8994), (103, 785, 8, 55.4218), (103, 148, 9, 55.1454), (103, 1081, 10, 53.7557), (104, 843, 1, 16.3911), (104, 1121, 2, 16.1018), (104, 678, 3, 15.2519), (104, 420, 4, 15.2382), (104, 763, 5, 15.0335), (104, 204, 6, 14.8516), (104, 435, 7, 14.4859), (104, 889, 8, 14.3405), (104, 1084, 9, 14.2075), (104, 1334, 10, 14.1969), (105, 730, 1, 42.5791), (105, 1204, 2, 40.4662), (105, 1225, 3, 39.8891), (105, 914, 4, 37.0598), (105, 928, 5, 36.8491), (105, 775, 6, 35.9046), (105, 827, 7, 35.0789), (105, 239, 8, 34.9154), (105, 1252, 9, 34.7173), (105, 1024, 10, 34.5905), (106, 388, 1, 40.7672), (106, 3, 2, 37.8585), (106, 1106, 3, 36.3762), (106, 664, 4, 34.2438), (106, 393, 5, 34.0664), (106, 1370, 6, 33.6299), (106, 180, 7, 33.5186), (106, 22, 8, 32.8091), (106, 1237, 9, 32.7189), (106, 4, 10, 32.7147); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (107, 128, 1, 55.9570), (107, 191, 2, 45.4898), (107, 375, 3, 44.9225), (107, 629, 4, 44.8607), (107, 366, 5, 44.2413), (107, 1182, 6, 43.5569), (107, 329, 7, 43.2051), (107, 2, 8, 42.1098), (107, 149, 9, 41.5577), (107, 1224, 10, 41.1816), (108, 180, 1, 33.2448), (108, 664, 2, 32.8317), (108, 4, 3, 32.5956), (108, 2, 4, 32.4498), (108, 3, 5, 32.2212), (108, 306, 6, 31.5808), (108, 388, 7, 31.3520), (108, 393, 8, 31.0792), (108, 1107, 9, 30.9803), (108, 309, 10, 30.8857), (109, 628, 1, 45.5086), (109, 365, 2, 24.8866), (109, 1240, 3, 24.7448), (109, 1248, 4, 24.6330), (109, 662, 5, 23.8994), (109, 272, 6, 22.8657), (109, 560, 7, 22.8377), (109, 344, 8, 22.5601), (109, 1263, 9, 22.5044), (109, 661, 10, 22.4951), (110, 128, 1, 22.4927), (110, 540, 2, 22.0972), (110, 292, 3, 19.8429), (110, 306, 4, 19.4058), (110, 105, 5, 19.2567), (110, 365, 6, 18.8969), (110, 117, 7, 18.8882), (110, 1043, 8, 18.7769), (110, 94, 9, 18.3422), (110, 648, 10, 18.2866), (111, 540, 1, 22.4003), (111, 349, 2, 19.3317), (111, 300, 3, 17.3334), (111, 62, 4, 16.9522), (111, 180, 5, 16.4435), (111, 1236, 6, 16.2319), (111, 1311, 7, 16.1350), (111, 494, 8, 15.8874), (111, 365, 9, 15.6063), (111, 1260, 10, 15.2139); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (112, 305, 1, 15.8778), (112, 63, 2, 15.1562), (112, 525, 3, 14.6523), (112, 573, 4, 14.1661), (112, 540, 5, 14.0881), (112, 570, 6, 13.5599), (112, 26, 7, 12.9650), (112, 572, 8, 12.8787), (112, 308, 9, 12.7076), (112, 1253, 10, 11.8443), (113, 315, 1, 23.3473), (113, 139, 2, 23.1542), (113, 193, 3, 21.3804), (113, 799, 4, 20.6144), (113, 1082, 5, 20.6117), (113, 244, 6, 20.2814), (113, 1078, 7, 20.2205), (113, 666, 8, 19.8772), (113, 941, 9, 19.7910), (113, 416, 10, 19.3614), (114, 332, 1, 58.1012), (114, 541, 2, 53.3732), (114, 577, 3, 39.5446), (114, 625, 4, 39.1698), (114, 401, 5, 37.0095), (114, 573, 6, 35.8238), (114, 101, 7, 35.3604), (114, 1296, 8, 34.7043), (114, 574, 9, 34.1416), (114, 42, 10, 34.0936), (116, 1153, 1, 48.1882), (116, 625, 2, 44.5204), (116, 364, 3, 42.8722), (116, 608, 4, 40.5580), (116, 280, 5, 40.4735), (116, 26, 6, 40.1695), (116, 1161, 7, 39.9383), (116, 283, 8, 38.9282), (116, 876, 9, 38.3984), (116, 1142, 10, 37.5724), (118, 55, 1, 41.6414), (118, 324, 2, 40.0479), (118, 73, 3, 39.8085), (118, 54, 4, 38.9898), (118, 565, 5, 38.1924), (118, 50, 6, 37.8917), (118, 82, 7, 37.4409), (118, 645, 8, 37.1197), (118, 306, 9, 36.6814), (118, 328, 10, 36.6717); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (119, 667, 1, 38.0984), (119, 666, 2, 37.7840), (119, 630, 3, 37.6267), (119, 328, 4, 34.2829), (119, 1082, 5, 33.0189), (119, 1007, 6, 32.5067), (119, 1391, 7, 32.1619), (119, 43, 8, 32.1197), (119, 171, 9, 31.9334), (119, 255, 10, 31.9156), (120, 329, 1, 49.0122), (120, 630, 2, 41.0043), (120, 667, 3, 40.9954), (120, 1391, 4, 37.7557), (120, 1264, 5, 37.7496), (120, 315, 6, 35.8307), (120, 572, 7, 35.5771), (120, 668, 8, 35.3836), (120, 272, 9, 35.1941), (120, 187, 10, 34.3144), (121, 589, 1, 46.3465), (121, 543, 2, 41.8299), (121, 588, 3, 38.0557), (121, 237, 4, 36.8340), (121, 216, 5, 35.0124), (121, 590, 6, 34.0824), (121, 138, 7, 32.2482), (121, 426, 8, 29.5618), (121, 591, 9, 28.2615), (121, 214, 10, 25.7499), (122, 199, 1, 30.6752), (122, 544, 2, 27.9341), (122, 755, 3, 27.2945), (122, 567, 4, 26.0248), (122, 597, 5, 24.2895), (122, 1066, 6, 24.0952), (122, 748, 7, 23.6565), (122, 52, 8, 21.9540), (122, 251, 9, 20.4920), (122, 600, 10, 20.0492), (123, 544, 1, 32.4610), (123, 790, 2, 29.6748), (123, 878, 3, 25.4822), (123, 1335, 4, 24.2402), (123, 814, 5, 23.7942), (123, 599, 6, 23.1316), (123, 748, 7, 22.6929), (123, 606, 8, 22.4101), (123, 1336, 9, 22.0693), (123, 1113, 10, 22.0122); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (126, 714, 1, 35.3899), (126, 755, 2, 31.2495), (126, 799, 3, 30.4412), (126, 631, 4, 29.5495), (126, 1163, 5, 28.2361), (126, 1153, 6, 25.8667), (126, 598, 7, 25.3525), (126, 1336, 8, 25.2601), (126, 140, 9, 25.1295), (126, 372, 10, 24.8957), (128, 677, 1, 45.5418), (128, 1332, 2, 45.5033), (128, 1339, 3, 44.6274), (128, 1334, 4, 44.0149), (128, 678, 5, 43.5933), (128, 676, 6, 40.5865), (128, 204, 7, 35.7617), (128, 206, 8, 35.2460), (128, 704, 9, 35.1805), (128, 315, 10, 35.1054), (130, 898, 1, 29.3173), (130, 753, 2, 28.6104), (130, 1275, 3, 28.5714), (130, 428, 4, 28.3420), (130, 633, 5, 28.0981), (130, 680, 6, 26.6811), (130, 97, 7, 25.8549), (130, 236, 8, 25.2212), (130, 1196, 9, 24.9258), (130, 224, 10, 24.7283), (131, 142, 1, 35.6087), (131, 294, 2, 35.4369), (131, 435, 3, 33.9588), (131, 1310, 4, 33.5497), (131, 522, 5, 32.9021), (131, 1281, 6, 32.7591), (131, 959, 7, 32.4774), (131, 122, 8, 31.8646), (131, 1263, 9, 31.1612), (131, 661, 10, 30.7302), (132, 84, 1, 30.7913), (132, 314, 2, 30.3978), (132, 294, 3, 29.7347), (132, 710, 4, 29.3975), (132, 9, 5, 28.8721), (132, 315, 6, 28.8181), (132, 184, 7, 28.7131), (132, 207, 8, 27.6578), (132, 1287, 9, 27.5802), (132, 293, 10, 27.3287); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (133, 755, 1, 57.4171), (133, 790, 2, 40.8707), (133, 594, 3, 32.3900), (133, 431, 4, 29.0941), (133, 280, 5, 28.5161), (133, 188, 6, 28.3901), (133, 179, 7, 27.8107), (133, 600, 8, 27.2532), (133, 1155, 9, 25.7840), (133, 214, 10, 25.3404), (135, 1228, 1, 41.4590), (135, 439, 2, 40.4265), (135, 1198, 3, 40.1910), (135, 26, 4, 39.9237), (135, 797, 5, 39.8198), (135, 572, 6, 39.0002), (135, 625, 7, 38.0352), (135, 334, 8, 37.6413), (135, 310, 9, 37.1598), (135, 547, 10, 36.8784), (136, 548, 1, 64.6359), (136, 617, 2, 60.5517), (136, 613, 3, 58.5898), (136, 622, 4, 55.6707), (136, 616, 5, 54.0107), (136, 614, 6, 48.1355), (136, 615, 7, 39.1603), (136, 162, 8, 39.0104), (136, 1150, 9, 38.9688), (136, 163, 10, 37.5302), (137, 757, 1, 44.2775), (137, 793, 2, 43.8374), (137, 801, 3, 40.6939), (137, 797, 4, 40.1869), (137, 917, 5, 38.2401), (137, 916, 6, 35.9997), (137, 420, 7, 35.6077), (137, 683, 8, 35.3134), (137, 924, 9, 34.6105), (137, 1341, 10, 34.5559), (138, 265, 1, 20.8957), (138, 1187, 2, 18.9177), (138, 358, 3, 14.4529), (138, 311, 4, 14.4273), (138, 1228, 5, 14.1484), (138, 293, 6, 13.9049), (138, 1216, 7, 13.1890), (138, 798, 8, 12.8479), (138, 291, 9, 12.6163), (138, 294, 10, 12.6055); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (139, 252, 1, 20.5124), (139, 799, 2, 19.4094), (139, 924, 3, 17.8386), (139, 431, 4, 16.0334), (139, 315, 5, 15.7522), (139, 925, 6, 15.1428), (139, 672, 7, 15.1153), (139, 800, 8, 14.7774), (139, 316, 9, 14.1949), (139, 610, 10, 13.8561), (140, 758, 1, 56.4259), (140, 1247, 2, 55.7334), (140, 806, 3, 54.8322), (140, 683, 4, 53.9812), (140, 807, 5, 53.6704), (140, 1252, 6, 52.9405), (140, 804, 7, 52.9136), (140, 811, 8, 52.6739), (140, 810, 9, 51.7592), (140, 28, 10, 50.4550), (141, 635, 1, 52.7038), (141, 691, 2, 39.8404), (141, 1143, 3, 30.3318), (141, 68, 4, 30.2620), (141, 1336, 5, 28.0566), (141, 572, 6, 26.2307), (141, 1374, 7, 26.2019), (141, 355, 8, 26.0284), (141, 1157, 9, 25.5314), (141, 552, 10, 25.3766), (142, 1393, 1, 42.1175), (142, 283, 2, 38.9833), (142, 983, 3, 36.4738), (142, 1161, 4, 36.4117), (142, 101, 5, 36.3423), (142, 1104, 6, 34.7553), (142, 662, 7, 33.7869), (142, 1099, 8, 33.5362), (142, 84, 9, 32.5136), (142, 559, 10, 32.0127), (143, 662, 1, 24.8882), (143, 635, 2, 24.8230), (143, 628, 3, 22.5324), (143, 36, 4, 22.3592), (143, 283, 5, 21.9236), (143, 101, 6, 20.5185), (143, 1191, 7, 19.8370), (143, 689, 8, 19.6400), (143, 370, 9, 19.6031), (143, 553, 10, 19.0571); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (145, 637, 1, 36.3362), (145, 1339, 2, 28.7121), (145, 699, 3, 28.6111), (145, 705, 4, 27.6221), (145, 1334, 5, 26.0812), (145, 1332, 6, 26.0227), (145, 698, 7, 25.4756), (145, 779, 8, 25.4521), (145, 465, 9, 24.9047), (145, 247, 10, 23.9278), (146, 779, 1, 32.2651), (146, 251, 2, 29.9442), (146, 1214, 3, 28.1540), (146, 1331, 4, 26.6827), (146, 36, 5, 26.4132), (146, 1289, 6, 26.4100), (146, 187, 7, 26.3374), (146, 1334, 8, 26.2951), (146, 728, 9, 25.7491), (146, 781, 10, 25.5585), (147, 638, 1, 38.4850), (147, 673, 2, 35.0655), (147, 1343, 3, 27.5843), (147, 315, 4, 27.2272), (147, 1095, 5, 24.8456), (147, 288, 6, 23.7359), (147, 792, 7, 23.5652), (147, 712, 8, 22.8047), (147, 433, 9, 22.2968), (147, 222, 10, 21.7539), (148, 639, 1, 39.3798), (148, 715, 2, 35.6140), (148, 42, 3, 33.1464), (148, 962, 4, 33.0791), (148, 719, 5, 32.8985), (148, 613, 6, 32.2916), (148, 958, 7, 32.2737), (148, 811, 8, 31.6128), (148, 32, 9, 30.9730), (148, 553, 10, 30.4406), (149, 1122, 1, 53.7935), (149, 760, 2, 52.4511), (149, 822, 3, 51.7872), (149, 739, 4, 46.8746), (149, 741, 5, 45.6825), (149, 1051, 6, 45.4678), (149, 1068, 7, 44.9269), (149, 1126, 8, 44.6116), (149, 1171, 9, 43.9303), (149, 1131, 10, 42.3962); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (150, 817, 1, 47.6341), (150, 819, 2, 45.9046), (150, 820, 3, 42.7633), (150, 1119, 4, 42.3110), (150, 760, 5, 40.7265), (150, 823, 6, 40.7205), (150, 825, 7, 39.1674), (150, 837, 8, 38.7694), (150, 826, 9, 38.1241), (150, 818, 10, 37.3467), (152, 910, 1, 19.5862), (152, 729, 2, 15.6063), (152, 1007, 3, 14.9604), (152, 516, 4, 14.4281), (152, 720, 5, 14.2511), (152, 913, 6, 13.2524), (152, 1331, 7, 13.0583), (152, 1001, 8, 12.6654), (152, 1006, 9, 12.6320), (152, 1092, 10, 12.5497), (153, 761, 1, 33.5224), (153, 1127, 2, 24.9147), (153, 1214, 3, 24.5239), (153, 951, 4, 24.2682), (153, 1126, 5, 24.1803), (153, 1027, 6, 23.9982), (153, 720, 7, 23.7266), (153, 1048, 8, 23.3666), (153, 251, 9, 23.3435), (153, 770, 10, 22.8119), (154, 762, 1, 50.6262), (154, 1024, 2, 39.8465), (154, 837, 3, 36.1974), (154, 1019, 4, 35.9240), (154, 644, 5, 35.7775), (154, 462, 6, 34.9243), (154, 1294, 7, 34.8012), (154, 29, 8, 34.3863), (154, 241, 9, 34.2767), (154, 825, 10, 33.3591), (155, 848, 1, 65.3763), (155, 764, 2, 62.6141), (155, 847, 3, 44.9487), (155, 953, 4, 38.8208), (155, 1126, 5, 36.7432), (155, 117, 6, 36.4250), (155, 42, 7, 33.8478), (155, 897, 8, 33.4132), (155, 1068, 9, 33.2035), (155, 739, 10, 32.9578); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (156, 764, 1, 13.2828), (156, 42, 2, 11.8999), (156, 908, 3, 10.4953), (156, 878, 4, 10.4521), (156, 844, 5, 10.4432), (156, 846, 6, 10.2638), (156, 552, 7, 10.1839), (156, 847, 8, 9.8926), (156, 740, 9, 9.8618), (156, 1392, 10, 9.7407), (157, 640, 1, 60.3904), (157, 725, 2, 51.0603), (157, 29, 3, 47.0623), (157, 909, 4, 46.0591), (157, 658, 5, 46.0517), (157, 883, 6, 45.3953), (157, 220, 7, 45.0533), (157, 722, 8, 43.9439), (157, 724, 9, 43.8856), (157, 1039, 10, 43.6154), (158, 75, 1, 35.6407), (158, 720, 2, 32.4621), (158, 726, 3, 27.3354), (158, 724, 4, 27.0882), (158, 881, 5, 26.5951), (158, 884, 6, 25.7993), (158, 909, 7, 24.8997), (158, 723, 8, 24.5132), (158, 727, 9, 23.9807), (158, 951, 10, 23.5195), (160, 51, 1, 17.2976), (160, 711, 2, 16.0365), (160, 5, 3, 15.8336), (160, 859, 4, 15.7364), (160, 1008, 5, 15.5313), (160, 391, 6, 15.4541), (160, 627, 7, 14.4635), (160, 31, 8, 14.3074), (160, 982, 9, 13.8430), (160, 709, 10, 12.5528), (161, 1387, 1, 37.9496), (161, 1131, 2, 36.5440), (161, 1117, 3, 36.4352), (161, 1020, 4, 36.2892), (161, 642, 5, 36.2782), (161, 1071, 6, 35.7020), (161, 889, 7, 34.4989), (161, 1013, 8, 34.1490), (161, 1172, 9, 34.1429), (161, 1032, 10, 33.8984); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (163, 894, 1, 38.6089), (163, 857, 2, 30.1092), (163, 627, 3, 28.9779), (163, 658, 4, 28.6912), (163, 766, 5, 28.4050), (163, 859, 6, 27.9890), (163, 858, 7, 27.4198), (163, 15, 8, 27.2145), (163, 1008, 9, 27.0545), (163, 442, 10, 26.7860), (164, 641, 1, 68.1474), (164, 730, 2, 54.8805), (164, 734, 3, 43.4712), (164, 733, 4, 41.4042), (164, 422, 5, 39.4431), (164, 15, 6, 38.9989), (164, 736, 7, 38.7555), (164, 731, 8, 38.3396), (164, 700, 9, 37.6485), (164, 826, 10, 37.5773), (165, 704, 1, 22.4124), (165, 685, 2, 22.0839), (165, 1328, 3, 21.4468), (165, 708, 4, 21.2241), (165, 748, 5, 21.0154), (165, 815, 6, 20.4948), (165, 755, 7, 20.0481), (165, 52, 8, 19.7834), (165, 265, 9, 19.7135), (165, 1115, 10, 18.7978), (167, 895, 1, 78.7430), (167, 918, 2, 64.2217), (167, 757, 3, 61.2550), (167, 315, 4, 59.5054), (167, 1333, 5, 57.6517), (167, 775, 6, 57.4186), (167, 561, 7, 56.7800), (167, 1375, 8, 56.6271), (167, 521, 9, 55.8010), (167, 1392, 10, 55.7286), (168, 540, 1, 36.2769), (168, 184, 2, 33.4952), (168, 625, 3, 32.7127), (168, 13, 4, 31.7974), (168, 486, 5, 30.9684), (168, 1274, 6, 29.4417), (168, 782, 7, 29.3222), (168, 1319, 8, 29.2623), (168, 878, 9, 29.0565), (168, 1068, 10, 28.8115); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (169, 522, 1, 53.0113), (169, 1038, 2, 45.6328), (169, 605, 3, 45.6190), (169, 272, 4, 44.7615), (169, 759, 5, 44.2897), (169, 1218, 6, 43.5364), (169, 1202, 7, 43.1434), (169, 123, 8, 42.7715), (169, 423, 9, 42.7239), (169, 801, 10, 41.9191), (170, 229, 1, 48.8389), (170, 282, 2, 47.5526), (170, 230, 3, 46.8047), (170, 924, 4, 43.3357), (170, 60, 5, 42.6619), (170, 927, 6, 42.6213), (170, 907, 7, 42.4795), (170, 1164, 8, 42.1844), (170, 1304, 9, 41.7427), (170, 196, 10, 41.3245), (171, 229, 1, 41.3767), (171, 923, 2, 40.8025), (171, 230, 3, 40.6896), (171, 545, 4, 40.6814), (171, 924, 5, 40.5564), (171, 246, 6, 40.2631), (171, 431, 7, 39.3420), (171, 1239, 8, 38.4298), (171, 803, 9, 38.4065), (171, 1243, 10, 37.7788), (173, 928, 1, 43.8609), (173, 926, 2, 43.0766), (173, 739, 3, 39.9049), (173, 897, 4, 39.8670), (173, 1034, 5, 39.3713), (173, 1068, 6, 38.9902), (173, 1024, 7, 38.9872), (173, 744, 8, 37.0095), (173, 763, 9, 36.4394), (173, 1131, 10, 36.2095), (175, 769, 1, 62.5913), (175, 887, 2, 54.6738), (175, 890, 3, 47.6953), (175, 886, 4, 42.5006), (175, 1117, 5, 41.8407), (175, 1172, 6, 41.7972), (175, 741, 7, 41.5812), (175, 1146, 8, 41.5164), (175, 885, 9, 41.2535), (175, 926, 10, 41.1498); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (176, 769, 1, 31.8181), (176, 887, 2, 31.3988), (176, 1146, 3, 29.4875), (176, 888, 4, 27.1154), (176, 890, 5, 26.4166), (176, 837, 6, 25.4559), (176, 742, 7, 23.9794), (176, 1056, 8, 23.5775), (176, 580, 9, 23.4254), (176, 891, 10, 22.5220), (177, 1068, 1, 36.0137), (177, 898, 2, 35.9095), (177, 934, 3, 31.7516), (177, 371, 4, 30.4286), (177, 828, 5, 29.9576), (177, 1070, 6, 29.5187), (177, 1287, 7, 29.1376), (177, 78, 8, 29.0390), (177, 600, 9, 28.3276), (177, 251, 10, 27.6375), (181, 1360, 1, 32.7711), (181, 775, 2, 29.7385), (181, 917, 3, 28.4576), (181, 1221, 4, 27.9414), (181, 186, 5, 27.0241), (181, 394, 6, 26.9085), (181, 131, 7, 26.7248), (181, 537, 8, 26.4699), (181, 112, 9, 26.0167), (181, 270, 10, 25.7864), (182, 941, 1, 61.3277), (182, 1068, 2, 52.6888), (182, 625, 3, 49.7199), (182, 371, 4, 47.9734), (182, 828, 5, 47.6405), (182, 73, 6, 47.2763), (182, 600, 7, 47.0785), (182, 504, 8, 46.4648), (182, 185, 9, 45.7824), (182, 270, 10, 45.4659), (183, 993, 1, 15.6418), (183, 997, 2, 15.4566), (183, 1074, 3, 14.6389), (183, 1093, 4, 14.2815), (183, 970, 5, 13.8056), (183, 1195, 6, 13.5946), (183, 173, 7, 13.1010), (183, 1075, 8, 13.0748), (183, 1082, 9, 12.7165), (183, 1083, 10, 12.6906); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (184, 1288, 1, 29.6403), (184, 1326, 2, 29.1763), (184, 974, 3, 28.6301), (184, 773, 4, 19.0891), (184, 1095, 5, 17.1597), (184, 1265, 6, 12.5353), (184, 1328, 7, 11.4242), (184, 397, 8, 11.3770), (184, 237, 9, 10.7569), (184, 374, 10, 10.7517), (187, 869, 1, 38.8329), (187, 585, 2, 37.6611), (187, 6, 3, 35.7195), (187, 5, 4, 35.3929), (187, 91, 5, 35.0340), (187, 1375, 6, 33.2853), (187, 267, 7, 33.2156), (187, 582, 8, 33.2094), (187, 981, 9, 33.1628), (187, 1392, 10, 32.9218), (189, 945, 1, 46.9933), (189, 92, 2, 25.3086), (189, 1246, 3, 23.9558), (189, 1063, 4, 23.4592), (189, 762, 5, 22.7712), (189, 745, 6, 22.2285), (189, 1043, 7, 19.8610), (189, 988, 8, 19.7135), (189, 328, 9, 19.5897), (189, 248, 10, 18.6933), (190, 945, 1, 37.4009), (190, 988, 2, 31.1787), (190, 989, 3, 29.1074), (190, 987, 4, 25.6652), (190, 985, 5, 23.8205), (190, 527, 6, 23.6989), (190, 314, 7, 22.2524), (190, 660, 8, 20.6105), (190, 278, 9, 19.9988), (190, 1310, 10, 19.5774), (196, 859, 1, 58.6476), (196, 948, 2, 56.2778), (196, 1008, 3, 42.9555), (196, 766, 4, 41.4621), (196, 729, 5, 40.9033), (196, 391, 6, 39.2751), (196, 627, 7, 36.7502), (196, 858, 8, 36.6664), (196, 727, 9, 36.4773), (196, 546, 10, 36.1276); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (200, 950, 1, 47.5877), (200, 1021, 2, 42.8704), (200, 1014, 3, 30.8762), (200, 1023, 4, 30.8158), (200, 1022, 5, 30.4540), (200, 1025, 6, 30.2373), (200, 1019, 7, 29.8552), (200, 1034, 8, 28.6798), (200, 866, 9, 27.8386), (200, 1028, 10, 27.4336), (201, 1021, 1, 17.9365), (201, 950, 2, 17.4500), (201, 1017, 3, 16.8005), (201, 1013, 4, 16.0072), (201, 1026, 5, 15.8026), (201, 1023, 6, 15.3739), (201, 1020, 7, 15.3575), (201, 1015, 8, 15.2302), (201, 1014, 9, 15.1564), (201, 1029, 10, 15.0891), (202, 950, 1, 16.7498), (202, 951, 2, 16.2656), (202, 1013, 3, 16.0072), (202, 1026, 4, 15.8026), (202, 1028, 5, 15.4366), (202, 1023, 6, 15.3739), (202, 1020, 7, 15.3575), (202, 1015, 8, 15.2302), (202, 1014, 9, 15.1564), (202, 1029, 10, 15.0891), (203, 951, 1, 47.2035), (203, 866, 2, 46.3223), (203, 1012, 3, 45.2710), (203, 1017, 4, 41.4259), (203, 1013, 5, 40.9058), (203, 1031, 6, 39.8060), (203, 1014, 7, 39.3263), (203, 1028, 8, 39.1119), (203, 1020, 9, 38.7635), (203, 1015, 10, 38.6913), (204, 1026, 1, 35.0203), (204, 1020, 2, 33.6122), (204, 950, 3, 32.5183), (204, 951, 4, 32.4868), (204, 1018, 5, 32.1662), (204, 1013, 6, 32.0547), (204, 1028, 7, 32.0444), (204, 1029, 8, 31.9915), (204, 1027, 9, 31.3616), (204, 1019, 10, 31.2665); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (205, 951, 1, 33.9748), (205, 1021, 2, 30.7038), (205, 829, 3, 30.3935), (205, 1125, 4, 30.3179), (205, 1034, 5, 29.5949), (205, 1131, 6, 29.0738), (205, 1033, 7, 28.9588), (205, 831, 8, 28.5949), (205, 1031, 9, 28.1428), (205, 550, 10, 28.1324), (206, 952, 1, 94.5465), (206, 1034, 2, 55.1930), (206, 1026, 3, 54.9632), (206, 1023, 4, 53.0375), (206, 1013, 5, 45.3722), (206, 1025, 6, 45.3492), (206, 1021, 7, 44.8716), (206, 1024, 8, 44.3845), (206, 1017, 9, 43.5717), (206, 837, 10, 42.7265), (208, 897, 1, 40.2650), (208, 844, 2, 38.5439), (208, 846, 3, 38.1026), (208, 764, 4, 36.5033), (208, 841, 5, 35.9572), (208, 953, 6, 35.6390), (208, 763, 7, 35.6192), (208, 760, 8, 35.3468), (208, 887, 9, 34.2979), (208, 1068, 10, 34.1233), (209, 826, 1, 49.7399), (209, 844, 2, 49.4538), (209, 897, 3, 49.1812), (209, 763, 4, 48.4402), (209, 887, 5, 48.1596), (209, 846, 6, 48.1071), (209, 760, 7, 47.4069), (209, 841, 8, 44.8250), (209, 1068, 9, 44.2881), (209, 764, 10, 44.2374), (210, 954, 1, 39.5281), (210, 1038, 2, 26.8796), (210, 890, 3, 19.8169), (210, 497, 4, 19.1783), (210, 840, 5, 18.2281), (210, 720, 6, 18.1381), (210, 580, 7, 17.7700), (210, 870, 8, 17.2141), (210, 640, 9, 17.0600), (210, 329, 10, 16.9037); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (211, 1038, 1, 26.6402), (211, 828, 2, 26.5019), (211, 1135, 3, 26.3083), (211, 851, 4, 26.2470), (211, 889, 5, 24.9522), (211, 1043, 6, 24.4107), (211, 843, 7, 24.1497), (211, 1137, 8, 22.7889), (211, 819, 9, 22.5074), (211, 1134, 10, 22.3296), (212, 954, 1, 19.3721), (212, 890, 2, 18.2694), (212, 1175, 3, 16.4098), (212, 1125, 4, 15.2037), (212, 769, 5, 14.8574), (212, 1134, 6, 14.7885), (212, 887, 7, 14.4310), (212, 1068, 8, 13.9071), (212, 531, 9, 12.7362), (212, 765, 10, 12.2074), (213, 954, 1, 30.1961), (213, 1043, 2, 30.1238), (213, 1051, 3, 28.5747), (213, 828, 4, 28.1074), (213, 889, 5, 26.5981), (213, 930, 6, 25.7592), (213, 1134, 7, 25.7216), (213, 843, 8, 25.3317), (213, 839, 9, 24.9718), (213, 1137, 10, 24.4397), (214, 955, 1, 52.4809), (214, 839, 2, 49.3543), (214, 763, 3, 47.6829), (214, 1045, 4, 47.1490), (214, 1363, 5, 46.6018), (214, 1051, 6, 43.6762), (214, 838, 7, 42.8950), (214, 840, 8, 40.9183), (214, 1178, 9, 39.9191), (214, 928, 10, 39.7183), (215, 1051, 1, 44.8963), (215, 839, 2, 40.8919), (215, 1046, 3, 34.8574), (215, 1045, 4, 34.1875), (215, 838, 5, 34.0688), (215, 827, 6, 30.3738), (215, 955, 7, 29.3378), (215, 763, 8, 28.6496), (215, 928, 9, 28.6127), (215, 1118, 10, 28.1734); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (216, 955, 1, 48.2311), (216, 1045, 2, 46.3154), (216, 641, 3, 35.8360), (216, 730, 4, 35.5878), (216, 1175, 5, 35.4312), (216, 840, 6, 33.6818), (216, 839, 7, 31.3740), (216, 1046, 8, 30.5551), (216, 838, 9, 29.3634), (216, 763, 10, 28.9247), (217, 956, 1, 61.4571), (217, 1050, 2, 55.2382), (217, 1049, 3, 38.0621), (217, 1392, 4, 29.4332), (217, 843, 5, 29.3829), (217, 1358, 6, 28.3816), (217, 1128, 7, 28.3243), (217, 1119, 8, 28.2362), (217, 1387, 9, 27.1740), (217, 889, 10, 26.9275), (218, 956, 1, 30.4193), (218, 1048, 2, 29.5036), (218, 1126, 3, 27.7366), (218, 1050, 4, 24.7447), (218, 763, 5, 22.7461), (218, 739, 6, 21.0646), (218, 761, 7, 20.9045), (218, 826, 8, 20.7729), (218, 928, 9, 20.2259), (218, 838, 10, 19.7646), (219, 930, 1, 26.1902), (219, 926, 2, 25.0102), (219, 1051, 3, 24.0431), (219, 931, 4, 20.6008), (219, 826, 5, 20.4203), (219, 937, 6, 20.3331), (219, 1059, 7, 20.0542), (219, 1070, 8, 19.7014), (219, 851, 9, 18.8850), (219, 633, 10, 18.8098), (223, 1062, 1, 38.4167), (223, 1075, 2, 37.0468), (223, 1074, 3, 35.6470), (223, 923, 4, 27.3530), (223, 1202, 5, 26.4224), (223, 924, 6, 26.2652), (223, 252, 7, 25.2944), (223, 1239, 8, 24.5997), (223, 1243, 9, 24.5088), (223, 188, 10, 24.1242); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (224, 783, 1, 32.6689), (224, 251, 2, 31.8320), (224, 677, 3, 30.6480), (224, 52, 4, 30.5933), (224, 924, 5, 30.5608), (224, 676, 6, 30.4494), (224, 433, 7, 29.4843), (224, 917, 8, 28.8373), (224, 246, 9, 28.2786), (224, 1246, 10, 28.1693), (225, 42, 1, 32.7909), (225, 671, 2, 30.0181), (225, 94, 3, 28.3130), (225, 1362, 4, 27.6052), (225, 1225, 5, 27.5638), (225, 1273, 6, 27.3505), (225, 547, 7, 26.9308), (225, 1079, 8, 26.7521), (225, 80, 9, 26.7252), (225, 917, 10, 26.2951), (226, 1063, 1, 32.2664), (226, 1085, 2, 21.4736), (226, 1081, 3, 20.4617), (226, 1082, 4, 19.8369), (226, 393, 5, 16.5664), (226, 329, 6, 16.5219), (226, 394, 7, 16.5196), (226, 228, 8, 16.3595), (226, 323, 9, 15.4670), (226, 117, 10, 15.4624), (227, 1088, 1, 39.6717), (227, 1054, 2, 26.6674), (227, 1086, 3, 24.5941), (227, 111, 4, 23.3692), (227, 1055, 5, 22.0970), (227, 1262, 6, 17.6247), (227, 367, 7, 17.3545), (227, 159, 8, 17.1826), (227, 1063, 9, 17.0527), (227, 1079, 10, 16.3693), (230, 1101, 1, 19.3938), (230, 804, 2, 18.2894), (230, 1065, 3, 17.2041), (230, 553, 4, 15.5485), (230, 810, 5, 15.0539), (230, 1226, 6, 14.9967), (230, 1097, 7, 14.2132), (230, 968, 8, 14.0165), (230, 1100, 9, 13.5723), (230, 1279, 10, 13.4973); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (231, 553, 1, 26.2728), (231, 1096, 2, 25.8262), (231, 1097, 3, 25.6579), (231, 1065, 4, 23.1089), (231, 1098, 5, 20.9600), (231, 1099, 6, 20.0217), (231, 1100, 7, 19.6080), (231, 1279, 8, 19.2642), (231, 1302, 9, 18.2304), (231, 1117, 10, 17.4684), (232, 1006, 1, 51.1735), (232, 456, 2, 43.1193), (232, 160, 3, 42.9546), (232, 556, 4, 41.7917), (232, 318, 5, 41.1489), (232, 25, 6, 39.7941), (232, 626, 7, 39.7898), (232, 1393, 8, 38.7528), (232, 498, 9, 38.4472), (232, 369, 10, 38.1323), (233, 302, 1, 22.5769), (233, 552, 2, 20.1332), (233, 236, 3, 18.8381), (233, 328, 4, 18.0216), (233, 949, 5, 17.7370), (233, 691, 6, 17.3274), (233, 1072, 7, 17.1749), (233, 82, 8, 17.0348), (233, 1199, 9, 16.8897), (233, 259, 10, 16.8596), (234, 1066, 1, 53.0192), (234, 1197, 2, 41.9829), (234, 1112, 3, 41.3204), (234, 1259, 4, 40.3015), (234, 42, 5, 39.1438), (234, 846, 6, 37.5104), (234, 899, 7, 37.0742), (234, 801, 8, 37.0460), (234, 1321, 9, 36.9778), (234, 432, 10, 36.8482), (241, 1071, 1, 72.7820), (241, 885, 2, 67.9929), (241, 890, 3, 61.4645), (241, 769, 4, 59.7916), (241, 1134, 5, 56.2611), (241, 830, 6, 53.4676), (241, 889, 7, 52.4878), (241, 739, 8, 52.3980), (241, 740, 9, 52.0628), (241, 765, 10, 51.9995); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (245, 1386, 1, 62.3937), (245, 54, 2, 57.0051), (245, 1185, 3, 53.9316), (245, 460, 4, 50.7619), (245, 55, 5, 50.2775), (245, 1192, 6, 49.7877), (245, 145, 7, 47.5880), (245, 366, 8, 46.5743), (245, 784, 9, 46.5680), (245, 435, 10, 46.5496), (246, 460, 1, 42.3513), (246, 55, 2, 41.6300), (246, 1386, 3, 39.7865), (246, 62, 4, 38.2072), (246, 1109, 5, 36.9744), (246, 562, 6, 36.7629), (246, 366, 7, 36.6052), (246, 1185, 8, 36.5570), (246, 54, 9, 36.5097), (246, 461, 10, 36.4706), (247, 492, 1, 57.0136), (247, 56, 2, 47.8180), (247, 232, 3, 46.2313), (247, 57, 4, 46.0345), (247, 434, 5, 41.7745), (247, 1231, 6, 41.0671), (247, 973, 7, 40.0941), (247, 27, 8, 38.1188), (247, 1040, 9, 38.0590), (247, 1311, 10, 38.0525), (250, 311, 1, 24.6331), (250, 1187, 2, 21.2536), (250, 265, 3, 21.1538), (250, 415, 4, 21.0607), (250, 503, 5, 19.9948), (250, 416, 6, 18.9952), (250, 358, 7, 18.4696), (250, 316, 8, 17.9814), (250, 1268, 9, 17.1270), (250, 798, 10, 16.6949), (251, 504, 1, 40.1339), (251, 71, 2, 32.1654), (251, 663, 3, 28.3994), (251, 569, 4, 28.3806), (251, 1375, 5, 27.9489), (251, 1276, 6, 27.8893), (251, 209, 7, 27.6283), (251, 474, 8, 27.5010), (251, 72, 9, 27.4624), (251, 978, 10, 27.0008); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (252, 504, 1, 41.6241), (252, 663, 2, 31.9181), (252, 1276, 3, 30.7156), (252, 569, 4, 29.4690), (252, 209, 5, 29.1456), (252, 207, 6, 28.9897), (252, 1203, 7, 28.7010), (252, 1303, 8, 28.5079), (252, 71, 9, 28.3101), (252, 36, 10, 28.3030), (253, 553, 1, 40.3338), (253, 1279, 2, 38.1374), (253, 1098, 3, 33.2172), (253, 1157, 4, 33.1296), (253, 1147, 5, 32.5797), (253, 274, 6, 32.4872), (253, 1043, 7, 31.5576), (253, 82, 8, 31.2793), (253, 1100, 9, 30.8418), (253, 1009, 10, 29.8398), (254, 118, 1, 61.2816), (254, 750, 2, 60.7083), (254, 341, 3, 56.6054), (254, 157, 4, 56.5461), (254, 410, 5, 54.1502), (254, 217, 6, 51.9066), (254, 575, 7, 50.4531), (254, 236, 8, 50.2478), (254, 656, 9, 49.7068), (254, 1280, 10, 49.5251), (255, 118, 1, 55.9632), (255, 750, 2, 55.4534), (255, 157, 3, 55.1186), (255, 341, 4, 52.8861), (255, 1280, 5, 48.2702), (255, 217, 6, 47.8179), (255, 575, 7, 46.5286), (255, 236, 8, 46.3309), (255, 1274, 9, 45.6682), (255, 656, 10, 45.6313), (257, 139, 1, 41.0127), (257, 904, 2, 39.0418), (257, 1082, 3, 38.1610), (257, 238, 4, 37.6096), (257, 239, 5, 34.6504), (257, 315, 6, 34.3370), (257, 893, 7, 33.5658), (257, 213, 8, 32.6071), (257, 905, 9, 31.6747), (257, 140, 10, 30.9692); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (259, 516, 1, 57.0192), (259, 431, 2, 54.1679), (259, 252, 3, 41.8607), (259, 141, 4, 40.6526), (259, 904, 5, 37.9427), (259, 755, 6, 37.0005), (259, 188, 7, 35.9956), (259, 179, 8, 34.7597), (259, 808, 9, 34.3538), (259, 1153, 10, 33.2256), (261, 320, 1, 34.8753), (261, 322, 2, 33.4824), (261, 321, 3, 32.7480), (261, 527, 4, 32.3019), (261, 476, 5, 27.3248), (261, 478, 6, 23.8508), (261, 1370, 7, 21.2901), (261, 1235, 8, 20.5643), (261, 736, 9, 18.6308), (261, 424, 10, 18.3748), (264, 367, 1, 37.6788), (264, 451, 2, 34.8532), (264, 532, 3, 28.7646), (264, 703, 4, 23.5275), (264, 731, 5, 22.6337), (264, 55, 6, 22.4638), (264, 1119, 7, 22.4608), (264, 293, 8, 21.9532), (264, 1087, 9, 21.9488), (264, 1192, 10, 21.8218), (265, 483, 1, 21.3889), (265, 35, 2, 20.8917), (265, 1274, 3, 19.3130), (265, 1319, 4, 19.1732), (265, 533, 5, 18.6818), (265, 501, 6, 17.4043), (265, 411, 7, 15.6321), (265, 1390, 8, 15.4003), (265, 1151, 9, 15.2588), (265, 178, 10, 14.8054), (266, 941, 1, 26.0403), (266, 139, 2, 21.4399), (266, 1080, 3, 19.6127), (266, 1082, 4, 19.2913), (266, 666, 5, 18.8277), (266, 491, 6, 18.3385), (266, 1256, 7, 18.3249), (266, 9, 8, 18.0668), (266, 1221, 9, 17.9943), (266, 270, 10, 17.5540); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (267, 542, 1, 25.6928), (267, 963, 2, 20.0700), (267, 1073, 3, 17.4107), (267, 586, 4, 16.8856), (267, 587, 5, 16.5659), (267, 869, 6, 15.4748), (267, 580, 7, 14.8924), (267, 981, 8, 14.7405), (267, 1375, 9, 14.4837), (267, 872, 10, 13.3378), (268, 543, 1, 31.8987), (268, 590, 2, 30.6222), (268, 588, 3, 28.2310), (268, 216, 4, 27.9021), (268, 138, 5, 27.0196), (268, 589, 6, 26.4984), (268, 237, 7, 25.1175), (268, 214, 8, 24.8594), (268, 426, 9, 24.2360), (268, 772, 10, 21.8121), (269, 591, 1, 24.7351), (269, 138, 2, 23.0643), (269, 216, 3, 22.4039), (269, 589, 4, 20.6596), (269, 237, 5, 20.5198), (269, 543, 6, 20.4723), (269, 511, 7, 20.1517), (269, 1155, 8, 19.0807), (269, 31, 9, 18.9135), (269, 214, 10, 18.8985), (272, 633, 1, 74.8193), (272, 682, 2, 65.3676), (272, 428, 3, 63.4828), (272, 680, 4, 62.7688), (272, 224, 5, 61.8362), (272, 122, 6, 57.5868), (272, 930, 7, 55.1143), (272, 49, 8, 54.2749), (272, 677, 9, 53.0647), (272, 811, 10, 53.0496), (273, 548, 1, 33.2465), (273, 616, 2, 27.0274), (273, 617, 3, 24.4768), (273, 622, 4, 23.7928), (273, 613, 5, 22.2685), (273, 614, 6, 20.5357), (273, 1391, 7, 19.1154), (273, 218, 8, 18.4792), (273, 1103, 9, 17.4208), (273, 315, 10, 15.2660); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (274, 997, 1, 16.2837), (274, 1075, 2, 14.0963), (274, 993, 3, 13.4610), (274, 1374, 4, 13.3914), (274, 1195, 5, 13.3678), (274, 1074, 6, 13.3553), (274, 409, 7, 13.3453), (274, 1093, 8, 12.5621), (274, 1350, 9, 12.3436), (274, 1062, 10, 12.2542), (275, 634, 1, 88.1366), (275, 685, 2, 68.9044), (275, 686, 3, 38.3496), (275, 572, 4, 36.9647), (275, 1300, 5, 36.8191), (275, 593, 6, 36.2971), (275, 440, 7, 33.4669), (275, 1290, 8, 32.9373), (275, 655, 9, 32.5499), (275, 857, 10, 32.4769), (277, 809, 1, 24.8595), (277, 758, 2, 21.9401), (277, 808, 3, 21.1096), (277, 811, 4, 20.8601), (277, 893, 5, 20.5255), (277, 807, 6, 19.2374), (277, 1177, 7, 19.1690), (277, 1243, 8, 18.8071), (277, 804, 9, 18.7914), (277, 1068, 10, 18.7474), (283, 82, 1, 15.3115), (283, 1217, 2, 14.9172), (283, 1379, 3, 14.0479), (283, 554, 4, 13.2051), (283, 813, 5, 12.4948), (283, 555, 6, 12.4771), (283, 1219, 7, 12.3424), (283, 509, 8, 12.2203), (283, 274, 9, 12.1611), (283, 895, 10, 11.4989), (284, 856, 1, 23.0873), (284, 857, 2, 21.6398), (284, 390, 3, 17.7552), (284, 766, 4, 17.3553), (284, 858, 5, 17.2665), (284, 1008, 6, 16.6480), (284, 948, 7, 16.6350), (284, 391, 8, 15.4932), (284, 859, 9, 15.0840), (284, 658, 10, 14.5854); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (285, 1243, 1, 18.5913), (285, 1062, 2, 17.5575), (285, 1239, 3, 16.5920), (285, 610, 4, 16.0716), (285, 1074, 5, 15.7418), (285, 803, 6, 15.3386), (285, 1075, 7, 15.1176), (285, 229, 8, 14.8543), (285, 252, 9, 14.7538), (285, 230, 10, 14.5984), (288, 763, 1, 54.4843), (288, 841, 2, 42.9035), (288, 839, 3, 42.3436), (288, 1126, 4, 41.1268), (288, 897, 5, 39.9641), (288, 1068, 6, 39.2221), (288, 1051, 7, 37.4252), (288, 846, 8, 37.0605), (288, 936, 9, 35.6809), (288, 1173, 10, 35.6102), (291, 725, 1, 44.6381), (291, 722, 2, 42.5714), (291, 640, 3, 38.7324), (291, 908, 4, 36.9338), (291, 220, 5, 36.5268), (291, 1360, 6, 34.8156), (291, 186, 7, 33.4115), (291, 1279, 8, 32.3142), (291, 740, 9, 31.5510), (291, 764, 10, 30.9961), (292, 767, 1, 49.0491), (292, 640, 2, 39.9812), (292, 881, 3, 38.6057), (292, 865, 4, 37.9893), (292, 870, 5, 37.2776), (292, 980, 6, 36.9144), (292, 907, 7, 36.8001), (292, 726, 8, 36.2907), (292, 868, 9, 36.1108), (292, 314, 10, 36.0310), (293, 390, 1, 52.7697), (293, 15, 2, 48.7744), (293, 42, 3, 44.4574), (293, 856, 4, 44.2622), (293, 1339, 5, 44.0039), (293, 52, 6, 43.4813), (293, 627, 7, 41.6552), (293, 895, 8, 40.5264), (293, 1035, 9, 40.0724), (293, 1322, 10, 39.6623); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (294, 766, 1, 29.9635), (294, 894, 2, 28.5073), (294, 1392, 3, 27.4222), (294, 1242, 4, 27.0053), (294, 658, 5, 26.3927), (294, 15, 6, 26.2151), (294, 827, 7, 25.7510), (294, 720, 8, 24.6558), (294, 857, 9, 24.6219), (294, 319, 10, 24.4626), (295, 641, 1, 24.9725), (295, 735, 2, 18.2475), (295, 875, 3, 17.2234), (295, 647, 4, 16.9511), (295, 734, 5, 12.9570), (295, 648, 6, 10.9996), (295, 1398, 7, 10.4351), (295, 950, 8, 9.2517), (295, 745, 9, 8.9824), (295, 551, 10, 8.8094), (296, 733, 1, 21.9331), (296, 641, 2, 19.9508), (296, 730, 3, 19.4662), (296, 422, 4, 17.9113), (296, 735, 5, 17.1839), (296, 734, 6, 16.9817), (296, 1392, 7, 15.2043), (296, 1088, 8, 14.6737), (296, 736, 9, 14.0769), (296, 703, 10, 13.6541), (297, 642, 1, 70.0711), (297, 744, 2, 58.0768), (297, 932, 3, 52.7450), (297, 740, 4, 49.6015), (297, 760, 5, 49.4695), (297, 957, 6, 49.1475), (297, 739, 7, 49.0449), (297, 1051, 8, 48.5931), (297, 888, 9, 47.7750), (297, 1172, 10, 47.1200), (298, 642, 1, 61.9075), (298, 1122, 2, 55.6538), (298, 744, 3, 55.4345), (298, 739, 4, 54.3348), (298, 1055, 5, 50.0111), (298, 932, 6, 49.5248), (298, 1051, 7, 49.0271), (298, 627, 8, 47.9929), (298, 1054, 9, 47.8992), (298, 1173, 10, 47.8703); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (299, 184, 1, 41.7806), (299, 486, 2, 40.0764), (299, 746, 3, 39.2814), (299, 966, 4, 38.2980), (299, 1361, 5, 37.9349), (299, 349, 6, 37.0794), (299, 1281, 7, 37.0402), (299, 1008, 8, 36.7487), (299, 442, 9, 36.7374), (299, 395, 10, 36.5688), (300, 723, 1, 27.5150), (300, 881, 2, 26.6823), (300, 884, 3, 26.1780), (300, 726, 4, 22.1937), (300, 768, 5, 20.7711), (300, 909, 6, 17.9810), (300, 746, 7, 17.7959), (300, 724, 8, 16.6628), (300, 767, 9, 16.2404), (300, 261, 10, 16.1884), (301, 889, 1, 44.2547), (301, 897, 2, 43.3941), (301, 890, 3, 41.2339), (301, 843, 4, 41.1341), (301, 743, 5, 40.2778), (301, 763, 6, 40.0367), (301, 1174, 7, 39.6557), (301, 798, 8, 39.4047), (301, 760, 9, 39.3919), (301, 1068, 10, 39.2575), (303, 739, 1, 21.2994), (303, 931, 2, 21.0960), (303, 1294, 3, 20.1750), (303, 930, 4, 19.8608), (303, 1059, 5, 19.0631), (303, 852, 6, 17.8197), (303, 1058, 7, 16.2898), (303, 451, 8, 16.2548), (303, 1054, 9, 15.9026), (303, 428, 10, 15.8345), (304, 1134, 1, 40.2983), (304, 1071, 2, 38.5530), (304, 1053, 3, 36.8913), (304, 851, 4, 33.5627), (304, 1137, 5, 32.4711), (304, 1362, 6, 32.4541), (304, 934, 7, 32.4203), (304, 885, 8, 32.1431), (304, 730, 9, 30.6343), (304, 1184, 10, 30.5384); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (306, 625, 1, 45.4673), (306, 1297, 2, 36.2450), (306, 1296, 3, 35.1435), (306, 401, 4, 34.7928), (306, 1391, 5, 33.8628), (306, 410, 6, 32.4604), (306, 1217, 7, 31.7766), (306, 318, 8, 30.6710), (306, 1379, 9, 30.4891), (306, 24, 10, 30.1855), (314, 1285, 1, 39.8852), (314, 1310, 2, 33.1730), (314, 687, 3, 25.4135), (314, 232, 4, 25.2388), (314, 925, 5, 25.2337), (314, 1378, 6, 24.3983), (314, 122, 7, 23.7668), (314, 1319, 8, 23.3642), (314, 1274, 9, 22.8329), (314, 927, 10, 22.8041), (315, 122, 1, 22.6816), (315, 927, 2, 21.9075), (315, 1310, 3, 21.8484), (315, 814, 4, 21.7474), (315, 1333, 5, 20.7590), (315, 225, 6, 20.2827), (315, 1386, 7, 20.2747), (315, 677, 8, 20.2105), (315, 49, 9, 19.6838), (315, 370, 10, 19.6724), (316, 147, 1, 13.8680), (316, 573, 2, 9.3683), (316, 371, 3, 8.5122), (316, 1080, 4, 8.2350), (316, 1236, 5, 7.7786), (316, 937, 6, 7.5792), (316, 112, 7, 7.2610), (316, 786, 8, 7.1713), (316, 57, 9, 7.0399), (316, 443, 10, 7.0253), (317, 1323, 1, 18.7688), (317, 131, 2, 17.5391), (317, 1287, 3, 16.1774), (317, 1017, 4, 15.5579), (317, 912, 5, 14.4817), (317, 72, 6, 14.1600), (317, 352, 7, 13.8378), (317, 1182, 8, 13.6971), (317, 1192, 9, 13.2369), (317, 133, 10, 13.1826); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (321, 1290, 1, 68.3504), (321, 1338, 2, 36.8326), (321, 315, 3, 33.4316), (321, 1341, 4, 31.9931), (321, 876, 5, 31.8503), (321, 252, 6, 30.8180), (321, 638, 7, 30.6121), (321, 121, 8, 30.3589), (321, 467, 9, 29.6685), (321, 441, 10, 29.4495), (323, 859, 1, 21.9077), (323, 1290, 2, 19.3484), (323, 948, 3, 17.7246), (323, 78, 4, 17.6893), (323, 747, 5, 16.4793), (323, 766, 6, 15.5623), (323, 1341, 7, 15.2243), (323, 39, 8, 15.0757), (323, 391, 9, 14.9695), (323, 442, 10, 14.4770), (327, 1291, 1, 94.1337), (327, 163, 2, 73.6641), (327, 1344, 3, 63.7257), (327, 1345, 4, 62.4145), (327, 1217, 5, 60.1859), (327, 566, 6, 59.3280), (327, 164, 7, 57.7485), (327, 357, 8, 57.0362), (327, 1346, 9, 55.6360), (327, 982, 10, 54.7465), (331, 959, 1, 34.7471), (331, 187, 2, 32.1283), (331, 455, 3, 31.1296), (331, 651, 4, 30.7841), (331, 89, 5, 29.6716), (331, 45, 6, 29.0586), (331, 798, 7, 28.4632), (331, 974, 8, 26.8902), (331, 358, 9, 26.5362), (331, 1364, 10, 26.4888), (332, 1172, 1, 56.0416), (332, 1145, 2, 43.1910), (332, 887, 3, 42.6221), (332, 1068, 4, 41.1220), (332, 1125, 5, 40.8195), (332, 739, 6, 40.4678), (332, 1171, 7, 40.1603), (332, 1126, 8, 39.3777), (332, 743, 9, 39.0625), (332, 1173, 10, 38.5293); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (333, 739, 1, 37.7699), (333, 1068, 2, 30.6722), (333, 885, 3, 29.6719), (333, 1117, 4, 29.4766), (333, 741, 5, 28.2295), (333, 1173, 6, 27.9166), (333, 887, 7, 27.8910), (333, 897, 8, 27.0795), (333, 953, 9, 26.7917), (333, 763, 10, 26.7680), (335, 1177, 1, 44.2683), (335, 1178, 2, 42.4977), (335, 838, 3, 39.2647), (335, 888, 4, 37.8721), (335, 1176, 5, 32.4352), (335, 1146, 6, 28.3561), (335, 1121, 7, 27.4908), (335, 885, 8, 27.3257), (335, 1132, 9, 27.1410), (335, 1133, 10, 27.0591), (336, 885, 1, 40.8960), (336, 1173, 2, 40.2678), (336, 888, 3, 39.5966), (336, 769, 4, 38.6709), (336, 887, 5, 37.8685), (336, 890, 6, 36.2244), (336, 642, 7, 36.1314), (336, 741, 8, 35.4379), (336, 105, 9, 35.4277), (336, 891, 10, 34.6624), (338, 1294, 1, 49.0648), (338, 1362, 2, 35.0521), (338, 739, 3, 29.8698), (338, 828, 4, 29.0261), (338, 47, 5, 25.9942), (338, 92, 6, 25.7403), (338, 1154, 7, 25.6653), (338, 848, 8, 25.5414), (338, 1036, 9, 25.5246), (338, 777, 10, 25.2979), (339, 535, 1, 60.5521), (339, 25, 2, 50.4039), (339, 1205, 3, 47.7149), (339, 574, 4, 47.4622), (339, 625, 5, 47.3608), (339, 317, 6, 46.5295), (339, 1391, 7, 46.3713), (339, 1198, 8, 46.2979), (339, 666, 9, 45.1213), (339, 1238, 10, 44.9913); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (340, 1319, 1, 29.0756), (340, 1274, 2, 29.0433), (340, 117, 3, 28.0245), (340, 893, 4, 27.9138), (340, 974, 5, 27.8338), (340, 175, 6, 27.4598), (340, 39, 7, 26.4502), (340, 1068, 8, 25.9231), (340, 1200, 9, 25.2811), (340, 1280, 10, 25.1979), (347, 983, 1, 29.0626), (347, 44, 2, 29.0253), (347, 1151, 3, 28.6184), (347, 946, 4, 28.5755), (347, 666, 5, 28.2939), (347, 324, 6, 28.0410), (347, 323, 7, 27.3999), (347, 1393, 8, 27.1311), (347, 1002, 9, 27.1137), (347, 123, 10, 27.0129), (348, 36, 1, 33.1570), (348, 1281, 2, 32.4330), (348, 1393, 3, 32.1391), (348, 323, 4, 32.1131), (348, 101, 5, 31.6761), (348, 35, 6, 31.5536), (348, 44, 7, 31.2216), (348, 670, 8, 31.1686), (348, 983, 9, 30.7627), (348, 666, 10, 30.7175), (349, 1221, 1, 26.3092), (349, 1222, 2, 25.6533), (349, 1082, 3, 25.0637), (349, 299, 4, 24.9114), (349, 456, 5, 24.7094), (349, 342, 6, 24.6294), (349, 236, 7, 24.4085), (349, 208, 8, 23.7385), (349, 117, 9, 23.3944), (349, 33, 10, 23.2849), (352, 1375, 1, 27.8210), (352, 292, 2, 26.4435), (352, 231, 3, 26.1898), (352, 458, 4, 24.9921), (352, 1192, 5, 24.5989), (352, 837, 6, 24.4473), (352, 461, 7, 23.8368), (352, 261, 8, 23.5361), (352, 472, 9, 23.2942), (352, 540, 10, 23.1394); INSERT INTO cranfield_full_expected_rankings (query_id, doc_id, rank, bm25_score) VALUES (353, 1386, 1, 24.4049), (353, 1366, 2, 24.0865), (353, 458, 3, 21.7119), (353, 54, 4, 20.6899), (353, 528, 5, 20.4481), (353, 49, 6, 20.1030), (353, 540, 7, 20.0432), (353, 653, 8, 19.9667), (353, 122, 9, 19.4346), (353, 395, 10, 19.3488), (355, 1130, 1, 27.8177), (355, 419, 2, 27.5011), (355, 1399, 3, 27.1582), (355, 400, 4, 22.1406), (355, 1400, 5, 22.0852), (355, 1121, 6, 20.0969), (355, 1050, 7, 20.0885), (355, 1396, 8, 19.8590), (355, 820, 9, 19.3052), (355, 888, 10, 19.0016), (356, 400, 1, 30.4278), (356, 1399, 2, 29.2554), (356, 419, 3, 25.5979), (356, 1387, 4, 25.1563), (356, 1398, 5, 24.1846), (356, 1400, 6, 24.0470), (356, 388, 7, 23.4844), (356, 1121, 8, 22.5705), (356, 1008, 9, 21.9274), (356, 1119, 10, 20.9715), (360, 1312, 1, 53.8124), (360, 1286, 2, 51.9933), (360, 317, 3, 51.8523), (360, 401, 4, 49.1755), (360, 236, 5, 49.0129), (360, 1296, 6, 46.3050), (360, 329, 7, 45.0629), (360, 575, 8, 45.0078), (360, 1316, 9, 44.8235), (360, 1313, 10, 44.4448), (365, 1188, 1, 39.1512), (365, 1380, 2, 26.2450), (365, 225, 3, 21.9802), (365, 70, 4, 21.5025), (365, 1291, 5, 20.3976), (365, 1345, 6, 20.1382), (365, 1218, 7, 19.6894), (365, 431, 8, 19.6537), (365, 1124, 9, 19.5523), (365, 416, 10, 18.8886); timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/gin/000077500000000000000000000000001525762047700244015ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/gin/concurrent-setup.sql000066400000000000000000000056551525762047700304550ustar00rootroot00000000000000-- Cranfield GIN+tsvector Baseline - Concurrent Insert Setup -- Prepares staging table for pgbench concurrent inserts \set ON_ERROR_STOP on \timing on \echo 'Cranfield GIN Baseline - Concurrent Insert Setup' \echo '=================================================' -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_gin_documents CASCADE; DROP TABLE IF EXISTS cranfield_gin_staging CASCADE; DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create temp tables for dataset.sql loading \echo 'Creating tables for dataset loading...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Load dataset \echo 'Loading Cranfield collection...' \i dataset.sql -- Create staging table (raw data, no tsv column) \echo 'Creating staging table...' CREATE TABLE cranfield_gin_staging ( staging_id SERIAL PRIMARY KEY, doc_id INTEGER, title TEXT, author TEXT, bibliography TEXT, content TEXT ); INSERT INTO cranfield_gin_staging (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents ORDER BY doc_id; -- Drop temp tables DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Create target GIN table with index (empty) \echo 'Creating target GIN table with index...' CREATE TABLE cranfield_gin_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '')) ) STORED ); CREATE INDEX cranfield_gin_idx ON cranfield_gin_documents USING gin(tsv); -- Create sequence for pgbench inserts CREATE SEQUENCE insert_seq; -- Report staging row count SELECT 'STAGING_ROW_COUNT: ' || count(*) FROM cranfield_gin_staging; \echo 'Concurrent insert setup complete.' \echo 'Run pgbench with gin/pgbench-insert.sql next.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/gin/insert-load.sql000066400000000000000000000060461525762047700273510ustar00rootroot00000000000000-- Cranfield GIN+tsvector Baseline - Insert Loading -- Creates GIN index on EMPTY table, then loads data via inserts -- Measures insert performance into an already-indexed table \set ON_ERROR_STOP on \timing on \echo 'Cranfield GIN Baseline - Insert Load Phase' \echo '============================================' -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_gin_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; -- Create temp tables for dataset.sql loading \echo 'Creating tables for dataset loading...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Create the GIN target table with index \echo 'Creating GIN documents table...' CREATE TABLE cranfield_gin_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '')) ) STORED ); \echo 'Creating GIN index on empty table...' CREATE INDEX cranfield_gin_idx ON cranfield_gin_documents USING gin(tsv); -- Load dataset into temp tables \echo 'Loading Cranfield collection...' \i dataset.sql -- Insert data into indexed GIN table \echo 'INSERT_TIME:' \echo 'Inserting data into indexed GIN table...' INSERT INTO cranfield_gin_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents; -- Drop temp tables DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('cranfield_gin_idx') ) as index_size, pg_relation_size('cranfield_gin_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'cranfield_gin_documents') ) as table_size, pg_total_relation_size( 'cranfield_gin_documents') as table_bytes; \echo 'GIN insert load phase completed.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/gin/load.sql000066400000000000000000000057561525762047700260560ustar00rootroot00000000000000-- Cranfield GIN+tsvector Baseline - Data Loading -- Loads data first, then creates GIN index on populated table -- Measures CREATE INDEX time and final sizes \set ON_ERROR_STOP on \timing on \echo 'Cranfield GIN Baseline - Loading Phase' \echo '=======================================' -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_gin_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; -- Create temp tables for dataset.sql loading \echo 'Creating tables for dataset loading...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Load dataset \echo 'Loading Cranfield collection...' \i dataset.sql -- Create the GIN target table (without index) \echo 'Creating GIN documents table...' CREATE TABLE cranfield_gin_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED, tsv tsvector GENERATED ALWAYS AS ( to_tsvector('english', COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '')) ) STORED ); -- Copy data into GIN table \echo 'Copying data into GIN table...' INSERT INTO cranfield_gin_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents; -- Drop temp tables DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Create GIN index on populated table (timed) \echo 'Building GIN index...' CREATE INDEX cranfield_gin_idx ON cranfield_gin_documents USING gin(tsv); -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('cranfield_gin_idx') ) as index_size, pg_relation_size('cranfield_gin_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'cranfield_gin_documents') ) as table_size, pg_total_relation_size( 'cranfield_gin_documents') as table_bytes; \echo 'GIN load phase completed.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/gin/pgbench-insert.sql000066400000000000000000000003611525762047700300320ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO cranfield_gin_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_gin_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/insert/000077500000000000000000000000001525762047700251305ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/insert/concurrent-setup.sql000066400000000000000000000076351525762047700312040ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark - Concurrent Insert Setup -- Prepares staging table for pgbench concurrent inserts -- After this script: cranfield_full_* tables exist with BM25 index, -- cranfield_staging holds rows to be inserted via pgbench \set ON_ERROR_STOP on \timing on \echo 'Cranfield BM25 Benchmark - Concurrent Insert Setup' \echo '===================================================' -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; DROP TABLE IF EXISTS cranfield_staging CASCADE; DROP TABLE IF EXISTS cranfield_queries_backup CASCADE; DROP TABLE IF EXISTS cranfield_rankings_backup CASCADE; -- Create tables and load dataset \echo 'Creating tables...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); \echo 'Loading dataset...' \i dataset.sql -- Create staging table with document data \echo 'Creating staging table...' CREATE TABLE cranfield_staging ( staging_id SERIAL PRIMARY KEY, doc_id INTEGER, title TEXT, author TEXT, bibliography TEXT, content TEXT ); INSERT INTO cranfield_staging (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents ORDER BY doc_id; -- Save queries and rankings into backup tables \echo 'Backing up queries and rankings...' CREATE TABLE cranfield_queries_backup AS SELECT * FROM cranfield_full_queries; CREATE TABLE cranfield_rankings_backup AS SELECT * FROM cranfield_full_expected_rankings; -- Drop all cranfield_full_* tables \echo 'Dropping original tables...' DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Recreate cranfield_full_documents with BM25 index \echo 'Recreating documents table with BM25 index...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE INDEX cranfield_full_tapir_idx ON cranfield_full_documents USING bm25(full_text) WITH (text_config='english', k1=1.2, b=0.75); -- Recreate and repopulate queries and rankings \echo 'Restoring queries and rankings...' CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); INSERT INTO cranfield_full_queries SELECT * FROM cranfield_queries_backup; CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); INSERT INTO cranfield_full_expected_rankings SELECT * FROM cranfield_rankings_backup; -- Drop backup tables \echo 'Dropping backup tables...' DROP TABLE cranfield_queries_backup; DROP TABLE cranfield_rankings_backup; -- Create sequence for pgbench inserts DROP SEQUENCE IF EXISTS insert_seq; CREATE SEQUENCE insert_seq; -- Report staging row count SELECT 'STAGING_ROW_COUNT: ' || count(*) FROM cranfield_staging; \echo 'Concurrent insert setup complete.' \echo 'Run pgbench with insert/pgbench-insert.sql next.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/insert/load.sql000066400000000000000000000050771525762047700266010ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark - Insert Loading -- Creates index on EMPTY table, then loads data via inserts -- This measures insert performance into an already-indexed table \set ON_ERROR_STOP on \timing on \echo 'Cranfield BM25 Benchmark - Insert Load Phase' \echo '=============================================' -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; -- Create tables for Cranfield dataset \echo 'Creating tables...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Create BM25 index on the EMPTY table before loading data \echo 'Creating BM25 index on empty table...' CREATE INDEX cranfield_full_tapir_idx ON cranfield_full_documents USING bm25(full_text) WITH (text_config='english', k1=1.2, b=0.75); -- Load the complete Cranfield dataset (inserts into indexed table) -- Turn off per-statement timing during 1400 individual INSERTs; -- measure total insert time via clock_timestamp() bookends. \timing off SELECT clock_timestamp()::text AS t \gset _start_ \echo 'Loading Cranfield collection into indexed table...' \i dataset.sql \echo 'INSERT_TIME:' SELECT 'Time: ' || round(extract(epoch from ( clock_timestamp() - :'_start_t'::timestamptz )) * 1000, 2)::text || ' ms' AS timing; \timing on -- Verify data loading \echo 'Data loading verification:' SELECT 'Documents loaded:' as metric, COUNT(*) as count FROM cranfield_full_documents UNION ALL SELECT 'Queries loaded:' as metric, COUNT(*) as count FROM cranfield_full_queries UNION ALL SELECT 'Expected rankings:' as metric, COUNT(*) as count FROM cranfield_full_expected_rankings ORDER BY metric; -- NOTE: spill + size + segment-stats moved to size_report.sql, run -- AFTER queries.sql, so the query phase exercises the realistic -- post-load state (in-flight memtable chain + cache + segments). \echo 'Insert load phase completed.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/insert/pgbench-insert.sql000066400000000000000000000003561525762047700305650ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO cranfield_full_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/insert/size_report.sql000066400000000000000000000021621525762047700302170ustar00rootroot00000000000000-- Cranfield insert benchmark: post-query size report. -- -- Runs AFTER 02-queries.sql so the query phase exercises the -- realistic post-load state (in-flight memtable chain + cache + -- segments). Spilling here ensures pg_relation_size reflects all -- data on disk and bm25_summarize_index reports the final segment -- layout that downstream metric extractors expect. \echo '' \echo '=== Cranfield Insert: Post-Query Size Report ===' -- Spill memtable to disk so pg_relation_size reflects actual index data SELECT bm25_spill_index('cranfield_full_tapir_idx'); \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('cranfield_full_tapir_idx') ) as index_size, pg_relation_size( 'cranfield_full_tapir_idx' ) as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size('cranfield_full_documents') ) as table_size, pg_total_relation_size( 'cranfield_full_documents' ) as table_bytes; \echo '' \echo '=== Segment Statistics ===' SELECT bm25_summarize_index('cranfield_full_tapir_idx'); timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/paradedb/000077500000000000000000000000001525762047700253665ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/paradedb/01-load.sql000066400000000000000000000102311525762047700272410ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark - Data Loading (ParadeDB) -- This benchmark loads the complete Cranfield collection and creates ParadeDB BM25 index -- Contains 1400 aerodynamics abstracts with 225 queries \set ON_ERROR_STOP on \timing on \echo 'Cranfield BM25 Benchmark - Loading Phase (ParadeDB)' \echo '======================================' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_paradedb_documents CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_queries CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_expected_rankings CASCADE; -- Create tables for Cranfield dataset \echo 'Creating tables...' CREATE TABLE cranfield_paradedb_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_paradedb_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_paradedb_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Load the complete Cranfield dataset using the shared dataset -- We need to temporarily create the original tables and copy data \echo 'Loading complete Cranfield collection (1400 documents, 225 queries)...' -- Create temp tables with original names for dataset.sql compatibility CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Load dataset \i dataset.sql -- Copy to paradedb tables INSERT INTO cranfield_paradedb_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents; INSERT INTO cranfield_paradedb_queries SELECT * FROM cranfield_full_queries; INSERT INTO cranfield_paradedb_expected_rankings SELECT * FROM cranfield_full_expected_rankings; -- Drop temp tables DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Create ParadeDB BM25 index \echo 'Building ParadeDB BM25 index...' CREATE INDEX cranfield_paradedb_idx ON cranfield_paradedb_documents USING bm25 (doc_id, full_text) WITH ( key_field = 'doc_id', text_fields = '{ "full_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Verify data loading \echo 'Data loading verification:' SELECT 'Documents loaded:' as metric, COUNT(*) as count FROM cranfield_paradedb_documents UNION ALL SELECT 'Queries loaded:' as metric, COUNT(*) as count FROM cranfield_paradedb_queries UNION ALL SELECT 'Expected rankings:' as metric, COUNT(*) as count FROM cranfield_paradedb_expected_rankings ORDER BY metric; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty(pg_relation_size('cranfield_paradedb_idx')) as index_size, pg_relation_size('cranfield_paradedb_idx') as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty(pg_total_relation_size('cranfield_paradedb_documents')) as table_size, pg_total_relation_size('cranfield_paradedb_documents') as table_bytes; \echo 'Load phase completed. Index ready for queries.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/paradedb/02-queries.sql000066400000000000000000000122361525762047700300070ustar00rootroot00000000000000-- Cranfield Collection BM25 Benchmark - Query Performance (ParadeDB) -- Tests BM25 index scan performance using the standard Cranfield IR collection -- Outputs structured timing data for regression detection \set ON_ERROR_STOP on \timing on \echo '=== Cranfield BM25 Benchmark - Query Phase (ParadeDB) ===' \echo '' -- Warm up: run a few queries to ensure index is cached \echo 'Warming up index...' SELECT doc_id FROM cranfield_paradedb_documents WHERE full_text @@@ paradedb.match('full_text', 'boundary layer') ORDER BY paradedb.score(doc_id) DESC LIMIT 10; SELECT doc_id FROM cranfield_paradedb_documents WHERE full_text @@@ paradedb.match('full_text', 'heat transfer') ORDER BY paradedb.score(doc_id) DESC LIMIT 10; -- ============================================================ -- Benchmark 1: Query Latency (10 iterations each, median reported) -- ============================================================ \echo '' \echo '=== Benchmark 1: Query Latency (10 iterations each) ===' \echo 'Running top-10 queries using the BM25 index' \echo '' -- Helper function to run a query multiple times and return median execution time CREATE OR REPLACE FUNCTION benchmark_query_paradedb(query_text text, iterations int DEFAULT 10) RETURNS TABLE(median_ms numeric, min_ms numeric, max_ms numeric) AS $$ DECLARE i int; start_ts timestamp; end_ts timestamp; times numeric[]; sorted_times numeric[]; BEGIN times := ARRAY[]::numeric[]; FOR i IN 1..iterations LOOP start_ts := clock_timestamp(); EXECUTE 'SELECT doc_id FROM cranfield_paradedb_documents WHERE full_text @@@ paradedb.match(''full_text'', $1) ORDER BY paradedb.score(doc_id) DESC LIMIT 10' USING query_text; end_ts := clock_timestamp(); times := array_append(times, EXTRACT(EPOCH FROM (end_ts - start_ts)) * 1000); END LOOP; SELECT array_agg(t ORDER BY t) INTO sorted_times FROM unnest(times) t; median_ms := sorted_times[(iterations + 1) / 2]; min_ms := sorted_times[1]; max_ms := sorted_times[iterations]; RETURN NEXT; END; $$ LANGUAGE plpgsql; -- Short query (2 words) \echo 'Query 1: Short query (2 words) - "boundary layer"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query_paradedb('boundary layer'); \echo '' \echo 'Query 2: Medium query (4 words) - "supersonic flow heat transfer"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query_paradedb('supersonic flow heat transfer'); \echo '' \echo 'Query 3: Long query (from Cranfield query 1)' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query_paradedb('what similarity laws must be obeyed when constructing aeroelastic models of heated high speed aircraft'); \echo '' \echo 'Query 4: Common terms - "flow pressure"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query_paradedb('flow pressure'); \echo '' \echo 'Query 5: Specific terms - "magnetohydrodynamic viscosity"' SELECT 'Execution Time: ' || round(median_ms, 3) || ' ms (min=' || round(min_ms, 3) || ', max=' || round(max_ms, 3) || ')' as result FROM benchmark_query_paradedb('magnetohydrodynamic viscosity'); DROP FUNCTION benchmark_query_paradedb; -- ============================================================ -- Benchmark 2: Query Throughput (all 225 Cranfield queries) -- ============================================================ \echo '' \echo '=== Benchmark 2: Query Throughput (225 Cranfield queries) ===' \echo 'Running all standard Cranfield queries sequentially' DO $$ DECLARE q RECORD; start_time timestamp; end_time timestamp; total_ms numeric := 0; query_count int := 0; BEGIN start_time := clock_timestamp(); FOR q IN SELECT query_id, query_text FROM cranfield_paradedb_queries ORDER BY query_id LOOP PERFORM doc_id FROM cranfield_paradedb_documents WHERE full_text @@@ paradedb.match('full_text', q.query_text) ORDER BY paradedb.score(doc_id) DESC LIMIT 10; query_count := query_count + 1; END LOOP; end_time := clock_timestamp(); total_ms := EXTRACT(EPOCH FROM (end_time - start_time)) * 1000; RAISE NOTICE 'THROUGHPUT_RESULT: % queries in % ms (avg % ms/query)', query_count, round(total_ms::numeric, 2), round((total_ms / query_count)::numeric, 2); END $$; -- ============================================================ -- Benchmark 3: Index Statistics -- ============================================================ \echo '' \echo '=== Benchmark 3: Index Statistics ===' SELECT 'cranfield_paradedb_idx' as index_name, pg_size_pretty(pg_relation_size('cranfield_paradedb_idx')) as index_size, pg_size_pretty(pg_relation_size('cranfield_paradedb_documents')) as table_size, (SELECT COUNT(*) FROM cranfield_paradedb_documents) as num_documents; \echo '' \echo '=== Cranfield BM25 Benchmark Complete (ParadeDB) ===' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/paradedb/concurrent-setup.sql000066400000000000000000000067031525762047700314350ustar00rootroot00000000000000-- Cranfield Collection - ParadeDB Concurrent Insert Setup -- Prepares staging table for pgbench concurrent inserts -- After this script: cranfield_paradedb_documents exists with -- ParadeDB BM25 index, cranfield_paradedb_staging holds rows -- to be inserted via pgbench \set ON_ERROR_STOP on \timing on \echo 'Cranfield ParadeDB - Concurrent Insert Setup' \echo '=============================================' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_paradedb_documents CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_queries CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_expected_rankings CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_staging CASCADE; DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; DROP SEQUENCE IF EXISTS insert_seq; -- Create temp tables for dataset.sql loading \echo 'Creating tables for dataset loading...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Load dataset \echo 'Loading Cranfield collection...' \i dataset.sql -- Create staging table with document data \echo 'Creating staging table...' CREATE TABLE cranfield_paradedb_staging ( staging_id SERIAL PRIMARY KEY, doc_id INTEGER, title TEXT, author TEXT, bibliography TEXT, content TEXT ); INSERT INTO cranfield_paradedb_staging (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents ORDER BY doc_id; -- Drop temp tables DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Create target ParadeDB table with BM25 index (empty) \echo 'Creating target ParadeDB table with BM25 index...' CREATE TABLE cranfield_paradedb_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE INDEX cranfield_paradedb_idx ON cranfield_paradedb_documents USING bm25 (doc_id, full_text) WITH ( key_field = 'doc_id', text_fields = '{ "full_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Create sequence for pgbench inserts CREATE SEQUENCE insert_seq; -- Report staging row count SELECT 'STAGING_ROW_COUNT: ' || count(*) FROM cranfield_paradedb_staging; \echo 'Concurrent insert setup complete.' \echo 'Run pgbench with paradedb/pgbench-insert.sql next.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/paradedb/insert-load.sql000066400000000000000000000101121525762047700303230ustar00rootroot00000000000000-- Cranfield Collection - ParadeDB Insert Loading -- Creates ParadeDB BM25 index on EMPTY table, then loads data -- via inserts to measure insert-path indexing performance \set ON_ERROR_STOP on \timing on \echo 'Cranfield ParadeDB - Insert Load Phase' \echo '=======================================' -- Ensure ParadeDB extension is installed CREATE EXTENSION IF NOT EXISTS pg_search; -- Clean up any existing tables \echo 'Cleaning up existing tables...' DROP TABLE IF EXISTS cranfield_paradedb_documents CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_queries CASCADE; DROP TABLE IF EXISTS cranfield_paradedb_expected_rankings CASCADE; DROP TABLE IF EXISTS cranfield_full_documents CASCADE; DROP TABLE IF EXISTS cranfield_full_queries CASCADE; DROP TABLE IF EXISTS cranfield_full_expected_rankings CASCADE; -- Create temp tables for dataset.sql loading \echo 'Creating tables for dataset loading...' CREATE TABLE cranfield_full_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_full_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_full_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Create the ParadeDB target tables \echo 'Creating ParadeDB documents table...' CREATE TABLE cranfield_paradedb_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_paradedb_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); CREATE TABLE cranfield_paradedb_expected_rankings ( query_id INTEGER NOT NULL, doc_id INTEGER NOT NULL, rank INTEGER NOT NULL, bm25_score FLOAT NOT NULL, PRIMARY KEY (query_id, doc_id) ); -- Create ParadeDB BM25 index on the EMPTY table \echo 'Creating ParadeDB BM25 index on empty table...' CREATE INDEX cranfield_paradedb_idx ON cranfield_paradedb_documents USING bm25 (doc_id, full_text) WITH ( key_field = 'doc_id', text_fields = '{ "full_text": { "tokenizer": { "type": "default", "stopwords_language": "English", "stemmer": "English" } } }' ); -- Load dataset into temp tables \echo 'Loading Cranfield collection...' \i dataset.sql -- Insert data into indexed ParadeDB table \echo 'INSERT_TIME:' \echo 'Inserting data into indexed ParadeDB table...' INSERT INTO cranfield_paradedb_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_full_documents; INSERT INTO cranfield_paradedb_queries SELECT * FROM cranfield_full_queries; INSERT INTO cranfield_paradedb_expected_rankings SELECT * FROM cranfield_full_expected_rankings; -- Drop temp tables DROP TABLE cranfield_full_documents CASCADE; DROP TABLE cranfield_full_queries CASCADE; DROP TABLE cranfield_full_expected_rankings CASCADE; -- Compact index segments (ParadeDB merges during VACUUM) \echo 'INDEX_VACUUM:' VACUUM cranfield_paradedb_documents; -- Report index and table sizes \echo '' \echo '=== Index Size Report ===' SELECT 'INDEX_SIZE:' as label, pg_size_pretty( pg_relation_size('cranfield_paradedb_idx') ) as index_size, pg_relation_size( 'cranfield_paradedb_idx' ) as index_bytes; SELECT 'TABLE_SIZE:' as label, pg_size_pretty( pg_total_relation_size( 'cranfield_paradedb_documents') ) as table_size, pg_total_relation_size( 'cranfield_paradedb_documents') as table_bytes; \echo 'ParadeDB insert load phase completed.' timescale-pg_textsearch-b63fe58/benchmarks/sql/cranfield/paradedb/pgbench-insert.sql000066400000000000000000000003731525762047700310220ustar00rootroot00000000000000SELECT nextval('insert_seq') AS next_id \gset INSERT INTO cranfield_paradedb_documents (doc_id, title, author, bibliography, content) SELECT doc_id, title, author, bibliography, content FROM cranfield_paradedb_staging WHERE staging_id = :next_id; timescale-pg_textsearch-b63fe58/benchmarks/sql/filtered_seed.sql000066400000000000000000000255751525762047700252320ustar00rootroot00000000000000-- Filtered-seed top-K benchmark (issues #434, #435) -- -- A filtered top-k query -- WHERE ORDER BY LIMIT k -- -- is planned as a BM25 top-k index scan with applied as a -- Filter above it. If the scan surfaces only its top k rows by score, -- few may survive the Filter, and the executor re-drives the scan with -- a doubling internal limit until k rows do. Each re-drive re-walks -- the posting lists from the start. -- -- pg_textsearch.filtered_seed seeds the scan's internal top-K from the -- planner's estimated filter selectivity so one pass usually suffices. -- This benchmark measures what that is worth, by toggling the GUC over -- identical data and queries. -- -- Two query shapes, because they exercise different fixes: -- -- single - one BM25 scan. This is #434, which shipped earlier. -- union2 - two or three BM25 scans of ONE index in one statement, -- union3 each with its own Filter. This is #435: the seed is -- carried per scan in its query value. Before that fix -- one arm consumed the single per-index slot and the rest -- fell back to pg_textsearch.default_limit plus backoff, -- so the 'on' numbers for these shapes are only achievable -- with #435. -- -- To measure what #435 itself bought, run this against a build that -- predates it and compare the **seed on** columns only. The seed off -- columns are not a stable cross-version baseline: with #435 the k -- binding is per scan even when seeding is disabled, so 'off' means -- "every arm capped at its own k" on a new build versus "one arm at -- k, the rest at default_limit" on an old one. Those are different -- amounts of work, and neither is the thing being measured. -- -- Metrics are latency and scoring passes. Result parity is covered by -- the filtered_seed regression, not by this timing harness. -- bm25_debug_scoring_passes counts passes: a well-seeded scan costs -- exactly one, and each backoff re-drive adds another. -- -- Usage: -- psql -f filtered_seed.sql -- 200k docs -- psql -v ndocs=1000000 -f filtered_seed.sql \if :{?ndocs} \else \set ndocs 200000 \endif \set ON_ERROR_STOP on \timing off SET client_min_messages = WARNING; CREATE EXTENSION IF NOT EXISTS pg_textsearch; \echo '' \echo '=== Filtered-seed top-K benchmark ===' \echo '' DROP TABLE IF EXISTS fsb_docs CASCADE; CREATE TABLE fsb_docs ( id int PRIMARY KEY, facet int, body text ); -- 'common' is in every document, so its posting list spans the whole -- corpus and scan depth is what costs. facet = id % 1000 is uniform, -- giving predictable selectivity that the planner estimates well: this -- measures seeding, not estimation error. INSERT INTO fsb_docs SELECT g, g % 1000, concat_ws(' ', 'common', CASE WHEN g % 2 = 0 THEN 'alpha' END, CASE WHEN g % 3 = 0 THEN 'beta' END, CASE WHEN g % 7 = 0 THEN 'gamma' END, 'doc' || g) FROM generate_series(1, :ndocs) g; CREATE INDEX fsb_idx ON fsb_docs USING bm25(body) WITH (text_config='english'); -- No index on facet: the facet is applied as a Filter above the BM25 -- index scan, which is the path filtered_seed optimizes. ANALYZE fsb_docs; \echo 'Corpus:' SELECT count(*) AS docs, count(DISTINCT facet) AS facet_buckets, pg_size_pretty(pg_relation_size('fsb_idx')) AS index_size FROM fsb_docs; ------------------------------------------------------------------------ -- Harness ------------------------------------------------------------------------ CREATE TABLE fsb_results ( shape text, sel text, lim int, scans int, off_ms numeric, off_passes bigint, on_ms numeric, on_passes bigint ); -- bm25_debug_scoring_passes is newer than the seeding itself, so it is -- absent on older builds. Returning NULL there keeps the benchmark -- runnable against a version that predates the counter, which is what -- makes the main-versus-branch comparison in the header possible. -- Never called inside a timed loop, so the subtransaction is free. CREATE FUNCTION fsb_passes(reset boolean) RETURNS bigint LANGUAGE plpgsql AS $$ BEGIN RETURN bm25_debug_scoring_passes(reset); EXCEPTION WHEN undefined_function THEN RETURN NULL; END; $$; -- How many BM25 index scans of fsb_idx the plan actually contains. -- Every arm must use the index. Read from EXPLAIN rather than inferred -- from the pass counter, so this works on builds without the counter. CREATE FUNCTION fsb_scan_count(q text) RETURNS int LANGUAGE plpgsql AS $$ DECLARE r record; n int := 0; BEGIN FOR r IN EXECUTE 'EXPLAIN (COSTS OFF) ' || q LOOP IF r."QUERY PLAN" LIKE '%Index Scan using fsb_idx%' THEN n := n + 1; END IF; END LOOP; RETURN n; END; $$; -- Median of `iters` timed runs, plus the scoring-pass count from one -- separate run. A warmup run first, so the numbers are steady-state -- and not dominated by first-touch buffer reads. CREATE FUNCTION fsb_measure(q text, iters int DEFAULT 7) RETURNS TABLE (median_ms numeric, passes bigint) LANGUAGE plpgsql AS $$ DECLARE t0 timestamptz; samples double precision[] := '{}'; i int; BEGIN EXECUTE q; PERFORM fsb_passes(true); EXECUTE q; passes := fsb_passes(true); FOR i IN 1..iters LOOP t0 := clock_timestamp(); EXECUTE q; samples := samples || (EXTRACT(epoch FROM clock_timestamp() - t0) * 1000.0); END LOOP; SELECT round(percentile_cont(0.5) WITHIN GROUP (ORDER BY s)::numeric, 2) INTO median_ms FROM unnest(samples) s; RETURN NEXT; END; $$; -- Measure one query with seeding off, then on, and record the pair. CREATE FUNCTION fsb_run(shape text, sel text, lim int, q text) RETURNS void LANGUAGE plpgsql AS $$ DECLARE off_ms numeric; off_p bigint; on_ms numeric; on_p bigint; scans int; expected_scans int := CASE shape WHEN 'single' THEN 1 WHEN 'union2' THEN 2 WHEN 'union3' THEN 3 END; BEGIN IF expected_scans IS NULL THEN RAISE EXCEPTION 'unknown benchmark shape: %', shape; END IF; scans := fsb_scan_count(q); IF scans <> expected_scans THEN RAISE WARNING 'BM25 scan count mismatch for shape=% sel=% limit=%; ' 'expected %, got %, cell dropped', shape, sel, lim, expected_scans, scans; RETURN; END IF; SET pg_textsearch.filtered_seed = off; SELECT median_ms, passes INTO off_ms, off_p FROM fsb_measure(q); SET pg_textsearch.filtered_seed = on; SELECT median_ms, passes INTO on_ms, on_p FROM fsb_measure(q); INSERT INTO fsb_results VALUES (shape, sel, lim, scans, off_ms, off_p, on_ms, on_p); END; $$; -- The query shapes. count(*) over a subquery consumes every row the -- Limit produces without printing them. CREATE FUNCTION fsb_q_single(pred text, lim int) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT format($f$ SELECT count(*) FROM ( SELECT id FROM fsb_docs WHERE %s ORDER BY body <@> to_bm25query('common', 'fsb_idx') LIMIT %s) s $f$, pred, lim); $$; CREATE FUNCTION fsb_q_union(preds text[], lim int) RETURNS text LANGUAGE sql IMMUTABLE AS $$ SELECT 'SELECT count(*) FROM (' || string_agg( format($f$(SELECT id FROM fsb_docs WHERE %s ORDER BY body <@> to_bm25query('common', 'fsb_idx') LIMIT %s)$f$, p, lim), ' UNION ALL ') || ') s' FROM unnest(preds) p; $$; ------------------------------------------------------------------------ -- Plan confirmation: every cell below must be a BM25 index scan with a -- Filter, or the benchmark is measuring the wrong thing. ------------------------------------------------------------------------ \echo '' \echo 'Plan under test (single, sel=0.001):' EXPLAIN (COSTS OFF) SELECT id FROM fsb_docs WHERE facet = 7 ORDER BY body <@> to_bm25query('common', 'fsb_idx') LIMIT 10; ------------------------------------------------------------------------ -- Measurements ------------------------------------------------------------------------ \echo '' \echo 'Measuring...' DO $$ BEGIN /* * Selectivity is what drives the gain: the lower it is, the deeper * a scan must go to surface k survivors, and the longer the * backoff chain an unseeded scan pays to get there. */ PERFORM fsb_run('single', '0.1', 10, fsb_q_single('facet < 100', 10)); PERFORM fsb_run('single', '0.01', 10, fsb_q_single('facet < 10', 10)); PERFORM fsb_run('single', '0.001', 10, fsb_q_single('facet = 7', 10)); PERFORM fsb_run('single', '0.001', 100, fsb_q_single('facet = 7', 100)); /* #435: several BM25 scans of one index in one statement. */ PERFORM fsb_run('union2', '0.001', 10, fsb_q_union(ARRAY['facet = 7', 'facet = 13'], 10)); PERFORM fsb_run('union3', '0.001', 10, fsb_q_union(ARRAY['facet = 7', 'facet = 13', 'facet = 29'], 10)); /* * Arms with different selectivities, which is the case #435 is * really about: each arm needs a seed computed from its own * Filter, not whichever arm happened to reach the slot first. */ PERFORM fsb_run('union3', 'mixed', 10, fsb_q_union(ARRAY['facet < 100', 'facet < 10', 'facet = 500'], 10)); END; $$; RESET pg_textsearch.filtered_seed; ------------------------------------------------------------------------ -- Report ------------------------------------------------------------------------ \echo '' \echo 'Results:' \pset border 2 SELECT shape, sel AS selectivity, lim AS "limit", scans AS "bm25 scans", off_ms AS "seed off (ms)", on_ms AS "seed on (ms)", round(off_ms / nullif(on_ms, 0), 2) AS speedup, off_passes AS "passes off", on_passes AS "passes on" FROM fsb_results ORDER BY shape, sel, lim; \echo '' \echo 'Parseable:' \pset tuples_only on \pset format unaligned SELECT format( 'FSB_RESULT: shape=%s sel=%s limit=%s scans=%s ' 'off_ms=%s off_passes=%s on_ms=%s on_passes=%s speedup=%s', shape, sel, lim, scans, off_ms, coalesce(off_passes::text, 'null'), on_ms, coalesce(on_passes::text, 'null'), round(off_ms / nullif(on_ms, 0), 2)) FROM fsb_results ORDER BY shape, sel, lim; \pset tuples_only off \pset format aligned ------------------------------------------------------------------------ -- Cleanup ------------------------------------------------------------------------ DROP FUNCTION fsb_passes(boolean); DROP FUNCTION fsb_scan_count(text); DROP FUNCTION fsb_measure(text, int); DROP FUNCTION fsb_run(text, text, int, text); DROP FUNCTION fsb_q_single(text, int); DROP FUNCTION fsb_q_union(text[], int); DROP TABLE fsb_results; DROP TABLE fsb_docs CASCADE; \echo '' \echo 'Done.' timescale-pg_textsearch-b63fe58/benchmarks/sql/memory_stress.sql000066400000000000000000000224411525762047700253340ustar00rootroot00000000000000-- Memory stress benchmark for pg_textsearch -- Tests a large document dataset with a constrained memtable cache -- -- This benchmark: -- 1. Runs with a small memtable cache limit (1MB is suggested) -- 2. Creates a large table with randomly generated documents -- 3. Creates a pg_textsearch index and runs queries under cache pressure -- 4. Reports query behavior and performance under the configured limit \echo 'Starting pg_textsearch memory stress benchmark...' -- pg_textsearch.memory_limit is a PGC_SIGHUP setting measured in kB. -- Set it in postgresql.conf and reload PostgreSQL before this benchmark. \echo 'For a 1MB cache limit, set pg_textsearch.memory_limit = 1MB' \echo 'in postgresql.conf and reload PostgreSQL before running this benchmark.' SHOW pg_textsearch.memory_limit; \echo 'The benchmark uses a large dataset to stress the configured cache limit' -- Create a table for stress testing DROP TABLE IF EXISTS stress_docs CASCADE; CREATE TABLE stress_docs ( id SERIAL PRIMARY KEY, title TEXT, content TEXT, category TEXT, tags TEXT[] ); \echo 'Generating large document dataset...' -- Insert documents with randomly generated content -- Using various content patterns to create realistic text diversity INSERT INTO stress_docs (title, content, category, tags) SELECT 'Document ' || i, -- Generate varied content lengths and vocabularies CASE (i % 10) WHEN 0 THEN 'artificial intelligence machine learning deep neural networks data science algorithm optimization performance scalability distributed systems cloud computing microservices containerization kubernetes docker database postgresql indexing search ranking relevance information retrieval natural language processing text mining sentiment analysis classification clustering regression prediction model training validation testing deployment monitoring logging metrics analytics big data hadoop spark kafka elasticsearch mongodb redis caching memory optimization query performance database design normalization denormalization sharding replication consistency availability partition tolerance CAP theorem ACID transactions isolation levels concurrency control deadlock prevention' WHEN 1 THEN 'software engineering development programming languages python java javascript typescript go rust c plus plus scala kotlin swift objective c ruby php perl shell bash scripting automation testing unit integration system performance load stress testing continuous integration deployment devops infrastructure as code terraform ansible chef puppet monitoring alerting logging observability tracing metrics dashboards grafana prometheus elk stack version control git github gitlab bitbucket code review pull requests merge conflicts branching strategies gitflow feature branches hotfixes releases semantic versioning' WHEN 2 THEN 'web development frontend backend fullstack html css javascript typescript react vue angular node express django flask spring boot ruby rails php laravel codeigniter symfony asp net mvc api rest graphql soap microservices architecture patterns mvc mvp mvvm observer factory singleton repository dependency injection inversion of control aspect oriented programming functional programming object oriented programming design patterns solid principles clean code refactoring technical debt documentation' WHEN 3 THEN 'cybersecurity information security network security application security data protection privacy encryption decryption hashing salting authentication authorization access control identity management single sign on multi factor authentication oauth jwt tokens vulnerability assessment penetration testing security auditing compliance regulations gdpr hipaa sox pci dss threat modeling risk assessment incident response disaster recovery business continuity backup strategies data retention policies' WHEN 4 THEN 'data analytics business intelligence data warehousing extract transform load etl data pipelines batch streaming processing apache airflow luigi prefect data quality data governance master data management metadata lineage cataloging discovery profiling cleansing standardization validation enrichment aggregation reporting dashboards visualization tableau power bi looker qlik sense statistical analysis hypothesis testing confidence intervals correlation causation regression classification clustering' WHEN 5 THEN 'cloud computing amazon web services microsoft azure google cloud platform infrastructure platform software as a service compute storage networking content delivery edge computing serverless functions containers orchestration kubernetes docker swarm service mesh istio consul vault secrets management configuration management infrastructure as code terraform cloudformation arm templates cost optimization resource management auto scaling load balancing high availability disaster recovery' WHEN 6 THEN 'mobile development ios android react native flutter xamarin cordova phonegap ionic swift kotlin java objective c dart javascript html5 css3 responsive design progressive web apps pwa service workers offline support push notifications app store optimization aso user experience user interface design wireframing prototyping testing debugging performance optimization battery usage memory management network efficiency security encryption data protection privacy' WHEN 7 THEN 'project management agile scrum kanban lean six sigma waterfall methodology planning estimation scheduling resource allocation risk management stakeholder communication requirements gathering user stories acceptance criteria sprint planning daily standups retrospectives product owner scrum master project manager business analyst quality assurance testing documentation change management scope creep timeline milestones deliverables' WHEN 8 THEN 'artificial intelligence machine learning supervised unsupervised reinforcement learning linear regression logistic regression decision trees random forest support vector machines neural networks convolutional recurrent transformer attention mechanisms gradient descent backpropagation optimization algorithms feature engineering selection extraction dimensionality reduction principal component analysis cross validation hyperparameter tuning model evaluation metrics accuracy precision recall f1 score confusion matrix' ELSE 'enterprise architecture service oriented architecture microservices event driven architecture domain driven design bounded contexts aggregate root entity value object repository pattern command query responsibility segregation event sourcing saga pattern orchestration choreography api gateway service discovery circuit breaker bulkhead pattern retry timeout exponential backoff graceful degradation health checks monitoring alerting distributed tracing logging correlation ids' END || ' Additional content block ' || i || ' with unique identifier ' || md5(i::text) || ' More varied text to increase vocabulary diversity and term frequency distribution across documents.', CASE (i % 5) WHEN 0 THEN 'Technology' WHEN 1 THEN 'Science' WHEN 2 THEN 'Business' WHEN 3 THEN 'Engineering' ELSE 'Research' END, ARRAY[ CASE (i % 8) WHEN 0 THEN 'programming' WHEN 1 THEN 'database' WHEN 2 THEN 'algorithm' WHEN 3 THEN 'optimization' WHEN 4 THEN 'performance' WHEN 5 THEN 'scalability' WHEN 6 THEN 'architecture' ELSE 'development' END, 'tag' || (i % 20), 'category' || (i % 10) ] FROM generate_series(1, 100000) AS i; -- 100K documents to stress memory \echo 'Document generation complete. Table contains:' SELECT COUNT(*) as total_docs, AVG(LENGTH(content)) as avg_content_length, MAX(LENGTH(content)) as max_content_length FROM stress_docs; CREATE INDEX stress_content_idx ON stress_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- If we get here, let's try some queries to stress the system further \echo 'Index creation succeeded! Testing query performance under memory pressure...' -- Try various queries to stress the system SELECT 'Query 1: Common terms' as test_name; SELECT COUNT(*) FROM ( SELECT 1 FROM stress_docs ORDER BY content <@> to_bm25query('algorithm optimization performance', 'stress_content_idx') LIMIT 100 ) subq; SELECT 'Query 2: Technical terms' as test_name; SELECT COUNT(*) FROM ( SELECT 1 FROM stress_docs ORDER BY content <@> to_bm25query('machine learning artificial intelligence', 'stress_content_idx') LIMIT 100 ) subq; SELECT 'Query 3: Specific terms' as test_name; SELECT COUNT(*) FROM ( SELECT 1 FROM stress_docs ORDER BY content <@> to_bm25query('postgresql database indexing search', 'stress_content_idx') LIMIT 100 ) subq; \echo 'Query 4: Large result set to stress memory allocation' SELECT COUNT(*) FROM ( SELECT 1 FROM stress_docs ORDER BY content <@> to_bm25query('the and of to in', 'stress_content_idx') LIMIT 10000 ) subq; -- Clean up \echo '' \echo 'Cleaning up stress test resources...' DROP TABLE IF EXISTS stress_docs CASCADE; -- pg_textsearch.memory_limit changes take effect after a configuration reload. \echo '' \echo 'Memory stress benchmark completed.' \echo 'Check the output above for cache-limit behavior and performance characteristics.' timescale-pg_textsearch-b63fe58/benchmarks/sql/memory_stress_micro.sql000066400000000000000000000125101525762047700265210ustar00rootroot00000000000000-- Micro memory stress benchmark for pg_textsearch -- Tests with a very small dataset to demonstrate functionality -- and establish baseline performance characteristics -- -- This benchmark: -- 1. Creates a small table (1K documents) -- 2. Creates a pg_textsearch index successfully -- 3. Tests various queries -- 4. Provides timing and performance information \timing on \echo 'Starting pg_textsearch micro memory stress benchmark...' -- Create a table for testing DROP TABLE IF EXISTS stress_docs_micro CASCADE; CREATE TABLE stress_docs_micro ( id SERIAL PRIMARY KEY, title TEXT, content TEXT, category TEXT ); \echo 'Generating micro document dataset (1K documents)...' -- Insert documents with varied content INSERT INTO stress_docs_micro (title, content, category) SELECT 'Document ' || i, CASE (i % 10) WHEN 0 THEN 'artificial intelligence machine learning deep neural networks data science algorithm optimization performance scalability distributed systems' WHEN 1 THEN 'software engineering development programming languages python java javascript web development frontend backend fullstack' WHEN 2 THEN 'cybersecurity information security network security application security data protection privacy encryption authentication' WHEN 3 THEN 'data analytics business intelligence data warehousing extract transform load pipelines batch streaming processing' WHEN 4 THEN 'cloud computing amazon web services microsoft azure google cloud platform infrastructure platform software as service' WHEN 5 THEN 'mobile development ios android react native flutter xamarin swift kotlin java objective c dart javascript' WHEN 6 THEN 'database management systems postgresql mysql oracle mongodb redis elasticsearch indexing querying optimization' WHEN 7 THEN 'project management agile scrum kanban lean methodology planning estimation scheduling resource allocation' WHEN 8 THEN 'machine learning supervised unsupervised reinforcement learning linear regression logistic decision trees' ELSE 'enterprise architecture service oriented microservices event driven domain driven design api gateway discovery' END || ' Document ' || i || ' unique content with hash ' || substr(md5(i::text), 1, 8), CASE (i % 3) WHEN 0 THEN 'Technology' WHEN 1 THEN 'Business' ELSE 'Engineering' END FROM generate_series(1, 1000) AS i; -- 1K documents for quick testing \echo 'Document generation complete. Dataset summary:' SELECT COUNT(*) as total_docs, AVG(LENGTH(content)) as avg_content_length, MAX(LENGTH(content)) as max_content_length, MIN(LENGTH(content)) as min_content_length FROM stress_docs_micro; \echo '' \echo 'Creating pg_textsearch index...' CREATE INDEX stress_micro_idx ON stress_docs_micro USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); \echo 'Index created successfully! Running performance tests...' -- Test queries with timing \echo '' \echo '=== Performance Test Results ===' \echo 'Test 1: Technical terms search' SELECT COUNT(*) as matches FROM stress_docs_micro ORDER BY content <@> to_bm25query('algorithm optimization', 'stress_micro_idx') LIMIT 100; \echo 'Test 2: AI/ML search' SELECT COUNT(*) as matches FROM stress_docs_micro ORDER BY content <@> to_bm25query('machine learning artificial intelligence', 'stress_micro_idx') LIMIT 100; \echo 'Test 3: Database search' SELECT COUNT(*) as matches FROM stress_docs_micro ORDER BY content <@> to_bm25query('database postgresql mysql', 'stress_micro_idx') LIMIT 100; \echo 'Test 4: Top results with scores' SELECT title, round((content <@> to_bm25query('software development', 'stress_micro_idx'))::numeric, 4) as score FROM stress_docs_micro ORDER BY content <@> to_bm25query('software development', 'stress_micro_idx') LIMIT 3; \echo 'Test 5: Category-based analysis' SELECT category, COUNT(*) as total_docs, COUNT(*) as total_docs, AVG((content <@> to_bm25query('programming software', 'stress_micro_idx'))::numeric) as avg_score FROM stress_docs_micro GROUP BY category ORDER BY category; -- Test memory behavior with larger queries \echo '' \echo '=== Memory Pressure Tests ===' \echo 'Test 6: Common terms (should find many matches)' SELECT COUNT(*) as matches FROM stress_docs_micro ORDER BY content <@> to_bm25query('data system', 'stress_micro_idx') LIMIT 50; \echo 'Test 7: Specific terms (should find fewer matches)' SELECT COUNT(*) as matches FROM stress_docs_micro ORDER BY content <@> to_bm25query('kubernetes docker containerization', 'stress_micro_idx') LIMIT 50; \echo 'Test 8: Complex multi-term query' SELECT COUNT(*) as matches FROM stress_docs_micro ORDER BY content <@> to_bm25query('web development frontend javascript react', 'stress_micro_idx') LIMIT 50; -- Performance summary \echo '' \echo '=== Benchmark Summary ===' SELECT 'Micro benchmark completed' as status, COUNT(*) as total_documents, 'Index created successfully' as index_status, 'Queries executed successfully' as query_status FROM stress_docs_micro; -- Clean up \echo '' \echo 'Cleaning up test resources...' DROP TABLE stress_docs_micro CASCADE; \echo '' \echo 'Micro memory stress benchmark completed.' \echo 'This establishes baseline functionality with 1K documents.' \echo 'For true stress testing, try larger datasets or a reduced pg_textsearch.memory_limit.' timescale-pg_textsearch-b63fe58/benchmarks/sql/memory_stress_small.sql000066400000000000000000000124411525762047700265230ustar00rootroot00000000000000-- Smaller memory stress benchmark for pg_textsearch -- Tests behavior with a more manageable dataset size -- -- This benchmark: -- 1. Creates a moderately large table (25K documents) -- 2. Creates a pg_textsearch index to demonstrate current performance -- 3. Tests various queries to show system behavior -- 4. Provides timing information \timing on \echo 'Starting pg_textsearch memory stress benchmark (small version)...' -- Create a table for stress testing DROP TABLE IF EXISTS stress_docs_small CASCADE; CREATE TABLE stress_docs_small ( id SERIAL PRIMARY KEY, title TEXT, content TEXT, category TEXT, tags TEXT[] ); \echo 'Generating document dataset (25K documents)...' -- Insert documents with randomly generated content -- Using various content patterns to create realistic text diversity INSERT INTO stress_docs_small (title, content, category, tags) SELECT 'Document ' || i, -- Generate varied content lengths and vocabularies (shorter for faster testing) CASE (i % 5) WHEN 0 THEN 'artificial intelligence machine learning deep neural networks data science algorithm optimization performance scalability distributed systems cloud computing database postgresql indexing search ranking relevance information retrieval natural language processing text mining sentiment analysis' WHEN 1 THEN 'software engineering development programming languages python java javascript web development frontend backend fullstack testing continuous integration deployment devops infrastructure monitoring observability' WHEN 2 THEN 'cybersecurity information security network security application security data protection privacy encryption authentication authorization vulnerability assessment penetration testing security auditing' WHEN 3 THEN 'data analytics business intelligence data warehousing extract transform load pipelines batch streaming processing statistical analysis hypothesis testing correlation regression classification clustering' ELSE 'mobile development cloud computing project management agile scrum enterprise architecture microservices event driven domain driven design api gateway service discovery circuit breaker' END || ' Document identifier ' || i || ' with hash ' || substr(md5(i::text), 1, 8) || ' Additional unique content for vocabulary diversity.', CASE (i % 3) WHEN 0 THEN 'Technology' WHEN 1 THEN 'Business' ELSE 'Engineering' END, ARRAY[ 'tag' || (i % 10), 'category' || (i % 5), CASE (i % 4) WHEN 0 THEN 'programming' WHEN 1 THEN 'database' WHEN 2 THEN 'algorithm' ELSE 'optimization' END ] FROM generate_series(1, 25000) AS i; -- 25K documents for reasonable test time \echo 'Document generation complete. Table contains:' SELECT COUNT(*) as total_docs, AVG(LENGTH(content)) as avg_content_length, MAX(LENGTH(content)) as max_content_length, MIN(LENGTH(content)) as min_content_length FROM stress_docs_small; \echo '' \echo 'Creating pg_textsearch index on content column...' CREATE INDEX stress_content_small_idx ON stress_docs_small USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); \echo 'Index creation completed! Running performance queries...' -- Test queries with timing \echo '' \echo '=== Query Performance Tests ===' \echo 'Query 1: Common technical terms' SELECT COUNT(*) as matches FROM stress_docs_small ORDER BY content <@> to_bm25query('algorithm optimization performance', 'stress_content_small_idx') LIMIT 100; \echo 'Query 2: AI/ML terms' SELECT COUNT(*) as matches FROM stress_docs_small ORDER BY content <@> to_bm25query('machine learning artificial intelligence', 'stress_content_small_idx') LIMIT 100; \echo 'Query 3: Database terms' SELECT COUNT(*) as matches FROM stress_docs_small ORDER BY content <@> to_bm25query('database postgresql indexing search', 'stress_content_small_idx') LIMIT 100; \echo 'Query 4: Top scoring results' SELECT title, content <@> to_bm25query('software development programming', 'stress_content_small_idx') as score FROM stress_docs_small ORDER BY content <@> to_bm25query('software development programming', 'stress_content_small_idx') LIMIT 5; \echo 'Query 5: Multiple different searches' SELECT 'AI/ML' as search_type, COUNT(*) as matches FROM stress_docs_small ORDER BY content <@> to_bm25query('artificial intelligence machine learning', 'stress_content_small_idx') LIMIT 100 UNION ALL SELECT 'Security' as search_type, COUNT(*) as matches FROM stress_docs_small ORDER BY content <@> to_bm25query('security encryption authentication', 'stress_content_small_idx') LIMIT 100 UNION ALL SELECT 'Development' as search_type, COUNT(*) as matches FROM stress_docs_small ORDER BY content <@> to_bm25query('programming development software', 'stress_content_small_idx') LIMIT 100; -- Index statistics \echo '' \echo '=== Index Statistics ===' SELECT schemaname, tablename, indexname, tablespace, indexdef FROM pg_indexes WHERE indexname = 'stress_content_small_idx'; -- Clean up \echo '' \echo 'Cleaning up test resources...' DROP TABLE stress_docs_small CASCADE; \echo '' \echo 'Small memory stress benchmark completed successfully.' \echo 'Check timing information above to assess performance characteristics.' timescale-pg_textsearch-b63fe58/images/000077500000000000000000000000001525762047700202265ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/images/banner.png000066400000000000000000015227341525762047700222170ustar00rootroot00000000000000PNG  IHDR3DdsRGBgAMA a pHYsodIDATx^[,,OR a2cf[ec-㏈59039%`{Ӭs0macd=sV#C0emsNC1Gh~[׈G$+ b-6 z|۽udvDVPC\։2Fp ey{V`XS89v ,e3L98#rl)UK)_0'xTw]%-'Èg\(Ƅ/ -Gbl$qԓׁ{ t3!vlԻּ%ٍE"Tu>=497ɡL#1&I6F$q<#xyϕk`XXbh5X5(cc${78'隿r!XY:G Щcj/_Qa&b`3ӗqXFXrpJ̵zZ{*P(׊.o+qN x憲ܖRKqs=aP ?I^tH= λYB'Q˦x )ݭ yRU 3l_"}ąlM1$vK,T*NN-l˺Vjw(d>Ŭ&{ie7kG$/6I'lu0mz:~p֑ h;9vjˠ@7Xѥ%nFpX9'?̓f%k.mt<]8l []|y<1Su3=:mBMĨl8\?uuס^@yb;ӲH A8m&=y|Ϗ ݌=Z4Ɩ[\o Lm-o0rLLGQ`\E6㣖џoiBt=Xs 3X5I~O-Utc܈e)0?[2 ~֝ڬeOSFjy',օ(m[vmeqپ\}#y,28(=g}5s 1#n87>|8tE6/^s cj^{eul'3۾)fq}a[y\/q2 GS\6FxTb:b~ 2r>5Q1ҜCmQ1)\mH&O$;p>o=u~rL-`6?̘.x74v{}fǕ~{C-|cyd 4 ~I纲1Pt W65:,̌e]R"H(ݢPl9ݚy*sT9wTPb~f->ϯρGNsM5<tXqwER+:T3mm36ʻC=md>Tt7RA 9gh#TlGYkvR\!x'7Q`[ژJ_ucDɕʘ'K^r\8pm֞ScUtenF$6['dr{(@Mi5cpU4-a^䇙!-\jݱRBEY!,1;#}l''>6P(&뚷KsMl芪ArLq,2Ikxs:U9| )Ʌm;)<"hd}8QJZ￠,pmNRNެy &m֕'9-]uqMa_rewuXVYu&ijb<Ź` 0H8':rrt [Uf"RĉvqSkiXV獖 ~9L̶՘{ӕ; 7zv0BZc)&d ZUD c)s0;ާ՟[¤0dg_n\xMm˚7S"|XFg엦";v.6^̲sʃ kc*}?&y~țhƥ>~JWXoJO^>h1zytJ(:`8]Bw94jmr ( _S8pXL!kot3 Cx .2:Q5v_-p]Uϵ3zZ 2<dOp9?Bn5C}Nen(_|bh9賭bo!CNR܁XKNinOV/mQn9RHVk]MހntR7¦Y qa:5!Қv\kn\hGZ.ƕ,uUPޛlq-d׶L+vƏge &%"}Ŷ[k\G:֔[-$˕6^s_Da.릣xM\Dr-CŐe$Ni?8Z8u&K@"mڢceX1Sc&h\ +'co)oem?a*xT ͵)[~>@z2GXBqp~X ԲLGnb78'nvr~- Cw-_L᫬-8. U:WAEqwp̓+TC0ϰLbWL;2RmWuO]/6JEIƊqHc:JlA'M5jMYb/ǖ|E[†A{)N{/d$dbkKؒ;=61޷gyg09U7]_"Kׯ, jmyb;/-9Xn܆(K3-8.Aga̎< J{GoJZD " |HiW3?A7(q,!rr~n~1d6Ǝcډ$N5g8 Vy~N=Av9%2(cfz5'aV ?Ċz^Wg(V\nXӖzHQFT.22grp|:H Y)РA}^EgWE@gsTx>-|cx pvıʋX<ҁ#1 9P;~)sSto/a Ǖi'SWkm9Ua`43Թahu hV, 1/,^!rZs Ι3ōÂ3懞3B$xLhW.n&bFP~?7r>e=Iղ Z뉨2R.#"26uc|ḡy5pw8/.P~c?aXT!QҁKyD5Ɍ6%5EA,; .Z:Z<%psB@D=8HӪ۠PEop:J~궜8fnu+m6%-@Qxʶ7:SP\ͪ7O&q$ [7y&|MpϸYKxXe7^^~2WLlSdQ;.`~'#iL2iձ_,7َo"lwT/yPK |=Xx/5r]k(T ; m5ER/-3+a?8|NTW>1FbmiOkhފ4Piޓs +KkFp!}ce]rg5\}|ܭa:|ZT Y?I~A6U5qoW_nP*㸩G/
'7/KlldU/ԯP-y? ~kwq0[zx3H7rLyr:[lL7z/s.16(En38n*7GBŔiaߓMOGnsR l >-yZLͅ Y/03nBmy7P~]JP} IbDLeB?ŏC<8oU=8dcs( :qv[rx~5r=Fp)Y+\N'ȸsoqOH?y@fpl5I/ 7 ˶.G=5(RLo[Ӗ!])eԟ:Gzkq]%9,yй0^*G}+/0v+n '߿`4rnEqGr/LE/ U`q # !m֪u?*z!@Eu};sp2X􉖝ugA{s%p2n{q4僑FD]nۜRgLyηΔCAK9h-2xlz4 k)xTkW q݋TW~ b9)!jO3^\6r@{D|ОLK+AGoO]P+Ra 5x37H9F}2T{` uOԐub. ;2VsG|r}P~bo0P |$Ѓd$_Bfh_sq!-Ed*#YK?F;Wyӫ7p.'Zs:9ky5S8"6[}tlZ)(oӺ<-~b} 41̒ݬ7|nc?yѱH>R̢Wv _co1,gˍi6a+|qAE(:rQ|!Z]hM4vp,.VΉMX'5N˩ҹ-SdZ\s|iNMRX9]Cs8k g bUvOA ?XG?u˼Jц#7|c$לFNxx%˽>(_ڝj1o9YD1ߠ=1SE d]&[yLߝ6/-zBd''+~\;]ѹzhbEʻ(ژ>gr@ڋ9 |{iK|l|_4N:oj1[_l2#lKQX-;9ƗzG M`xEȆ}4rۜdUُBՔM~<H%ʮ7S%wp<ϳΫH.yos(P9.@i bK}{8 =ƹVTqA[m;J^.$UKern:'6랈 M}2nRýHy|yDA7o{}7z خ#Yo˿AE cx|^ot._,cM<6;D^g1:袗mHRo@n:pU(XAԝC ci6Bñ^R6 _.\g'{$5?gi[Tn;7ֆZ~qB hӧs"O'7lg`L~ 2 6qUvĉWq 8O8&]h :̖M1tbaEP`.C1 kkVBgᰀ[ ?'uU%S}O|hl,n)aӦYԡ‘ֲd7ybM~vK~¼kS$ +_+lQDaxʛOWi$P\tH el$fw&;+U\ ɢF-2xMyPsS/Dt.tmeæz}Qs' j ˞|NEuh϶N>?f݈_aד_ -8B jJ9}Џ~w%/"K!ڮRIb9.dJsA:wDc\ N!usή I1@k~5qP羹{'ׯpQ tsG=ϺPc^XkCD|P=P.c-j ]x>Dq x.n@PYoIF:ZJnMvY >~`MpUvh9nD8GsL7V85Z5ymk}Ǘ.{0]KnjF:B,ۚΈڦ~3mlk^;|pb?QXV_nlTdSDۯ_rRt@"7QgsѴI?aֹxƲ '~Pmx/Pmjnc]F/וatY+htmZi#({Þ(8Y%ZTm\ c_V c\n&5F 9 ׏/w;R(OlĉC1HŁ^!ׂ\BoΓגoV1D}Hʽxx^QS=Q¬-Wy6_C'e%0/!sd^0nӰ,qm1?^*A՗fHM5ﴛRwX&" ]ɯGmq;mQ̃qy -E k]M`i{sO5@-rX:D9&=mof.Ek;F9u9jp/_րiѶKj^;DNNcdb;v}JpQNbx]W za13H;p¨P94释~ѠT`6g*Ozspx@7S'2nR9&sGc և',m5K(qwL;Z>俫nsf s[0D yULgq.g_fWC6wHFXܔ5~tp H(yzilC [2H_'ղu:"Z PdIf4OX)a31;M=jiM0֮` UMQIusIiF)圀t~ܐ^oUx@o넱eګ.y.pu5Y)_!j6QbuXvc[z@┭?=3WU,Q_|u0hj+a9Ɲx8a' #njkm xSlv$BS?^ux[/(WIrVgKm ӡ=> _x=+ k!0

;ֱ'Tñ&zxaB=22jƮM;=jfQxO$bW1aMIZoH߬r_jEgKBCu퐼Åj PB,?I]tx-XK'?g;PcXō9DOxџ1yysN>uä)QkU¨4硘|@jn re:c͙C 8gD4/4At"9G4"t>6}mK1đ*E{^d^^nLT}W{α8WTYdqp. =DXnQ -_X.6bX϶Z]}ӧ$7z+n#ͶcL6oC `?|?YWgn.n#8]뻢o˙m0lKk9'tzCcAnBJ{'ї}ME%g B>帲l_<8ՉkEvvm|zEfK<9F/9gZ֥Gy~=u2p"r'(#5G5ksQH˲*b8mO㲺q[5F"wRd]yF\Ma bc~Gpm0p7/>¯s;5[9ŦJdG(G/>uM┏a,wΧ_lfgH5Lj){t4IQSmP퐘`rH i!n?AP;Wi +,.3E`X#23\w1}^|.Fscm.2WDq|1v} ObdhVu=Hho(5ylˏ66HE^?|ۚ+}%;{|{bvk?o6^LO5P5 h *hk/As+XPpYNt7$;qOo-q]re׸ ٧ ~R=߾BuEra s" 7M#px 1~nT*_V0^!+m>V H~r ݋qy$l]X~4GݸC^.gm9}Y,6 Vw̕Tj2X@$dsmLåo#Gά~͸J'jRQpE)7x1K,{v8pO{ (,,%qڽeg?sްU= X/`]\u<-ۡy[LmGSIVqـWyrT1Qxps5[=F~0ϲU-#X7/`Y[tc@vpkC^N#x[ŞK}ΟقDXu1`IBYnl.9.Ii^JG]uV<&X]+Kފ|C8kL1:RسH?_22J h?9`[-k~Qd_mS[/8)b et#l勻E9K5Y3(UPe?cHGL*L+hYxy̜ɖfvO[**5fzPKwnA~Ks{aomָةͲs^+Q)鞻P89vYH!`o`^Q -im4+E\b(M\ g\G!Z|s ?<1~|C8hE {_"AeCGc웼^82ׁCi\ 7ѭ-`|<;_![~7lE4!|,sp5Y:nCv:ڲb*ޙmbzS<^wօ;]mJ3-`~mTa7[8oo޹9*o(N\Epq͏.|2kdYȶ+ &lV$kԞWד]7n:ܭ;)rJR,U4{_vb/yrdTFש/:op84'b8X:x6ӬC? %r^ يUnDͳ/@-׏`툮3j=>0/I-ds21.eY(`*:{MCtk\]FW !٪}m( lBWp`_{ {ec^αTb HQ_?nsI{H$ $uM,F4zP1.[i*vLc&ht{>^}Ќ¼k:a\t9~,zAp=W=dBN[8Kp*4Nxl[~3 e)H ƆYț6cR((Uq~Ec..1w6RCާr8%˞vJ.nXo qs.*י:""nv<+8`,}Wtǘ|srT@Ǘy;4u0dt&CI[Fc?:(dmyZqS['j둡1: \xc,|0 %!҄L=ߢn5QjVz7? 4E=Tzr8AM4Jc"̂.aͩs1.P}aFǺ֓hYSMV`篹WnVS #U)p~]/țz3[|.y!"q|>}c؈\A߉!?M9gnjO}~KY?@yc?c3wϭxE1`쳊:Ű~j_o>ϥ^d:Q+&6o9 cx* ?Qm^6٠tݯkuo!EѕQ(mL[dRq }c݃SGO~96ڜ3'4[Q1F=Rz >FW-1XF"e9&8n"IWKe7%30cjtLzo~4EhXK٢Ţkc߇F+.)2Z[;j]z,o?3 _QBV)/mA[FqIӮ=tqs}'c:oh?r ~dg՜"f>ȺVl/VvǼf;YviWQ[MBYK(G|#YVn^!n]d7ۜy^Ϗ?ӧ6?1˺͞g1`V #=;6b=`9U0}՞l~ÿMRw&Y^*;ޟ Ŷ_LmVcE],۪>70du$l;-eUGU%n{%&*Cc=6Wۼ/z*?e1-\/ƾ&M͢`3H.)b u=6uNp<ƅMd%vN ԸQgؾI8e 8򒺙qǴ"d15i3ɲ87ٍ,#aga(eE u^r(knOoђWHegxϗޟ7Dx>l`Лٻ3#w^81'nF q1$rx29X*cwU79&)>>>bnԦZŊFUZ^[p|֝bff#ڰoN&PO Uʚ](Qf-ly<sf!㺐Fv~]N;Շ.t?1I}DfڼUS\б?B՞p<>WzlWzHkhX.^tOMY"_?AP#h4sn^@Ƈp˙5k?' J\#w/]!nHda_k%Pz: ^Zx lx`1Z[1wvkc!̈́c@NY4sHVڛ1c$Of@];mV0T dW`V)(pz /I|;3ߦq@ae5va oE l@Bׯ:+۟_)S[~%K!l'^P|S|tR2@ik?C^QDtV0/ 6!Km{'{XQsݤv}Ԗijn،4J輂rR>v(Wz{ĥyv7{'k2P0f~990RDQZ8r:x_q4tu9i;jN=jH)Շ?9J \*# c!۰SQ~@c;]Lޠ\TsyGdF,$~cro|1R͖̓SϵAaN`mTcM#R-\0Q>G+{D<_qZr>}r*雙W4`v yIVvd,zz0 2nfO~~A|{"oYZI\s]w7% Y\J K6#ʼny9ߜ:#aXHBz nd^-g6u1f)89mdy3Ѱ6y{6%Ⱥj9 D*KK`č.$%VcqbKmabښ-ȇ"\uɨo4'd\7)g@X>ΣFl;&r}L v*.{H6byer{{QZг?34"-#zh̾ ~;k+Z-nb,/kR.|POswTzlA˳k]ka]R#K 礫;xwwYf[3J'Yλ/o_H* _.q`n?vp5u1scEmr$u4iNYu1/,_腈Y%874Cu񹾌+{>*5Hطe徦9`}aJ֠҆9YmѨPs"s}|{On_eJ2+^{6΁?hkZD7q$7U; ۥ0Dun냯΋?Տ*,\>8.fzHDԬe/)Ʋ^`[xuP–]i@5s q*x~}0L˜Didpx(AXfU(?s .S鎄 D9++yr\1hN]@m@$:eu-{!ɚm-/pi;Oڄnwt&b~l7:-"nbyT/Jcm?N$L6FuLM1Aq [m-Ճjx]#&R7X1b>mvq|c'^fCń7Rbc^;nQ7Q XgjFgD+dȵ4cW~x'Ȯ,hN$nQ?6SEXQ ).Y'|^e7޲6Foo3*]M VJޗne%Cჷ/ 6J{YCwUWҞ ৘czY_]*cGo#(4U @{ `K8_xa5]'k#T~.qK[ZRF|:^1TrJZC=/֪ _SR/d<9D^6f`HyR TcqPefd(<}iT `_{-6,>9d>fM9]s#gfg>rXnZ޼D¾$c) 刮4;s;bJ$1Ιܰx0||ty@~23C|sǺ4y4[6?!}@M ȱM=c8${rZ깊{O8YJݼLZq3m!,+/6ʹ\>QC?3MR.S%#?cFA&\ /~~qIJS~xĻNkÎXBP|\a(_@m._[k؉c%4@}_=%Vº]ߖs<\jر:z;i+gu;I'Gmf(_pӥ6>kq)ڲc ]+܉z`oYB7\vp˯n>'&REWU jcW|ş8K-":qAp(2/65鵵Qc[ \T/_٪W+%ФL3/.ͬ)5_UU^En1#je{_W%Kw^j#v{̛Ӈkh\0>5÷1nn4m]7 Z6_{ͬ~fR>RlD۠oΚ}鷏,GaYPp뻱D8D>w'm(~ScpW"MHyni}|^Y:q1U" <?+}=Ph|7^ng\A::15^r-+@Zcv]Jvmֿ:3ZvJ7,ܠ<go9lwP~dQdEyb[ZeWI*n$!lj ލ.sd@&K US.\?ס  -ZKǁkat-qȽ3Ʋ@I}I,^}=BFq!LILoƿE$u![kv 42)=\]l ~1_1}m|ߨ2Xk ։oj"^9Foo 켑FUyщb2g(%V9d~Vw;X승I *kMﻭRC-[G L -ǀƘ n6R>T]; oi\7"9q!J IB@uGYMqH(9N.akIYcaCk}xR;&5_ ӡ]pM}z<a1ۘx[27E]騞;ґXWSg/ڐ1 r*u%Uklp?mGĊ3ߴ5(DrPk dIZ7r]I]a+ϝ+'+[>VԞ&|e9ij zv@ٕ)7{u -TsEܢM5&TnA2C[s<͝ -x&ƛe7L'˜_qoU|1?_P;A1]!cG%f ZL1!ZM3I.%N`~@۽;'۩|e1x3 ZOP SJv p||/<#PT+mtu|BɎm3NǾ05d/hc'\N#IC-$C\| Z>4j{@Ѕy|K~"eTJ`s|k $jq+'Ǥ7A$`%jIMWxb[k1« Ca3G:4mnFrybf&SΊnySh2-w[̀uvBw̘BiX^녿) |AYoK7M[5wK!g}iJ6􎗝|cw&Ǩ0*ڱ`ՆqI+0s;Fc:>lOMn` lWo-nNʾ:6n9:%īXKUn&zaWƨqfGe)/*mHQ㹦čL?鎼g$9=|֟\N'Ӎޥ_kl{a;brpo;42d( UǸ9^ݾcxLzl9~'p7.6J/$߷Aq2[ؓ\dxד7luq^DR\7WnɷZE` Z'} ^&ώ=Ė7Nv4I45B&{ݺifg0l#J*r9thZ?;Lާ0_p[3.NuGCly˃v_uK.n\ 8PYpAU(8R^]믏ANeE\5Ɣkwki!.*}x..B/uak!1SD+%#i?:,̮v{ŵV=䷰qa VWo:?L8+]I3qjߠ, `9~wR#k:?ٝ]- +Dp,LUSi߲ۆsH:J´Ot$)2i-NYpVEc?gJ{({7jeu:Zf.ctWȪ&$Αc_OY~%_j~,y֯%'گY#/h5;b 2~\eg&`XpH^/`[Rf Ӛ9ٜŭO:Ja0-DŽ۶=FjR7\ -LiTk*NG+M}=9׉?sƨQ\8&EXǷXƜe}~ۚe\ 5^  5|r& zU4r-UC|c-hyIHi+ۚ35b1 2,S6(fhhx%K bDo9r1F0ۭ/./uJ$( +cޜJdzXo/㸨Z®$5FZ) V蘖qO 3L,k>γ/bo 'zjA^vğtPvp莪 L)k Yjdrp/.qv,C5ֺ0_ٰS <{@hŜ_ϿTaWp~kŋ'r:9׾Q5#@+B+~O `p,HYyc ^+yGs&?l8qQ6Y܀՛l4$xF8Z#ihS\\~03KT:eiY.el:cV2ڃ&lAB_`Yf<`[N e_JbIį z&I}|BRfmd@}u= ˤs|d%Aa4Ox('][\DŹ9n=s`MP3W2iVIJ}An<´vWXʸWb ezgc7ڮVT͜)0ӡtƜL 4жD*!K֏gʟ+iwT;D]5`ɵsI&Y8Ӳ~B[5 ^P9s<"/A=%e=);0Tx jlU`[Nxzt F]KtY [_u[ffVdII@F%!_'\:^lΏI"-(h!s`0X/6{~[ջA Q\op}}kd?3W<v"@XzqǾ#0VIo1M 1nh~K%383-rUl[=mMN\ͲngF6^оG}XX]4H}A,Wg]0Mٱ濮7V,'HQ7[5qRa s7,*[lt1–_67(6՞q[C[Tҟ2gW=([m3G<9Zx*~ ?J=~gԼtp\U cR:p[C~!c#Ͷn,u 8FzVd\ tq= VǶ9\ΙrKJu"/fFz9NO8&} [ﾋ®jV`_s>2_'ehr7f9GÖ4O^vh&KqZaGc3NLQ}p>"ԜlO@@ TcR9+7Ng'6ʍTͲ(t6Dģݯ.s5F g/q5rȿ̨1P;yȷ$YȮ0MKa/ oKypmqC̋}2_5ݖ}K'a+լ[؞Z 3;’o0T.n2;_|k/ˇ6ן x^7xSpAjvNϑ_\}Țk:Eߩ V5o6q#@_S3(-I~s݀)dr;mE K?0_e>dqc Os7yYο#͹E`M;$$sӺvOZ;\wU=D#UJ=(qLq:Q/dʢY,#~qL+N?YWp:=*c=M>3n xLn=ȍ>|7:)yR:68>|k,rTt@?ꈶty0\; ]|rYIwKD uLt[cLS7L/ ϻs\th,!j_2mhòg5.>?mXg:_OJ gdUz nη}6_Q8I-9Ӏ. l:׏@#7Ńmؐ n#we~*KJ  .ĚpӱY2R@T-!PpN67 ;nJBF潾l yڃ9!;(Գ/ی_YKKn%*a=G#<ߍbf|s[:%k_70%{9/s}3s>^#Y D,Mρ$,&kӯ [XZoj&Ao/$)8lLJo0݀mʷ}p{|~A xG7lWos' 'E \ zc!".Bq6},݈XK~/q]q>s[¸Z||:V9?[ @$P]>E kۦ ٜE t.%`ӫ,5`%pY_@\4KsWUv_ ʨ5>35MZ.ϳo?a.¤؇tNywR'kP"Z=y ~A.tk^*cڲ-;t:_gH۬ CZl0WsO #v痼|mֲ S\kfEwZͅ&H̪ "ܾ߭N Fس8-ҴR/'5]x1_b'$%&U\εf^1Z Q97 .օ+,u%]< |q.ʇе*@-ɴe=zgZp,q3v䘁bnXnDgB;aPsq» ՠ1cvIJ͹fz_d:C!C=`ۃm7RAι37}塮^8Ӛ9Pvر4wm\Ѷ&#}roF!fOzF`?\,h{hxLY-˪l%)cA~7$76sqᇙxG^b7V9 iʖ qz_`RȸZUz™ TџZ_Hks UR^ Cr}3oP"_?w*(PזuׄjT\_b ؟& rk3^Odfŏ`<7L}:I0xxw_%bZ[ fgA9xl^s|w}`ٜ+`SlkA2'd\׈=b)ϷWx՞=jZF7=\%%&M-vY6}Tړ€A墮49|Ⓩm|IQS#V$ŵDm9nq1߯s`^6˯Gz6zCWAEn3tE+AʒoꚔ-FP".пDηM= >`Z]y2%ñ:[0=Bu Qs\;Eû<ͲT0q lALZH֝ٿ1}r.ɯIF5c,~~r_Iyօ`;v"=ӮsȲinZqc)t_0uN4Xz&υumz?Sz!&aԟr(.(IvZ1OCoaGN9qR/{LrW 0cgUJݯALkI eٳdKZ/GPqΠ\y)H恤!89)v1?nQ% BGKԪ2k_P5%xV֊i.Ƭ/lGt/uSV*,s@%GY4!pYU[bz!)68%9`YwM"6cX3F3rW=%d&K1r;@{Xpq4/&[X,Er~b{=81.@ ;~pD-&GON5qmɿ `]}5ދiYim|蜾5 ^rqe%0OǓ[ė n'ad[v̇Cxù9,+om".q3;mw&^Sm:B%*'imA,Nwä : M/`Y{.+iD܀e^V*hjP-8!+r9jӣWr?` R#~I'TznI'oX0">oENP[:Śsô l7AM?@3j# q/("nFkp^{9R1Ni #<Ʉ7Wyc yW|ihg')C¯ >+D3s/=G (Ck-/`1~mxPrk?˹UpcK[V ا2H;(ο&}^C4D8!t9JRz}@r"H+ijHK5g,iMBq\K -cEG9jF2j2z96d-vAZ ^p=s"2P Vw79?XNEs!nD(.Z`M;^)AEx(ܴ uu5xG7K^35mA1lw: :w!.z\C;UfPqr ?؞7!\i3y*)c'?䓾dWv-y>\ bwllwT\\/"B* (;W[&\(fM IskEAiG&-a&st*Lc$Ѵ%%=!lOVzK"Tܗh׽yiL<T^ .hQ:OmCz3^Vsc5 ,|09Zhmyt 3}Ź ױncxʪf̖< ykݴPL77F{A,C=b8AȈJUYZZ:aߋ. rtF)F+蛈)])մ*1JK A|+m!aL1aocQ7{GTI#j2kJG co{Kyt ƮuEEBDy t_^HXM~|i9pʫ|S'f#1AJ[;8@|մrPz89r _6ユ@#O`S/5Hug T/-ޞ?Aqހ68qR*KA;be j.,LsA"!zF/93<uKC kﳢ{~[c I?g{'tK\҇&+C&%.5WwB!.+ԅ]j"8#ꂲW؟bE8/dm73ewH9upxbG5=-$@.aCy(25؃ 'z>ns1JNG=(\.FOfA1S4x }Yylop"8cX6y׶ YQpw9E?-Rkne3?>H3-簮S@?O)uFV'y0ojC.K_jUrJ"|^E{'S=1-'}Km6R3/ys~duY6@j8͝{ lM,;a?r3M< =LwƧxW?m/` #˞?{"sr]@S(nXO]N *9:(3*h1׵ٕ (_`^\qؖ$ ő}`xFWKTbVG8w ??``Ye~S3UFX MI󦺝ΡL55OTcz7(Jn7D-\OPnǂCԪc;x]:%&HiXf=k=~mΩob-c} c~e3n+k%"&gpڒF)i=kzv1|ZuKKXϛ},CKaSM9aBZǏwO_5{&8?/u){[c{y=]`=:ާh ^|kfӱ?k,ǴeK2q!8Le6sLl YVvFݑF SGNr>(,'^;ϢcPRIc7('&M\5'|bR"Jm#ּ]#T G{'5~zf}!O9PysoX"1Et=9{y#~y@oov qqI:<$b./47_E%$ƭo6lsapw5M^bBOSu``M׵b@O|ie/ n[L6#QYi9 yY\97fEYs |cb`e7?Uwޔ6- SӅ>i jsY>rpB^9!P8-YsrdZ,z9z#4O}3=f>Q^k-])o*8Cע:|8-\:ER%ǑXQ({z`C:;0W=1,^s-Uq-Hy*WMYNScoqQseIT^C@|OmNwn4?|.M}/U)f֌K]xzug+$^",2ϜzZ)Oԃ%sӍ5G)ah h&_dK·{OQdx\˃b.[ $s݇hN8Os7cK]6L)C[lt5Bs']3C]?;],0?:iFNb'q~1^hof>5QT)bRVa?x#'E3;20 7m2aGu6lK͖ތu PQ`zT>KnINR#7P9vm]?7H P㐎Yr @S.N%smڔs5 ߋ ҳXBU9ܢޤq地i:'f@+|Q~.OXb?|ZR сq! *a Q+Ğ<6GٲMAfe-m?9?&94终Q8N3Lǰ;\-y`.Bub+nkf|\gmhvt_>e?F'[NJFyH_uOb6'^dFr'ϡ9GhΩG֜z cӃn*MW}eX!Z։}55>v ي'Qͺ+ϼ݇'b{^y}Ε> S-wQwS0VGjOySB"ԇ 3<%A̟/O6K7yO(`"œ!k<_s[/pJqzՃCiSC^ֽrLj: #cWp]/Svt 3}yK;`N*;bneMA]-x~Cn~)EC$_n>b8oQx <'8.!]Ŗgo ~dsU>`n1K9:7-x?SMK)}ts'eYumpJl&埼g"Px=s\4vC` IE#k4^{ ExѲl_9rc1*Oq1q8Ըy2Ӯ8A=]fV *F> "ĸ^sqDaC,fg{դ50_k`lg|&z܏x m{čm63n_`z`zD}n~p'&~x.U5-uy6g'GG ffz͵vO?$0Gc>/0\YVr^vQ/L/Ͳ}O|ݣ8Βn_ 2.Uc~-l'"@{բ/59lO=rmN{3/X!'qm֒.W=l{/aLiuj&G,P>eh 9f(;I5l,:sE8K]*nEE/+\S<N>?C/"?xM.D|}bHUO! JF1waκjhSy9C1ƚsAq(:)'Őcqݑ;qZ$mw ߀7Ǐs\bcJoΧ7y1' r۳?je{!q}K.uSdp}@HIp Fdt;H}AW;*H [StI9uN۹'^ [uN,Zq+\ ?hŵr䡘c1>c)ƚ ^u{uAFYV!AYG۩`R *y_凪:_2kQk<r /Tc{+ 3!v{ F'OK@>Wq|΁uun5D{!ιu_ 4/'cWrKm^Q?Vo]L}_ [>z'*̓< YZEaBá6#.KM;2,'E%ᙨ }%gIDuQ@W.qe\.ߘKpZVpY ,c?!|lkZ䎋bKYgǿVwc J>Hȗsl_|X.UujƬȞB"P~pNn }bw$m EQq|"B GģRULx Mx\@F]M'MMXkv+s@Wa}7Xlyy%aXp)HD |ezi-<|朹 N[bp? qSFR}M4|.E _`9_\ p44Q}=*W;m,[Nb"OB?dm6q >D |DeRf q.Ņ犪qzɡ_y?Ϡ`&O_0OѭFhyEs oI XqNzDLG{Lq8:w3 - ̫waqTL])~N; CKS!}8[UeusuŢ/V4_2&.5 OAq5K|1k/;93SL7?:.y7fp9moMf.,hCHmF-nڦeX>!GM@MG@z2{d[Y 0-Ljq K0"*ZX8]Їb9ƎY&짪`Yfy ^y]ʑ\6~ ໃ75~D86og~]*k %wʚq\r-Z3o1[hl1#Ж>f6mq3ylYb  C9.[ r,h`]+d?Gp;O9S>y^}i}Z(kR 2J-ϛb j󑛥/vv٨ Sw:m)kO9d0/笞7hJը}a+kG[|E6n3[@/ڜ?pRj s*b9 #I m(+(9+某Nh qHCFt}I4-/5(qْ4ZތE7؊QVX#4>d0/kJ_-"\b>C Y/ǣ䢍9)US1%MP@r8.ӧDcm#ҦU~pYj.%,?T-h1B?c^EÃr@lKE`z^o˔7^ g҅"=}w]-uDP10iH׼=6unr3{k>1BV/q`ލ#˗{8UPA϶l?>-yJ~O^۴qL8hօy9p^H?3&UcTQEWV_&k Q'('|vZ.ib b!zP7˙7T΃*p}?]]?Vɮ_C\.1,P9't#<2O6z J`@(޵ބo_񪵳)7sUN7xtUjLUaΜ Aϖ3a]*,|r%RInl9f9\' 8^<ưtAϸj !o%מ5tשps1~>Ds5[q4;ub䯣Q9 8^s;M( ~isOÅ{cw?ql͠~C,(+O-#I5^c^$/+ =m(@R$-BD »n!MD/vJJkp@-OtثLua踼(7llb{ S2H/p~M˸l8:"K :JWA:2>x2a{0_g<(N ,+oF#p~#lTI@{]W| F*<\Xm9(d\nH)w)=,/xT4F*|.C qhНdBDǨsM_ E絷?[L*7,WCdا|kϖύu}ݺdNS~cK\sŬmPrYOfT?E3k?w [8㺎)(7q,7xu炒x:3&kfMv{n:m}%DŸUTCGWqGdǛP׸@x"6I]BG ߃?ưfށq0|8!E#P 0?6sb_@̓}(7̳`B@?,s(cXR3Q[~3s:|~{')l*)$[|gM+6xwW{ :_xͅ:^qҏΨx lO}za$w8H-QUKsa,_W86/`Ǵ#Q%999.jp K<}Q%A>Qwni [ݹVfʺ#D iu9 }cGY3x=aס*esY cUPY6O">~BPv[C+QmLū_tr9],"8Nׁ@Zcf9 䇈iVJqMhT#B{ْ'( i]k?[θRӎ$CNbXkVm9c{Cx<+xm:ycaGւEmşA'D|Yy8e r`n$b1-oH(i_>S"D?Z0u43s%)+@ƇIޕ^x[} -ϵ-a[Hf}[͖p\Jωk)-EsY3}+&(ֳY9fMtmyt1`'G)AEۋ-8ƓvCyq̈=gq>#F5-:S3;uQ~ h$Y?:-8~6>0_b{-_;w:P,ik΄-/z3B_vlEv^kK6ʚ\sc&?_g1׏Re<nM,<|]֢=1ăr_HT]xߖx̶p9N;Rz_m3P,e̍h=8wP*?? l-1*L&?<{J#)'LzJK7aenvu> iWj۟}7xY8Gf'^p% M?~Kuic RٰTeuqP@i}ټ܀G7S,Y?9AZ<|C(yJG[@<߅n4w\[~rcn4{D?6 {Wj%[\}PRL{{Q;QǗG\/8L6fg;ι6?j RV^AL3eȌof|vy1KsDA^Տ9& VA8nEYkmHnP7t.7U_l1ߛqrȘvJ6/lA%no/(/l[_Y١Gzf׎7%c _CŅ\{mcy n-i3?xy~Z0xXӥ_kBH{Ma{8A@5vS(2PCg]ƒfspkoZvY;b_-%k%E~/ޮsmo$,qyz\bخ[}(!2No:dBo1;;W6mgב?:5Xeqb0lA'A[Vş U!g5:',. T~8w6H19/gn>Z >.C6-tuY*|%rϗ׸;AO,YNd˄OHHnZ'vIScؕi%I1^.K:$>kc?iS#Oր'50@s1sߍ|ilp]SF"d/M9tOE+nq)5 8ߪ> B·?0(|Iuu{y'Stk$׆lI6 Nh~MG/?1,$f-d1č1ZS^v.8遊?F~1[ъEQ~pNs<āΓb$ʺO`P2DoŅ#=^ ǣ63_B:= my0-GmxIRNxr#t‘N辧S{U,׏ fڬFࢢ?u %6Fo?F?R<.fT>/6X6 _Ck/Y{"r>.ߝ]!Ep72 b| Κw O5ρ] {R0E=yAkm'cyF1m:Rs1zv!ƫASdn>@wNV9I㗟seBp+[I%<9"||/Yb:(T )ѺN1Y܌O 7H1۟g%p>f63Lr)1p!v݀y9vA0>weF}pr-v5~3PjF]8peZmby-h1,`}Dj&?>|xZ.Z$ȏa4z4~u"tTMݿ▣>-ޗ~lo.Wy^Ts1fz7I5~)4f8!B /-_Eb7 g G׊ ^-Rs>?I*fJ976o^^@a4>Q6WX?砳f`L:O$&%f^ u!!϶ڷ|;x1?\RE$_]L%P@HjЁ֙LˋaxT)]Z{oRݍ_If|Y/[Pj?NЋz@Hw.]aÀjrh^hkVz^e>g2SԾ5'KX?Sm^uk_E[%ö 4f0i&M'g)͈-3Vy~bv\]Ηi(e.p^ ?Tŷ}峟\}G^AsQ خ[$ruF">dAdU=iHuesIb9sܓ(p4v#87q\E"˔dn58hvXh>yDEU`ǰS+J{ e¡:ëv2o0YR_X _I~\p/#C8pV"w1NC<#] i}Ɓֹ}(ՋauZ(|e1 RwA{mFhٚԢAeU. -*8F Cǘʼn^(g֪g@/lx>UH'e;=3Zr'ڕz[]K\X>W|vLA:|o~Msu'tqi}mjqgv,7{zmrX) ^;ܟ%ΗJωv>YsC'ǔ .rIyL7*G҆ʼn7bsŶg,@>R0/uzNNKhMMAf S]N羏HuZ8fPZ}m](βL(yP!B'db&r ]$g9FL?SV*3ZH.DZ1AaO9=/yO";LLk'B'me5tut}wluUtp~0 Z{&-X5s.sL{͟]|IXo,ࡳ*s}YكS"}~r|r Ů[L'/>3z joVy2g)-G&W8}>'KkD˾nòr3dMXQl;+)U֋\{dzHqTr<G6^A taGlNJe5U>~~p=_#_&+m\%ԭ;Tj7u:9 :kwG0z;±~.({`d,i+ij?8GX T+DWw?<>eN7R4R{`-f!_CUuw8KFZn1L>o)ޝ.#M1M# א|ʻМKN߿0XW'Fe=3-HVs5պa_G*};x9=M4KG~Ձ1c MB2%>csq>DZ9 i?$I8ضmt/'~z¹7ןz0:vڦG.xB.R 4+]nDs2 eڛaIO"g ~ :`D9> }7[SLm鯙89/^a:v ݦIA3DnG>s}P̫Pk /߇ط"pJu(^2]"T<l"H=k3s};j0syS㼸ϽF3QydVyNFŷ@|Ag3رdPxJv'$NQ_,"+'ܡآc^7|{tڔ266CULe>/\kt9LPܷ3V(/LԕV ؾKl}0^} xKc@o.f7xu1"ȑ uFuUoKOK&NGw^sX+KeTqPj>> 0 d;A}z `CBԇ2ؙx.2eAfWH\nO/ sٔ׵]百ێsiq]0קG88"_7W2UjTfP";<(p53_f\`iFƯwsʵI7w'z3gh$'3{i/֍(ut,Bz}ā )t5N LRZbǎ_c>v\^N ]>1~D;> S+Oɘws9p򆬬g@7F/""8 R}0-4Et2o8W2pyި/ɗYQsЄ,D^V(_#^dܙ`NC]oЉp־Qz6ߜ)~@+eRuuy񮥤^/5\Eoȫ~%8q{%\Y}"*Fug^KFN ^Z>A|QM|%N,)^]/9:8ەZHݔOjd=ĺc]$_D߀>smyquFt?Ed=)Ft#4i.j$m`ٯ5!S-y_-)R 251q%cri'xNAU1C(nsˈ&b^VPbsYw͡+ l7çV+:6ۓjaqǑ(um;^BC lBuZmZvk/1 liIiXIZǗ E 2D9mic{h\1aS9eGIZc1u1^%vҹxX67)9"lU}Fq\{밑7|:ܟ濈mgJޯ |G;9|B7}.1K53kK>7h95Zm׌8$su.C>*=Db,`Hm\sC#İ0F`c'Q\.zķ5!xnC3&KpzC+m(df//1mɋmg* TPJaukK^O̳UN8je? kh~#qP_$8U@ξPr|:!+:.5k(=\but5 'Ro}etca{CCDYς\` /aIZI> l֖Kd=InEBFfx>cbRg2>V*5矩 bp󽲰[~850Kf\)dE!=,1ϊuDDF/ JA$06urw deP|]kukK-֯uDqV~ Y_toBۄ3:=&l/?8naɉ@);P.֚9iӀ au$b }J8l~$<[i8aΚy(yƤ[vD[jEl拂T$WTbEyNYH!sFՋlrF_f쑺䳮lFnrnmٶ˟0}sh]j\W2z&p+8mP sd]<<ތbevlkIKYXĎ w_ YbU_@GI?5ik=h^^欃B8T{73S;~2nB'J3v3|3_pRʉI J6zCIl *nB 1܃p:J_cob6 v? _lP-kI6߰Zځ2ǰ-{xaVZg9.hc:/9%NYF1ns(ܙdL*9-^KΐPxrK1ܛ=r Қ9 8)DOs6mFLɦӸc0.uz6q xx17۾C~sՃe-:׽ybHs~ GXsa !0rp8nN[Dc' ^k"v$Ϲ!kkY53ﶚvIrksƢG<\p(aـ:3bX_4b]+;$VKڙdo$hgZnޠrq(c奵O髿pZYgg-gPb]3Oh- 2T +Џ9C`]7Z#X,?\) *uχ]&krt"%.0[ $C!?4묧:'\9FC:ĶĊ/o3$?NHU U_^g3(n,かM5gg7b:&|rL .-zQ/]}嘮fn+[ e>8.a̼?ejۅ}4i*mCXͼ(y91A1b={*۸6 嗜'żO0m2׳eMDS(?@"Sky-x@tk0ԃYLk7ꉥitQ6atľqr.M՛=lMe:V8ŋs8F_ ED!n [7 3L8x]g)pLSmn1&E|m󹢫WGE2x`Qc'pIEVueõK@{9̬TV  ꌷyFjG[ePfhVg "ݣxCFnTU=sw?ZWb2qγ#bźG׷W R<|=*B[~"}|EsVnC&V2Ҫj hU}p9+0L[Vrgf::wA)cԡGkps 4cٽ@hG`A_]EcjDC}S .nxժ"o|%fu/u~\ ~·7% ?:a.p/9ztgpץ7Υ3-L/5fJ iUM2^.0惴N".IO7=E)>)p9Sau?a@Ȍ/7V9>IcWMCѼQ琷g[ծ h'Ɉ\3SڊF%@A+:6p#j5?ۋjYc LkS $-i ȡ̴:/VCB,MGsim7>rӗA2:HVA'"KL8q|DƩH_=6E~,W6niדbVGyE6'%}Hkrͫʲ2,v\cf̠uzUW*/:U tFb6ēBEf#?^F;{^YbN ďm2+?CvK |hq-c-ލk6F04sʱDNZY·lR+~gSQmM}rļmH> mdh<&#y$:寔xPlј-$h ;HyuO:;?"oƻ_%]lcoC*(*>Uj(LkTE"|lExX(a> ރ`; K2eO!󹍇qJ.t2us^ s'8qiFb̩5b':G7o"ְ)ೣ8fZ,4Zxܰwʬ5i_G!DeKj/6ckx73ȯ9?6q#R!޴?8w2Ƽ9ӊTZ*|F,%6r"|)i6DO5o.?2_,U==Tš tmW׍\1/b#p{up$^uyJM5|k ukZ}C,Ve?2"Gu?oCzX&`^UAײt'Q볚 9֡X ]ʝrzI^ O;o7M_] /K Rͼc}+\{pq:1~1Ŏʆ栎>hhd;4 "e;76#Ip7eN`zmKȿEG9mKv>jN9wNLqUZG1J=t7oei:qB/.>B.}2.ݤc^ mnW(2'O7zHmobJJ{yLWy.I ơnTlq湇}ZfB6.=GV.P@qE]j˪ʁ m͂0&?Sz؎$yȖ72"+zµ7] <^'5jј:AU Ѱzz^Cci]OLFIQT11src󨺷/xYs c#Fk`?ʧnVq i4%"([8ͤj`} lm;◥UQdgZ8F=O){RzT\Wu73=:q9(<}!j?>y jr}źؾɎOBXH+G) ,S*NQ'wP|Ҋ "Gz|NO .3}>ן;QaVb_n^̶1Su9(&e 0!|3{Ȑq$ Bb5z%t1X|m>sxRUZSM.}k๩ߣQgzs nn ս;gwU7殭mQ+l4M~ܷB^dGvohR_rXt Ct~eYv}o|g?6:OvL=Eof9?!!BcΑl2|k?%Olsgf7#:kw:s_+8ٟn1ue\Y<ɇ\>EH D~lYyΙRik9-hn^+KUf[+Kò ;jBY\W)(41|pm&P_BV* ?|M=^q 5l l"9 Bm'jȾD}$빃5ʝP xWd4vE7~c|F)mxӸdP;'p'Rtmyω_WR`mu}k#?&ZVX[(BH],mC8lhP]> W+0;^{zU}oI<>Q赚Ҟscy~瀡7't5"9\BM(kVGudžʼo.QBOCg^!KyN44XWmUC6BwyX˝Sl fۏ2k^Xnυ$:z?>^ǠzDz9&PxL8x$sOds$epFmǐ;wYThܓ_n5pģW?aW FoB,pO 7ONBfahͷot\#0} NwM/7MyIʺgkaY\h[IJm3U6w^V.F|$71۲Q)Ymz,5b{Q|,)kxo~7 )v\ܜH xovF9bw }eoPu 9z|ŋ}eLuL?wi;F_w˲5UXO/ =GYڹ/yQo|INt~}S]~)s%X\`Lt5GQKd놂U6eQJ` #OVTҍ@eEֳߤT:O[`K}$tԯbࢰ)ӶA_הHh4h&VGX{ō8ӟ-~]Qn ͱ/fDJ}*zX;ʧeWwƋl S?̲NiEGh1ګ*znz~Adoڭ:~8ZL> Lo>c AoJ,*:>0}^>VƉh e pnT>5Q_֥#%9%bHe&NV۫ "<෩ iRmJ *u$?_.lKQ:Ul9@e'=(*Ks*!$Xo߃}ʈkڬtL.k q<@<|~\ƨ󫾈h|l>5:g}0sT B@v]xZÐ~QHTjTՌӲ_~K-(Ţ%iר=þbď kĬKtLdXY[W:T 5_Y뭅aBg*;X+ i 8 0: ?sgbT`,9Ue?p// Og9_JMdb@xg̻W͊fK:}=M=Zj"d} kq̏'5D|{~OA_wkz2"ڞ 2u|h<ȟŰjOZΣƖT` 9W#q2 /dlŭ4$<=/ikno#b#MVZ ِD|Fb'01CDĔxM=c/^}ῴz:㪲ʍ5~#oe'Og=fH>)A{>.Xv1aKEǯL}̹S.bbZVu袤"Mp~tAG>Ds-#pɮB<ն>ܷZQJim]/fׁ}6} 8>Fl7AA>Rh+ :}/V6úE9`w5UaC4.̺Exx4|[k6,]i28-(*]Ir:9V}F:I@_Zjy҅:xh4tT]׎U{P+/z5R d&(MWcf5҅BxpTV[*]7rK.[: FXO?}SBY}yn|C2a6Ёedw*Fm ,r}sټzIǻ-fޮ\HP\]UR60?LׇmDm6L@vNB_v4޴&7НGm* @VUI4ނ+yTr%p.2](+(IPբ&9i]*^%XJŧN`ǣʍE8O nfF]%}Wo3FO<aCzpaiW[ezTFu<~~Eu&} v,F+WBCJMsq#'m-DPx7E_OOǖ?3 iB,ɽmUۭ\m]6XM|7 »^h-@D*-yY+fsك}ף30=Oˮ줤V#tQ8'`_BV~rM6ݬ_d:]QUz|Aymme.+T1D~(n(G X 1wb*~mʽv`Jث./{lq=п VzYʇd^F;'~QyY·<ϱ%rt=ؽׯ5:%˷e}*_Ag[c=BVTIlnۈwJKT bLS_/ېJ*3 SAuɔ~3}olі$͸'f"6`*I~z%~׎X7#JSHopr \ ޿JVαe{ا'ˋhSo@mh{FۛCQO5ʶPO3{T|;Ӟ 湆hnEz~n57< J\xDG-(4:9wUv̴Z  ,~‡Ebbw+.+ [gk`IOuÁes7r-m"9 2ayˡx_uZT~+z^#_v̿hs1z` ۛ\|b)g:(xYWkcLrᴙ+]ӛF⾜FfwԟkXx3X甀zL.!,DIDAT@=6+=kaa}bysRU^_'8/b (Y"OCrL  6v~YM&7P<#*E8\tF'ȋƯxg.X% b$J^)/$5zDpӴͽ6} M.WQc$AR 3VU\#k-˳1F[n/W+] &dehbNTZ%mEwMQs,~J#]:ߍz|"JY}l8̾07|\N_ݹϒCN^yN@1S9%2 בlAxzH3מ胉I<+U4n񂟴KgSɱ? L'a9h/gҚ,Lo:adcNBnm7Μj }T MѤEy2izkFGm`y[5>&S-$Y7.^"=뎙D+\^B6$6ei|JՉq+HOCni+{mËq+zt9ہ7}.'U74O\&XOԈ.;ޚW-^qug[eܷ[t0r>A6ZBEʼ6!:0CocG73pB#& Y5n9ue?ȳ'%\7>sr3]?YR:ٗ`l0*;uO>cCO>\lɐ |ePݺ큯|Q1uz_ ?+M=kdԟOx#o勮5߮]l΋?p֍|^|HH]B11գ 2G@}@} nxTɍ^bt99_/.؞'v1&w%(Kmƣ4Vď15RkK -lOe=¾ߟl+7P yQK!^[9r-其fs>]ZE]x+F4%_AEFBmw3CMt5 mSJ^ɾ|̣yIɒ8hQ(O5笽Aƨ =gɒ;%bQf8 t\^v :>ʗ{DO>yɴ+JSLL"uGa,@tLGvuߓŹrNPpsP@dFK[3|U.[ShݯH/3tC9+92ΥMmWHm<^3q#M ^Or̟%i:6XVU^_r3F-aMf&wq 7ϳPgN^Z[A,mJ$54.'ZvsIBP ԹR=߄z}'-Gd*w0E&CJ8D2Pp^YAu?Lmecck^ԑٯgQ9O1KtN9/׀I.:7Kk]qXt47z ydG1d4ƣ%ox uG_ Q;[1 '1`Y%'r2X=(^\Q*ÓeiVWy穏M J}CR'cB=h\9>>p!U>k-] g/@v'54)z &Cx[UxmU@:8|b?5٭7:>azɹ)-z-$<,cYLi@r@i<)Y4.܈hr42CFoe5\tk P݊kʢoD*=s|q>ҿ)R٢yxT%u!Yp$HMӇݸDzGuLֺKB/u]9u3GB5U7MeǾ=PVgJ~ ?!VZXwT~j~Y)t:D۬@3.{"B ""ސ=#$rSљogK*r9ܱܵzF^O͔ (s(DHw桲-7,͐s<~ǧ3ձy#0A5*-+n<Y/Z%rΘ 0?-kd30 ؍#asmEs2~A\zlWDc倧<|NߖEe#h|%^zWVefc|a4eU&Ar.cyxXs{z0@;J"$ _xR|,&l@:_,eV^,N YvJ' yFِnw{<#q6iO'OCOF*z0+)&oT_GURZ\ڤk^B7ImN%kKmӗŨequskmb69g2+ ߔh}ѥV7;>G>Z^*,B\wi:Z\ւ&A?K| %\6 @CͳE R̗k(OWNV{cgן+0ݲ.ZSKħ*VݾVyn3V9f3 [i`cs5POH˝>V8ڴ!\ЬE%~JRƪ,n4*"hkW(v{]>8T9]bWr8*ؿ5ޜK>,J&atB\z8lmBTd]E6,]}^)װnQ30O.>ɭ-~^gu8}}/Qbft^s|3&r\Oq+\V'妯i ~.y-2x#E@d:s̬F-fw rO* ZW 95mOajߧW rvN4@m pf3#3؊2 ؉Oɱ^TTF_|m~.Mߝ'4J SYKAog= { ~܏)Wy43mrBZy4\"/tBċrrVFJ٥9HZ*;fvJˍ1s,a\0$ZX2d$eb^NOx_ٓnmx9`\Y/mq1gƿ})gt ح[W?^A/@ҷ9Ktf> ,=?sN/t"*y+̓)-TundqZNdo8F;׆JVS_Bp YEƭ}cwE`/S]';+ {It3~FtӪgZO+ʸEȼYP)6t)NXu3s_. K%.1,e=-i9_N0TOZ6 ׉*CNNNժ9\`uK6-4]_xKˑb1:2ʥ DC? j\}Hv@5&E)q6-(v( *&T|OVmwOS[+[=jsқR֐ZS3-T6}ڐ }_+%=_|T5jLWKiQn0 Sy7]q\ؼPZPb.V{\U r ֺD4ֹ؜ɹ9SxuRr|od)W.D8c n>1'~g1n mvE0?'IMkqM\`,Eq9&'+t˺o5!?Ynݫhw;ϏB]1-甆/Xfw`OfMcs #a&w'oǟy/,ɱ#K@(v @€q0 #!as}|/}euS/?<.GXښy}`;ܔf=5:|1[H2Қ5c+"J&to;(^\f6RXF#hry(E_4Qtes=M_v󫨃 rAWWE$ LKK4! &>& Ȍ asw~?(kf] pv~+{_jbgJDRYXBr5_X4> ɦd @{Ns\5J3 A9o15oIg`f)܍ͤoTCq4&Q]:zs-.LAVXI zIo:kÈ}Y|ދhح|_>(.1dqp ]O \+,N{9P}*.`=eq{⳺u|ѵ~ ]Nzy#QۥGwV>8\Dv3ӄ95 Rj=%D}{9TF rf0]{?0*mNṜ-|~܊"'!bIk _zlfƯlZB4<okz2R,]D1SDqkn@QЋѤsrn)'ǢU.* e ֎6)ǿ9vt\_lW=`f~b>DzDR)7ͫ3<(aXѻof7rzoU);) lQ/ Lzނ,\]c]h 6ThƎc9/Sƙi*5"roCYG.:B;x>q6 줜<#x~%B8<]##r$}a&Iؠ73q8҈4mwz)Oz ~/lLT&!<a)żK٣Bbf&} RZ7y9税AT͏ -!26wc+o'Ei œp9i&;lAouڤHwcj?QuP¾R$w1>6~ QG' )q6x_w|]WaNHs0,l)~nx߾uߠgz=o{/Jff =ߺ}gM/1JyȳfNr"Rp2髌]]ˌ'/.x2˽Yn p1f>OuπHb÷/EEd5.&, yu5aM) z㕋nI 'ȏtgV?\]#O;;08$ qqq۫>ݟri9h!R|7G^c'Xh>:YQcW,nXFc-^<ư\䲁s '/ҹщqC1rkOa@&KnEc.A4@tm'CѤa W l(HjU~b^(CS5љCUЃT(q#+gm wy3 9ؿK_Q/[H ޘ IfWPTveSO/90 tZV 3vْwcYW MNCRp,0fYj )EO #/Wol YU_^Sh"pn3BV،;'e#}DEu^ oLj:HC#S K|]dUoNgk"DZ:YΓJ?"aq@Q//ޱs=ol(ϙefգÊ-m0} WVpP.7;𧡚[rj+&[3yxIM /:(yL9&hD]uձ`)``pc}NZ7 奡6![ٯu>㙢ʏ(AxM.` @,OxL0u+[ bZgZk3&}M?yfFN^^EWܷOַż)g إ3o6G~gH@a>HӴe?)!a@0^Z׾/=˿ǹ~<zyT9܉A@ֲ,/A:I/$?2xsޏeI$<|Sw2 e\nnp<ݟ;+LxC / 2^v?5V=L%d0&+1:y!̇K(-ǐߣߴ9cg\FՏ|O?qcDҬO 9w WyzS[wp9S ӣ}~LIֺ_zƀG.X:ګ%lj~@BAPޒtlR*mElkHjrǾ# kqų$v/t8%zm̷]r/>ü6y!y fIх K]Wń7/{筪?e+Mо_`͆8Gj'A "ߴ.e6ϕmR_`/ipzJX,6?08Ӊ=uЛ'9KZg4gVc3?r\VqΧ`[]ӣιjlMίYۍ[ dyPs8_3"+guފnI~XCDیIE$#>Br\H?6rt[ӣҭ"^\x7zL]n.aceD˪ghiOZ1Ce\ŤQ*PSp# o^t`T->UVc&LSP`y^w?E,9ʜ_*wJ>l:ؗݥ5y.SՎM=>u\[! gǢszo%9ئO3MTIisWX  ֻDe +~]Ec(C|c>p~>/}L<`l&74|0:s<я^Jտ _8v& Omɏ`0[۸zq(tLlXܟ˫vE׵.D-NO|-3ki'}NP %<39\likqd,<)T'յ~)6ۘ S8vi?~ |ݻ߂ WGo9 2Tfe&ƌS> ^+/h͖!%L(w6!ו`5>q JJ!uRY}<3>q-eig3x.ᙫ2p$6p)M 1sNA%`qV#w2^Oو׾,5$% p=#oq$ _M=w "GqHt3ψGΝ [w?SsOa;0ɀ:dq S00)g\_g^4arysTs4oJMsQ7"TCD'dz~@O߬.]٢Ɋk蹻.6e4$?cczTyD 'pLWm ^EX 4^ư\cX Vo}fAX_E*#~.?hcc.'VEH'w)73}ʳYKjZ.H5v>YKGWڠ74w?˶T.n ka']?F7s=(8"[[|5pz̳fP"A?/4󛿙~r4tȝ5a8aWQ+l~&6Eo ?_8vzquޘ]Ԕ{&yysuXMd}%-'g'`7>u=e}JXqeiofJx$ݑ<,]*6Ԟo.s[П7hq NRvc:38쒛Ƿ~vp6;cU=NۛcҰKyi(d<;}y1*^n2][FΣLNS.e8l7Ɨ 8^A!'rDQ/z -ݲF۝k?-_Kq}}nC;k6 ?r~ crd`5mw~n>W7_|K;s߈D8!wNXCnf =MUxqY )h9 Q^=8y /8^e:s^33^1/I}&^39cH}D!hh(&QGuzti!y+?/k "śn O偁yzuO.DG#4# n5>?iqҌ@vCfAe#QSD͸ [4$|*Ѱ;QֱM^pQۋH]} a=܍?lk]m 2.m'!f# 1kˀY7A;3mQYD)l*.P&-2?u/#fA=9IB{'^d*te= ֝;[hN^1U(΍mZ̒d XF1,Lzz3|ib% Nj|+@#Bݰoݑ>ͼqCB;\buq9sZtق@ChG> <>^w0fڮ^ME=i_`do"nB.r.U8(=R4כ1΃m:shi6f?[?X0Θ F` y7 Iߪ<#C5giuzҵ -t̔ҨiJ3Inn^<0ᗼj}Wi0Oד2 XlM=68nUZ8R@ > w[D,Ov;=^GpJ9n+'ڼo/ak5M|CZh՚?8_}w󳿎/l#k}%1qJ{J\8dElJp㢕 TPMu^v]u%-ӚǾ^f=eqV͂ ?܏ wӖ/q@uټy}@Ou"r_ dتD,HR<(<51a.D}\IZv)Zy2x_yh2fd(p@· ~+x<o#0H_ K$9<'M.>>3.Г~fVMH2|^'+ӊV̔ xjO|xƿy\nx;DŽI/8)y:I6z}r)78=$H71}Pn p5c>f e|٧^7_1$\E8SՌKZ*BX qx!`^l#IZbLֹdq8wo}3>188aڍ ߭s2Ƅ^s/y٫3vBۚ%E)7~ԓ;4$\ ɏxd+A #Yow3Q[J=l{UMx>#ܤͽg#xz4=zS2NօrKGaSoʏ^VY(z)^-Wnx5dooIk,ϻu֏uE٭ sڴW\^0uMiYqp;cQ?ńl9ϛ꘠C+T0[ezHnA>%1y~v7?o#߾\G4M9|_Ah l/8*٤Ǘzoӣ Z4sj5|y!* ʮzle(e7beuZ[x<ܗhȥu8q= m^Ga_L]ˋ tc7l]*.ukBºn#U5i='1۵?7vk]LJ|51n/eZֽ#b"_ߓ{_iܺt*y;3/zqhd y8qq~= _|;w0ԯ{Y6~SvsOG_K~8j4#}X5 F%%/ynIaugJn>uh7 ?"*7Z';Mݺ5ln hYt\P^ޭN~B{EHň5͏lۦ:kj;5;E֩+y$v|&˿F KMX%u5`^=J}pŮXq 8_`9dc^ ݋>6ُkA<:2)`iƷ'x)gq;G䜑72.ܳ)y|>.׾ _k>*%nf!u-[|q¹%M4vOk JW5@B6^IhP$Of,s~ `q}q}k<O} $|ߺK组}aʸ:fm8tjFhƜ2ʆ`ҒV>~8z>$Z0ҥ6*v{̻=ҀoG0k ׃ ginH-mJ]B4N I%Go. aоI?k.P{7qRXŒ6k;Ut>,op~1,ϼ>DuaXnl{V!`ۀ8f/w| [><"ku7Ζq?o@!s0 e,Z< ( L[V^0_|n{;{Y&K^q~5 IQfy2>\vg}t (sGJgөxrIpj/LҲXLPW@"3*c7'R?ÐpC8Ux/|{+p+_W/9.^ ^r\8c81\Q~쥸#Kq#q8-8h]T^~?8|\^Cd8Hr+ozV:ؙb)L磔;Àd_1vOpOtAD72˸q R5GB1L>< 8OǮ&~K3~[ݸT"OodGv AI1wp%GƝG_/}w^RyazazLc/c/í裸CcpÏb5bz?e^%Xr72e连 C&:dGd[k61Xϗ6>%=2#wߓ`tΕqȺCUГ秶E(:<;H`y*U*?=tsj'j1`yD<ʟv} H.WywEMrRu4d)2; Rg: flbe6YMsiHgCs@Q}Ң_\BRkcc|r_e /+`OR@YGe~lXd9 q)8տx>GHiJCZgYW8-C۹yZوwFo(sX%iJ%Vvm`,(Ap2lA([s @ZV'W+'kqvW"l龅pΨuvS$,xD|0A6[p6 x_ķH@n:%L=g#Lg7-kgd&9z:ls;مnSV|qg8Mi8Nc8'~<=SNq[N;E?[WLwзrspt#. 6Qa6^P~|k'tS/M{ \KӁbWn++;թw*<豁5p^WVf5>.kM© 5}(*zs)9*lrsTpX~8 ,_mn+~ԭZ!ĈesMt1{ EVDI_,Y,lr(iLZr`̀}Q8lÔARmXD[H0Wwx=医W3.f\ZÄ#/puw^9ܽ|;?4o<7Ry$?& 9xgPSʶٯNfJ1T =WEMS\Fof7HusNj{~:1e}n2Ϊlk:3\ls'Vu>cm%CfԖ2rp&\޽ճݻ˻p}uk\^]#.&\^e:ǫgqu9\Ȗ,qޔ1lؽ#NrPr.Ggowc'.$ ?dHU[5~ 72B>c^J *%kl#%qlh{\.Ae*VT? Ζsv~)*So[Ag;eY|d;%9 kG&]f =!o! 7Z+{ηqͺO r]SY/}V']) Vh[~|c7aOƮ|S" 6>q [{ [{y4CQ\վ>ly^m-ծ/^RUtuꣶpaǗHOϟ-ڛfY.CBҧ҇4bwH㈔@WYN|K;ŦY~Z>gi߹K|Z{4V1J*Ǎ"٩<^?=+79r}N R/$tސ !ZOQe)0C?. 0V;OȆ:FDk|[y{ʍr:xj,E9;z5O| '~&+a5\oDXw,^,_74"a<,uU\au>wFy+\zn{裾kPxّ5xZKTz߷tXs4Ϙ E 򒌼e[^Y^!8kW9m>&▏Љi##zY_ivaIcuc],`kF&,?UV/D[)gjt] (+=$+.cOÀ{o~}x|1c8\!KOr6m妦Ȝf86a&SƜviY9xW1M?9lTw3䜐g{T@YORw? _p<:bx 0%L3F>3C|KOJ> W.AZαq>\J3 )ۘ00#ӄ&~w#rN89O0An[ nIQ5M7#)js#nvY?<ɧua4 g즌pg߁G#=v0$yT|j[GsG08Ct /Me(PR/=LԤu/<Pٛ+u ?#h t&{%y/ǡ=ϡn?Pr1@X2Ӷ }N'Ơ|h|^S] NJ3d>rr5ܱ>X c6?=f>*z. :W|j+\nC*W?;.>ͨ6ݼ U_gI\72}TR YިlY2IO&cɩRgsD=8>ҸKʠ0[H߲)<"u#\X+M*B=2ϗ\cz&͌aͮB-Aۚ~.1ze+ێ?SRfз lg,m :֞Q¤}Th5$ eKfu!<9P|u9t`5q%3C]C>>'Фcc%~J~Ρ -B d4qOҀg|ۛr^΃:S}͖a xskn/&w} PV:uV|!r]D73.^K|NKQ<"Q)}LKWo׏ 異Rr|=~X n>ϓԛFQp%Oz\v"KE>`(kW斜sy> Urlylm9۸ESH fyPй)i+ev4MHt8 , gſMlnX-SDž_t&^=cluDZ9aC8%<oWしӷ#nV/m`Ѭ 23AŵVrӍ-rUndھ]tSsKywNL8sG}R{5iJP8]7F {΋WJ};BJ1xh u b(l\֠owes87:R1=-,gNUf3n~RY&I=/yU>'K6~m B 7K{1}u*.W|C,͠?F[7^[Ht>qVqq44 m%JŲbwA,S#L )z;nzk!ٝ+|9I@;T<({ 52'P7< ɯulrh[XTN"QU«Zrj]9JSgrȢU1g<0,~t{!662|R)?,煨cem{{2],ﳞ|kؔ4q^)r, Hƶ\[E#LvsRqF94^u>sNj7"%Ud9rl),倿!DԍRգ݀Y62  #` n9׳0x|wa1jc1m+Rl'z٢zQ[/0mlG!=p o;/>r_;[8x븽 OSjǡFJog1zmd(yF@~qQ_v|YByF< ݘ3qez| V~ƅCD*#7)}fDHp7}c͎k>YݹcoU'TNP 5]=$oɶNjW,bPc0߽`7O yb'ǟzOO>q ^ɔDGm*Y MohG!| p<<ߗbFiWu{ٶ 4x^]o=p}ww9гςcIjV`EusIv¥R[ B,oa.RDi0H0]ӾI`[ᮂJ_igb,%g-7](>n. =ڜMЛL+WZ?+MFFcn71XucQk葃 i-jP\xDz.>gC+# #rN6v9?)˃K6 IE=lf儢K/D>)܂72XLC|.q9̯C>D|qǂhF!p*۪|nʎ 7VNerdruIkxe@x1l Z> #6ߖP\t[+6.Wʑ|bGYgty/I_(;tB厝\u*sL϶ooHptxz/04]%ެ;TSB;q6U^DmB)ʈ}k؊+Ѽ6$)ua_8:v˿G͉a`ZKT;e0sZS8Y "o/Ey^<60g!;L9=Ά[[?2K5>}f>lkaCTҗ!/p#P@2?C ?(~Ku0A?WW OSc_C9r*1sE[c-lCvݷumJ~c|p~ɞSeeUaп?sO{G[㋎9c-W|T|X^ k`5zyu˖)gG0[{,OL;a'*(ѳ6ԽAl@Sf9SBj)qnVvH OQQ{*lnEZ,qaa|hLz@Ĭƫƻ-{ JWՇh~o x^VVw 7KR "zM$K tXjaʧ i_#t\gYY./xs:򏷗.6b>~ڈђ8UW;{7ٱy!?kඪ=׾s&1/47ҡ Gd /Gx[nx4mޥ곦q{ݍc2Б< sʰا+sX7Y[S݂+ukxU󂃏v=۸.tVSjG4?e, a0&=ќI)a'Ɵx׳|+k YR9XPܖs6<$iDN#>~).}C(,RZo)|zЕʷbQ6猋1]O?,~׷^?=yq܌w?!6iuu/\60\8];L4ͽxtVVyQGB1IJǩ3Q }0*~`7@M2:!uZ'(~^kFϒ.~nM[V{}*[b˸t)-2-y.X5*le_w28~`߄~2z޴fOԉy{.6j7ذIۺp9bk`zEXU$4~\;iWtr[ٷ-%kSμJ7O6Xa|;5o*'mW̏s%YVˮ=TM441h)Zo6l-}r|yp߄};hp*R\4>/1ɭ"9 |b^%o8wa API29HveS+]I)ɴ .l'Eه7pޝY}Ԏ.$̓ ܝgzqw~TZ(XԘ?'r1p8?9| kfp;n6͊f]j=!gSV&SAkWm3O [ɇȇ&[S\oB6~s`\9媍m޲vPsHXnhY[eWcCҥPmqS7ٗ v= M** 0A.2DG,.H;;ݥL>Ɓ^ѾTlo[t 1E _aBLWB+`^%:`z.} oeX;4QgDtDeaؙU',uym.A#[IAn rCrg1˘|C_Kp5MإF.%iIw?(ٿ#&iw73K o>|<8^f9Wa DM.TExizg$f 8bOWx|-ypyuK\.q=]0_b%2"K E%+\N׸<q}8Wt1'{]yv6a뛦UtM<?;g8 YRrl5O>"?d}uK \,Id_c#RATi6<[\A>f[e6]E=H襺&E+yc2UpnqHW~(eT~KtśS~c>9X9K|@=EvcgWStߔIJX\! wa9:;װ_GWdn0FzG-69L[e3Ulڷש{3K!(Rao]NzԿ|_SI+jP.w>\mid%`y\roa;Uxf ȡ|X6Ȗ=Ґ-n|W>5TZˎS&Qb޸Z5YL`s]O' Yts)o~9cEoJ@JnsXb`ĨO8mkȋC$Y~9m Z`#҃-Bזío'ţJ`#LS67|9%qN߿?Icr9j^"7/ҋLUb6Dv 'gt QxB[ɠF8bfXӣJF7EAĻ]*~tӲO||ggQްwn!Q&$b7N708 $n!W=_xv:"&pilbPſ2%r&}p Cqu/?p4]#3iu>aa$D̎z/A~7vKs.5l*-l(%D7pCGY\W֐˫!}Us]ΏcvGu[ !{X=R֧T-[,H@IL<޸ԕNۘ@݌gF so1r~Bd+.yra~7@|KKBM34c71g1 |ʸu>xckڇ>>4[&O3>{z?~O=xxigm2]ƘF n;7~ݖ TNIo"2F rư1 =wʌox:ԭCvC\gG搘!e.";Ojθ5WO=;.gy7MuɨzA;;IeÌ!& 1cw8`ĭ;zWy.I,2\Zйj_UoPޱ$՜7#F_/Rq\Rir;㽂پ7adrn>&{Cf#ϲ sA.GcZ ~<'|fw+#N녠  l ;q-wCbVNBA76B΂ayq4 o]|?W|ǟx z+dssVI3$?O0c8S%w7?/O?<〜3a8GvE{0 _UBsa5>vs6mM{qov[=#s6dmaK^I;T@F g;?Cx×Sx6$kgȗ쁚 "˹\9 s%Oߊw٪TI{oƯ(/d'@\Xٲh,QSsSȵcs?y9-ƛ=oLJ&ĎA)\Pj_`J% ;2Z+KG[J׺:TAf=ЈߒY//qt9|67mxϼS4ag8gv{`؉<㧛n16Nix[8LG;mLz3g=يeFniXW匋1>?]5b>aJ>{eQ/KAO`IRn#iθbk~_woW߇e~m|ݯi1gl;v9 \xrׇ1]_z/[C{?>w=>{ 7>gO|4i8L ǣ9$`D<ϒsaP<'DeqV#Gܾ={mOd{Ʊ8m_n)z_ci>#y&Mh}4=s.Q\<$=Gi37~g MUv|_ld=/yJ n~ug^Ut$oT5;A>֝>vޘ~ ^Ǎg,Tlö zLnkX +\_uv wI6u62\_&H}VtQΆ n D-9Cҭ2"_8> 0,ks7UnXiz.XwIdoyu@S`v2r`yC`k N-~#ú/AB%yȓ$W!,gWn^J?] 'g2s6`9F; H_Glb؏ xDtafɼoO֡ oܟxmCŃ#S|&XyU= BeF_Wn?#%fDvlQ·? mSgB7htY+~S} KewwOуWF4M.G3y8r3K#sN _ c0cN8 wOx0kYwn7 #iC8 sv'n7D qH g!a€iN/?%;/:w7ޏw|9p0 \e VgܥJ2 8g\)̳[Ol|٣`wL["`)HXƯ=->wwU o0'8Cq:~'<5M8͔gvT.9p*\y MDj1맂Ŷ[.Y ?uP#@0z[{rkK "{.zߋWȏ(WޔXTM(MTp zyyܟoBt|q';d]%f\0|pS؜q,J34U|pL6רOV֚\6T1r'XP/WkuߺePi5 j .EѬzxO;obcrlʍ 1]~]68k4>Z݌ rv,uxGQ2ˎ^q/o+~+M)[ۘږ~]R#]Ce#%J[ǥ7ɝSaٓݣ ^5=:?_|{u_Q2^LDviOlX}$~ kYGr^xrL+\orLEDkHC!ykpO[toWTgX5JiS钄z`*QrOconEuD>궰01=! Cg}9WJYپUA4m&PԬ4y[ڿTv9XMUh9#Vuw$Zgl(JXԼ!%SYp·x.~}x 9Gdܦ7HT#!O÷V. 39@os'&88ml ,=N }Jxj|31WπaӀ~wМ7%)⠟C >fݞ~!/]oL _s}وf\.v cnˀ;T%:r7$>Y/F`FgNx*ݏn;~O gnC#p}G CAt4$ƒU> Y>y$7GLp ˄o3~_}po7qr6KX$omM r_&\ ҈ykq߸@3qB:<)=˸n2oV i{V/8yO;Lw⏼g4֐p4[?BE~y6g=/Y'D`9!/gȱb.tgdyҹ+lsU6SHY)> C&P,WmEPs^7C9k*Qoh퍶^EvD`?o[ӣǣEWocyy՜щ=/$@5Vr9+'Al9DZ?sdG{5Tb?0le@yslʾŒUH ձ r=mJ 3@ ,SIM-_KBfꘘ $b"m`r.G }]2^#]#wǶ$I܎Cv Fz5ѶfWv Jm՗Mtwkc>y.yxE`ټD@Gp!k؇PdQ\XtOtlL[]o{1 c}3F/s( o鲅5Y 16[6uU0Hz̋m̃~G[{48K7Wjc:eam?O^)u5L1"z2nKtXf͜nfv.Hz]Q|هoRD{75F qQѼ2{]P=B`Y\ir71{| LY_]SSr{ŒΩu3l) :mr U"W՚ '2Fiϟ}_7|}}vSJ0Y_̹uW[dyml3qB>gryn8O_b2FZupϵ) 7Fysln O_Nxz\G 9-UgyuJEϨM< ӀcC~g97}%' MLsƶnU~-D&0HZI=fJ cJiO%{=ΏavW쮮Ơ74a} Stckm!cNq:Ni0RN:NsƬ/Gt#/V;enan5Hyz'4MoP#*f`y, ncރrd/fl /9,gf|v,royDQ! =>=x [̛7F!ѓݫ{x& G۠םePU~yQl_#r7k| \G9/Ml=^\]жǼ 랋5tid#vіaNv%@lܹ*>az0=:/pBbۥcΘ 1%|SO6b<$a4 ͔OO<ۋ 0u;c}~_My5?܈٧4ȧe~9]1y7M:omrf=T#1,sq8=3JKARVedA_]N Yq-,ڞX* xánt$Un2WjԜlN)mPjS W(K|K u c8ٚu\ȼ~N@RgXCgaA/:H˘^k#OCY o(R]C ʝEx"zJhB;RhX%>>qW*i]>3MTjK}V%׫ھ@ά)?("pcf{VX۪d8+I?_n/(IkFqf?k]ԇ[\C,i 4 (4yW򶨆P+ i `m93rǔ)afѹfůo%~31ʿS$h Ϡ˜C]V&nH]MɮV: )]8 [ǛL2Ȱ`׎5Sp'_45#3OgՕz4o&x ur+(~y:ޭ]X)ǝ-+[wrXF0+BxJ rSׂJrt4 3#}Mw/Y4> R0'ᒤElcI8~m -{CfYE #q?C}_X[>apk>KH+C_GnVbZ^?؃#$/]̌Ȥ,3>yڝ0}xMZ|IſF'ʙA^W4\J<{|7=H@O]ɸbX\+Xu!Ȳ" 0r0`}"B 5_qbG:|P3ߕZ?}0$<HN6Liơ1]_8&0R GB4_xN|`Z`s~iUvRk5w=e_)>Hjy)CC9ļrau2'm ES]_Oe_6mCs87Դzj7̠,Քpa9E~HKaUn<8Y|K ܡGV)r^LAdkӇ+h|,#-{TR% #[]Y!#J&>OSy և N,EĄ^ X{ iދ`EL`tQ#ޝ'EKNhw^ZTu)-! ZX㎺=Ĕm,['Seȏ. )h?0gJ0}Ba > f La`Ak޵jVF1'y ?UE(7mO5?ہƄ=SX%6ik*HxLZn3htV6t%Sjzzز֠@i %wiBx_4))MpB^ԴØIc(@_0%/M,%N:" I:fMqwDy ¬ ]~uQsS!tt.hapߖ-heN6Gq$Xl+kt R/<^zȑq=^Z&-t"\r>Q7Ӵf|ж/Y9  9He#[# ){g0H6q'ou{9O~^Hu[I*K9%TNs88(AycDn/'{|^{Z޵N_{0 Z2af}3|z% v'dQ*k<0_sr۰uSݎd kZ*SNV}tcNB')VI}J ]v3 YA>4\=4"mNpW'561u|2uIj)AyY Lso5+j͐BRu GO<.9]gAN}72sBܿh> ;'(YЇc Udf2+xB*B&l:Uw3hRgAd+-yZ.~#upCKKgE]G:3&p4|d]SqW/$7y ɸxS^{'l s{`|-R=$|o)yZ{Xed+$Ӯ DYIP2L;-]:Pkv92LI|Ex!I#4bS*xQ /h5 s.<o15lsAл*.h(''}Px(wo >{p 밷iP>iDZY|at?-syEW 6S`z|Eg P,J+waaJ> X'iJY>"4M]7(GxjOi Qk~tS̢F=~h1Uוia` YB!^BrE~)x1F+L%= Ĕ@"bmpWXOX߮=ˡQ,#DZoWHv Q# 6#τM(x4_:4Cpb@ORPN.wzʘ ˘ T-~{242+YyG\1*G7۾$T)gݜ˶(Yد(Ⱦ+3rK_@09}F.ut XsghJZ}VyhGxԵGӈNHZޅ@M=O7o_*9&π+Ta=(t"1ϥuD/Еiqg,-`Z7U]4M=XD/x&zMNUBML0 e<G݁v6Z|qhe9}VB%ߕ5uIR5F]X41~:雤flx\=q{AY訐9rběcQZYVTrN/-x]C -YNlÔYb͗o)FA BHv%U]ӞhO$`A<%p-Zz),qؼ}զA]kTMJON޳6rIf짐`7oy~m5c98稾[ihSo:i!$i;~s3%Xϧd~P${ʰ/.oce:N㤼D•/ʰi9U~8eC=3MW/䗦B>PylTC<ᱧB3'˓:wxπӒd{ ;h>J䤦u_H<~0Fz:RΈ|kn:ir.)ږf3{_870;fЈ=BTJvf]O&7\yFjmeŠj{+w{/< ,wxO[Z4 񰦁5 ñ󾹌KveEh-zL΋눮,<2mXKcj[_[?Xmn8o}<@azTNMԊT2`nxz^ L'8Op+̑œxçv\FN}DdnԮ ^4U芄,/z$@/ʩMf-3cѣ #甧e:qu"tb\9i1}. &DVvܾ"/~#āhx't#Zq0Hc4m@“!ir氝EY(Jk,9ԍAU&Ur\!4 sعEQ(\it")OBN}%蓉9s^LKeB* ArWpP%Opx=Py6 N87}}'[vaEC>QʲQ;=.aD4x` ۙ\Dꋖ3O3dd_ sJdJ{hSe3Iiy¡r6VuOD)(cWp֢(J(UL690[+A^N@`gP;OZ&CLGurOim2눜i2/J< \T[!NpdZ8W,Dw%/6sgcf՘OQV(ԉ&sިZ4K%ztc80 bEB?@t'Z5]^^hoUHjJDP7r:.~aSG1;֬6(s,98W {Q!nK(3w&*:8}4gAN^1$L*y.o+qmt t%D!$6տ9 S:xp 9]e(ut{=ыx\GjֱD Aآȫ+~t> Y9Z'aZ?LTW3_־/t sbKO_75ɸ7g A9 1 0( eѐ't jBW~AiS}$_Jy$҉]2*"9 0J|{>'Qad̩0թү +c 6Y3 -勼_o֭[G>Eu]Z+'lv1͠#W)!͛<cH OX˾ ɉl=`\ը mQr?(  JֵN o7ZʆT[}6 fj:rIڙNJ|X#=ڠ,-{^ٽFp4pPbPNv.|lZ \#a%oq WY`#3&.+TM# Mj0W'hANSYk10(EAY`8,12dZV*JЗfV_$ԜqWC'ϫsTdFǨ2l m6Lwk!sHt_<-Hz'yT<z2c]qqZrP`47p8h4DY J Jx(Y̝Z+~c%i< _1t<ҳSt@q7)%^'i{6&;⧕z^K0f$>r4 gU5}ec TH䩃2 rK1G`)E|Cܓ+,(LA=Abf[d+MdQaX|1a&D O xi乳ϧ,1z3%PU >XcwO/br:YAJ/h} >wJ6]Rv)ן4D D,FE~k au/= ?3)Nڐj=2=\OzGȘnRɦ.Ί{BO9΂4'>g}Z<ܸl 5ςUM*I7ib'0Tصg/98q^KX^&9aTCrc 8qo[H#qӛ?P8g .W^mۯx20 6Pil3!8vG-qء7k`e IkϘٚ~KI|bt$mDfJ ɓQNLn2`a'ԇ E.9~ic45pgg>&0l1.=#d`6,,o9F g) 4DZJ7PHCg34qRH>OIxm儊I{pW⬳/]._y%];v^vciyI+ XÆ y&t8n#8rl޴s# 4d"ʢ8[i-ltXХu Cҍ0*}~o&X4Ǹʼn]ν8<}+ؾ\{ j*9A[ X8-[6[M8Gqnyߌs!Kу:jVXk~y{ t tI, E3]\Z+s.ƙ\/W\q5m;vݻzRS P8huظa6l܀[Q8Cq`E%e i'ym~,i5 ҞLسOAs/pEat?ֶT7PSL㾓ܑ,7NH ˋSa΋5Yw<'&G!sUUa2p8 q .ŶkWoǶbǎ밴f\!['0Ƣ 0?7M〭<&ّگ/HAdwI>Oy^)9 ? zBTȳ%!Sgޣ8-׋k YmsF\ %lcIvxQ$QI1uq`  ֠),uq#؄'&AJ>)te3O} SrܾH3Mx۾pqyÒX1`,y+ŝy)*8=b}FOjx0MM!0 Ȱ9( t !`wQNQ JX>؉T|?~y*J`bá"p.)#pDLk_AQS{4U@Dw- wnyفȦ~W"$tZ+G=<4"jc(L+\~vSpKK@YrƵ/0M@=7xͱ~qeQ xwmkV?]. !TrrD/uDλu74#S(` a"t[c+t]6 "y}ꨎ^ND3&~StPI~r&%8%Xg~>j{EXFO~9PX+'};׭[N%>/b=(G#PSˉLG7tPsp gy.j4;caqF+`K iiMCTh;7>w-qwqa`qaEӪtIR)_~e['ُWjߣp4dy8^dp*i)%}Sb*F@܂#nr:4.?e%M#\vNfmPyC6(t#5Q_8k}i97=0eFK%_7s^0*4r7ݤn$d PhʷgPO*|r2AwB+R lQO*FxOb=p` m/^L0c;9b [p -1a_3{I7G3څO*l߱N/}.rT|)Z8Gp M0`NJGCquĝVoMֈIcОhnn\Ƀ-? ]:vBON8=fu=H!I߀ۮ.i5>XxO|Sڳu;%q=Gbӆ ʈ0$S0:T5Yr>)z!jH,Ld/P7K19R_?u.λ2;Dw>9g,:I rA-o{/p;79P_GQ,Vr?f&ڧ+ssW,Dw [5_plEY888lݺpEA=HD[(u1]{`ضmvދIU 6n~-'l!Si}hQZ9W^ !"8g!v-~q_Lw%n1] #ְuD"î"2Xnn~$vw=nv#qz ˁ;(BG ژe%zB'v1,fr<ـʞQ.I]dUWf<;X/F= @6MD!«{pg[ǟk}}ŵ@0p-;9"4LQbq=Mpp7m ^lH3UiY卿cdhiyF)$m:shߧNNgŐ:MSϢ() JIdS5/tC!}5S|BJ؊VNį+pwDYqSHi҅ժ ԞF9au]cD%`-;³桒D 50hLsftm_\v^\T'˅U!m%+/< F;0|`N:ܱ3ReMg^,,TkeO+}E2 RtZW{Zʑq_ f)*1k7#xkn3| OMbQ"KA`[L@*t0gq8S_׾{ .`\ OT"O+NYO@ie-,C0eԾ`Fo݃p wޕv-=Meoziɱ,fF|EyJ_r|8Ņd=K83~ڋsv4_OEt)^r2Tk= 11 -ַ so&2\X ^"ǻ˲[oxkދ0NBȈd `` ۢ^y:&c&40tt3W`Jb28v*b>}az0\}8*P !lmE}+^&4%R@ @5_ ;? s@짤;eT6<ӻ ojFs8:]ϑ\I{I6r7ڬ -g~(JaS~VtNY/;CsEP7I59]ωwi?u;0pv83 ]K!D?cPo1Ic,jfzRiy/nG=~ 7A 1lJSEUT ?ݸwcnn_{<ٿPn1!2@p|fR~d6rN'q-K"b˟@OeBC-+{/%X[0'9OߞsLaP9zo4ijI9~˼3XФA5yga7x==-oŅ8gQZ]OX*h5kߧA yr9~ Qs^wc? LJE8-`hiy{ӟ87<O&'XGMG:` 0i*\q6WOy8Kpہ2jXsyD P-_?8ҙ w!q|OHp7L& .j|_k;Oy(M! B'Wa71 LjSZ|TM'5F%?p!xCʛĚ,NT+!S~Wy:!әt68}+#oďO/fvC!:)-zi}>)vV fHNozIEd]ȐT0o?L\0c+ >Z%J4q#)4ɢ[b n|SC1NĈޑ+g(=fʍ: :CD0,,>s0a/@`Ѡ̬` ,1#䪓IUX1xF2 Xw+2* ]4mv9gZ W"s_7K~LvN$azPiٱL_OPd 7b*xʂ<,5q ;:?xz{7ϹL;դ}rpte7`h-sxyFi#~]c m7 f-a |&L`{@|~P 1_'ʴ[ z@'ċi9ڮ,\XT2V<5 `j3'=k_NZly >FȋgEc0^pC4:pkRA1S&@ '~s~_wmG9na(xA JYD:oZcS`5e~=o#L,2lHMt XM7!ox/D| ΝS~LQ.. jTI&u! "bO4p0zBKnr^'ⱏ}6_t!U pcx,n'[|϶8# /҄`@vf׎h4Mc[{SO@C ZO ~𙾜cMJY@YtwkKGjp}0 ZxOu`E|#k_.,!א^+<-SYhm,`ʔ\D?N laa] ʄA@W:4pá5Hk TEMxswyػ>kdbL4sŇYy|zH\c?->__÷Co- E!'ZO/d*!d!K7B2أ^@⶷9F 7+>wAbJT:TJ`O|xS_37FiSrfc9,z3n21vі9wso|3Ǣn LWm˿!|'圵(%O]&I;aɶ".#BL ]' Lk+^nYjhH7<Z S|7\i}'?L61M^+j" 0[QT X9 \?`Z?)?򍄖 F˭^/|Gg3NЌQ9`]nQ.QZB*^d '12ġ79{w?G`8a/Ttⰿz@[VWg$lJˮ'~ lC(JB&L mi Z-JVZ{4s˝nz1xi5=ȡ7Cr;+l+q D]{񃟞8ǿssP$c;b1buO]lmbxZ @'h1 sm?=8wǁ[Ga Ǿuo2LBQx>w룠uaׅ5u,"<&L0=vG|Fc)+U2vgA {>o'\|)o)`CppPk֍ҫG>q6ci\mVQFi Zk 3z觾|Gnnn4`ZkNSdcKo}c}gD?zfnIn!7 ʲd]`8NԿΙHf9 ޙFNfCb&bf)[& ]KYTH\fA{^:Ej0X"_`=-ìr̂<<-]WL\:ȢWMss~&ad- 06xT 4Q&]C|`2.X8[` o7{B!æf[ѥq)-}+ CE'gcxWضOqtp!W7LGcLgWilJRxCeH1rᱢJ]EJ.yYSU]Щ7n f;!'I8)Ym\gs9l nq`k|rj2$Ӥ=0mXԭ0x<6X|]8nke^Ki=GsR1a96˗ puq?q-kgLHnd]i +QX!=k P7|zS~rnEzACN& ׹8IZH||kb`6ކ3|G8aqa1 iXF˯񅟧eHc{OxCtس47cތ Q,n]XGr *qGx<tr_6YHS35yr~;ڎ|x|M!n|ȁT)-*4uTM k_#4 oBHFzoЈ^`'<8*ZѿyR|'9ߍI*jަi3׻JKD"dis.~LߧʕAN@w/~}pO~lNtO ;6'Y4byR+8{x;>?`:ع9?99]8>McQN8xq rr`n{_|{8 q؍֭, X,5,R(F9xׇ`\kNOP!>yK }@"7,#!xoeo<|#ϾFh*pE_W mֲeKr% 1`0෿? ~˱#gS޷9r!tT&Ɨ,P<&-ns_56j i\pWK/ +} t |:ǓJoY> M:?6_GzB-m0/+º tpŦM,Sh~{;\%aKB<""`,.="|;'1\hpQbaaW/sJRZ206Pmc4TO~79xCHdnɞe-ȸSl!h-[9ػ~x+ލw'1F_@E!e=|Ň&^fx1M'D/PG>ZayRg'xkߏ~y2 6%[>XU.ZlTSk8Օ\yނ7$z9/ķs_Oz!X8ge#b :1TNBjs[g}G;yH٘ i)ɐSn(LQ-[F{W\u g0+1ܴfX BjoE"n4uGnVn"O͡[EG6/?$˻,<ӊAQNSى>AFӲ#NM@S!?,q!!vƺ8 ةːUX) $ۃw}ThStw>yƠ0@ Θw\q%q6ރa(@KlbD@M&DBl6/) P7?!5r]rsH.aQ2H&\}OBrB.eQ2;NlJrhw-ja}EaPId'Seq`G)G3kƚv C*'Nd'?Jp}cT %{A:ҖE7fM1Sv{O  ˋ {pmhiHAr~_a[ u փћ G# z~9cY gI^nn 0LpA _v2N0 zDtBTwܥԷȯY0+Wig1dN=|UHH{{dn" -;yCQ򀭠r|T"DӋ JbD^';2A0r&\t%xKߊó{ C?=ݡo712χ\_a5Y'p]./EY>܋<4 q?Y5uN.OrAuH-[YneZ!BS:̆E폳G'2dv.B&5+h(ܭn[ }طm>qYI=~Sp> xqW`p(֯hMI DN=7BŸ,a:Vm@MP7!Lv+_2yzGdq/ :iwuf5{/:F݈ H]Ǿ??v; TuIeQ{ekc捲9)ݝ,u#٠.550ףشgy!^x?pwM]7,o٪*|?e̤xm tlJj8E ,~p)x^S( ȆEa.k/&2ө2Qf7 ?Lro`y< sU]G Uyvy}y:BΧ) αZk(X/hO7秸w]xTCCl o{a/śEפuH2A% 8Oyɯ]qIub>-B7rR1HIJCGn4zut[WuDVt:/ o DԲe&L!6jemZЎK:ЗNWl$7M, |Ͽu / ZRVSr30ah6`@`AmJE$.`y{G <  LǍQSߊ0T*xHEMY8>+@΋!.ȯ?Ev!F)=|A(m5(1SޡnH4TÆLv[YTk€5 @"mnE'B@wIFuN! Y32 uY}ʃ ;mHWO]d`5PS!^#uP%P!6=/&E%a ιK'\fZFe~a[?DO63;l1|ZK4poCl|!`]@:mưd uWl[ pxܙW. g nT]\OH\ǧel~9 S<`.s)o{ҝR>N!sЗV}-$WIqܙ-Hm~|%ʜb4}#iX`Ppmij^V㟋m_qRT"O}S4ȅ '-N 2%(ęZ§.tSs;v[M֍&{1T2mkXyJ*:J댔 rcyu[{Q<\ W2a+h%D5a9_{N.386ºd[+䑂ʟ~!ՁG^V +[y+^2дf$ס&]ꖱ}iOY Iz'P%tF6.?Z]LH& #$p'N1EAEMkX&}q&L "/*6 j;otܸdLj> @Zkq= ̺҆ilDy ҡؼ?%O翋 pepK7gs~/XdmX>1j-{1mـf`f7z𝉼j_#4KI /W]z a8(AWd*Ql;R/J?=i7)7~g[ōri!9KNK-o 1GhJ4(HYl.y,P׼PJ+#*ϹGN9dcW暝x˱{=EiP7Bެ=GuaǚSm'4ɳsz;(1X =Oy+=z, nM[.{f[TH[D/t&y7 qcy78T2xmYOC~qTydVg/5f3P.翊{?I$>8notġRY.[ .KGuN ;w,j9M\W۸i8C^lSV%,,W8Y~È}b|HmJ]_RHAupd{8>X:qe8<0 gC$xjԐm4cA KL 7lƿvx~[P3H*bN(Qy9˞3abr`]8kie "wDr 60e@D8(m!&5a͘@ KKi=@-(Lm<ۊWLFQj.^%SXӸOX?G]y [gI.?(o".Fތ "05LܜhP!O 'p JQ"- A<$,Lg5/{J,ܰ,6{aB% \L!$g$}=I9N{rH L^ iSCB.Ɂu:(,|q<euy³ixabQ%V<]L_tFcM{]Ȣ6ohO 9&~q]`bJ 6+. Ox缁7%\Q ze"WdZG1~}u@lFx_|TK5 `4"V\".t-M o4wE| ~DY>Ob_$ ~<1:D|_@5I[' 4snư& :Y(5c, /qgcaOnLZ+ ٚ4+Z9iTkRj 8? 7V7R%N@sDރΔpl#=e/Sz%BY眏{=ix{ޏr1\2_ff[v$0ZeG71Ll &40Gj)P5`Fƫ^v#'[ %'#9>IMIj{.$.  K [0! |/mPڳzˇ>*7D5h6SD) ^-SiLF24WPtA^6P ^K0#`08|E]7ɑ' rʓ n(*N%Zʶa}n-Ȳ(<;?k'.:f/lb.:llҴ-XQYT8Z> nA8?_|dRcPLC>yhvEA8#߸&}S=GQص:z-uX yO$9!y )m<#bN "ǟ৿43F8xP$;{@lx˃-&xAy֯6{Nށ .pi>1>z2>e|W)c| N.k h?2㯬`gSNMJ*-tUA^MQoY|s(7{XPDpw1S,&S̀mnnO AJci1B)$xclu 0hdG7oǑH6V5-s Ǜ#( 8c1 €#[¥e<|N F+UH%֏͂HoRG?gKYx4L_Z}~)NYy c :d0Ɩ-\-iI₦l4u}ݠ:N; z t!! %mjIWѺ̵dLf|YjoP#o+9#~ܗdVuDZV.L2۷,}3pҏOۼ5+{.i:qoZ-T?#iBq#ΰ45 P Z^xկ@_N`γ}kiHdraeޮTs Tk>|t}Wv h2O gJIVW6lVNA!%>'bػC %m [L#p<)(È%$.H%Hth SH$[]f~D^xR"/sBcj儂mupExK^^? lm.Q2] UZfh8tEyD^c׉ŕ`1/~2}#^@m뙶-]Reng;Ԝ=<1Ҷm(+-Qw1I$VX)PFx?K.d vrB`{Ȟnhx(ⷡ%2,UlT,`'sI]{҂ܕf h^gi`e!)$GL 4։+*LtN0<"l]y1'* glNA'!"Vs&r\z b~2PCQ8e_t*S~21@ 4eԌ^dtl{ $1Ovmi?@f`> l1NͩCw}3(,0ѬrQ71:LdՁˠ'Y xx_w2L5tc }^ctBp1ӟ .ec$܎iaY{} 4h<K|^w⺝aYr. j{h[9[ưx/z{?_G=gmBcʠ Qro=8n4y~.NYlY^Ui?ߢIcֻ`^M?MhFh&^\LDx+& ;(%i ? :B6:߀*PYF0ݻgE a-qut[}ꑵԺ@FiHxQrMӭ3ʠG4 #]HMi("yϳs\;|_+`v~mɆ4gw:Y blLpTrG/|;cګa̓%TN"$D';N̈kiouIYa_$I%d}X  ܺ[3ZǡZڶGurZOة ˼1%_,\{y(6-~ X0h7P2UkGej,ਧS;"ƀC;/0e=@SW5[*aZYPTF]ޝH as%^V\Ht3u$եIi1J@&S9 {idxl`*Z<{.5ցG˘ .@{5/ /֍NVrIg1D~j)}5WBk%M+ҝҳx;NeMIbWox߱pC9TjK:I80_D_JayC[w"1ߺZ)^^t`~8 G=~ \Wl'ӴmCLons 9hn7ܺ^'oju(h 'Q6/X沫TwFgJ. 1O伖M0N%-pZ6⻟wa9 Kذ Gl!ȗݚYE'ETaOkg<m#Z֩MmDF:H !rU2L˖iN~P\ZaM gu@dDz]qH@obaa6'j%Ygg9i tR"_[MZZq# Emjrvߔ B*f`u&fji{* 78kf#Df;yxq8c\g/;`5ՠDSXoaFt ICWdrrKsG7!X`;XgCiJ4"gǛ4} QEq.PCp(,W|A#O8|u_͡Z*6x_X,*蝚`IwKuCڪes >AOa\: xǸp4x' ?IKjss s5 Q\a9נwxWpe\Y`^T  eߗpwy=wzw[͎;= ?>D2Yj'gX#nU r@Cv0;d2F2/ibdɓμ*>V&z \Z:v*Rgh%DqӅo<MPlEӠ ,˾ILMrv;,QJTDx\4HbĘV~-7`T_ƛ7O`.Cf}Si!R=ьTyG4A{Yt3E5R+z[ 2m< XdI g EaPe͋xߋ׼Cػ{ A I'+оD #fW~ \3p:~/ ;3p-!Y\˗t5P\2dp52 ,sqY/ ִAAcC.' i4SlBfЎ$bY!$'tzJz# }RSF&/}_[gH/OMX oAruHӛOp$ /K2Axt0Ni.BRg mm,RGR]اeQI4G뺠ӅLٴd"J24&C_+F!λb燰+&R)26YSL 8#M} xc(>8Ծ@\=hFEo(ʂi֚kuhRĄJSe*Q,]hφ'puLgXz|Z= tS*,("_3]|?D\\ԓ %u-BES!; wyx sjn^ "Sܶ+L,8x1&d5OGb~nOeZce]K>myfR/rԑ2Q' D=ʼ0)#ɪb  oכ!̡1sh0hn~wg-qNXke5Z` U$inP>!z_z)hfL5VTT.i({&D?o c0p{P .} i_}_JG|g>/UB_$w3[ Tm(4xы^FѬiHL{vBZ\n/p{Y~]ukͤz9ᯛ(M zaqqs#_W4p7 u jMG0m;;s &Jc7M[g@ 0iM/} o>r𲑘e]e\o[q"}SVG*߹?fO?NE'NI%[MMh1r#/x8k0(hB^ꔤۧW@늼%d|n=m[pjXKeoS`:(1mdH|Tqm }ᱷI%ednF'3EM #9z}8^EG]q0w)pDq *@7^ǟГFsR sy\.d9Mג JrWB;0_2⌄A8"D5'ţ )KrO!J)`,NܳtqUbM<39,©Fvő+C7(dl*aZ=hܰ,d*/ Y1IDȀVJ6(}t+C? U"HK< =l@o7*kr53e^y! =¦yId.4ذ08}m9vgEg&4pn+1G O%*Qp@aaFf`i7Ý?Xe?\Ǝaju])pծ'Wg;4K5U(h [/cy o kj8[zX,ZjFJ=p6HK%$+{SxӀ󰆻 hNe<ŎPwY.!pS۠䯭k,5l`{C s8f1 e\AkVtzmPRkȀ1 C0઀A1=O«/1hwgi˥BKLcHS>sFKe:?6k)n)r`%dxS )< 0,>4<ρ>b/'s,d,PLZiJm95khy=\cs=(` LV㽟īؽgc}0lMM%z(m;sy9r'2YZl#^򢷠0@XX`EF,ya`Z毨ttn3K"+яq ƺ$|oFS7PIz5~˓;xk`Pw ez'ON)L8Y}Ҭ=4[ P6)) ٜQ7<)d`{?-Q 9>D*Wml'H RP~h{Et`NDLkBGھ * OfF&-&=Q IH~O 0MtCY(\Wҫ'cSn0֊,2~?az~`nr% LQ۾C`זmmӥ4ܩDzSQA׆6_Ov|3_CLS@#2]w$Cu+ӹ@nDz"2wo_ OfǧO-" >ХcNHxgЭK] _:2M?RI .B9S`b%o]Wb,w ͸1%.^ XPbmp䩸i,./H܂ѸRB!"$`@+G>89 f &YWwx*~*̹ۛxS^6҈Q-Մ&j,&7#!h5]"IyriV^d Oit])Mە)qn_]P'gXL~EY ::+a0_P7R/ƱMkqԧT(>ɢ5K/^/{cQn(Ùc -Qj[4vLƖhLX6ˣOR92yûY5厴!wch|jm&(l-x?9.`rK[F2o Q He8%t*G!'4!;'97>}ׅeX`K^88BkWqVi )A M3o|*a~0/]! )ʼn;D 6X53xE g j2R^btK*.N\ 9?sJ鎻Xk!:)g/{;LaN;41l/k c"" 8s˒і, 1wEA,|5N;4{XZdQZ|"Q6Hd'pӀdʋZ`Jd2utj`6bmd׿B⼭u4#Á{ڭl%5LR7tEyO|kr'92$}QFMKLz%bcZoӉVW>J:GROC[>CHL>$0S@xc^b:XxX˲#׺5q׶%YsXotQ˥>cN)x`PƠ!0\]o^Z8Kriee<__| S/䆴WzB_%p\O7幕(_0TR9ɶ^Dq[p |.%Gs>;,B7Y? BY|j5]D9%>_a1 Mx1#b##D'}ꃉy5Iȟ暦W/mÔ屧?8 ڥzpIc0ȝ/h =S^0Hj6BwD#0D>|8I t _j/_ k>mBk"YT:LuhV6؂m#[M'QF. "z(?=wxc3NZ{/g6(KNhobVKȹGyKG-[I1P5c pY~aG2Wu%}Γda2ΕEE| {p7"xjAGPєG`4H6H ~h_4Q'Qw!w)炞mLuBfYq$uG*SJ6Nf$m$6+z m lmiL!eJZ0s+ B'ZAg>Qמ+|q9|x֙=pTnz0z8_d4JCQL \* /9|#84RƜb3:ʃʛI< Z?IrPW_xx"ޑ-ǽj[mLCs6ƻ&! T5 S˦ ?ˠٲ{2g zLOX^4}_kL#1P^`dPn"B Rm4~@Jʝ`=ȧ%S{E8"8ihڪ"6 } 3,Հ%ҩuǂE^5].]hI~$?P X yS[p(vւNP8< t27o'|Oc; ⽸{Ёu,=zjf thEu%VZŬvx hǷ|se6V@H]d\wuFg3_F\s9[UecځU2tČH@ :K Y¤ d&guw}#A#w4@S'V|jmgێu]9M&B Ex 4oxДf`K2O;aJ[a(5g~iu.^d8x7ޓJX\G?~NԦA)@wmwؒDNvgCZsGrνOs8{I H-@5!^lL  tSBpw?{]cfڽ>v~߼s=gVef4icCzZ.1zgWVWWXr}/+Ν[S{Ca,$`Q d썁fI>3@1QK,g 1KO01;;0Jap]\C|Yɋ`cP#eJ'L+mf@!-I2!},)f42s z^gz薮(uk.:uѶ殸LdJu%3^ܗam,icEKMGEs:=aa6޻t[ACT{?y/i)śS{Omr濽u]Qjqr/epME>, ||KS&ș$D1%] hixnX }?.'."Ǎ(r.y@t9J (Ycm;/>˩o߇Wr-[蒮h,N72qfH GvE/seiIp5ҸeZjZWQnǿ;jJQ\7ACfe&%- Ax'|pAі9U]3k;Ǽ}+vg_<&*nuGV( D;s Idh&9̘YPL#Y]D&{ ۮi;|Qů}>\to(ɞR87@$/} \H~rf\Ze]:> 0.0Nˈx `LRhi# rNf*yӎ`t{U0qe^ӷ\D+k_51|׹!/ 5+L@U^DQx\_C$k`Ws;⼥7"_yeO"nguj*A*Qb=˃Idu"q)D?}S4_"iø|[-NN'D'^cv[E%ơ]?G]Qǹ2 f %֦$'F%d./eD&W) M${L$jLƜ*y^>iNYr>)GKoc4?t]dR<5R܊krs2I]9%,7VH{6;f471>;H˓}&]R>?$[r~=x sLoLXO6D DZԕCL 19dc,y9 ,)=Ϙ,\o 1|c SY已m:HP9~\'1r<\>mXFg}ElurM'N js ez42XsڬK"sM0f!mvS<_M :5 KǃK$.mdbK<7Xշ'cuxUr*,(fg4 ܨQ=RMd9B݉׃ ^zy;>¤u!MrV1&ڍ9bx,%]n|;3;BFFӉ:FD,&C(=美={Ѕ;Y4xN;lPwDG //x%]YPϐGa':Q|Zt*nK+iʭ4r[ܚ=}6x[wʽQx Wq]$ ष K2[5$Czm"r%ohH]ʈe^V.! /$.{;qi (玴F K8lYY[VkWh7)Ӻ[A?ťAr#sB]@׼}8-NeX`62e7}QL'o𸧼˛HܤwrY̨5yMmLMl=jY;5g}P#JMڎz6~𸧼={1NTu]Q*1Oe`4-eYrUo_^Ï ͐n<;tOop($5\?\\5,ڪdg#~q.WPx(e򈌍 ɪj%2ckWD]ȕ}PwFԋYY;旾K14*k|'=ՈIonx]S%T }9SIŸm5_Oxc[a"ʼ #|\yzKЂ%R(PO8G"+䩥TV5@% K)~:ᡏ{>_eaXUR̰owӇA<1Y{4зIETlAy:7ŕa6Z /~PAڡe= YMCC_a|% 3кR2]u͞^gme/BE !gN49 spV}HJw92 aݴW,#܃9r0A89S9h:yWjÀ7W#p"t ߅H=2uyp?S |(0slCnØ3Nh[ f%yi7'' J"mQf\c7"11NF0鸜Y@t]i?طbG44xgq!#~|9x˂[MK7瘝'wk&,i͕_]DM,hg\o{7}Uĵ6x!f/Qd?#&#ma].xecܬgw:zߔټE(<%S8IAr$`O&ۆbwx_2py7B26àJd8ػr˯1}$ZA5֕wE . 3 )AivhY ٠1q] 5++VV|P.9z4( +Z>.STfLi7b ;fK0ơTG$/CU'kbUѭz&_ :@y1y!dMjRB ,M'\zv|7QG5N޺4xq/i6l*;|(}##&~d4Z,)m qo -vHVǾj 0 9R#iC^:źvdlnq\ޏ|g\'XnjoL4Raѳ x['7 oN} We^6^F-(sAĈE$R)T]\ L,l7fPwyO$ ~y9C0ylOy⿾="G(c?] 9oY]M (.c L[;=@Y 3}+s:G'?i|忳6k., E17f(Ahsk5Gl#Ć""PN-R/9 r>3+ [f3\ٗˆG\C-p|*BDԫ;t?$16-MO3=|]ǙMPfʪ\{q0, ~m1av<Gw}3٥Զ˿ #NWz?Y 4edPzo[!`z,ҏZnxض̰;(뚃F!! DV E*P/'luwBZK`,!x+QD#r((r/Bo샘゚$:`b }0FGXWqEˆrgzrLt(xE_=k(|-*C\5.)]AvhN'0d?1]j97 O9ZU׻OMtL,T~ ["QR`t'LotD6,iF\l)bCECX+<p4sADŽqbljۉyi̊n[.ĭmTWK\$7Qw|-oiZ `O^2y1uk*](%ݻC qSj6 R^E{(V>Z` 9{3e|dԴ:NNmD]Z%Ta60^u$NbȈyrK/0`7ix1fb_p51yBm\y9 /*ڶqz>7Eτb^&6dAނY? !~b~/<; F7bRs>Q4(ᰎFm#zYD.Ìvz}dpqER0`׿ ^w eYuyݑŭ9A\#^T^r%O_#)+P鍗F>¥Lvy\$2QNҐuqFýNgTUMF<3| 1gs݅t^Htˌ` 8WW%9 x;?(t9mmz^A;}<疢{ǖK,lt%/<(޵OK+XCBCkY1k_! C72tpl Zů};_rnf)'J e%ўGײ}&#}\)_Nu}VʋyJS~{NwƛWNy6tL.#O!X>0'ݗ:h;{ɟ4jH/?C+ )_ū VɞbF.E}1'"c ڜxBM-D/ v׽#VE<_8Xۈ =};Ć=\dD4^W =7`z%U_[EGʁc4u?Ɤ] h%ry딷ueIJKЪm]D!CވMfkQ6 eQtuo7l2d fd|A=޾|^6 Q嬜*7R@Eb pG&+2oD_At=3KsS0 1A}6yp9L>$~P|>2Ճn%[oȃN I\Tl2SԖd}t^{6y$IU&O<&R1VbKhR~89g6Q^'^HJ*|{ߏ!ɨp#s_gUMu^ҙkYq$fY(q>Wi K1|+m)*G{'yDD*͑TCe|)6O猭M;0PO~pv|vyI>;ۓ^lש.^ecwL*93L޾C+Sꔐ-HƮ<$#HlAJZJP:,UD 3O9e5~OtLQ8)y߇>Ň?E&uMT~Jڐi`)JrIY3NX[=BVO$k 925&]l񱡌5T~NmRqlnxzӺ3J ӆre ]GkGgwUYxۻOG##Q٩wGH27)# pl8x鋒q]3h\~ϡY^ߴ=>EwLV.7qz# NE63,)|'k{奮%$â"O;!JKJ¨|wd1d6iT9'gÇ-t^(7ot9wTSR#w{9ÅF񾣜i ?tQ3fٜF'˄zoCڞK:P:T2sSuWc.YT&sO@ +&$i_G z~\˳ct0,l];O_ޚtA3 Z-qd."ÌSL25wR}G=Q֢g&̙A9CʼG[Bɿu]Pmle|y\Ħ)|H՟+eƩL~=GIkəQv 3܎z>P"scC٭P+Aʰ* |c r#D>-%J4I|$/=µEKVng=&ú|Ar9ZP9p߂[Į-Y&6j3|MLUqaUϮ GQ^U%uUPԥ>+ʪ( .lw1Yd֞CIt{JBe!~R|$Ԕf6(TLTkeFzHN,.%ztՃ?g<:lւٗAԇEYNwV 5y ~pv蠐cAVNp,( ʪ+Jl;;?b^kS:v,0l0 Mrػ!3*>sL:7yHyK=>ELC7&tR*u U휰g?]̷ogv]̯%4ݳsJ낪.)d\#29 Oŷ謉$@o萐p;)ڳ0l?[v$Q!ci;_E8t:9ח}lL+X뺤K&K`Ӓc.y&B¤`:L&ITPU#ĠG%OIg:#E%1dN#ӥdQtIcyXZMKH=OY8*5z%Y#w Ad t޳2kҲ,^< $g Pg}./RӰ6gc7Wdu*mMH6uGw*[:t8՝]v.vTæN;D׍΄Uvm+*bՖ#h}:U8Cog)t@݉ec MSs-oBJ9gL|O^ gjXˎ|d:aZL5U]sڶDp|azq.GxT7<&7` 14n.];x޿<߽MtrWkrLP@̍aY )M˛j6m4a4N'ӊ餤!(Jr?vhwnh){#x)KtR2JN+Kɴi3:S֑$2Ӻa^cRΨ+:Us, Jw|?__e3}H:nRDb:鬵ꦗ6Fs헳c7+{X7gu*v.fWٹ0[.e <21T4Tz9E轣EY@VÜݻspsNvZiGD+ve¶bٯ/EgfnX>f6s6Ѳg;arĵz2DwWVڕWs?+y>>n]fA|lꂪ|ieU_I?#"sTzfsӴl98>ޏϜ Js̐<|K!ԚnRO8"gT|U !|nEy?bl#\Zam9/2,ЩKOv{0߾+vrVee>VY|'{hĀ_r7Cːmnx=thE꒽;7zRX ܩ\W u\_ƪvt0Y+S[{G.r-}:8%n2BH|ZPU0a:,ӺcZGIE!q{RTּ=hڭzGS*Ƿ?||AdBdk ;zz@}nl}{z^iuȟ,.jR$^TA4 A'bA^XG|-߃%iuD15<ܟ+1)(сbrrpWNϜCTЪv c|u4o MeQvrzK;˂$QӓUgPZ; ,g]QEyW74߂k)e. R:_q v\߯xhsjs:/ANklK'P6<}3erUcGg?VDWkDM.JC%) ">:|WOm ]syk(燒vYm{T4 /~N_G,'DysUdZjftrS6]૏*8m4 4 J)ǘ|[(˸fnq \T@cAsc~;*ZG=RFR3=\*C%(Unij9CPN9|,6vx_5ZYNՑWw_]ғ%[Qy^p&qW{WWR(+L$$~{7'bc[f'6,/s̱Gq1qGm&6-Opcn.߾;w*6om;Q>W`'zڽ7V v)J=|?`9O ȀKrQ˜3sҵ])Ҕs}1~ʗN]Quvv9~pu"NtMvBGo?lU0N9d~krQGefƗڜ+vK/w| _[6CDݗ)ҭkq7L1蠪fl>>vMRڧAߒ?`CLϓ[X ]QCg=oyɕ6M(7-6e W:y (ىB 5Xee G^Zcƒ}Hw .dK pγ:ky%}'¬鈝EPZB3eSݢ&sMnN} 7>׿t}esWrN;b~~}][kъ O #DNXyɔfny󹏼SN5c:Co+8'.t H]ysrogrn8"<O(J9s*.D]8O1ekY/+rx, )Qtc=~i/}.OyXV4,H˘a$h)xî3꭛ \e. >TQ[+p,q>t!X1WՔKKɄKY*fBTl{'k@|xSk}xqImox$#X~ hq"!: c"3U|p5`5FH9 /<,(ZdCǖk++<'?*l2q94/xجʎ<t1&7kr3VnY;]UQZ{4SZ_'ō:'cN>Oq;XZDB2 4鈄Ѵ@ӲyV/~k#FiMGyK]xrN_\A:@ +Eh&Q#QuPs8:fufGpab$K_QԱq~Unus!dU0>IvkweQ\f7Iz wp̱G0k*Ѕ1W[ǼiX]kO>U>/qѯSoBKRc&蹬/)fsc'G=<\c( 1m'PyHg]S:UUg=f nfy? jbLK>u8/Bˍnu[{[N8h&kDGʦkY5\zN>|_|ԛ6ᖷu3Y҅^ 2&T,r׾G?>I5XFD31ۑubH1ii_΃|_g1S|MlS /+ aB{_9Ʒ={qKܐ9MeINoڶc4\~. ]?=ʹ\DTr=CϞAOj)N#yʳK8 `b3ҍűUoZ桏yz˻IW$m_[|%ѣ,l +Dݒʁ> ꘵vtՔ+v7s쑇6(K8me au#ѭ=#;Yh*r,/$],$6ّXV8"ĵ۸'r'rۘL&8ݼaݜE\pՃKb`A#R1R!KY1 VZ;8[ Nc6o4eA3ViwS(j7'&uSN>ؿͫ;׉Sׁ0㪬:#K:%f+51)uW \(uYCwq}yۆǽ')ٴYF]',:`.LʟKT/jK+YYٵC[ Ј ^\@;wlv-O_fވ13j=w?#h.o7:! ѹ(r8ׂsx&<ޑ3\)S)kzaڿ |284bu1"k4s|>setEA8wQVz+Lvo;Υ>F|Qq4-׾.grf]G%E&E8Qjc4܎exb.u_y:ց r+nCR6FtYys_FN˘p<|5 Fi([=/]/~Kp]ʊNĿs[\iqS~a|PޭA֗XԔָB- dޠ:gce^O8L^]Wxuf |8cx\Q'* B4mrL' "p]MK|#柟Bv)m|tuZjl얌Fxĵrئxыw@|$ܯi$v/́LE'z6zHCuɤiO?7;63!.b5J~NWWk-+q;܁o-˽{.H_bBFDŽ }(16cU>_˞^D'Xre-*Ih=зWy3(k[,nH0z H+:#8x#o+GN7yxXil16x"A}ѡ.M\[G :db/ qn˝na63֠:-MQ2#h}'_z.,u,!u7Sc1T6\NT9"E]þ;>{Y{~7 N݄5m˶yksoY>|9 q;h6ZF^([6/|,K/}`}IlZRxPDSoIrh~l:|+MN..6х9umMdPu+-\Zs.MN +12!POj=oyaټi2TD~Gગ)ub ]2/fM⤬y2C ^eV4N/sySmZoBۨ7-ӧab8ZI::RoEc>8m|97Nv~ocq;`r8{0tdf^ysSO"wu 1#0b-*6 Âo 5jfc[Y//wGO##RJ8HO_rϹYaix)SOe3yt`F6ŏ|r5 %1MK[ كN<1G ;sa[Ύ׀¶k>HYQ⺊vZ?+ ^lZ\(XWک N$)]Ϭ,e">z|R ?ֆt]Jň Ptv@v$1) t[s2&r$P*2D*U߽N'̺l$4/QĨhpH4Mǧ(K=/FV [9G򵳿Au !@1XkI4%rw"[u.}Ao?ێ=<_yK]VsV&uE -]+nTmbwc }8{)׾)|G?g%SO8Pp"j`XmSI輔eL?hnۛyyJux=gֹ2e Nuc,ymZ|"^胘{tck̇?y5dE]g~ɓ|iA/Y>vg}y<\{]tfm EЖ!_)N|fN~.w9xmL4?(>= / ֢gm FttAQu;=TestOK/0YN yY0|UMt mw={[c`Rȶ4VP&|ªä,ԕ#UA]બ-FS+c>O>YZ?0" 4:zm()JW?=o9 ;.C'z݂k\QW%G4ĖM[wAd OIJs-EW%4tQ<k|&˻pQR.52yi+Y3siZgxJeD5t.IڷKk@JjtI< O-} ٶm.W[ d~x;6//q^{՟pp[WBLg!0>MC.tԛNf7>eY0[˭* ͌+rE9p%(' F2c>(/Xݽ&gOp?Nqƨ^$dq,̄,6mZ7HE.Cܿµ{MچRϫ 9·WڷCF):@QVU=<=W[ق(hyYL 1Szm'`8^)q#R6y6o"wCB$8Yk*6*{¬N%:q&$xMTudRuLIRWEVW.l}P7'Xe)(݂6ij}Ꮨ̾oV Pj\,u6|dQ\}4Qӟ(B]E vˊEJ/+ܘRϪr9qaw J/1aqz8*5T:icF܌|+^EQνVi HӠ"dGB֋JGAuW?BaQii0_ d#wy$r΂F!uOTZ_Ϋ`̛cX,<$< H41Za|fJ_:QH~57h:{s{ܓ~#k1v\Yeuu I-t¬ɮFEӝ*m+ư*9m<zYݽtB qK@U!8{YU^HJ\䂶vh? 9tyҿųƟ9갭8:f4eV!v,JѶ-y#c<xƧ@^,hq .dkډaS cݘ^`7yaúש= Q<$\ɛhW>k-*-mϭ4^خAWn4mҶ|ɳs &om!! d8;K h-p[<)_& 4 w F|,j$?Y.8ME!EpgK-/53(cI]`u7^T&xbm 8=ip12qx2}?euer9NcZ(hpoRNL0Ң/ vn v F_h@hF&#ob#Lx>]ZrZhaNj6eM/7 \5B}k<IET#Y]VG ,:pNLQrJ׈>gx_GOk_L͹ ۻC9-bG}7M?|GntS)g6vUY2뤃ɎьX-"_Ѷ0[^a8aN' lWܔv,u p5N8"kkkG wbN:?rmm1rI o~ Y[[G+kL[d<""ӁgioGA a]2t|Yh4, 꺤k[n{?o~qn\P҆p%+I2*FVO|ݯwUVVV7rJ6^\nټeumF3sq /z/y*'Gs*Mީ;6eS+ YOQUtM*^9k`BIg巵'z$g0|{Ǖ夤 ݽe1|3d3G ']z|ruoI׾ǟvs8ߡ:48ݍ)UHl40o[^‡ =aZӭfEРѬ^N*҇$tHtPPَ+o|/|[?H3om6%~n|N׵LSN<n|䝯o}-ccKD5ʂVZ(4QdQ,"SՎ%|\B6Kc#>x }UdeIe8 s<-bbl>6u6w஝,NmvX^K.^NB._f]WCetT0%uzZ ZG|AlHA%qo h^y,q~i3SC 2;HKrf <78D^2i39C!@hE]KM^O1Y:Q1+#!䯁%}h'M.+Ӹ6 j3wCr5X\x|TmlA7D RQp]5k-4|r_W|ɜH#MZJV~֕j1 797DQ&/>ϟ]$1<-KolV8dU;4J!c1tDK>ttU ks979r>/=Ga!(9-E&<[_~i)KƖq,Ib$j5׾#sme>|3S?-n$wBcSICŜ)]GYz?wռy愣#Y༣,\ aHO4^\e6v-'txٿןAӶEK s1fՑO^&wd|6nPWZ>N1ڤ()S|@!=}=puU!?(zHFڳC4_!r\bZrܾ*{9_Bv]@%ʻBY4#78{R;,;S ^)| "2p_^?ҔF']\>a;HIOXu)(il݈O| nN]ηRϗ]Dd<34 _5]g<=t++( 9RG=uЯ4Φnv.WbmdBw7_ܗL'5Ek1ZۜNuҗ0۹r W m,6#>2 V~v"׿qх7C|E}XEU|XÅ-1M4Em]fvSq 'r oy}fV/ڷ!-H&>K3KAf sm.(eNYAf.dr{/lpA}` :* Q=k\z.5b#nhzGqAkVѼ_Tcx#ZڣL9_Xyoeb]x^5O&;}lz5놥Yi*/?cy:D+VF6"Tܼ/};|ߦv8m\Ep!{~-u.kz+EIXsĉ'X^vڹ/tNNXUlSr .4۶n!;hBIGh+> 7%s< qBu}|kM !7PLz>͠* _p-&>w?G+ ELL5? kp!t~+=z9ʜ,U|'~bY?X\:|Φ-KV/4b@"QY2֚[v<_2>7Ͼo~˟~;g|Η>.^Ya˦)m bsE6FWɗef?utg=/}u÷m⪻Ӊa+CiVVڎ#;<|&w+Jљ$M'49dͨ|KBYӲ=<ɞz_HĆ:z0M?cބ֊w[&FL6yl3BV(|K;=;Y|}_r)]0_:URݝ*En%$6 UYp9לϼ%bﾃmNk/Mt S1RmKeWÁ/.`tz.MѦ2Ǵ<{8%rՐZ;P}:fDy:x>.듌R$ Q@ B{#z6`8O}pR,x_8pةД-ߺNi9?gg "QCwn̓Թdv9o:R!=eitLg`tLq]y v}4^t]nQE= 9dUﴁ1A5"gF &+ ʌN>#0h &dE-Z?c1"m۱6[c%'~85կ;̥\3N\ X:xīDpEWPe2(EeƆ 1J[!@e]{ʗIwŴ$6kiaJ>PU}9H o5i(;~Ǘ%݁U5}Lٚ`D'0tb:3:kI|2UWWٶu>/9?N3Y&WҾm0iFhyNj\-@gՀ_d! dJKv"r0`4Óar>_GA1ixG?+p64eH zv GpuKػz$J 50ӕ%u.?? bUv s5]99G0#s-%[_<ʴ}&[;sitҡ†MmI AQc>3&<Օ^N٠+չ$$f.)1ctx*y.vO}n[DhD_\t'@ '#B^3eUЭmڗw I{ujL2䲺ifF F>1XEMx^J7k QNtg2/;WQDKheq];'|\k7n۷i;MdZsI}^gޝjSrNKӊI,T%ֶwx39h0|Q[hZ7 VVlNQߗg*G%y;r\SN<O8N8ߓO0ٮ< k{s 'wg={Q(OΙd-ތE31FMKQxnv=ڇ|WiI.sI}e]Qn޲}7kgu/}Ng=f@2%WnΝ) ˂o/Z#X.dd^Ǜqz|E_#/0Kvf`BCy RzCni?3|0}G?} DRz^srI)GzB7jݍY2/I^NNP) u0wW쟭rx׍n}>#sW$p(=ZO1,:r^iZcdp(Ik$pl3ŷ"e!v{LMzY[֨s G>ȕ;Z\73|bʫv z4;(}rj6y=={&eiwxռSe|[y= G5 +bbik"#mEO?;<{zYo#8 es5+$7 iŢɌR a/O4 ߯ ),¿Ï%cNoG쁄 n^3Î9J#&S޶E|@+.ʍP$NtNVV5mU ~C~}u[T2HI 1-ZZ4B ٣vQ(u21齥(gX0>om^tk+DIjd+J!T|%J nBpb5%6\:qևʟյ.00!?H%̩rTL,7 E;ߒ|׿p N$͖ x'f N\xau+_OɹV~˕DA-C&1T[1sWm+sf8Džd.mrWΌ{8u_u ͒մ~L[nav r5Yz=79T+̍`_Zf1Sk*K. |oc^m[s4Lib&Gi&? 2MVw)i2b(6AV}CI~% [ģA4oۋG,F6{0n3KKc]!qs?h| bNF$QhɹDl7t:) 'aNǓWr_YiM6BGTIOItIƄ#T3My_ɭovmf6Al#X^eI۴lݼ_|nr=~B5U"+b\N|A63)W^t9_wq9j7DhcK:Zf97igHvU%97e!/ge`ą?)yK罒W^ m-ur1ky;]X8YQ^>yfξ,]JKƯQx>ݏ ~d2M@~ɵWNq/™P}AT&&mimF]f\kY:}\.+2rًDCIL@IʣR&RAIsITS{@TgȊp&몢,Eٝxdz$/Xa;=|~xǛra]G5Pe."s q1:~ p,u/Ҥz=3~z|¯, ͥptU6yc߁յsV$=i3~.3Y0;bmZ17-1DG?aqE+uQZHd}V ].3N4kk7Ϟ/)c:6ލ9vwK]Wwt.u^${<)cOaeaԁ-鴤ݻk\<iڌ;͝o*oAsmcQ*ˣ ^<呏we{7f1NU%e֗DsŨNgz_s"_nnpۄȀK Wx|`!_/ ]T{wZ-u9g/-bȿַ.}KK5]̛ijB-,w`_Tpq4)eJk=i~=.D H׿u?mԞՕ5VWg46Ʌ;ӽ^P]1+ /;6O8(^{Syc(q=Z3O*:g2-)Bdm>Vağսyǻٟx+O}O*eUԣ8LEXdW -/ܲ]C74h6xfL,w/ͫN&sQ9'nuq7< sцs) ²w6oY7cIGW%cRq`DJ\^"8d|t<'i1_A1,Ohn v6F&.y*e5ԴK5KBUBYI/ƍU\6O|P4|hbmrÎ׿)w9l^_xGinRqӮEϮlՀ1/$ԕg# x\=)92(I;O˶˖a &iL~fCT;7I$Wׁow~-<w@Uc͸ם O[y_m7鱞a[ ˥-ca;_EWg`%Q;uLّ? O>ͩ!rz2$Ff'Nniz~!/7)KʿrY6\)=SX \п+sHV*G"k]`Z7q[\U.Lebrc-" _]۲2oo_s:͛a6UOtPg!uO_!Ul9lT(1=?ʝCo#y. }It%Rz}3:8~N<`|mon eW؀AΨoa6!}sm_.r,OmzQăzQF3돕4Rй)_"%lx^ȝpB8t;Xl.kt'{3*fs}q<fۑ0;ҕ IgcԉO&F]3hNΪ m#߄9?9d1HD^v)UBϯ7xKzH~ؿ}d3m !V(gJ6JEMpSCc*>'nfbB0dkOg,8I,8?EΈ̘9ʀe 1:WÁx2,OeU}NcYd^K8-¹OȄ1 -I{PڤC_rg {qiu楿v[Io7Nдc2_3cZ)KO]N#//xSy?<[x+~l۲miVv! ڼId-(hچ{JʭZ\(щջXrrOw?;&uVY u":jgB7ZxE!cW=Ee(#e E%JL5+k3˘lFO1A,bS/r#T5 Ӓ[[+{p\5Fw>ػJ(Ji_NVvm[ynăT\`sG*jG4[Vˬ-/:e|3"w?溧tR0ͥ.:qsi u13j m 2՘΃ ƪWe40OMĀ- $O˭ysk^(fs9>Fό/o5yc=խv56oY⑏?׻Miv^IYy|h=-:}s~Zyiq|ٯ.L_s~<6P0JZ^d0ܔV"56<ǀa@C˸Ѭ+_ғjDxZ_E\2d Mye[z``a=WYci6*_~){f~"XT:`=XS`n;C-w(@5-w 7۶KtX׸ރNeW ;%cZ ^o廜O}+o%-u Q~z@3`\8+˫/ eolluUֵ*#Iȹ4c^ 8qEp"kX޺Ƌ4^6(9v.2ki?]Yk`<2"O7={,GUuԜս ؉1=Ȋ(Zdh흮pԅ^G>U e bۥ嶒V <^ eٽv<iFE/yl@: V}Ks.5qd.>VFDvH]Y(#kqz 2ςM.a2s.eUѵ[5OzB'-KΕ{y} ߨ 俆.e`'r]y FX]%F'N&u;~GJG'pm@&AHˏͼ}e0NP.ivN[uQ !j]3Y:ywu꼀aʴ˓C^qi@ i"!I>iaLGtge[d K",FOnC?E?)ԝtR󘿿7<)ޒ÷N̦%h:Gqd&>?*]' /_rW-R׫iMIBe)DG Xo8}*K  EQ(N7C`xO8I!#[7/3%M[Qb Km[C§0H$qMs宽T 6i)׮{JrD14 ,U{sni2C2 (}> 01E@@QDg}}Oy( *2$AQH陞NsZkUu/gVU*X]Ǐ?x~|[}?o5|w??}9j7sŗ˛SuP=c]X?~];X",r%yr"9|_.ّ\$ 0pR<;Zpsz9|j'cwe_oe<ǞݫiC$'eMFse4K_sqw$1湝Sd+ǎM g87sϋ#c`IV:L'!`cc3Ÿ__ю4y3n1^YƸ6n=]_{w j|ۃMki $ݜZBw) vHF;I4CTWĻ2=Snh >s.s^YGh=Lc纾;;FW2g>zim5#ȘCwy6iһ罨7<' W"52>qS Y`H*1POA-A6!ӶPD5=!ƐX0 vnq9Dcu(fl|Q7|tװuX ",]NP2;CqK_׶nS=q,_,^z<3er,:W]7Ua6|fOiڼv5IUt $>?ꁭS8Bh[Xk{~9{c:#l74c">ix Yڌk3dl̏[]@G#Dm `ʫIVD"W/]ݚʼ)Z %_C^ g>4#DP>(?Ó<su"O|gF&zc1M1Φ躙tFx$wxl 22 ),"x<~ȋsJi^-<*K\]\f@?)εޔbe41Ά`86Zf Cc40}{gPţ~/W2 KT$O?۵oXD7"LI)N]h@k}R%*'?*7V~ jpA|_?hix>\4+xWO <ԣm  K$Vû\DaJC&ΒJI$ Ǔ5f'-auy=1sO`zxӽQ*kFV.-}[뾞bG8i( mvֲzc 9~Mw&^K? 2hv .+5#e\A%+\G8~4~7-+wMK~EAOMAoDNJMD2iJZ0~\(;ON/s2F6z, \_q$xEp;,y.şSNo ~6-fHeCc.o^H/Yk B-}U̫^@kP$Sp =QavR(OUO$3[}˚VAC4e{dz|=$G֑'0A< )6|K7{  NHXF䙏+F^dvᗌkO ;t#VOtZtp?13rȅ;^ _KZn@J{] i<@\<+ws:>i(O3 z*tQVaZ~eDӲKZ5_桶\r|Q FD!bB λ|a<oZ4KA#kZ"wNvь2O!:E׌&K8Z_KnwŁx#(mdⓑps5(jV ݂gO u:^ԉԺ`ӰqJcǂ|U8MW褲.Ce㇟8,!Bރ[~>i>h4#}+0iՠ*#H|;WY^B x<.ȽCt_͍u\r׋WKK#@ )zSZwg ̓J>pXv;cgOH}Exԡ! cxS_qHt fAG!SS CPaߝArpCGC>4АxҔf(_R\v9tQaN3F $?|V)@:11Qv*E^P6,_EB'WiחEe4r ᬟ|tx3ZﰺkM^AMZ /;0HH< 3q[-\v67/yq';n~9hRl:)Lhz#NEzO.>>t=6e?ueXӂ3uGdN~?񬧠?y˘cǓ4_O<]/]t]l W`{H6zOTj{ Rr_ TTdW}ƾA;#>{xc&]NjEMQ|7/L0N&uriL?,ۀЁ5L)?M.wg2?H!JW '=ֱaLj|$yb?cp4ai\ <9iGK[cF=hP;|% Tt9h%MڵRH9+3dEYa |81="y@p4MfJd:si~z)5u.nӰ$`3γ u6V}l8'C>|uG<^~㒕d,eo+<̀Ea4G} x7轇k8tW7OhF) !8CwS晅t@}X߁R>52Hs5nᲓ]N 9&Ax!Bf^~~/۸dʭy1EP&ˀNdhG^u]?ƈ] ϐ,`j 6Fw@10=O'q'0n[4-p4#ݓWrѨũ羊fi &ǫ0-Ljd`Ap kOxcq%`s:kM1nĺu~#wuzm>r;?Q#\>-"zL@~;ՎB*6I}fv*Mi{4ǼK,5`q#(Ti-h[Ml̸@Eeq;U{6q/Uhځ{vwvх6r\ `>` д(ilSg~ sgޑ54N? n6 U IIE=9iwͦ&#2tpCws"Tˊ2sLGx⿿ |d](;(fcVB2ޣmg2h9V D"-Gⓟ,O%bTTd'Q'F!ayi}ྃsZkֵAd]k6f.mn0Lʡx$-@=>ñ'Ѷ2aC z $gIM]Nt7Ƨ?E^RkN1e<{@ml /{sxگRA97{6u{ӸwůK!~ /}s&66 _2][PWg&?_5 P^S*gw(0 @|\}wB`[(_s>}"4m^ph"i{4:@`x=&Gd V|īar``./`α] jٲ}g(B+;3k("(gg ~KyH6)o˴Jaws+I_ n,FoKmd֣ݘ1BEUc(Y ߵFF!`=FMs/~{1oڵUzJz*p"Yȧy%R:d`a(SJS-OeD,!YAs8aʍU`e҆GND_;^;:\u}:hܣȖw'>ҕH&FYl 7p,L*RzBof@A'we̎׾3hi XF#؃w jsgנv9<)ip; ]9Zn2zOuW&w8Sȇ=w|Ĩ˃یJ Ίm' t\t6Oû;Qr\M";)wۖ'v[;81dJ]sfl 曨T:!Ft]3/pygÝ>N Jwh*fdnB΄Ofy8hk=;pU^LLi)'\ZTÃ;[@f$;HrI'Wly )//sAIܚyA4@/-4⨒uV'2c>{9rOy@4\yWfn@@G׶huLY΀3(g?Y^ƃg ӣz"rQ1IsEl(@Gtμ\<{lBK3^ ~2$qhzJ>Lm;WZ<,w .,, >/'S`=T=`|QR*ڝߜ;#%P-Pmzl e2%4\(O6cF'.NYbɳsh|.`4(-/`}cp|_ozm mb%!2́ ln_F| _C*odg,,K;,.3=hMoDY݇q{aԶb^l7au =]ٽ 󛿈7o{#~O7Jo%1&F0W ~r9oH~B-uZ8ރ"b =-!q\}0u9YTˏ}2~_֫"(Fwx((thڑmyw+,JҺ\OHmI;}!ƝS;ksN8orwɴ0{d2+*Ўf^FQOI ǡGnz ; ^oG< ;}j"Hit ʗ/(8Bn͉M#\p^fG"|> ^sC;{ |TX2egh+ɧcYxu=2 DiBi 'V &B2rAjM/yi_Fa ?i8 r%J=Yg=gT7deHvO2@*Vٸ Y=pnsfƈS^~8{Ja>,XlG YSpVeg4>ls0o֙>8.W|KLg_@,J r!FW' eRMLjP2yD@>^[ӞhY]A @ӌ4-_ @{d7gJ܈ G _9zN^~F,#oZ Ic@ _m9)~oǹF7D> L8H 4Zru${N8޽c7js,C~҉XԆt~i jt(X~w#c>$=qdsˉm.]a@3uRjj Z%:NN@םe: Lw YZ֒o$h9]8uzuC'(iy: oȱ^WqۑЬ.vqYZc6Dt}GaW\~!&u P51v3RFf.-ғ'4L.#\FqcpQť1XY÷| ׶B.PR(Gu(ڐ#.m, ϵ F:vcbC8~Ÿo<4ퟡ!u5!' 3ve7 IDG{"Ls>w(O3x3Td]WNO=xwf9y= .ArOٸT]SPgp "`eXV 3~HX[ޖ+NdW1ruqA6Mr0n2Y b<monjq)_<px8=*p^d9rVXl: н6Q./#W RT|q_͟f;Nթspۅ%CD5Øܠsa 59kx\ V<?v^y =Q4Q;PM?Ʉ[#Nă!dz\p@eG&w\R<4;Ι2?wh&K8~V|RL]nf}ɘg9lC܊[nny~jZY)Ur5˅g8SW&7pyg}/=Wj(P,ScH!DS3;.ߔ 'x'A]i\5 Zo 5-jj*^Qc쇦cqY4 o hYNKq*RHC K5|%r0Υ/)aOvC?EuBT7%^V#5TʡXmRZݚ#_'m1jy>Y5+83I]^"f]c61:f= )fCqxutcѪYٔ5lt)f]Ng8qjkp#:6A:p$fxJ;wy2CdaJ:ϙw&5gv/zUxā#h+Ѥ4?+K3JRB"sI>&pqmGs/x̺n^N[c]‰+jL6E]|b'GW"*~錟Y .xf,]|(mkp[TebtmlWq_W׿ ~sW>ayb'c]0N>icV֥p D28HV (}a<kv=up O|TI]k(]2o}unN.O4fuf3;ΰ9c6];LgSL7g7uXߘW\ݠ4]igi'Z8hˈq 7H)guBs@~)_.*a۔-pZǖ?[m9k!Vv2qmp^#bȅ>-15/@4@ ;ViTǬA> ^|Y#D<C :fMz%,BR"ց=NO/}NѴ-A'^_iBSˇEmL7<v-#NpS{U "%K*-qxAt}>}|w鼆Ԍye] )@mA@'maoAzįn&7٦솑g½K.<d/o)"W:F]/kg9oG2Q,Sy7N$>Aq0t@y0/"P"KBY7̇I|v`Ng E!aJֱn4($d,ԁٳWx[6uZ1fΜWP3ACjQS{qNQ8Q`U3DęG@Cu=F0:e sŽS睏Bd oToHjRV)~2(w$ZƹqS2pdDחq9MRo㝉=&|"|r2pq[Ɗқ`GwD ch9;Y  eK=`o"r1BOZ0` Vsoc#,vupRoQHӲ!)5ZQM(Ia ֺ]@C}78Xߦu~].&E rI;"Lcfň{څ߻۽3Mt.0 mDr]@b&2/с@w~+Ld%1qAukZM?`NI "`(E~% I~_2Cp> PpE?izP HZ䍳w#n,ۅG>1z')c9S-Q u_ BIvz_WC#kCV;rt-/IGVWҎmbZͨ+gq=; D~{f]heTY\.i`va8{~̮ >D\q9xЃvd$Otfv,g3>LU`y6 Jib0dnZ2KHeg;50bD:#r9yZ(K$YbŚ *9:*P\% D5m}&y"*_&ϝsr4I#Dc7o?yK4a]ػwم=Wku,-4ꖰd+-N&Is ؽ}{w{q{w*ڶ1fMK_(7#@hF l[:;Ba #&vFc;YKc?Jc1wbɥB _HH~NSˇC-y1=v;t-]8ڮh9.<(Nvn{vl7eKDw3p p}?\@&#"c#I_]ڠ/tue_!M?tB{@-uCPE_U {VIg[o}\}OWO}ˠexчfٴGPc{\QH+|Coֶ;/!z복aܶ}sއ}{wݍ{vc׮U,/ci2DgѵK6Xn<VVح,cuukVg.۷}{p5'.B;Y蚴xкq΋3|AyXIDATcssjD'6MAʚK:UxBq+$rdj0:t}.dO]хYu|{C:gs|9Ce jGb1R0GnOǙ/"nr3N7١\&ǜyt4BlqzCpZ}Hf'GlV2 J5RuCm؆k_pqUQAr2Z4LI;==0.׽|767g'[gMTq˰]ZC7d̳/DYIG&QÌ@ݖcˊ{h?'^HqY31I8neweh[ R=m4o5=5V^a3؋;a:CӴZdWIջVGYLtg!Y.fLslQM6wS&j0v Yؼ<顦(G(+tk%2lAqGٸCP-=fe t{''@;,&ПN#f9m[bxg!`#|c Swg D?%W9B*Pd^h!P˗R3p{ wGW+WT:Yy'DK*F266 n#<4tȝ9R˴啖qYi,Mq)x>ZJ[!W9m pm4yWvc6%WQecgpq4biy>Ah^ fwYk3 Y Q!-y8>7WLU NMrGѯ%s a3]gcɓ cvS,v`d#en^.C doaoKM_vP׺Xǭ[֐9y]?A)$ "D߸ͬGi;ȲLz4|!΁ RjU,o?/|z4HdReG. t:'4e0эD_ȷk>߽_g'O}$?q?\15_1>}'惟5Dޫ1{'~ow~{qˡ#hz!X`̕Q+"'9xNmixsuBUi[* +z`\qx~,`M Р^V{A*&;B(Am%Exc"crp*u>eps׷kCA>>=E}0_J5,h8-ϒ&,wA'16l%ĩ~'=Pft/{&u|<>[0ױKFR,hJ%s9 ]ph&s׮/g4⥿xǻ?[n9kkX]]A۶<:oSGT3kRYsKС_ @eqb_y$ z7F1й/w\k>\O}4QmxrO8=s w7a>~];!o?}8! %G+}zNz5/:oMGR-9yb"u|rFɯID1= tӄez9f[BiRg}]N^|eox<rs-M>ZےY8u*y $>m( 3u-ϑz@ 'KGi3{;!%;4_\W1AUr7|&I:>.wqVیؚ%<߂{ǘݳuWPǵ3&J|/Ǭ6:J:]:yfYd]70mchT@Mؽ Za=*yې$iezu 弳n|7R]WDYIhI4Դ ,c5vU]f]LG-`~/-ׂ8XVHafq:)aa~U? s`> n -Cc8x8|pCπ@AhZĦ%' ;8X-O< w_ۋ7_z7I\!} my q6}㉸Gލ)M`}?0"+)KjD;CmpD'i|eB#{=y$ȷiS=r_g4Xfx[Y?4< {ȃz7X S|iE?3:ηA;f}cc=Ƕ}7?j:?[/eCηb؛hxSW_΂. ;&1Ӓ^Օ%__ǰo/ S]auulG<Ïl8=Ko%O}p|TcV/Z~k 7?/[hnGHz}s1.Mp&P:i{i>CC"hh<>< ;Čp0qq P(ezMzGp@",&k8u߆Sy`]O>6겝A0[Fk˿_t:<yԀvSO-kl(ȤRjkк SxgP1|a2S]'ɠTZ&1R> T׊55幄S"=>g?C7]k8k>sy {_o~pg{fm}Et|. ENpkf0n<; cG}=:cp(ڈ֬ ϒ#Khpn4. Uɇe[r F#CJ@rv ALhC$V,g!˴)%%9?;/CxoIR|[hJH`ɀZ/G^4// n`lFǡMƃ -p fN7Y8kJmT 墀u-|lxoYv9گnt3d)j27~W<9Dtq4}x؃ͮ!LqP91;tV̱{G-sa|/yM*|NN:E=NvdJGҁ<>cw`4Hl& `~S8t( LuP)U7"\v {`:bӉ0z]Pl5sF@:ϴ%:oem&ְ; )?O L *9eV5aHF"X]d0p0:}[•‚Ay/K鵸jdtP_+1I' x77|7xnfp㍸qM7n 77ᆃq-qø;x݁#!\wa\w7߂o> ƃq[p!<|ơ#Gp#1oO(v;n鑆= c8~9`qjxH񽁍'~{qۃ.ztAogySc2~bQG38\»!wok}s~ ~ #?÷_F4ElH>33UgBPÇwZ#Y|B /0~NEϻp׌1݊ttt3C^onGp݁#VGp͇q͇pv78<ނn9o9-pa|7><h|jKDS'sqzG8M 1z DbT+mOgǎOQf6b$,//a׮byy߁Sx-~u}oY}%O~~/78i@M<Ew; ۆclO*D.eނiu$ejo9SѴ-\ zEVnwY]uݩz咋Y)#1@o *Z>n]y3wŻ5:W lW8Psf6)VWkeU XKU^5qá㬛+s' j[E6Aڅ8}AB4YTǵkg ^.EGUn%_9oӣ.cw.T\,3*V,H Q))nü0fH(A޷|zӂ\hb =|hb6{LK.ݽ]@ۦ-'ݸOKIx\msB5@q*;NVnn $`3d'NtdpK 0]#98 8yeNv5éiĬsrۧ[Q b -P5^:w*~IT4hpG|iG^v{*Bz1OnX^U*nnF*~sjq`0.Z{4T>90HeԲI:~g  w("Pɾ~Kq37;xr} '83z=G`J'{>%j#FhL€)\eMebp@)*ZCZv3MU4Ld&Tght%ݝAD;`6%W\K.:]3.*DVrΡou;X[]FLd^Lxx5v+[wg wnLva{ƻ^hm =ًveYR\P̗Gp)?G T⛓Cu'>Rs:s^^}省x6(QC(>G55~|#1B)broU9:k%:#l- >O6TQ퍱=NN7 Mv RLIAgH^է?!x((C$B8L& q#y6UIFbhS]x+uw=t(D|DZ;3϶KGV*u-*?dG4|{2Vs2bWnr$js|Nl⣯Gm{0vrRq؂}61/:rCtퟩ3W T`O-STC֦Ub^Йsb$BH,Ua_Z_ysK`|,!+Ռ@b*"7r<ĕnAw܀hY|Z=[9рt@ŀ:lllnƈsIcPLrigbR;P!is-$.ez0I A#’a:}nv]wYfF2]":EJrKy&@< | Y|(}VPe4"]`q<y1l,O.-a?7ubYW]\Q:9 U;sjM]"D]Xוؽg ͤ3EyŨ˩m:r'3[Zt*yњds)?yp@4) Y+V3y ײcp@XLpr41u U:v1j=|OaO? *$\p>}i"ScV&VMs~)3pj#lx贐:郾-^S{̀QN'4U'nP,LwkvՉm *Je<,9ӤcP1ziNљqFqEx86s7tPEI|Js/Ƶ,OzY\]Gunm0ì8]Gt]L\}ҭ~0Qkdwzg8Z_035hn  =P z/z$ZYrd;mQ.'g}7î6M }i-ﺞm{х.Dt]ߣc Mvn>*Y2QD3jq <- Q;N;OZS:4i}xbmm~W ?t g/Wo{>ŵ7\'nF#]^hn{ߍѾ=h΄ٷ}hC v@LD?؝y5edħ:+^?|̹Lq^(XY MB[$ "wnC 밴o/%iGvD4h,2xOO|w8!9 Qvdknk9fTY&8] ەm!pb}nN8vrl XP;-*lzu9B qgH-eP@g=cz:fTN 3iM*@ ?Жn1\ؾ2qZ1eFUkx%D`)rh6 %KT9$IvYB仃hƖ /ȸ#a?JglU 3fN<6K11lNO.I(ͮ!$$2<.W,1b٥;3/2cf!P𙘤Zן h9SG[X%w񁰝;m  | ,7 z9_8: ԶVQqΌF\Q<9mZSI Xh԰/u# < 8 2q;:;3ӄecc :5<жHpi>uc;::srSw%;Of9Z]M`QlZׅz>$RCd"0ĈtǍnvHĄrR2휋>fAYaKW˲1yma , f f];?Caw 6V7Jв=d(&&ɉ e^ *|K:Srq e;* b8|mptFߔ^xiІS8r]7d2 :S9yBΎr8Bt~k[,}W΃t#TgLDD2;tӛnH9E'yCe:"\p12(&W@2˺?~rI!E eezARؽg W\v1bg.dTb9^k/wB:oh|ʞ{]h[7RN~/$޵fղhO}N(XX?# *P `-$|d5ϹpFuH QxkuBUjE?*OtѹK |ަAێ3%92Kfr(wtjU=70l7\y|f:ŪoH\'Z6&}$ۚw5x:mcl%$ܴ 9'`T{cYrDSXcQʷ=r:nUjkN3GHѕEX-e`O,yd٧xL(k#Myg/<3PQ˫o֮$ 7rR2IӶ ?/W9Dpe!>K۩Ue]k;ç|8zRn{ur%tbe u;4X`,471}\5|0ݜbs} _1HiD3.RV4M:Mlނ%p23G' E؉dĩmM*SR|پ+$>pcıc'ږqg'msn"\Apbd"h3wmؚQA|.`w[ Ϲm6>ctrmGq+dFfv4u#ֽiayyWB6x,!f9TL~6aS8V>O c[~/,YC3 £ @7#>1@u@J|zj {4f1J:-7+kUyʣT .R]l:f,*}5[dD'}VW8t֪DkLB1K#{g[{8).-MiTZ {^\vX?Okc*AǺζNeR%猬S:_HDؖj\q6曅*wfXk**QXsbo kmd zmQpMф9p#{>&btŊb˙:̢HqecBͲ7ﵺ3A5P¨e^ׅKqݪ<3F\RF.A;'b (p'mCb䛔Qn(yvI.?NNe&KjC+rj75::Ϫ2ţ6LOeܶqr%ӧ ZU؋q3SS57z*|4&?ƠE,+F,^a*#NkZ\Lst9bvYRvo&e+1 " ۲3 C-KBWu#B (v "̓*\63;Pyڂm`"d{OOp O@K@JK}|oB;n7K} j=b+E$kY>ow;q9 Sߓxeo)ɮ#vZrHoq6 V'ۊ32-E%؎ ([UiZW&_iҢαsoҶ~ v2/+> OJYMrчyrx FQ_3(Oy'vpd,fy)ͤZ`<({k3;kbJ(YǶh{{cAl8mdx-Q'xcUeˤS4p";/}`D7i:GwupVӨkЌ&w8<߅ 2rIypgoZ.T-A"L%'8t5"+'8ȇ>V&Bm3tC& ɋ)< c\Cx̓Y -M&ZLЌ'5]XA )|rl_"ֲ/i2N&d3ƀ1Sxm3ӧ}^_ӟ '>]++pc:I܈>}jS >666p$߳w5"Ok@2LcѴ%D'><-6FԠ 1/ FM,ٙ+ n6ގxuҰpq?>`*>?+xًпa3Ѯ❗ӈ.8.W&͏scKh,ʸ}C,c28i%ZکsF69[(ӇJN4i0r"ʌwx@j[zkY 828 l^$i!/* dH&6bߞ]1r+hnAі`-J`Q;3ql EzuPۍ6Ta1Lf:5Ӹ2Hi ruM6tXf^;5}5CRXqIۖi˟$o[QnpU.({J|AD)YWXf;J!;hEADˬo1pU1"QBM)< {'cݡ:5:+侎Դ};FhG }8~#ޯ4kTh턘 [+삆<łV(>%}WQi Şiz8b l'5Ux֙{1j A%}#hҦ2~3G~) DZW'%y޼da|$aV8Ko:/塈"_}h@)uRW)xxKyö$+oN|[DY %`ʘ>h(D=愸XgQUe='>-7u b!Diky79!AٕJXL@3w,D8D}2RʐGȪ2J(vnϺYW]eWtp! YJ'%i$nhn}1K3,&5AͿE|E&e=B0 TNI00t$g g%&Up/K 7<@wki2S'8ۑvY+W2?cDևv" x0YjMzTQF .6lo[0E()GE*OYF9[e9m५<r9C.Kk#_ NIn }.`ʯNQפ~~$RxХNUʼn5!Dl6Eϴ ]E4oϘ+GVSi.G 6Foܩ%4FQ+0sumi(<=B X깍_恏ݒMӠmxE{`v|hw! ESAt29;] mF-|{ayidWSf7o{@t;Xb q(N.s wkӆ#);r2?Amc,$ٓH1 Q41#GGx_}ϣr#&fp4C/m!;BnXrYHPՎXɢm"cb0u]L^cGog{ ]+xܻ5g)b@y60jO ^?[8.͞E- rیeeO1MZdCu '⟡3Cn'r`=Զ*C Wr=By^SIHi.!ei(?nA4YC'I@!4ټiPx}Ճj| vf;`4"օDX9yj{AZR TyAv,[##9r{⯶oKx1utDt8J΂G",!1dѥ|sșe 1N5 #fᶼ6\ar.-y!2P)b,ψd.F5pΏxǐ BVPgn;H<6=kk&G^[}`Pmzb)ml F{ UASʳ&|}9Dy!_AMgsz{C@3-_YhkhR昔!ffyʓ.OђqLoSmOHWy@=\ Y%|_<~7|w"'(Tb@9ixRAFFwX ^'27b'8 \0_]b6.?Ħ*;ʧ:]he9MO~V*XFNϸ$b'<83ވ{2i% lH wV&cމ=G? G{+ʑ}MuH&q<ܘXf\T‰]5Y;RΖ#RtFHl.;M`i7UMm!)u K,#ۺQjy ${HVI6Ad9;YT՟gkLWM#ִ ]Ho|0Ж&Ȣza)̘(K8QGx![(^uiCPYE.tlN@u!4œ 0YrnIӂ7ȋ3 @"Z4"uG.Kr;sz^A]  -`vaV:@FDP(?<"0;]1BZжt('HxZ2ہH= 4i};4k+z>@=R;j"K$e;3,ߜy8Oi?]7ek@FRQlUW_xX=uIU\C[@=lN$oZ>WYWߥ@6.˖x`o1骴 rS.S&-aAv_ T6:~5y`g@M3rr[̣XPV׭ -xCy1$m4N j?ʴs<Ș:F٬A)W{ whG++tp7ƣmM hb'+ 5 6mc sGׁxrc/N&`+C ށ0C| =$oVB5n(y).>,ddKvf ݍ0Wo9S1I E]88&M3BM_rwD>g<8ɝTU6PR.0MEo@R4u15=8%Xeu#xq#|05;1S=N2=kGg"(乡 _r,_];trJBa̗y E6hQ:MOt6b-7y2J7h|#=WSvֿh :^0] {6E48iWm "_y[܊A@d5ےgAˀ-ZCMwrFǁf}!m]#U+IHKW7pDi$E³`uMS_}^*oھ ,;dYX;=h~h~_FG< I4HlPC3$^F-Y+D:QK}G͓IaʡLTQ'42/K'8s*1eHmUJ nAKmWK 2(:AM|x~KGS8^핮1̌R}(K^#/'׏~mG6 8fRAfsN췤|eo;\c[ |-Rv0eN-!6&XnsvBM㰾v)~7^NV1MZ4ǍeM˕6˃8Irۚ"6[h&Ԡz )&{1ڷ |~>cue ]%{?V9S1W * I㟓-xNrv>'J,U&t6:L^eJ ^s~1W;&V b֬_B\&g9 g('}n6dx+ހc݁ɹaF-Рz7ApcDД QNpH\ɜȠ\-IUC|Qdaų2N$+]Гj}lp:BiXCc[ S_Y̪NÕ"uWr(%?EKqTu,6][^{6d-.X:*HtiY0Ecɺ/_[CsvCiY<]uU\~Ðm"Pdrx-KF%%nyz5 /D{K^i:]aa+.8dt<\}C5I 9W gV%Ff5@BtSRpa9c =abJY);Mלit~-*8?Ľ(˟l @>;in#ɈW$,(#%Vve'` 5 @E|¥czDw6%)UIaG7zct3ĉd@_ڰ>)V6EԘ 8x NDٴG:4Om'1,O #ºSPcWqZɓK@LJFb ci7U_$n({8="ݕZYQ & 9nDfˤ䈀Qp!P ^E85)"D6ϓfڡTVԈuV>@'|V%zc@é:o& iW%;m2YM|\GN29 ntꎙu闿me4)LHR;0#ɻ|wo[F D[151 ei2Ҥ:6CO1t-GnԾsZik 87 꺴|k]\BI^,b,qJA`XX_a&Oe?Ձ +B">kzpF2[thym^;:S=PZ΀oC<-m@g+myz@l!i3K,;E$IXE%ݙ)us)\ s~iBYet7wv! w$T` |Ǒ~TmJWQY0Xtmk]$5PwSΧ9> RHM#ů~#:vu mQұ!Rޥ*1Ip'wґ) wI  8~ǩXlخT v0gmn "J&di/uut"vgnep-hڀ#*M 8xoo?۱y6n;.(ݟWC6/괲R ȎTϻQJ1e Q.΁O6r60Gy&_ /x`ue !Y: '`/y1V۶6ueiSYD&(g-$;W%LO!Bo3>lXT dSjjR>/P#ƈ1j|3_+^F~0=15 -Wє<)KQ:MF>fgNCM?7^ttZYIM" N`IӜS,ňֵ|s<:7 jhp/AHe Ψ Oǘ朌;d h^MZ5 4(/];`кcRcCyMN?nHmopka? ή? My+@!lŢ_Ml/2F[v]x0MƁ$%EP1ߵ{@]saz5ﭭ˻`8u7 DUBZ25lUPQw'\wcO\%Cɺ pi9z$!{P_9~7ḮOjac UX<um )ӫ |ŐD"'('5:,-`e8"nx84M6`y#3;$ęVN&M=_Q"vP )}GʗǸ;+Ovmo3- $M]̮ ɔiy涯>*3xS',ו[bT7脾,JɊp q}>=>-HuWIRWS=:x7BG;= '6-`"[C;-Q{) SYj))j҂֗Ljw{T6d*w EY!J}1J H Gƣ1x{9lL*32D=AEP(">o 66ѹ(v¢:SAZjY搃/DoJd2#bMyȩy8$?('e6)hY_92"~g&mr5C`s^+)<Y7H9be a. q!ZLVUol7ne{iˮKˈ@O'hN@lGzqbFӍxMVV `$ͣ@rWDf-\B,a@V WR+֡~Z7n.WBVi:*Wn\u*n1p dv~7S e}Hw`mr:b&,Dfcx )Z+7Aɪ610jlmv gd`̿m7C;ޖ*Vuss<a@@E) ϟ>L"E94T Ns {wwVuu}3~ߺSsvWWWZjժUT EYsk)KKvU$)\{*y[/kWk(J5 =@yI@r0GVqѐe4X.KV镗+1-QX9BJ)If)nI:X:ɎN&;O>4F1Zft &}?^@Rzth; d;@4_ykU3^e|! _ʒv:[on- Ycc˂."Y\:aw¾n z[ Jͼ/t <܈ pb8 ;wo;y(R}%L ZvWA7ֳl~m±>]k RkadyvlÏp<}.Dao>q@-(4d "w::uhKYPiGl9dD\Pw6釲1E}3~{`yuc1A VDR/nWO~hl0!2$CSd]jhOF#9i$δ [ Ueh2P1TT<Ɉd,aixixxyxyIDC=K&v201b4IJ7K##eyO&LR:y?T'q*(+Yr->`;g; !FNG[4>- םIm+ 75]Ht sv9[Ki2w)\KIME k,\z_7.);:2wavyf&md U0q8nld{#E9=Gz)ކ .Je6h@&02*=z 5efy7S%fHmL6{=?S-:Ӿ&1Ks/κhNkGN\fTG9RBl֜,jivCX4v <0 eHzi2ZGS+)Op+وgױP}<0 ca گ$6<Rs0M ~,6 nW'IW;'S./%MMAgخ/r.^@K(}aWHߓڄ+a8ʖ铸rY-<~Q3QJtAO i@]T˝xO"#_}9z%\YהgFZ=rS3ӐiH+e|cbAU`r K;Mr. !^@3#?7xw=E"[W<Ԃ:z7rܐFNzD]K mx3A+SOسq >JdwHa#e`m>@{<õPO.%[ c),1Qu1^$:zdbTPy(+f|۞XĠG}6[8˝x))GzPh0ꂨXYFU=-A˳va "kd)}u\mve2P{G;hcqE[UIJs^:aa>@5"kqw]q~w^Q"Q&Y=0 42p ǡ2;'Q:yvM߽LNdE]ʒf}+L'xJ҆`A Tc$иKh1&ԑHYx8UW]'yXa2ق| Bcǖ.+',!嬨ɡE XX qWFfYVH(qjbd8zܽj&y[oXmY!o_oėÀd;6|WM@1sw\, G4IB9G!_$wo,\Vhp;&l<9¹N14}ӛc[=g΂ivE@~Y]DgH`|gҒ$w6`M3{l`^ rD8װqYkYZ֯k5\kY:{u_w=^Y u7F 7 Xw=c\ _75ky ^\uXnc}l\#a8|z?]CB;Ocw]hpm`dmdO~̫߾g<Ghz&DGm03>O%H|b i5Bff5PkјCO{%H;\l9ˇcc@q9$C3Υިvl|}9w`67M=UY0Ng4mxCy㫞2~9ë-o!Fey`4QeggѱK uGyд/ess@A"0&Oh;vs}{("ͅZ=ҘVL۸=k| qUNdkYw=I~_uױqul\sF^{k~]N_{$q>7 7~Mlx7ƍ7~-p+}u!n erXjUZ^$3x^݅>nއNxP,[6IRd.0Wd:' eh#Vmu\x;`>_JV $G_Kbmك lz\p;m6]SNOɋ a<Ck2 ᷢ4y5qB,"-#L%(I]z:L U?z ϾιF& Ng +Z.y$)!1\g(_ZbuZ~y]F֛&yGE(̎9-a>7O?lm8|Vؼ,]KN t}$iVD (؟z*M$E (Zmΐ ‹F!rxv@k]ތ6 TIߴ2?Qp@ b*(^b8qDOL[J|1="o?|n[[JA'gmOmthYw S C-YxhI^~kayh@Qhu+Lط=?{*'j<ԫ(3UjV>d{rrz|ӶI(\~{t-ϵNj(zi*yRwԢκ{\\1/7]SGO[7'*1L.(<8"I'i`-ܓܾi+rwM¼C0Գ hWd!C^y#]!%(8ݡcZ&43gf8Gۊ,dצmMݧMrwso.;| $vZLӴL_?ZʲѕL|P"+';h)*~amw Exl#x"mW%7?,w@u4Teʥ캵oAKJ8P?رc+MPpAvgJycʕDi\gAk a~ z$ Ƚ+gDC3:ci!Q gU%uxܛiҮW{ެ^mc}?E魌c5<ا4IVFMx<21fㆴ^u]7+9`vQq"ٲ<(Ox3wlg(إL"S~Du3O{96 {^;so# )Kٵ3nyg頯h?ѯW"eUqm_<CSwGG)u&c:  ۴rڿYZ9>ǠAƃEͷg=?14z`*'y{]8a ʉLwXO4 {x5[]:KR(dv0'l_ ~~{ڶ3n.䰨N4w<>Ӌ9:vm[ʪ-.F>s?_|[qþ}4P-QGc+Z\u[5ܮ׾4J̉TGu9<\9p?՟=J Njnāпwo=Ky߼SOeʶZ m;'ƐtrBM{.wf%Y@s"d֯kt= NdPFW ׆wj!=j5 9f`#; U^{'r]펞(S'lxȗΎ9XZʢ O-o~[ KcEFk]?Hm l?\"XlW+MC42P\xtQgڵ(K+h;[92vgcD¡u ?Ϳ5-B7~nߩSS*x[s],bbKAv-(&K]?G_ƝO?uUB (C*NzNf{Y%.E]ct??SNX%DC0H jZ!:tƕ#={88|d(Y _ !UOP}gg61ۜ`=euox[ezx AJ1R.Mظ~~z)w9D(o& Ў:bj !PwO|˧L3;6D/F։U]y$(;]C'KķZrDl%WN%W6OHhm!6 n[F/^K`UpiH`Et!˹c׼J4!R!(.37~:J<T!} q.=DnH|ͿX ZQLaV>\FsșZ}'A>$ J\;^Q;Fe3dh,xb G 8B:D8t}:VĠ (]yÖhŸ>X]/MzR f879':-7qޓy݇VuVsJ'3thmϞ7P9yz;.Nvt~-JBtĢ^ٻko}KQSFERRwsG$Yxd13h{Nmc?heO3sE6⒜zz'JƞOr6~=ob;y:|gaYePƫ$5¢>a~N,Ͽ۬0h`w /wA'>o;Ҏ4Ƣ?&q_ܵe;/syŀ=X46/Fjƨ* Y >SClgY>6אNu4pNdl]+'Bq2O9-KD.u}mL)/pEA'3>3?$#?Ze"yjd30_]"wΒ 3Ox֙lYg8/Q`)F'|1s.fzi3; eCƓa*Ԑ)c¯Qe5WϽo~D1aoX_>Ow}LL# xJڤ6?زerW-Lk9MRJ.Z~7y: hF q"$aW$h,P:e R]P:xF+,E,c: _9[Tېc18"773k g%Ƭ(#Q-_-fn)ىGw(zin>6 7%֯Gb\e9o]$̮bu7 O|6xG~)krs/fO-:+\SSPbC,J6m{ԯrԶT$-I%/m>u9{{lґNq*oaOo87wÀ\YdVvЋ{0% _Sq2 KwK3$hr}m&7s--pd(OD :đCi": 3}EmXٲ‘j4(vǁׅ*?Lf8cߴuM1FvY i>-/3 -#_K=;Ai.$tgUϖ&DS͵Hw;c,(cB~ͳnΕ.Ԉ3 *5#c\p)wS_;GQH?Υ6|ww uӣ|#lQC+S%]z5xɱz|4'{5σ } qYAGa-[;JͶAoߜ;Ͽ{[$ti r֭knLYŹ$yDQ) 8))8Gy/Z#1Hp|klNSp9qVNTKݏңJ{ggq急-S+C_IphbASq5R1,p~3:Vb. :vfިgIz3$NidU=RVTܴ3(-B>~ /+[җUU҆@]uB\{ٝͷROf S#bw-m[~V8ڹ[سs;Vسk {wm9a629VwS*- wg[‰q޻>YeI{EF@ǀ=;tl[b׶evl[صs+;vl(ix]rm.qEԲC2ȎD3hΩLO6P!r ;2.:f7gLrƃ~_QgW׽ o|%o|+x㙯Mg3z%gJ:Wr֙7|g饜77%ҧ?e<SF)n*t2ٟNJ˹.x/2|e. 2~w^qDݤ$y%&_HfM$ߎ%?{Ixe0@O۶gnkTDe4j. 9߾,M:.Al)r쳖x13I5 uW`üMKvNy8uftԲ^t;8,# ,UV}Z? X p;u ,j_L jYAԞvu9l?[DWko: tY tn`xͲȂvUJ!J\qR%F^LB9YM^e9 %޽fdjsm^y'FX[%4 |(93gJA @[8/mw|֛.䳇Q2 6FZUBbH=2)ؗu0MnZDW,)jxI'-dgR(!*EUUQtд'x"Rm\/8@UxVvڑ6),.}ՅO[u*('#džwS3>i(*W(i܈UK `vZia("48uSnc.J:Ȏ6Һ /=O~q|qrw.g ))y 8v ?rq!ਲhdɇߦpQk.6FhܕϬ}O|5ez[eASeI(JB13|cFi(fbR4 )k~56xmQaow ]QK}p5}{h{zdzK`x[[{nb.21 c:E9L9?X!;G3&wo^Z| !2F=K@ػcѳ)&֙`BOCt5ˈ 4ڱzX$,E[D$Ա"efܫ4ٳ97=׽93΃ܶUVo}70;tL)K(G\o&Y gM/&@lp!6(%oS8¨`v4{ĽpUE;Iޡ ;c(zSn[sdOKYz_rmBٌY]S-MRm"28TNgVb2ki}-gM2xp놦n!HZ gmu/|ˬ:fBM nE8neSӲS( FEEs!ڃOzzKxÙ/Mozoxxۿ3/?~_P~ǿ?y?z ?%m:RQ%DEDHdH}_w G;c<ݐIxdf0 9<8,m.Vry5nEUϻtN&ݢ* h0<]Z~s{a Iܗ-H;:k:M R@KZCRIZld2k Gɂ.Էևk B luGaXD$U^KHܢXdgۯ2ײ0T@H ON8(۔(pp^|WBĪӂ7+Yݻ.F|R-zۨY?<޻3vreMyÇS-moG-=Bs&jkq.\4(}I7}<"1Szv^$E _RWz"lieGhFf LFﻨ9.jNר:ꝘdH#p]7?t=Hk=IiCp޼x,BGD/f5}6JeW b'l "@f !缋/KEFx}!_T F4dN;;a{pN( Q;vQ];ә"}1K1M]h_D_|)s +`\rWv Kv݋%8N|P0$I Q1͡k-G8=F;':<Rv·Ԁ;Q cѢ`>y/0P ߰rٓٻ q?s~aXf􊃽pc:mY@Y`U )2i;zMZ%aCM7w//b׊:48Mw%{14X+ϥ\'ueQRJUTUEYeIY08'0i^ hT14a]|JP,ѨjTdevxNDќd֐דDiy[~,r"ʹ։Mno_=ʱzՉmҺ=&})h܌50;N߼T7_Δj rpL //4~mxey7uoF!Fe4vt7wBk>D;Ө[tօ[b*Wq_?RJcXxE|۷4Y(y1JDJs]7\{ai2L*EWU&*gcm&'M[*M"ObTVTeEU2Y,X\_k OYBM2W{oo(fk.2뢦(teXPkz3| 'o{9gxo?ⴓ⋆ Y][)h}} 7llLNezc:cc*q 44-nl6&~yLװSxd2kQ:>HZS0x!!XamW:7AÑ侜r1LЇ|oNZL(KSO ML|-@Rx8i.˸k*@>LlCAWBn tla5eaHsQcdm$Z{*s^}nHtc1 },YiI1۩2֥~'e2fm4oS22ىC6^IVN4SHtN=^>2Y߲=Уw,>'L̎г"n#GGv]WȝNc0|ﭓI-d&t =<CƁm&%;) 6B˝1{ޛGL}Ψ A5L :=ҬȦ1>̟\r!o,9O*j;?BDC|!c^cmOyvX~v &};N[ Ntg_wDbO~4m-v]`:Ee$#'JݓU"_9N@PBkKj/u쌌lU|;!̈́MGr3㭈RE/u%fOb2*/Iti1[SO=Iv1#i X-hX q%nm&B&w XoZYԈ"@/8BKQo+}q".˒3zЯ@7&yg0fK7 =Zysy6;f,zC_f4tya=lTJ`?'=M;Vv<e6cu`cScGs(yϝضcmХRkyG\.|AA1)+[QdO@tNGo?YҤzo4ٔx2]HX7ly|'8ﲣ16-tESzśYٹ&(AOٗeN`kRMicLG٩16Pg Ӧaӟ*?ʲEoOY}O.$K= 1P>Hd.Sd]/e)=Ze9.Hb]}ٔb|Yon`D%o3tܑ4I ?W8ʲd4"kk묮>e{[x3zyk_=)u{݄1y? Ũ*MtfNntqb?(bԣGCҘ .K;>Q(y:ھ;wgRfhƶСqGPrWz+]x% /en!J\ys)1n.`,My8Ȳjo7Í<wюƃv2eWcC' WAy0gOa7lʑ#k9l^G#Qd&o9z[0;Y)A's{']g!tc<r JU,#w*y!Vm 4)K8[wW 3J`=8ߺ̏gBȱe:vl{VFʱRRT7%^ٚ?ۆ>m1%2ȩS2~Q0>y'>}}`e"մQv^YӲ108"[к{YI\ʫn+ĶSSҤ2 %̦/EԶgIJD3/FaA3[$7O%-Z#KEA=&|ѷ%9NQ9W=6[N*OvX%wLy /]R[2ǒdW޵nOp;<ξ,m=.8bt2ZbJ/n֎N@~]2q::qRr4WQxL MEbLd l1=xdwwQUe'ɣ6 NuAQMhF $}ǷI0h9|a 9avN=i/nS-o40ˎ 5.p#Gno|9-/g#=y?#SfUkz{åpe"Ѷ5EUK.i)J];z[9{w.w:fuZL fo'6V9=ĕD_ #(_߸@hoݯ6Ek;[61>PNyV4I=/Cd,ؘgvI'a66HX5>{ԁNu!OW8xh2Y/8kx,+DEwFCб, Znx3}$&=򴛴B#a{Jɲ_qs|`g68vH שׁ[9s-Uq'XlBc? P+G?^CXE2zEc†! uh0Y8L˗Po#u.^.C4˺4y[,}Q\x]u_&EŴTx|3WCqpd:{j̦,;ǡzs.[ ,!@7My2=P's0 [TyhȄ!+*}ȹ,;΍443|ψj"uN1g^mo|)R1ŷ3s-N 8f xkt/RΔqxg@AX8_|s:ɰh DsֳzGrp‹?wNyL93>tɌ]77\|[%.5Z>~E;w5i=0xՒvz0.jcOƎ-#/lgg|d޾WT{\}yi0($ބ Za%z㇋lN?xn@ =i>Z,`^?e.]W6h[F6kUGncK(6(B:O̒Rp*$$Q( u;ywH& tNir06ÐkOXdEi!?G=Wz Wkbmu995HjL0Bҳ>{OL '%:Mk ĘUjfCleQU.2\9Z\45\wuKE\dJk"7K%G1VCVQXe6?my>Vb ۙXhx |m7$ߎT^OI책j׫Q)bvHdƋ:|wC90 /ѶeNB;IL4vdOՙOnpuD>ࣟcc6|T)uNr=>:I7vt-:SWiIj㼵'a~HpAY'e!ũhQ{.99 Z,fg/|vnNI˃!A>twP`T[̌AmARmfGNGϼ64|H<dl!{3E"lCO\qBeu|Х :EcO.to;;r~[&r=XtKh (;C1}uJ%bi2fTh8rdsW ~Gϟ?7xY/⬿~O'pݟّ Yc1pjl~/E'E 4\{uܺܛ{ދ OO9{n{DUWW& ȝ>nbnRc jj ׈|YtLI^1aON*dM>s 5?GƵEl24^di3DӶ-]wF|$] 88UWd53 6(”`]<㩏woV!H#.]_u/t8=[sOu}5~VmӚ+5ZY vL3پ hC#W*̑# k㺍Q@cpԺ/sdYC98^qLv0O>N;$Pԙl!{Mw8q bù_2W^ysjoA3"y$5QƟ!H6]AE! eY OݴuPӉ:Otz.ݎNy-8;N5m>7*&R4,Ry}qYCS*hXfwT~l'r@kyUu4[r0CO?LkDPsV8unC bT;a2J  98 U(u!vV,0lp˙&?bwSp hKFxw?{'aFp3 ˃MÝ0]rϹn 5ۼxWM W(3`^ǀLQ>X+~2q 1gy+?w1[v8k7y[WP)-k׶|FڲcĈRY% [z)x]4=\%tljzY>Ch41QK#}~ؿOoXv:7߬޾#f}4K ƔT+F9NAwnKy9[8ZWƒ6h{p9-u)r7nh9ok|t?~a^)T; N[|53;MSQSw/(e:~ֈ&6|Iq Oy[?RR̫R!=kMk,3b5HXD`Gt^ InmUюJQFX>xa&a~$` ,r[#ddzD K\a΁wN붨 h]PmۖmiB*~G ^udË)jlJGqbmp[&']!&*QJkɚy r`$P-Mczh<_D[\ ^`6pN.˒(}3|[vbcH=mQ6SҸ7g|G~;)o/$c9ɉ Ea^&Qݦb|jȞCSiYi?/(A1~D|`QAvⴙ+Y03=C \h=i+5xIH>Ω[_飯ɸbeiҨEX;C̚?5/|2oyK?kTeI=`Ty )]0=1PĨ -DDIƛd]Rہ#uӰu=qK\Kqv*{~Ơ莦ù\IU8nK/NRC-xE݃'L3qc֡Zv4G=^R8ا?VlV~"dPT{c笘e4rȕZf8W)a~~_bc:#Ӡi.;ɞsFț+"wC{@:\ Iv?؜vSq#y޺2- 6!X1 -AyI{wq{628`gdע d&7^q5FD R5ߑs&s]5gt4_x)|G8o 鱦lVF$+3 pPPP}6ad|7,nXae^in1sXca2VFم#0}"(G%+79ZqSOnQ~s:lBա,Mx{ߟ8) |I%q+9cgNC4ĂuT-|籲c ǩ:° B<g,lAKKc}=(H5R#Gnq@d&*?ngG"rPNh=^pل뻁w#E9މkwݲotY>brC0_wS9pJ$*T7'DYkVĽYv&:4C|\l3[sĶ/-q?CDʲD@(˱ck1#v=Gʲ;@vk]rt P1+7cc"|03ck (U%\v5}|̏HW >"9cL4K2;VFv!7巶x){Ώ_*h)i(UJx3^7JQr-jua<"0.6뻋C0o]C`9]T &vIJ(q`%ߢzN},NHNpEmȽm.2쵯iݍ[Qxd2f-i7VE&1e,]bvNN%FȀ'cO?}SN4cOk9ļ?H[7_=<1/<qWi;y=ԥ&$7ݝ#㺎MncX۹L2PZ`L#(H͐CHdwW0pJ{)?a4Y!q 1ymGΟ{Q%[,_y"USAvi ܂AnkXr]|Bw'tdQt4zƯp*oq-)A-E`$2HTڦe` Qz}f5Ut\ɟ_u1WϦcu41lnokEEyTw1la.{bg)#ԭClj9?7VD6eЂpN>DFeT<3E;& kgx~ hXѺGv.! KyT3tDhlLv(;mPmr\.UA jQ0󸖼\ie$rĤ%ҩZ,m`%\q cB{G>w^iZdBByN9VVvĴ7~'<5*y,&L Wh44M ׏,8[o7|ñerʖEy[ C:ޥ~/uUa17 c1)7mo9DiEBlm=͐.Ή%EȲӴь4zwک{xGvXzރ)o&9 R[(VxRL]=kB}oK](<|ʝiꆈtk6ke-DqN;|Eo$8Ε='O}{aTxt2[l#o/-  ![!94Z8u߂ 1Pg(=Q'ഝ* 6o$^TY3*헁%0nmuP 1BYUeALN(| gĦŽB?޹91WlllPU8Ѻ,'[^ZE5]S8phEnYͽ*8BNQ}Iu@: r;&'ӚǸ}UI4֖ꂝ)%a0KTϊ1(#\s 3*ʢ`:1Ke2Sn: E"*}q r8qaJ 9 P&#>ÃθARX(gpEndx#\u>.="dI^e9['UFb5ty7N @˲&23GYfP" ]EW0H>ڥNݽaL[ ) I_߂HU@Tp7[_4mE!kztD幫g&cw'1_<W~i40*.8Yr^m9؂=$BIni `EQ {:L t;8tc|%ΎR((ԁbo6lLvx; &4QvOB=XYg!_Ĵ+ʣCO <' +da/:Aͺjƃ9'MwHAC&n qK.TO&ҦW\«/=ޠLk|l9bQʂXDu&MW2k[~e)ykMC.ia1DGkiy:MM^#H|f ?VaB&ɡғ w{O->6xZPsUW缚AEYtSNe8j1TO:32ANkx^]D9va7pa j#4h[tE>a^%O92q$9 O >~l?$6V7 G$!*AVSπ+;[_Ϟ"&i[ꦡb"|M[,Mˬg4,'(lDQBw?tW57G1$ݏy͠,oX0A^0!, y >r<}{`HVs οܢ,h,AEeq^N`0WG^_ڤnhʲj,ɟxMXɝ#q(M*u>moe_;# i5.ayA}wHa8UoJb1Ev,w~|"֧S&`6̕E1ef8Z|Y 4jl }'數rz͋Q&Fi~?w?t'τs[WJA&vW}ˌ#9>,m]0\3ڐ_匰;0 aL6@2A1*mvn!3]Qb)KN~.}l9ݿa? ?1UݥG οJnX޻ #KHL!>oOwBSNeY2LhHӶ<᱿ϸ7 uR'BS[SdWOؐyF%kk ItLi'}t*Wt.d1 %>cCeᠥ 5nRpUW󼗽=b_zi+fbbFny _^wo^L]7i FMAc}Yp!\dR^?s3}mXX<lmgj,98U SN3r,._|s~ѵܝ%HJ&ydmcW XEwTӣ]s,q]@r ?:d̼I͘&Svl7XiqE:砷hGr_z)1әcvlL ӂ94cD5t-ޅn3V_plxiˢw=ᕗ_ Џ*{Tdr.sDT S<,TV&Eɷ)EG+M)߆t\k.#k,6Hl#OSlH ~Իp}ݗF!0. A!TtQ VeK>|dh?;-!ܞ8EJG>nf~jӞ}+ݱYRtdm*<`d ?wv<0ٲ4۹k$\D>tx̓ Aw4@ӊuE_u6'pcG委*U\4snbjƄΚ fnw:]i&փhH9wHwĩ&;#9t+ =oocXt(Vg=Oy{|Ke ׏Oy1_r%eQB1Y _(qWcW4tRdey#5w\5LvmSJ'hV E> 6 b6-Y_f  fGwt;t!G#j51[Va;ug}A[vP?t~\8ݴi>ˇ#eY?PnֺJ iT7Kɘe؈2+#];x՛ '=99]LlZ&1*Њlț=5&jw󦤚]-|B !.v107 d$!ݫG^F041$'/GwqwGꩳ~d' Lt6n"{:zo.o/MJ^\dx 9-0,cS9tL|j[Cv=_{2 !2P*T19O$}ϲp:i#P[nw}!_]=DY7L:|pHҎs4H^kTDׅ/ձFVu~"2 z'*bTNI1XD*+p7{܆C <v_qI"m0m"6ZK)\;ЏgwBgO-cRA(ӛpؽr]Ç({vLfdiua峽2ST %c1=_8Z~#OxhE>BLE)YBy>*d"ɚ bߋev D%ͰH (xʻ|um^|>"1cjMhJOer,ڧBD!P5U=jfmMxiÛ}_yEڳWPT%~49/>ȧXY^R/t=Q9 jh:-IY#ʳ&Nz sBWtP%Ʉ3_x+@૊mzDPblaS 8mfXGew CS}3ur,=>G{\"U_%;#fu \G.- b, )n(zħóeejT]vɍѦq3zTiq޳*^7G4hI4aLØ֍/Z[u&jqdF5|ܒnCOr07A.w'nQǂc}3yY~>Xg(_Ey7C<2$ӱAw&ڝD-%9j 5^ 5_"tkG:=Fq{1;rDp6h=]@_c,}!W>bgn;sr|f:ZGz#{<RAt.:BOF6M`V7TU|'tvqեʗ_'>3Gue8XYYf4Q ӺfZ10?cAx-B]}uczu5\~ nNu0x8pdž"q3uGV9/x/{sDؘJw3HZ<.WDW?!1X99䅯I&&,/᜜~1f}&M?jDSψNsZұ=َQם|%TGb;+XN|7nI=Ĥieir9OO1+*o_"'@l( 6 1lG3; '{^ ׋Q~G*zͽNBpKd@,"}3GYy/`~JEL-fuCYa,Q5ie|̷#rTRUrwQwz/z͐xW1F>U^ʷQ.!8fqw:Y[]g ,֓Obqx{x#y _G\}퍌 , BP sG8':YXD@3C}<&~1fB?MCy8&ݏ]NJ*j;\1H#U}sN%]zih+;BǔϰX{V8IqqC{"9*gɆaվ 9cr3. 73I $LY~@V%I˵V,b( QXmZcv8:bx/ U3/ v( \L,Eq x@GOYh^j̴zʢGVyܷ ]Βsu53m V';CKWDļXR9Kxf}:;ˏazE1w<d<ʂC_fA\/cIeg6#}5ȱn՗\εxTCl:W': 떷"}gc-G?I3[k$L6idrfNǥ۫ :{v/}ۙjFUv8iuuՑxg^</c"D&1^x)Or7}koLvoe۝N_<oޟ׼9q/kkmP5wZb)^4r=܇Y8֬l]KƣS!E N[5tB)S{.~;G;]g)'Fg үlw˽ibl 3A:q13dgӵS B;߈/W^hT4M[ 4ʷi&"/֡4ei0 +AN~ӯi m#3{Ynyf .H'#ny?Q5ʲC|'to }];:߱]F_Jy}VJq!!e᪮m2+UL\&R!A-ha2QVQmwU#ϸ?qbJ}YNI2[K6z:PJV /zɛx?}ELmC9ξ%41wB&6uL&c.ϣY]ihJV fS7𣊕N`y翘_?Oxo~{y3ݘ1FU:\1gH\wZ=n]I#rQY;ʞ綘~)?"tXY0z eM1 5Pɧ9U)>;s8!峠ao XM˸g m f,y&}h8kM;8`Rk[%p_9$? 0yGw N{!,\F˨yڝi%aDS-d3̂Z[9ұMolƒ\tt1vrc|H56OR\9|jB\v(p8Nwoqx9d ȶB*w0NQm-冂wײ"c6:gD:.>JN_g>rZ|id$u2ITdK۱c[p %5e񮑻)?x?GĖZ†Z_=G4Qĉ<9W$mYCB'm#b4h]I[Tdeq^s{2r_3?(Z~mlN!#ZFdž;#hvqP 1/{xre$E (^΃p督t- 7ﻎy> Af !3u.D.UP3kk oz?':PlۊSGU* @ H]pŨ[<0bʏmbގv/3= )˸Ud4Ϥ)&+d6@leWR 4SBXce2o?{z2pE1~H o@Ӷ{&BedV;fd/VHd; 1hl|>cVa(0E_r=,a&>r[$_rW.fTUDdru>!sΥ2f|ݼ++HU8[O98iZ\qZBeJ`Ol֙K_F^wdiBUx "/sGM)9ysgTT#%v=u 921 KYٳs5/Wy^ϧ?Un]u-w a@f#ݺWP;}L[zFYrJΟQGڳ]!9R?~??,Fyo;FshPl-n:-h83]!}P(:1% -9qtgjWէ`ǂ`ЏޖNyn8[3uyYn~{/uv#~"Dg1v>s6lZ4@ѝ/!zx̍YDG]XٻkAܨ7ԧ뮿Y>[;NqN+;_eGz*nŁNeHG 怩-#i)@k5{ٌm<􇾟jy;yvbJ2䯞Eˈ&h8|g=լme˲'s@N=o$29GUQU1}c:[>a'3)[x菞^λR1w:.bH'(/WSE J['>cENzTЁhXtaNAѨko.bW#FimO ֧BAc/G/z[ħmSU #" TmN^F9d p];!ž;0,Xk0s$œQMC=qza_^Y=g?ϱUp gsہK6,~Go1=䷣ˎYUGc4mg{Rة.hAt<,Nw{`aQ$)J{&GkX>*"!cRdT>2oi{"!By=4!:[Yd0&EZۈoZ>woyɯM8e(Z7[FmMXa5o&Rh Up|fW:^iy3FMb#ERf!6K\ ιmx8(^H/ȡ< -,h{ٍ::1k{Wx_~'f!P]6}r2.KQ*3ô뵝|'yuqjfeI Ȃ3Z¼ܑ)FQg T۶&I^Pe\N'If4$E{QNڋ-Vϳs}ϻeR4C뵁Y@Nq0*FW_3zgBC}W+~b` >JF OQv l?+xYEu";mXQ#mӲ41ZYᣟhe:tYωH܁Mx!;Ʉb2 /_ڹGGZrX*> 7+>^w׼ ~K۷1ڹ٬m *YJib|k56X>=Ӛ~ON9i!v92SM2YEYEe}'yu;cˢGޛ?1NDK+cOLdHIMAe'N /80'~2?&HYe( {nH+{JS}Q^ئe̅\͹U Q=M 2D5;g S9JD F8#-=ٲ}w6-BvMpH ra ҶAv58ǹ\?Y|_ڵFL>h[h=,q[~g mde˄;գ@33X}.}8jg&1_u/ .ܽv}MibEj<J8,:PX4GVzKɢC???rpAs7L/=!-D tWà+:OhfnZ&~?֭[!(uG-F*_7OaXshl1ʲK)lh{C0PrZAC!7GRTo9H(*~ 4:z&ǫ<\J.? :S>q]r-L/Es5ħ™o{7O|ëGxՎ=4Մ6;au~"7ď++b 2*GEYPdC.D1pFe#m*:}o&Bgu␒He)sCƶm_i{P׍*牵V ߼b.?!;wm#[CZM-YP6hM/;{gJN Bp뼩*.>\>O𥯜uVVػk' Nr< Dv /OY[ߠZ8"+$$YQ U̎ᄽ'?#ޱi$:1ⶼ!GdIEQ}'>Eo'˄YKV+ީ^g2 K$Nw7Wq -](c@e?sp%d[=1- u5!WGV=/#ؾuEAP y,̷YVG%*}."e&;w0y >/o27SN>=wB$;6W%Eg-|7ιgh[HI7iܕm uJ9t$;\ `<*MHIsĿ}jVB[K;Fc]Vf٤(wڽ?~oPw$7{f^}s66s:8%r:tDstM88r Nd'#q޼ޛ?VksVRUJ*ɼtgE;ukIv:XN텴q{3F'BW9$JFtr_>!,UbifIX}yڅ4As%UtՉ3~WaBRpP9Oa \\god=c\-0ŢI#6mUzpjM3aenĺfVH &&+nw-ʧ]~ktãRo6dF埡vp 9t2-z"9XYl#Ǒ`~sM6cy!ױNta*L]q^eXȹ~pM%7ݖa(*=&3 :,';3am.ktE[gsF#<`ak-YwUɦ[ *\j X O|:$ʄ|BVH$t`Fo| >o/_׹+OYCTR)m *uGO"7NJP:CK.0z^'3 >\+zu9s &vo\Ϸ皔V7/ݤɗKK7_$]ݽGEcnE΍7i?:MPLM.;уq\#'H>4W3lǏGlۺv<ι8y!*^5@3D| 4b:Q0IT&1Ͳ,c|kMzS6&jӱ@2$mQ\ t2Ѻ1Yín` sZ]JS#J _ pę?9zћ_;&@ۥpojSL1?5_\KkkVblsjLN',y^Q0W?>yZK5\"$fI°Pm+ggŝ>X*L3}}1&'/4iF9iT]"ἡZ~k5S=#"5(L'.tF"2􁮹fNl߹@oj eb\=s$kd1bP3Y[o=ݟNnY1e3rpO(oAc4o(zE^m]y5<'9v>@ S9VP0X9/~1LMɮ琻T<40gXI+93//`aqD?`ͪLLٌڋgp7h^7i?:S"'dz^|#M1i`3jpN䈃a<cR_1mPSE¹Z,#]%W?+׮.J5ab&xY!5~\rytl4!I-[;K.-f01A}TxW8[(w,pȡzp4f.13 AFuȝ|ps5z&~ɧ,ȢAkIWz Góc8@6]E܁xxJ^"^tQWS:,SS—: piMI,Ї jF105;p~:{=N ySղs8&+9Z\|l#s_>%$,DF,ϩo1}(VδYグ]Y{ LK?ď ДB13$"i{KʑnvΆ ͤmzh'JZw3Qė2MP!I;v 材G'>OW@Y7\t)Th~ozM =#"6e12`zY:ue k]w:_檢r>fkDcgوѻւ,M!z RsD)(?Z&&]&OejY=Nw%–ƧԘrE33*woRHO#WRR@JK_sY s̭(rlfubAΞ VUUM]yFʊ+Ok5BVWuԵجmvI; exP晥ޱg<lC/W a>a0Y˕oԺU8j]1%#*5̮YE/[?ŅӫV27;M';ՌӞXHF/L Z+r3yk=0Y=ԶT&G2 }q7p.W%(,R|MЛ*'8(;y<9Sz\M03αw>888M).ѺHnG“"ϹuV8șg x2qƂTEGs'GNۍ';6l 3+XexPlZNU^VܥЦQWt'^D9Cˏ@aBfd% {|w[wn?Fհ)}1gŒfJ HF{ nz,=yѰc&pRBNGFNd4 d @ƕ6:+n/IIFq2(l>^ukJ_.>^ = vť{mw vצw]A.m7,eF E~ógqNU#b՝E݆])v)5%nC9|or9~`:nzur-̭cŊLʼnX檪er3,hY/ܽ5\w͜{;_0Eު91Ҫ$?!1X$F\Sg9UkMn3~wO?w$epzTWn#О8V=\pe뿾[nln7)?L ,+Qu,"o25XmH#g\ǧ>XSYi}Ϭߍqo;Q~ƝT >@gP9!d۹Wݓ=6,+m`O:)eZmEOˇ ƄxF}(r\3뮧?;MBњ\%-HȞ4uR"fMp//NAZ+,XEU e +ȳv+nc߽6<>Y>`*7ш3:3[>g_Cj,+ݍP:YScg=>.O>lXA"zV-mEz*wS,F>) %5A5b$ vsܛ>L3 i?l*>q,kZA?dFʩXZ >C|1?;m7ȷ}zlr,s+VP,)^"oeCXtVSϰ6c"o{2T jrghhЛ^A1;˭7m쟜w8۶sܝĊ҆Ez4:W8O?ӟOWkOvV¸c_xMyQqxgt2 1-!pWuS>Ys~ͫ_O{-PsNwfoujFh̡GBԩIxE|gs@JI%4ȊX+^*Ec!1SvNzū_<^A=DBћPn԰:elܸ_se1C3&i#vظ~6so Nq7d47ZM$R ϼk Ug_7{reTXkĝ[pffr 'xn?p9n'44&WÁQcW$r)΄;0}7,M/Lk3h̔: r~XH#gxtVYK!2_jgNM[t#?YzOO7m##ƥ5auv_drFz:W|śQe6)ʲjOfEi"` גյq1[mcg5:augkX=6o1)܃r8T nu)b a*a#홨h̬*6Y>V9`}lㄿsw\ G',s4dxm'+,5%T+WILM@@~6i΄v"jNO+Nxuc+(`vdF[z+gtӭ|{g2ZXdf,^Y&BxfY5qYeNn6.R>/y'<cAYfuUu=E;26m<8̟>5k1hB;IiѻbL Ka\Em+.1;Xfiff \ބdȲo4s}w,@^ed/uafCUdkړ׭7/ "SIsSΉCYn1y G,^][3Ws"*E I,,lܲn3">oپm᭥*ŕm]|X`_,0CyK]y*ɧZ-%{ذn2_ɜ؎f|GpWӟ 13锫}3eLh)յ7[nkn{8cȀ7^{GGpc/~e' /bj GCq UӖ/ k@ iר#O~ M[ի*qg*`E\OKתӇl!|ЀV ^K&;@/ M}!mcgQά!,U혝bvn!bŜ.T=\bOCY>+Go ͸7_lm 1шQh\J{vbm W^sk|?_tf6RcײVr7iѝǑ9 EZѲBh )c<<$3U};+Wm<jZ߷wf)cAC]f*K:3}u*fZu1R^q y}+Dʔ|BG^ <D"hRZ e㷡wЭ]X.|9h~R[4R*^m`lȿ\[n޹dsYAe imBdTZQ^fd{ ʠΉARFL,4j/ȼ2\g,rnR>֓p>a$ņZ k^VsȞ{0{Sd8cu%D7c"Lnr9u9h߽8ozV帪fp+W_wzt㭔ؕ+\IU>Kʤ!la2}qqb";2\ð&S|S )#DaRngilL2 r9obܶ|*6g] OBY%v^]L@~L3zӤyA ȳI?E ܷPIzo!Vt Ѵ,Qy'n(}+t .+pۇXӾqc%YX亚¤$mnڵIt w)$~L1@nNMoIX.$< !"\t%;31~ܗO|SdVLfI8TWKbodEy{n:^.>SWù)R +xmEɨ75km]C7'P;Ye#h_>}G 诺3Ü+2F/{ gc-usjtGW_uG:o E?m@pЁ{B&%*vlW^_2ZXX$~)/{g;!&nC[y_g j8~Ʋd]VjBpb.2Q néG=>^q~ kV1iaad֭\}soõbjLV"4Y Vy%)Ǹ虈x0EF0&+ij,.Ŭd6}ih¹tx~ |Ƌs\UQ͓QYl]Cʗ^e¦PUw/9 ?h;pw3}EI*uU8q-er.犫W] ;OO_5'L?/&8Oxxt!c8ˑ-dd!3}܃*4 v^tI-BW} ff y+WՍA~"bk)>+'x,Y\pLěVY2+gXN ^V,Y+`>?䊫?UN9d~`n.|ecAh|L%/FP`{;xXuu9/>%o,X[n>aHeH$Z 5O VGS~l`aqVd$L EK gxkNjLmzcih*RMC4|i-kLZXa$Nz"@ k;LSIN]=)Ipo0@,[6/?nBjjFYk%nSUXFge`Y-fd6ckGp`1, rt 0.%<_n4Af'/Tyj UN^`0cn}r,65=bHiWـ0a">z37P87{~o6ǎq4ܭSY0qC\z艹fuy}`T1jTбWn@/3\p}o?\qxLg 6[!Qڃ<„!L-tY;  O+^vڰXz9G@ݟ Kttd's@&n#~$8u!J磷t9;"yf<齚gl:񝰤\ƴ87Teu:(V UMfG ѵljRK'Ca'Iƴۗt,0K5P2_jܫ1fq<#)\J/ool1R?‥.z%\R;"~܂榃(DiLlj|jeUs7erW%Ӳe xj 3.NlZdgo~91dw;NI%^2 K˖#{ fzW0\M'&cQh^67Q^g2ɣ5y&5rFh57Bo {tGu _ؖf"e9nǐk9pŘWl# >Y noK`L^w}τ8iM3مQw]ntY6~At#;q7;3L*'uKqE"xA_.5E,nLM3z5Q ClƖ[`qLM15Co,>/γm׿Ar #ݵ- /eb0 xidဢl!314};[G]^wpFDh(wVHȔ# y-njvl.taݺṷ\ yQr\8mnݴv<&q%GG2 %t#Ma{}&++W-u-gy/,Byt7̂19Wgr-Pcf'wR`ļdȨ_PkyYf3]Luh냞忽3;qzN4mDwA5b2|la {{NcoKK)]wbM@{u~ K=ӜozL^cNŀG0 J(sݕ{'rG0x,sZ~W;ʺbo!ߧ73SW%~Oz8_[ 0EqQ'e7scy:v/D׬Vd-Cd :H2-v׿O~c܏r45iݏjU@r0_d%1Rd<ƥd0ݓE$7H;:`8 ,zf0>u]QWc|e^#}FPbZ#GGUd^XAyկ⍯{3=q% VZv;a"L ~~K aћ1 ҂sI%Nev,l̾l/uLI(o1i ,.ؼmWzß/n4գAkrm|RijCۍq&yax':.6fc1f3SE~;"2~YQ~mC&AӦ-w #s{Sp3}ћigb߽HON+wZ>iވ< =zBѐrNxE#M77x*XvaiK+i%'+,njko}ym>-?=bvvօ`鼵)u9E78%5'ĘYg8%jҽ=`m6~9c&Woozcx-YF?.9Ԙd|hFѲF0*Em t#&Gav^.ݱmBob4qXI@]))n7 wtӾ#;<אp-@Hnɷ];r7 2`AFvszi= NZzGµ.n`wiu߷ nӾʉLHk=%d#xl dEf15G^(w]/S{!c\9*'d\/;iPCqri_:McEN2 [xvaW(' *X),k 7xuw8z}wPR[G D^7WK*.q 5[ҡnl١sI34ro``JÆ==}\砪ŀ Ylp!فK:AJ>ns7~74WЂ 'luwtӕ`+2,UUbvγ^@(!❥e@YQ ! vc,YO>#KE=3YȊv TT_:x4'_SpR=u 5" .Db@D0!/ 1y[2e)\%,H_q1oQn{-A"WiCCX-al-y<}_;@G$Ɏ׵?`ƛn/z=?ᙬg?[*gP `W`WS _b^1PKX|#pAXsX ƣݔx<<Ϧ4YɝP鎦fr_0< b˸{ZScs7ذ~M4B4 3L-o:IzO!/ 3{177C^ױcV+{PeK$=}@Dq`sýء"'C٩Wk=29.0ė#29ijiRKw40</o]_ÂxV@)Ec&]쒔1!FB ԎrpC}07r*ըVA/'YI>;Q$ =D:<ő=q=߶gx_ʙ>QG.z/' P .'>\+(֯-1'跤uP߅Wx;ɱpSW"wa04g8E]2{>{0Phy6p>XGހ |c8 |d Ph~৿~k72XZwvA*z/ Z+4nkugf2SܒߌG B5GS6f[Ǝ'ElۼEq],³c^1$Dij!WGʼn'=qsxn^܁rړ\qI4&,#-K)3VV;|5m;8}FNZ}{AFE Ŗ,SJ5 "<U쒬eu.X=FYl}Wc(G,\vaX?mNu GfC$5PX9aqY9y!yׂ'rC=Gjw_êՆ+2VO,s07Y5X9Y=mc57wozXfP`%ߕAC0P2s'칏gf(,E!8,FWƤoԽRUT㊲)+GYBY:ƥ*U-aChn' ' _"WIe ([{!W=HF\ TwjdmC(ǤwM]t;dkYY>*K֮ZGƯ~%YPW{YrF NhiuQPmL A%-wAS\+5RհogCme^`M R.-,6e`z{ĈB+ĺs7$FiUM~FyAqvp2Jd;vOݲmKb`ùȰY9?7\'+u嚞Mne b4B];\USW{-rG65E6=ENeYRGԕ^Y\]Gc>X(WV)41~ }]W"ø5vE%ešGQB'KW[t笞#x\"2"hވȒuM;Qq`Vș'gU_ :~$0RH=s 7SSӜ1)XTO"|oijWxגuz^VPU2j:ڿ@p42A?"<6#m{n吻݅[=KUUTueP<$dR$Lwy΅\?:aU&VhmlN]cz69P臲znm"JyjQY X)y/;NYVjYvIX!c9K Ru%ԅNK@7I50È&etr ]Rgp.|<"m"H.!`B(F LM ' bTmP$\ҢXcpq9kDžmq|cg->YXu!NsAY0a`-,훹9LI2M Z'@ ߍMxn-8ZYLfeuks.>kubYq7MQ+ӎˮ9Sq2ME~ބّF#Nf E&x\z-wޣ=>ϼ{Ǟ {לG }ל'ܽ[/w)xЁ902lF# M k)YCu{ڑ#ix6HHg#ݒm)k$ݎCSV`{ D$?6!dow!lW$0V~O:| \U6 I8TT}gvNc`MRZ<)6 b'J-av5Jn]CYBzɀ3G tԂidOg|3x+3L#jNaNJ8OβQ{._ĦkD6=+Sk뀦dr $z!Zs|M}7`b[ {Ԙ[1g>>|L ?>lW#K %\Vq`zg> :ye K`W @Nz-eT7~x}yܓr}T'v8uwmv++T`7%Xbu%W_lЏiɪ`QCҤK2Mȡeoa2VeNKRuUw.zхұGޏcf^wLLڷr<@精gꕬs~xx[c~)o&t)e4](ze _WJvuFV&%eLebWM0Waza׼_= dVvʋ]q7<21j~?y7k0 ȵG* YxMcɼRbW;}OSz֊!;$"lmSh>'O?-܂bSyKgB(ϴ#ט\=-]k+3S䧿^'A4L-"P>Lʖ*hcvuWs._]@>=bY\qn })ԠiT' R1_ZXQ5a?U)ӈR. ;7c {mXώQ?>,ƪ<2MQ0Q9G&T&0OaH9vpVqF gjjHW 'JxyXoMEyWRյ'/"4&s${vjy9}qN>,SzRt1KwFM@ HwɎ:ȟ^av2VIhցWy 2>xh6^W>Fu1@/Z(z1e]{o] ނnKzr-.Ȧ(fW`3ÎMrɯ/;9=_K9H+(VagW:N;Kw>"O#NYy$Tgu#i1S42չd,.@LutF.1`-;8xc3TUENd%)n!l&[,wErE%aװwޏ[lQGBC%A"aa'aSBaӑd+,jkv#tȒs; 'SvLLwOϾmp5|]Vup d+ev_~t6R\Ăꓤ|a<ʥ&eyca<1a,uD*o[Е&4J=Ix 翸O;ުq!>:,Fkr]SլY G>Ʋ4e }B֢Ku3 >O׭w4-D'/vՠO]2ɚ`M0yw₤ҫP+c\;zE+yr[APitTdsG+l5I|@%*x@+juf| Fܞz=c;')_㼜<5x`*?x\Ց52{ui ٌq7Uw8~nyaY9G'^qž?1mXA B il;F܌eGSw>Dj%Ə0~tݠVO"s8hIl5%̔%:Rev9%w ]{{`BL}%ꦝjuaR) wqD2`3qGq҉3Xj87*ƀMtv 4SL'{RszZZ5z#.o1A>l;;/Ե>x<} q .d}6׶*.-4<Iޱ]itjz"+J <}RHk~rLJQ)ʱza8.6SCdAIpSUut`f-J-S9nTYjgt'ӈ.^1"%r1ssWWVàu:ͯj |ka{PBX+aGY##%%"dޑ ^w{F\nC%i^VMutz60YC6(B[IOˋLzƮk#''x_pcpc![qŗg2ӫ{}߫nZN~s9fcMMll1=uRی;F?Ե.S=QW^]M+M#vGo*BѶvS͏Ȋir؁{1eaCÇZu\2~H<eU133sw~{2c8X`ܝ@Nm_a#IgrTxLNO_=dzs/f܇bj=]K>37r<.Juy 4W4{[A |H6q',dJ4h%&Aj̩9Mi} nCzdCoI(S4|'@mcڲB䋕,#~_9>f( 5NO$&3A!T&6)WӗPtN3&RM!-}X`7U ~/qRkn/}> *Kٌ,KS(W³@/çK/ -̟"aYXyb9ηe=jf)MvIƭz܁?D2$&]]|j"BE;;x 8ᶭ@&CZn*6f* Ws7|Վ!m*NSIvaW7ַ(S[vpW J|i&1?G@' @0Ɛk'd*{~i>s̉u &D>" 1Oy0IJ~Rيy"3Scq 8`8èd4Sɮ='&0.te+[V>!!ع.|Sco݌: #M"GiY5)I0[jф?ᷩuR1,׾z_SUrj1J\M'B}D$)A/&eUZN$tr6ϓb9KaR)tURwdw9twx6!~]|oN3TFnN馿+܀D7yHX4sqGFdY/TZ;iQ$_F6D%mu*Ĉ.8wܶi#c";+ bR޷hDƩ<䙥j>Wy<ۨgo{-6t,oQE'ּ,J"IMwOҙw8ۀ'tOѬQoh^0O]:nM0KNT0VuEDk n)~wѥ׭VہH&]C9A,JcF*h< cjUUS77,Eyx??uw^.#tQTS^=SȢOAmzF5OQvlQ <ݜ]ѩݫ'5R[]Hex='[؉Sx[יP@1C:$Tۅ.B{G f4ɂ h~J1r)Cwg"0Lf6쵞bEMS>Еҁ aXfK^銿HZ6*/cIR] 7KirFdڶ?Cm8bH6GJ_o ^$[?^}p!8\ ~j>6ӲɂkȭydZ7oN6^A]WdfaIA ]B{nTߣ*#ڇ<^dŏ“` ;zmex525jAq9rk齓vyF㒣ޗU},lDMEa*u*ҷU8 ye^,PCX-E9U76~gS[P˂Y?&cH\USnqE@rh3+cCÄi~_)-ވ)Y-W;X~"H"v rmO' d=-G:oѼH%V`ݴ'hj .;6eQ)cںFrnb)cݢ>䗴)uO{(C@v.u& ɒ~RH*j]O+ϵ\F]oG+t$ >I3^1t!h kh1ꚾ}WVx AVŲu@P;aaD1Y1e7շ]zRj%OzWQS$y37RHq|0ݎ ̏s Cf 1y3YW|g6{z`Z;֬zWT6*E>%rj q0d3HtδaaMT BZu Iޅ=1"3 TeeeӲYqQ4L|||akr9In}>w^BAĨ7P`yf9mvp-W3pz\c ׸b10'Qefcbx\}0˧3OZ=B]CLDŽɱ~-[1잤#nD'@(?abQEǪ# ]h}&@w/JV_#O}V4(1Mc y3=Q/.k88];&aJ/oX ^+▴+ {A:ުД)GYS=QM9׾'<2[9YU>R<Gy>\8 1`Et|Ï;zv\my&f.u@bNi(Œ,pjSfhՅPU mI]6(\nm=%ud, nxi: mD(~D^nIA&L e솥k/* 4g08ཹS9F%Y f+J"*zZP&ћ1ʣL2~S%1XHضM[ٸi/^J/Ya@ɵɽ`xSĉ̶wā`$&-H -YmCTC0tsүA߆ ݰXCnuPQo1w;o~˫89(.-ሌث+4j u>uVr4Ok.]hxț*!aB/M##NZ/Ǽʁw9%d*VH7w0,r릑RyK 7,7WO{[L9agDi4A>8އـbН3"kdsC9*1•]lzj_Z,KxiMM݀0I/E#Q'4γƐ mۙZ}38(RCl3hYa%,疲|]N<ƭجӰxVΑz]D=hϲ5O u $oSmEn\R:tB0!Mif<3;C)刃d<.qz*Zo 4':LDy1v<Gh6rnXv<3;jlQ09YJա CD4:#Fe1pIXd3vP h,J-!A6Z֛Wg^F &/hx=K@mGzIkFt_&x5.ti:4јh>Yv (fBנ}dE5#[C4\ _415 BcoLOkr_#5n=8'rJvWJ Rt#r{O0 S@Aބ P6pGn_E?&M !LY#ְ@9־SnhXMH#:,Meq=sE1BȮKg^LtE[D7)E& -F̭o})o| c*odQL5r=*ہjO.n 7orIy$~$vyI9<^NHe\z2~ɍ cUaUJñE1X"ffvҏ gb@ W$cb M&,;v[7W"K&tm"\&]=E! #7Lw, O?HФ; д,J[o'QW!FE D_p+;Qآ1m+G{[Iǰc ]5fz~*E{OyZw1haPO[&:/39űGuy)ܖd˜ZwJx=]ARA Fk@*ԧ.ryhx08{sz[09d}yOjQՠ.ŪtG,Fepĉcȋ7I5%֨8 ~iӒ>szXBָ, ;z *{E1QãH ܎|Zwh1³T%h'an6 =%*m|;W ߆ w󮫬i.  ǒrI~^"R43VWx8vy/m&,3V]gzu)Ӝ &@"7 n1T#~>{& Hr˟ys[6i*.ƔapGr$6KއTV fb)Wlx)g,1cg=o|sÐeB 9yw+3mVImfxr$X6&LB6XҒ9Džd')]w7s{7PoԖҰZM38yH[gn})^֔} _Ki#y'u'A=Wc3͂CJ{L 1Ø{$vsDi'z[L ʺxtR5`1_JZ2N/4x44NtBf9Yaˇv Byw>NdyrWЈK{˥uGDudQ@S#g^Vyx7.bG5Oћ=_osH쁆󨝎dړU5yYŠ=>d^|c8fdHt' &CG"@]an;Tt.oLtx7e%]\S+@9'tXZ? ihGBIxFRtR/P0N}]}< j A.N`2:?Bw?20&%ǘ[x?=|KY1ݧe&d'< m̻ˮVynHgј>3~=0㒲%LvD_6ZS#:QGǴס} Pե++0YlN9`8o~~XXa"R~Gc|X EX :y]K%p4~vP*_脖\aQ=FF+CLrO2e|8IS1 4ór ;JeߤC%Ϙ!\i&]7_~ )|# =~'zu, :)H{ss^<, zR[͚}gIK"|4ɄbHRh07}d[$t!v d9x2)*u'dҬ#?mR> `h謕t.@n'g<ݑ Mc#3|'?=N8hD'F,LwMv*:*8˘s?_.?Ʃo`8\MNLF %` gE.'o} GȾx PK&4F^bhQ/ܾ_N^,_"5\t@ܢ#hZu<ޫt!aa,FȬ%<3l@sNzN:.nEl)> >?AE6r'ՔCI󑒝he/]Pr ]6΋{e<|v9˚rX mٱ:a/iLSX*-D%@QFL7FrIBMAЮ q M3/ k#M]GmA`.R.6t-0Q.jKU ydOPk%;TW3t5w,Ԏ{?>pԕ9J[9l]ckz5(ŪG%SXιzsE\](GWKCg Z` h)]T,c/ñu󱿞GX,0ec|-rky=dLQtfxէk;vsE_gGI g,$⭕úXd1m.yV-ӑIk%$7 k}"u['y*^DV+ti}HgB1^vS*Z'߯7_z1%}3-ö˦q{yq -N~'SD<ɎL[r/ka_<곣vްWtRvAIB]}[6'X78@*X!hIP*AU&C0 aKTu ar˂f!Ѷ)& G]hBX 'eP,)5ƍ~HGX7Wyʲffz~͜mX+D'K=;aqe 4`qѸivzJ7aq'pY/anh4j㊪eN_x-Bnc TV}_YgfgVQ+#w[(lMN-g:@)+LDU8Ei ڑdۜY 7~ޗ),H$'q35PJ5Wx_`* ݱZyy9zU;azj''3b]spRtf:Mq*p7^.=^B<@*rr^.MtqjO`ԫVtV^X9,^fN 5,8٩)]wr~OKvMV&<Ȳvz><`=߶[UWW^*x ɍuHsifǁ ) Nĥݛ51ٸisokх 8u)XkɊLw{PwWU͗?Y*3"d~QwƊGh&v%4A]v+#S%(=J.mĈu&p<|1GPv{wWfYFY]k5`y<W33JקrWJy,6ļq%czLo oϏ'tloje 7Av:!IYfNѨd5sw:ތ qssZcH7aaA. c4IRiP/cN5v{)! ^BCQzBvekoo}-s1ctbJo"7-{wTocJ'&=܍_q@Y,޺N.GDUUEAY"rxa*sgTZegVr\?1؅}!BhTIִh>ro8XaC׌!rOw>ph4b8NiD/~4@G4@+ <spgIm{Եb.c-ɩʼn; |CX;=>.,Q0$C` X4{k0.11ݦ@,i]J)cR5!LV#Ve/4 zNLK~=u]%?Hz& 2KD"皿EB@PVѸb!O|ʗ?=6P]^^z ibM/-~$z52]uWyߥxhW-Cy̑[pUEQ,k. [6Wv%kl&|ٝ2=hegRwYu5<eڂ6e^Zx} Viz9'G}qu3aLa-b֔hiDaXP`YVrЂ̃7M+l$)hnQE:-c'hL THS\DzMd۫^Mz1t urސB8_#,+[Wb~ \ X43h#mLhbbQ[AKLk %͏2^}yA/g\VX an]5rxÄ)s-Yex}}wlg-Xo>ȰעwDƨ$jzWcfnŀ{޼Wpa2@6eqQWʷ|Y'1vyQTMK d]e!pbzE֭Sʲ92Lx/gRXʒkxfr/?Dd6_4Fq,MP%MOy)np%u=[j"oB(rE.tn"g2  ڐ~gLKWm(ZNtVs'k9t~cJm`r}/uR erox1SӘ^+ʉHGDCn!\ "x'EzcB'<_U^gEHI(WAmF0 W{Ϡ}N~۹ᖛEuC8z5VH:+!WӎF cT7* X#;__0*+#A5OPޅ,ibASۛm '݋8 G=_yhnQ:2#ե :N^3_>{)fg1O5uTmYN[Z_^JhdtgGm 17IU6GbAY 9: /xѿ̟l NvLXა+~965/NmyE43x!j]vn{ݝ}؃乜`ۆ]qEت(c_El?'^cLkYBЖX.u&ί$1cP2]@wG dW uvwkU5>0x E$cuhQC@@Qrol'd{pn> #x>A@n N|xw ͚0h^Y0>@ 79u<=6m\e__WE/8E11۶嵘82S?Z1F#mF&U6AER]ai#|3Wc<%s?%N.& nh%7 z,._P|\:OZNah>PN m }ڗDY/Ty=xCE k")ɚ0'~0GO O+Dwkl3ͤ7!D@1D]5t @<&E |)0*AU`hʶFgcJ6 pxgLWuڲPt ۸IdŪƄ]Fž x5ȅhP"!cq|w f tbh&H/1m}+b @@]hyͻƟ /@yHcǹd]>q[~uӴDd׮{0mj` >O==ڵ DD DGA]iM@Ql:p_OO]i9U U|P~o_6Rb_e&Oت.CQ]ƑqOlVU/>Yzd?apM5U|*Oc 3tg^[lmIma?O}Nd=H;>ǂ[HWYBzq ( `dio<E%.KԍL\ ]}k_~@muÃi^Vnx2L6JqFpW: BjǯʳƿxqTZLd}cHIϋ4 aqq\}<V_(^rK.JzZ6QAˣ! gm* &~$:H<ѠZ>#0F ,BجRŮ?Wa#G ;q};gzˌVzv'@EP%Yt; yuOqN]ڀSGg52 K#4ԙL-z`M= N돥5L!V^J,oߝa6% Wq%@qcxb1>{U[NV`a@!j 1JȀ4`Zn;BħG `ϕ K5\8J0 6@~\Xj]T}q:ڢИmJBcӐ!Eׯ{aFw?LȳL(~{Ϥh ҍ[O? B Hl~^";tXƭcU]7v}Qhu7p[іX~N@[O_)j~~s(7oDSπ $(Nq6..4sL@]MJ,TzV(VUf;(< c/x8l{|h@3!aC5㝋\&O(&#(2 rR3V[oC@E\qʱ ||,1l)ӌG?y| _F ǜ6`VEyPQYY]G~ 8|'0&3܂,]ғo++\mx ʅфP.F`@/F łBB~ E](08'h0ߙIhoѢ.%>K0Zr h0^p-:Si.0('qq 6hL`a8adRa2j"<vtooRiݴ޾]SCa'횆'xpCr]K?d>A8q0/֝vPo=f j%oGl9ԣPU|o[$WPJi5ܕ*KKOkp+<~bӑ?~#7k:Ԅ_UMm^˰F;G=Fh`ӆH9Qe4,?L6l׿yzjPP,&Zʛe12ejGIY >n}^kW=z+3Ḱq@8z̾. p'O)\ w@YF@B0wÚQE ^] f]X]FC b_xޯ] E aT7 ydqTl,g N;lS(VFX9‰}ʗLdBRʗsח轡|< #TSpɸGc捨&vu2qiȻcT4X&Q݄YUaaa'=a}o^y Fԁ{'Grc'Nm8q<2l$)Р%(Ah 6zC/KC) d<^cyXϺ5Mx|[j&% `: 2omezxm70Qm]`1s }I-< :xtrfz k%cţPDxرܧI~k._I.5'9,χ{Q B;g=4n^G>Ow; 4 cIFOsB_/͑:t\]^ZJaC E㔶agÔl=(qy*}~*oCd@S9߹ afTF%|!`r7{ A|(Qq9ƍug]mP ]+QSI;F/06]WZL}n?h|+X;,U86|hBVkyWyWrfk" \ eq WQ(( ߻cؙO(McDqhiM>$8qb#!e__⧯y1500ZYVWur3u)rf2A0A,PΉg'_7(kK$K~H;|4Ҍ|mUIr-Pu0$/@Eadnpx'q[dpn3S\o[V<', |7P:l/tup%:,ȱ7l:CU{\_?.0ZYEZ FTAqeq04$D];H Q1.< p+v%̇ކ{*ƣfBEЧsw=,V*J懶'9r:]g\kڧ{pY`CJj}^g |-:B(} 4u@3QLQXBZcx鏾MScV7pe -Ǖ*} s5 Rc klٴdzj ɈuFURNl@S450iMV@5#*PU>Y0LYtwU٭F: N6͂q f[Iy͌ iꗻP˯u3m:Vg`zVjtEKx#U[y~:Q8Y!u*y;H ę?zZJ[œV7 F%/_㗽]AuG<)VP P0m؋nn'$(N ϐl4U$4~NF.f4qR`<+|\O>IX.avtpĻEx(nm~rȶfvL'aaU0E!wQOg 33 ތwb:Pe8NhZht!lX\W^[ ڴ_h06[~)ª5O6 'Iy Ż=u@i1 ^.zj_>UdvuG*ߑ(8L o׿u5uLxAT/kϵܷ%a++_~*ณOBEN'>J4h$2ws]SN%;:ڝ+0bDBcFW BNNůUX^Yg?~Yf5F^k61XD^rS';u' WȘpcT4(UVXYHȉcFX.A 5i|c09|[Lݴ NʆqζW4ldBUDLteI'_A O[uq5Jq˹F~nMmؘE@T;0" {n)v,x#仔aGOCf9eAB3UZ6lȌQ @aG}]\%hoqxǠޑ%dS?J\ɢ޴.7b<T6-19pϾ꛸ΛQUj@ ڄYYE #RXSu6Np)K=ӺMzd' ־w]wy}VJ¼:i3M…n_ +>e>?A=' uO_%|3EWl iAJ?h'YeǕh` xγ;n7(~tRix9ذ(lqx Jڣh nVf\Ůfpu6ݻp=G]Y?=ؾmjٱ̤ek6s~i}W_]1w~J=Zg}[h-n<^hlۼuSV)O8a@%C~Y;.xhPmcTˠ1t" -h۟?nBAr@V#@=};۩Q!ǴOrN/x&#.ga|'oU]-p]M2 <⴯z,A`m88Y?瞏|q hJB(B9ZpJ~7zt  <ï{{σnAx'wϑmg`Q%.8G'E$6|T5ahλ7^/^8]*n_ ׉ٍ5߯K7,I U^V&>Bv:B進ʂ-?r<|lڸȻ8^kzٺΕ0Bt{s*n؋kUr!)/Czka\o8rkkq ;X2 W3`y۶nӞ3xO?MtC&o? w%Dvpo38qݾUzݭ,ÓvWx~pjRfMrQ|`SeX `:EYy:D/}=+|Ӂx'K*::ґ"4-w{~g?![Ѭ,q<Vp3CW4+"xij!x3s3)pE@ 3 X؄kx=;&x3nvz3-GV5s8kʻ1ڣYPN?[KpsOO+#_ڇ逕YY$I$L~F߸J\KZU|}OZOWBv?F$ͰGEY:pv ~'m[6 V#JGob'c_]ɶMT010"^0MOAvD#wn~)AV[ZsTs?ȜWM'"MՃ#3ԓ5q]ˉrbO/=-A4Qyrn1Tj3_" #9 k12S>%m~qY+ܾw'K"FDXmj__w d %9xrЧGJCRsxՂ.*+YN7zFD|i3ɍ<} :*26 )Ѫzה1s =|pfQM(Ѡ}#FO@~w r{ <Gmq(M{ 5[nvG!PR)h$>}̃{ l͢a__s3߹:Z(5@EVrxJ<1VQ dXT:aAq$<N#0Q? qCP:q'^x7q=pn Ph@ <2Bs~D O>3ՃI &xig黎J]V͵%-h=􅲁ukq=͵>Z.qjgrKfy:{(:ßo𚗽v-%oᶆ%noksa:1Glg>|ozWtBa2[>kވK?iL95>&~ jvl G oYIh1qQlgl])7!ͬ8n7ix3Ɉ§viMf. J׎.F&rO~kw\z)Vn~QɻG Zx7=gÒ:G ~?Qp~X4.pO:BpG Ȩ_Y @…m}UW8̧>1jɲɖ<1$ӧ`kzr4 FNr>O|˻ߟ:Vn UY%,Ge(Gr-oYr2ta BY('>ܟ1Go fF`0ZY&=<^'eK0; _p%ܯ.:idX>ClpE_| V(p~y܏fd#6,su< dVA = eb@B)'+g\z+O\>/tL' 6w@ru|gokf P4SLfVc];t,ށ&}xu8X^Y7R&b[8~ju&fqrݎOKo|/4 P kZyX,*zr5F|"yq  ܴxr^g=غmf胎 ~= qX?<j,.Nw3~7/¾UL,5Q>,x [QD'(&8'<oc>~.OƑ)鏉>슂o]W =_9Dkbx sasN7>< >_'asXZ?Y"`VHۆX+HčSSP}H_}ka`=LK:s5D:!7Z.?bXQ|N/ 4RI@1ZxW~j\?W_d&#lgPp; JA6"񪄂0kj72$N'-Lu\!Lf*=r+.8(ΉMIҏF|pO%8t- ,|\(s +$,m /bKf&-`'Iy4'✻ Ȅ,毴J'"t0Lùwvb/x2# 4Oذ'u&x#Ogu"D5Oz#ܟ}<=x\ W?j8<7+0~ W߇P@QA=ED= [Q[+8!wnVaT(]r$.|ðe'ygk䓲˰"WiOyaFY ^w)'J&e;K?> ^3\+o _uȂc\fSMccsxO?mT /*Q[V݌ *}7"=vW[<>}77 .{~\Fp߸v'rUG2+iдF -oX$ݕr|LW“Yx_8r% [y !!D|h4¯K7 5MS_ ;25A[.^ĺ\*v<?w8#:%c7VfDoҿV.F0N_(*ʳnۏ'w\uw0cjj1PvYho- aVUĦ|yK|T1xm[~ }O>!x<[A^w@1.z^&37!]:ěr݌>x)N86 ֫;C^z,KobQz.P<O=KOd'40{o|_xǺAZ7e-PdҝC4=Mk/qzxp8Ev@շm52SCS);S$q9")QMh6b JGx \}5 S!;|_NLeGj  h|ێ>'I$>SK+r&IyHO 47O⛗<6 ]rGP$8drvNy  wAin~jF 6hv:~#78\$ڱY*\c]A8.(lj] bɛm9K?lo׸{PhCx _nnJrDt~^N}̓s@Kq Bb{0 īnXW1+@u:ՂHwݙ)@@[wM'eQC-C2i6u51Ζyw|:8Vq-[ArKpW7 q' IM^H˗"F-8SO~C͛n]<$xLG"Oz9/SAy1gm5->VN YZD|WDuVu#_8xasftB4G|3_S~98+GWeA>?< 2 Ԁ|+݉?_WT Ѽ:ؐ 8((E L"Z.1m'3+a?x@Uko7<@*3ټf}$m9_j`g LNiˍa9rJ*/S~F eX}<~3$\xwf6|Ĩ,qp}/L^ 9"%-3;Wv(֧C'F/(e8p㍷?9|׾~v3tU&V*m9NX\.my眉>aԓv2ƒr0"͆]Kځ_%I$Bwys__ufU!p#PD䤽m8BrT:G@ P,,`Ǯ]83G=\{iذ"4"SܕŔ,6xcnV|h!BxTG]7jY<"|9C# y 9 P:R~ K xm[b:;I3K St%H~MzՍfYܹoC-ФCh-qA99qFpW5z?صH 2929@H/RF0dxQ:ݻ~]/~ks+s@ ^G$ 7P 7b9?ⱏ8[,!' 2!LPކL&(JL }O7>+{nE|J@]ZT.z!T!i<]q9ݷO]e|??/~?ʝ{!l $nDh(y' #|@Qm܈N<99n8nW};qe玥G7 ,P9CH[tE<F#_7܎ח _e><%!W@v oB@rvrxCq}BY☍AoLJ?q r4r/Mz0yT贩H7'Yd;+9փׯ‡So_u5q; sh OnI'kOu@y?19z轱sf7l@_xs,;2Dz|+W`eBG;Pq=xQ{+Z'=_vqzCK+I=(_n9"\u]7GdTHp%v Bp"vl߄ д D2B?ih`ӆkA~Si{'f'ȧl?3ԅ+&9N>+g}2h:זo̹ܱ~X$ 07[&+ "ml#m;jTxrL Oq_:83ǁ:6zIH{'`+/C@=aaQ[NxP~0#'"p'kTؼp1}l#ؽ ߍU!?ì[\vK/wn*LD>,mtP£#xc 󓋼M AQx@#Sf2*;"SHVh2\H!$n/ޕWxn@Ϛ8eԧ{ <ܗ7ey@3 ?@wfi?}?} &GF=[Veڎ DWd#?O8u]s;LJmG7̅s+}\oK_&qe뱼w$4 |ʑ'xP 8( †q.v=>8gc h|x!tr=&ȠO^XCCs8if*9z( e21a1 X\eo;6h0IVܞ;4(LF@0]Oz܃qhyU:7 =K7oko6o!TuU-ք_(gN4߄|^04;3C3(xg F33X,8@_(?]kA0, PQ[g2|W|spw`uLlݲxJ(ADy!Cm/EFs!@ud;#]Q, Lg57߸_Wnmʡͦ <`Iާc 6`8]8p٧8 LuF@˦ixcm =gOҿ6N)o5.a rN R}4)wҝHO n7e b_9z6axJ-Fi7 3h'*ƻnN|_%_*+p͵bcxϋׂ<"7m¶۱k18qϳC{wuƉظa"2z/ǺM|Ŏ$s mCCe9۷k=+ķ}%nFܹ̖WQU5|ݠgswWPF/,b˶-8p9gsĽ:w;$l۶'(G}tHNngEGo}wqvVQM7'; clݺ N=TqOùw; Jk`tB))1ܗ>={wDeP=7z9o{߿{lyluU[u(JKزmܵz p{N?x,-. G+' ,3 )OðR VnwʵyPe "ڥԢî\9_g?})8BObd8EgT!x߂o\~5+ݫW^[o >oUUMwl{c& غm+//Z'qEG A Xi]^G'4n1'huBMʤx0=S|^>:슅(焘n^yfhuZO_&O^w-ɒ≅tM|޵Fh@VVq)'jVt /<ۢZ,N&)OmZU|OOA J&(ʟqbx2N&3C9Au`{pC+(]M\EGA3 8!|,NfʓNSJ+^2^c^]ęG-ai_.$߬*e)r7,1@]zR"%qLfm ۇʕ}>t;Ap+8Qw2nvzv^N?xM`aaM7bMؾe ؊;bU3/XZX@4b8+ljKXU *9nۇ[n7zc?nc/ػ+U+K ؼioل[`muܱ۶lh$HفWS|H뇴04Zv[Ht= R7G@Le/<*3;4y& N&o`pA AAzj|;oͷ=wCPeYذ۶mGb[8dqYKot/z +W5!,/ۦa[zݺ߹;bXN -M7`˦عc:rvځ[60jy`_p]ЩG4l='޼sдynw&ў<=/b2C˸m>y>s?;ľ2*w.M&ذa 7mu&lߺG;wn-K(Ijmv*Z)0< /C# LyY*;C|ReؓNuY<Ȃ38'_ UܶNr}s}^w>6o܈-7aMؾm͋XX#mmŹWT<}%~J!@꡴ߦkѩ#j@W,?rެEzℤ_8r!!5-8E;Ğ_NlNI^IX㍾21{y#-EViu`FԞa/U89,#aX .Lh8n`(sǜ8tlu%frꜧ$؋q :w:<ΖGFn EN0/rȠWL9@o ʻ!H2mni?RѧG=gbevaVI!Lc4L!}'$0 i;4X=C3O.ھ9mZt0SK=WKo 7$|#&Ntj:u9):O@bߎlnrLew`N oy~{^~*O>䣿zpMnh mC UF rĭgcqON*X:tｏyԖ۾cN}5}4fG~;hh?:)#4<~p a=Ч}~}@=Z4i +s`~hr\,xAqP|!7 A0qX`A1-'E+ct,f~C!tdɛ5D ,#o}܏F4G'm?@+hX hP3}1¬y/,X*?v=|]7 Ix4;⅘KCx' Bb !CәAYu g.HN<< F E>ĝ\O5oG,E7a"$+-e쇣6tư |LO,'?{nA $E%LJA{u?.W{m bNu xަFƼph G1;3"֥ e%ЌK 'ώ?cW!R]ԆȣqPS mY9OA3~Nu!@+r>XW|tV o+^:,ljb^vg&p Lf G܆O8P׍,:ٮhdQԧJ4+IBil&vZ.H,7r~9>tAxi{Vz:CЗP5DS~ȹ~!6}J`F8=oD,h<~7d Te\D"::ZdC nn<ꕿ6au:e9Cx'_\,XYx9QcfvC_{ _afBUq#sf I[ڎk#0sj:g"+yg oQ -Ĩq61<t{^{T4d!e\^7;zzc?r`!: !p⿖¨6 j=݀e{[zx&X:rhQi}Ɍ@ r0SDzu`r3OA7[Gw,}f.]Y8sNh'ںlь&mp+S-Aav[m0Ox4D?K|qR:u?{;x(ڇ*wmzpЊc.PVWO9<^˷sPkZ+ius-qw[f_yӸH[>8(tb=i_9NoNI]4T m$yEMPq4B79$:.k +2?k3~\t +37qX iuUя!/:q\LbOu>2avm8L#Mlt >|@or`,K2L{gpC _QtgiR !uT|d$[asLu0%~gA&P:t+Hl.!GQY[{?(̖tQf(g3uj>,:K+#W %h#(qۧbQVLē&,HfDă9&/:$41<ϝB'4 n[~)8~ZIDATpԶVWP<<|h 3+x&1RcbPAR+*׮@ލF_Zjj7:@4[:w<v!f$k"T x!@@֩LSl#vycG S\ׂ̫VjR {p[5ÈLQ46 5zr}У2CAE ^\,N< ypP6Li( kmm:Z~Ի 1>(? : &yz!mFB6%u@t:߆ IoxbH(MuC822];W#介4(vLx@Y((\1msau)Mw*. 2bY[}zI$ze]6i;ˉLQ,Py>O0x3Ds^'u|c4<ۈ1ws}fuw(DRz"сsX^p=)L2uR\WL!p O #;W'Ihe0.p9,0'&gqqh`@Y1בĥ%[g,D}dEw}-\lG@QJ+ư[vD$,ݘ޼.2aNO 2yZnTNk9w.=#Ct(~|[ƛȯ`9]eBS"{^jAҠ)Uy΂S.nY9O[P<8yJo|8-ɵ,ѓM?'T%NөSX0ztY\A/&G.Bq S:Bi+gZs3$  'g^ᖡىO;>0Cߔ$s6 SQq0 55 NZڀ7\<}ј?1 @M) ,Nni#,$ ㉵'2heVp K3j!9(!6N+r܉"$"ix "jEv >cVzTM|[xQh!ȀOjBdW\$TAO=ytCU(P58T@KTD݌P7#T~ʏ/pX Ծ@Hm'@dbh# XIrX'2}0ڙV}v&uK7 LyH>xp!| * qрPCn 8m"( ?Yvꤓj6`xGX>2b4O.6!(I8ϖ@w=I9BӜÅ\&S,} O}{&)WK݈umFKm4)X0Av0);HdRrn%|1@j,~%+56 hYZ ~jzض)jFrK}Z50ye뤑 c|3_e_*AP >DeDxg( #qYaTrk>*X —(`WNTl%#-j+1v1Wtͳ~Ծ6~B*i[mLFXlZlSdi5e#x^!I>C^_[@ )'r >l4mp_$ℊ̞ ?Anr\Wocƻ=Q0vLq@N$In&NKqߣzZk.IXkk >xgmr&oхXrR3j&BԗpTsw]ksZr4];y'Oā8r DJGV9踖)^:z탑X&ohsOuvG]+ͷCЉJuTwjC.%Z@0v~׉cq 6oYNPo#S8el{J21!;,$L55K ċrԯf-=/qwhHGy@!k֒0z:>/o>&6ʄ[mz-%0[_y 򲐖Mfq ;W)$]nw-~s&Q`lÜ<<>^9v_.)7?w !cL/<& IYY_$D]zk ]+;C5HhAuwN`:;]g R|rƵur;0}uIn$<;/L2b)S2:;$߼`ѴĸRt"C9}Ee/,;"qxCG/b:*GTH 8 MtnY,dV8&XݛȆ^ٛBZ"?1PudeV&d `$0'TM^_وW>'`i2zh0X+c%@3--L"[uvgLT'HGc~YZG/%F֨DCDY7ş9I9x"oqGd4EX=|G7`ԁP˳G:`ST VrU`<5F|6hA.J2m)l9I lKAl l#%d2.  ]eޖsmIȠ\n9qިdeDۦ2%2$`AP%+4@eynx#:n-'wa.gnB?/l}6W6]Q auZqs@Oyx$<{y.ۼ 4{~m">&ɂI 'S%uZ>oC[|[(2ʵuB^gՎ*n6,ŋa|؇;A+9nAK>7ѢtʊxA.V4)oͳo.|mՑ?=iaimp&mӫLc#0σegh~%8,\&ԆBr9z)?Ruti^0k|+e6zZWeVg#Mmx]|\K-K !]5hN ?@s_NV 䉒d>re B($'qiJpS/Q/9DMNBdnewp ,(*(: 7/"?AҠ 4E \cgIv}TZi El9Mkw|eE<e9O337(_ʌ''vcqjj~ǣO-\S4zA:mQf,+'5IK4^|ufEjA]`W"ByPYq/.o<{РSʅvֻ_m8Gᵅ<^YtGWN~i9Z NKBh'K5!o /cer@%q2^ t;IgQ¹~}q;7;:YR~mœU6GNJQm!x4#5(Ԁk?dnqP ScFܾM؍)(*8}V҄NJ"a oO0 ([p6:^ Jv[!-nq[Np^v+ZC4" ]Rp' 2A49ԃ+2t ғV@#e'2#N&xڪcK.LzO%ދeu#jҀ5"9G8!ējȘZ쳲dɐk oǘ`SYb";g],P힇DtG4IF{,X=|>tΝ B#v5;1ܻu " JuQVHo3ʶѕV 9=}q0aZ#)$yjZ 0ё !Ȣ KƯ}|arѬFSUgڊK[eBi}:cce jk ػ.9 фI(Fg-0!sկ~@!Dev$+pWh:6o#l\ r1OC3je %?5J>q!wgB|D@#.vN+*ʩۄ+>O|ICEX$e:<d97wFhh:u*K=Ӣ XF Q>tˋ00쏟l0ZǾB&6^Z~,$Vb,^f;24 ynۑ*WqPvGq$d`3+k`&p3[`ґ[_9@ϐ9P|"d(9ޤBFgPK@ГB.ѶP"F}cyy !2ԆY]Geg_N3 ySuOu"Jy`sNvx=|yiO$^LEF=f24 Ck鋥#օ>]BQ;9e`@6f P<_) c 9Z‹> ~ dY3O!d&$B^@KB#4JXQya!dUrq54Ziryt )g-6d:+WE'}O 46Z/S-tWw@\Ӆ8*MخLWx?}{ 2wGOw=`Z a<aqqx,p2p}|'+x[?fe{:dæo:7qɂFv%qO323^I >ɽ9ŏX$bD[ڎr||]I좙~ErV>Gg[䓔$MڠdE ;9DQU4G~Xfu{D}ڶT*-'::>N2{$wbꟹ߭$׺)BqL{W/MUw$Gm_4Ct@jq|qr?r媞trՇ'LEghqW~<2͊qb6FDh>Y<W!M-q) N_ҟj!`w6,oۓ쯓7+) i D_u4 ]Ŭ"d|ꄃqCu`2e<.$]H҄-x||_3PrQ7$<<}(꒺@H"Z%?WN:vHk#71)6Vq9}x~hAz'3s qcU8(Ub/2 j2iTh3M&ۼCB࢑a7WCF60GЊEUK'`Z呇[z^I <˿g};( w4޷edQo4&loȪ ?+/_a&M(Рjݍ5A:H;Ñ@Ox#KM}n+!N(le'=d#,HLu%Y!8nc_Eg!!@(xS>}̓`FӴ7# #;v?zTƛ')%Tǻ)-DHE9 u(+ ׯ\4Geuskh>ҳ?}ڨ=.!s@Bfcx}=%fgĖhxw`JZ%fVLު w|+ 75AR"~rr-V_!x$)鋇b<"^"Qe$:8s!ڨ:$(}=R#/ퟑQLJWOXX-CEQ9߂k|NxndouKރk|q$vWGIklbp23*=4.} p/_ 8\ӝmb\Iᥘ4 &~9rrbBr0*0Py;$ 0FE3񤛃nO>j7>py8q`BQ7 F2QSy8kl=\Հ TQ 0MHeҖniIxnYgx~TE0D2I|l递=[ ^y;cS[:@D(K9O;IK0 }l'+Ka0?݅fG4 cb\Mk, 7_OoI# `$XRK>DZH`;Q?N}"%Ofi65 ۣaW'6놷xʣPNWQV1kK;m(PaDR,aL-$z^Ze8Á![5mVB/}wOHcPI m*"T/m۰z;S<ȋ0$0Ne=dq IgzގTm~ ~׿LM+̪ uUu{W8rDu(ꒁ>sf[?|IպFSWcV5xo|^,wCd"$ Qv|xDrҶ5++$y{6Rlh:2,R,rٶ2࢙hK9 }+c]o@ "2@(iG^t%Ayus^y՟9DP\֊` ozոz(8&s^`H }zJem9Plz 黭/%K>~W TP:t.&΢Axz@![#8c8ڇ6Y }yor5'L_2zt ~6ВթgDi,w'chdlbSZ'2yE1#p{HyyIX(:fq+_"N9F$xoG:\!Z<4oo[RZ'^ =X}Qy:D+w`v"fH⚠Fa2LH )vbGλ 5G $artu=㹼S@c=|#v/lALRkBAhwQ=U3168K#j>P}hKWs@<<O02x _+Pzl"g.'LEvtJjJ0(kkʸqgfD@Rl434q[&)@NB.(CQbR`45@CX9Xaˉܑ”眨Fwri AXrshl>'K`k!# ~}y9%>4w46M"B>/\l=h ,{җx K"f,o뾎q g3m 9+trnxqG.Fi*¡{$uGj`]wE b})@.LAaex#O=Pյ-]$Iy%-ڱ&4!CQpF W B x5x_* _P JjPG Ͽ2Qgniq$MjN8xˣGA^pUܬ (]`j"tW;WwZ\0֓CA$<`e=UQ'U s3^ ^j勼rHkؓ'cA;}XY3|%.$!g0-* Kx/^?rص8wIⒶ[Ld$.~ h* ĩ-9˒Z>au/rmcTt+N Y џ}o|lݾgK茢$&79jcW G741W GSy2]ޣ\6籉2]iDFZoZiNmެ7g,/EW/1\e/38X\r;G)` y1gNWt00^X-ḾLz"ՉVFy̘VC1i-4=zx j'rytB{+OH/z$Om>/8+ 3I΋j 4 $ 񑤏/.IA]hy&@$w/KMB䕲0m S H;fn (#IL y^T`FA'ȀzCW#w6Pj+ez QtyY=]_՝80jUotE,md*$kK(/ x8VVg|PA3]vhym7klV}vG”Fg.$O1|ߝP7z:e8̯-[+cւn v$#ؽ$ dX6#⩅^3@̏܆&X㙁$zŖF;&HQǵCdžvt`]voN0u@"ڲL:&:(:4~g(K On> HNT)&A: ҩ< )=Ra2;y/-?^u82-wug* 1e[Myx l^|"uIkV؏$@z<~ݗ+z'?}n\wZzO=C9-$a;yiCqNڦ3^lK7އy֣y)fn$sCg[~?Y-.kI"]iM*c79txC[跏R tdQ~4ZEf 4 TW¯_{5.޳KCU{ Y!%P7q >~y8;ԸZ)xO 9׼唿GDӴKWlvR88-5>E++C1 F0fC@ywbuPKW?xЀCԨay@RHY)w+SKyBrj@Ļ쀶}Bs~mmdѧ_ N[N,e%uh$ qAx͍OUU`Tp rs@AT/O+O<KE eP; u:@NK|j0FvuW\rye:,?/o䟿[66Go_>yӒ¡o^ LvF=[k2=hթLfN;xf E @v@:yC?tzeZуwi ?y2Xqo/?} =nn[Vص{7~ijS}x{"ȱB&KAWn ANy7hx߻ '{DwHRBnLq;kuH )>YMM[{1 /Lfe'(Ll>d҂r"0~nbZc7|R~)|fC3 E[}3{&ܠh/T8݅: ‹/>?]\&JL6|h= {Eb MZLNW͆E95A24%*5U{,O=I-6h%ktEW+#p=΃ш,m lݶ8[nxkiU<l紆׆w 0Gc%)66Jw)# 2>mv!hJUDЃ\D&:%`y<7AHt Aeqȋ6qbztw8˳Sv*Q ;tlXlr'ۇ8m )`*mu&NĞK3<::!#yfL%0фBtShm!}ȁ 3>.$"ϳ0 Ǘ=8ʺIܗ!.: eHؿSzi5oemhh!ZOGE h>נ>C-+aȐQ!y0?rTHxY&b3tz4@1˟ C_uKY'>7LTpA/ Z5XWn;@N'atB1(ds65^{ xŸ|h|o@]Ѓ'j]oϸN(@ s\*<{HsD<|yie!2\{H*fT\[Kk<o'kc*Vvv"'Mf d ;/[G£x17xF̀PKܥGx%}:)[ KcR <Z~ģ̪!+,ݚ}1+C%ϢD$p)<O\F ߹% ?B b^9jqF@&ӯbyAH3P=N`zO6灌 HF-̓ΦćDB.5.Qԋ>+ !Zt.3r=*Zb=iq_к-/9OZ86RzmOׂim'ЫO')$)E m27}|~W$Y:9O\/THtQnDT8g=չ-H啖>x͔/aKKZ-Œ),jfaL[aE&c`}\;yx̓o~9lOh |<7z h&r{4hVkcq4#=y&Q/8d#S4u7M3KgjBvvU7܄O~ ş>q|$d*KY%Vo, q ѬmX@@1{WtƧH=f+?"RcYh'fY40e];OgҁEQ(']_:hj .d0Xo5}eYmKM%efy8"al~KV&>! IWݩ<>8d~Kk4K},zh_ryG34+9zSױ5$w ̃HX p&))OٵzWH(My YIuG#uM|idH5[(0\T,mé_v(G>zo^cXz_g9v=Z'AVv>dWUS%nZ&fk2γ.$taS號М1A3Ls^KyY)杼3v9 mmztWS̳C=[X ߱c כ@TIߏ=mk.ĉԇ->TsӰ8 Ty=,=bl7ۇ@M&(A $x#k6N %ӋLB[^~ [(@GbN\ =8}=a @`cN|6,tlV`/z$f չF=yueYMPZM8o6;^ǏT7R^ϫ V_rYZq]D:R6`pX@JtX^R6Q6Z7wnjpq͝{XPy3>Ѯ'|9'0xl^ 8rKwz}=+q%Z",X=O) BWsmD5d9R?sk2>:XDԸvo܇:}AE'.-Fl*J@ÖLrtb<$[7kr ?L뙔͏hr&XOzݭy<~tҘNr8 ; P7xpy9vd&!v1+mqlB3yw?CZc=_~?><fk!zGErfΌDG7 }(j~ү/Kd>5}c9L|[uN_Ffzkwkki 0jGU?d˾zgJ_:杸'Ernx<pΉڒ`s(5\I5.ׂV򤯑.iIu9 ?tN>/KKJ +ƃv Qmۻsoҫi1TfH^)}Dsm(͗ib羺.R(uL1B^-OeYöO~b 1k>yI@1f%<~>;t;Uo"?MI'|ɟME~'nc"?477yu S^=|({k/|0K ҇Coxo!у|E{-+_ y9:vywP7q_}.\OjkκoOՌ bwΛ'y3omʳty!? y߾b1 %ۻ+ST9 Niurwz^+Խ|v?+X>_$A4k0O>7oćzq4XLa椇ֿ s~yf8?^{I(y~dWk ow&o)È#V~/7~4賝/p\icss* f5nhљ,k^:>8)lu~)jeiD?%2_kY_/6E;ॼd Jc.5j|&G$Etv.?H{{k yL9&m kg7^<9]ح/Gۚ{ꔐk]`Qo?f@'H>#F~/v>Hwϩ`!Gk 7^vz̮NƘ^i}"A'1ԉ e^[@5 DF :Νm"?8?vϗx:maCrO}8Pl9Vt~ ~5snut> \]RsG{xB6TfVfMRlH8& 'bH|:y7(yMgE~Wz@$.DϢypÍV?gGE􆵞 /bdJ41OX84Fǧ 1HLm7҈9*q˼Շ1AD攧_Vn&"2k^W![þn=8};kܦ~ s/*Iw;B68խƣiiHM94:1Z_# mD|RE6Ē˺a8bTlu'K]vvt;grciٙfN?! kf=E+gJm0sSo-?S0]IGGl&4Ƅ vrsM=yemLev]ޏA7/ٞ͵zX2.ڇT#:i|r3B֠syPW ̃"?v]:~[LgF)kR8v>+[4˲"=> F؇|.p?/[*WjM'17qh֜1~ = 8V`MGNGzCϼc+Okpd` X]AWf]b^vk'X̂\ gh4aĕ4'Sf뜈ﲖPWs9M4zjgN!y}t2::y"ͭlv>;뎭짪Hi3˪8+ sA9/Č)g"sjrL'&F Iy#8")'7( 1-L'H#>e~T8^蓨L؎z7=ʼtyRV>x.G\n CB !PcW[0>݄ƒ=pjr}"OOS~Lqʗ|GC "_hȷ~ G?|oA|"?nsmm|!W #n~[p备bw>uC zd_d*3R0k--CWxD[\Lh] ku{y^:D5;^}X'O:>A\?2L{ 59&۹*!1+j{!"S>ڏoO~|m*=$Gw 1pn"gݞVngCɫ>Ǐ _P>"~(d| 2%y/$?<|E}(< n~7Gd mO1eÐWRϑ'y~<==_ۖ\>Ga&X|VSדImJ"RDԁ8u똖Xܭڳ0̬Gưzydž؅`e@@޳:*ޥ. ^'dM;ew yv~@"t 2//8bv%$3JǒMަ1w@Y,Pd]mHK^6v G( o}2}I1Aý [v:@zb>o+NA?<|=!mK6l]"{{\VF_w`8/d W4zC]8'p|Kny.>s8_9(sx]ﱎfk~dxOQ4qk5\k=E,VS;_(v9yiLf%riD8 55~?>Lm=M<[T̜CMMLM<ŀ;_9/^+(CTɖ>1=`IN4i%Р%5Vk%C` '8:[n#"Nޕy'1i8ac6}: | i`ɣ!up c>ݷWQڡ>N>q鯪H:Vo]~ {@aWwswd&u瞠,F4/30mik0yvrf^6|x.;e'3<=O某X|2>;Cn 2Y7ewA~|3S?}2}3]Gu5`7ǖo.̛̩|w=+|՟W%XbhoAluC^n_hbSioOW|g'5%igm1}LGmȔ1n(7q{_~~H_o_׏V~t yJcʵU/]+DN cyK,vN`_}w8u> =/]"jz;u=1 1)fl(v㈮SssibG+ٟ[89DoX".^]NΟNPd6*cqO] 8I7_'>d畯 ķ/yis X_pFY܏u6Ǔo;(7~pY~w8 7`mMA4R'2Z밋!F/5.'q,w/~k!lw\čuY_%2s-b`]<[ 0 _xN8¯=(5mp}Rǵ]0"v&.Fq?=MGOPMwrA%V t?  i wLw  d9=Csw'Lq텐Px~RH1p~CHx޲r\6Ӱ@%8+y)wx qwOs_wp]ɾ[cʫ)ƔcGy(^ϣCEiEthlx0_NIA'^xC꺌ܘ. Qϴzvi<i{:j7ekJQ`z_A;Ӗ>HmOhwrE^rTd+.K/˕:x } !}uup]gԯSGyxxW^?|Mr{xOnJd|x|4A9ZQ5۸7TEܜdb˙[2urϹvDr*44;|8z7P?A +_ۺ~ù㸡1f==z)<&;!bc<]liD1f>#M7MVΥ{A;Mʿvi.UM^-O!ȿD\1utkJʹDU"vz XNaݿ.3B묲,O@˪R~&"h4?z 9=޼>Vd_6uzMp=s#kOVNcƼT|)SMܳv6&9 c}Hȁ$:OM[ADd >~~+ ?w)g<# Mwmb^9͋CP[qcѧvO(Ko.,cxB}XtuAp?2:ؗm89tm s:MtA' گw\`;,ա󹿣8&w_ 9Єsv9bS"2|5nn x1ȼkOoL[~o_M޷|g#3D ,󍼒ODSyO{%wko~2o>>W@t~jjPb}ywNL@_R ɪq~qy"gjМSnͩê\qAu2;nF.GS@|ʊN/9:UaCEJ}.wcC`_f5:]+*h퍪Wsx41l@6W"{0Nk >5`/o`j $sr^<Ѷ z, >W]I2N`=tܸГ2.Q^DM8J v/2@B_8P]rtIkw&Fc]a C%xɒuPgK:v> ;dimFڣ8L˼qfm?C!o2Q^nw1CÍ14\{ =ՏZ[cZpU< 0:zHswn8u];Ν ȟe xeh:d<™ON:ٷZޥqRԏq+eM1DA?|?,ov9sӾV~ou_ӗrl :}n%+um<=g݁ttuOI?1z{oaI>|J~O&goo=$Esw` 9=Fy"uŶp=sO(e]ysÎ>=Sz?7GkL/ͺرBy|Ĺ$fȺ8[Uey<`ya&FB6_ _PZ=E)+3Ƃݬ?nm(Yu/ mEMz5zŨ%| D'`?? zPXzyt)}|ڄRE /5z#]sa-h-羙/wAS?b Y\;>-NzgRe<hv$ ?)kl\/ЅtUY+>X>ssyxuՙ'ɦ8 { ijva\rϿs翍n?| 87FSW0l)5Qsv۔)r W[߿ߗ/|WD<yoEY`N=C؈X-49Ib_;&%+* y=$ym }@_w+sܞGd=|5YMG= !8! } tB!+\"`߷D0o,W֒:g#B_zxdej^ x7j}k[:yFsYvO"MGWF+k'ehoM|S1C|ntx}?$&:GփK_#J_SO|n98\kizXG8ڀPI`?R*ב1uI ~Uϖ`zJk ֊ϳv }EYc8?C|3Q4W >[wۺH;N|I9dGWGLjm(`{a0=өS OaƠ[X`?l"#~h]Z@c&3R$ayyA gGjiRh#VD7٘{ ʝrxTflc}kD|$^֎)sv ':'+ybκsL\?ԑOz`?Ι]!p18 7Hte/ugNڎqΪ))W"?SejW&E+2k~|%Qn_l'p竮>nޮ?<,sK0Ή'co]W_/>x%O+yW$s>؏K whe:Jk3pz N [-Gע?їӳ<ːgy'yJʃ<)W啼/_ZG_+??S޼yy2< ݷ~E 5ڟִapU?K16]7`dҵoq!/ehqWDǾe12U}2$^`Q' ޶ ip/:c;"tSy|87bYbOH=a7rJ1qtbi9v_ Z?슓]86eN^ ab~XBq"wPmC`_WQwX_ţu~?xY'It8[x>EswӜ0% w; {ksyϗŢӝʿwa7}3d_IV7 90EڦL d{g3wHq)[aCړ}f4rKUV'1U#HAڼ/)OK:;elསaS|l-iw2)#\3#x|5<}q=ķprU!f"wJ ob@% EL ;'؁KkbƏym2 Geѽubbml vnXKVw҃wZ;O~a1$`,:D?.2unu ~d:qbڻwR~]m4: x\~zď f>>5׉Mr">x__ֿY_{OO>}#sNyx < 6hz9?f8;|=͠%:u6øMMz1?`ٿRwU陳zArO4tΗsM?5)@{/ ~Z?#;|S"? >Jz-?[{?S}N~ʵ+D3j4N'b:km=DSO&H*2Ard됼NWmdz椣B_`aZ9 kt:NL `C]1֭ksj5Ycv~颖O1Br>W>52bLΝO 6v8]׷ka!QiXO0p0Aie~jpc} c/{MY9pAЃۧGmmv\W+ڤxKd3vQLI>`{b@OIv/Pb@|ض+ (X'Gef]g|DqwA͢ATD 3PppL?x,nv){űmYМ8np׏Y'ԏʷ'ow䛾|ɧ""İDd8fc7~q/pN88β `#j}6e(z9ֈ }Xd +`.7<<~pYC}k/᤮WXsauL>Kzv:>`qf"j5z2o=6̋RΏ._m62)N|ӌlR}8͟Mmx]W`"s:4$6eE'?q&I36is՞@vŊH[nݨʜ!kLo( >s苢;/n9ЏlKѣ}w͛.F3(6t.vP/ Xz?MH2nXgGO2=!~jOGYϾIf:pV|R.rcjp)&dvuZ=ʺ$ - .Wav$[/lJ"h(2Œ.yW%8;~yδW>z-eڤ%.t ) ȷO<4`F܏h 0=%0cJ#>f;ywBAQ$OfϋAc19d[= ƾӑxdcC|ڤ_eChOMq,?p(c :lz>mIgOJry!CJ݉V#BƞBwyƓ{{?Bs:z\zr`"Ã<=);ٯ͛7PRDy>L_+K?wMy7Cw77yz~9M~0eٳTw[ ^#~S1x ͘x aC6_mTM<܇!ߓW"ɏ~UE'/o/Eyz &CW9Rb_6: Iyjnh85(D%*8g\6wK6l{_xjwxw1*S-n?d߲.jⵜ+ɁƬw3lg;pKʬsv7͐&! ^b dW>lo_"sWJ|d`F]P':  /R^aރ"Y+^uj8ȀoW4 7..Fs3|.g^$Ε XwHtsإ[pȲCY Xw6Fw:Ùaz $Eϱ _f T&^~fV7ס^T 3j=zpm\aNd5pΫw>!q\~Vroht>5W4'@0^rf~<M-z;/v_:X~Pb~ӇD vX,vVIhHeޠvMKu h5s~]"|.KN,Eny50<ѿ͉`H=`Gx;^cc/flDVR.榾ڄ8ۼ'_;o|R#rE˥՗2~ڠwrr$LP ߍ躾6 <ktN?Cڍ3v~vLm{xxg~?)vyz3t=>χt&`25'aу= }']%_}#ofG_x vE.'Z[xKsOiޞot{‡g@o?}=M!"Q!Gyɯb|fN27 0} ۍp2ո!^9 yɧ~+7/;Edo h.u/ H]pFyD]o'U{ܻ isb:Gw;%A2D,' caH?]Hz7r.!}sebEw*kpx ^エAGX}SKr/z\ ;:"+GLk`eÞi"9ηtC# ;Oq* uod p {d¦o[KJ5`;;YN9"852@>sQGm]?u9N]t(f>@{,,cϙVnEWn7cmRve[U{=C>A{I{[(yGl\v+WI^\C|k/]HևyBGF0~Yƕ<<Ht6'LMv>- bJ˚Px0Te5O&&yȌ f=yreKDE`Mw3up|'{^qE4c*bUR `x!$?msl;S^Hmҫ|vg{b$.^~ax\GΉ#-DŽ/[,]o_Iӳ,bZa7_}v>=VGy$Jog9DD<ː)jgviԿE8aO~*oy?K޾y+sܞ+Pkх_v>ٵ<swL`z=?Ěyr;ڂ7)<>Wӏ?O'M>7G_|oY̔vʘch +kcezNq^Cn-_W˘C>ڏO_?K~}$>G,.~pJM]">'9X$_ƻҳS0?*-%0DW=-ӑxwjaz yV;*n}>8j8_}%u7~Qe xY^ucn=:X7 ]ؖd1-F&!,']ZY1b|`^ fL||5K]rg#?=cOPHjHJ4~˵Bj|{[>2R6;^7pFxP z!}>?:ޓ^;Hv?yN ݋qSn{ ,nmQ8'9qu$K|¨~~~kKh~jӛJ~_D! E농9Ъ0vx|! @tomF߶mX*0J -&(m}v\Ρau؉7c=WR޿WAnQft0>%S.8C~co;毓_~|歞ro:f&n˓G؍Rt:F4bBQyvunn_ށ,\IBk}zᘾ_?> -bW98ySXt}߭tsSl(u5MVE#Hwp dc,Zz#{X޻ytSb:k$K$y]fOy7CD>|_SJ>ØvQ+'3o7Säyy!O"2ʷ}˗7_/}?S޾};{P?/]ZqM;aND㶇U@Ώh꫾իW1@_wȿ~{D~GE?H^}}Gy,XC Cd<59yӾ8iM}9Daëyy|_y$ ___EWem_$/^yޣ<==ɧ!+oNP s*O->CECȧߎɎtעD_|,#Ģ>Bev|}m7hU"ZC`,ݫ+l\jrױ,OtO).?}Ut4L'+lD鑚&mmay믫'9)A{VE=.\R qRAZ SOQyidv55= z?@Y]M]ޘ+w471Z#w3VרQt @ ^hMzrNhsusD(9/ͅ%yh<3_~SӤ|N:ԁ".-!vNuUM,Iw1,,Nf~8c 'm:Yd9!#.d]/fϨk.xtɲ…㗈H~wg8%ţ)^R.S>l;u‰/2uIۑMt`&_K~HNcylkH8>Fpye}A۝c#w3.{CȧO\28n"QQ̉yϜ"fXeIL?˷}woW|yzÃUa2OuǶ)FVӱM>}:z=l'x3e((N_|9oQ'?{&??}JDDߓ/}J^ې3V>\yc 8i%ixi?cɫ<>L&(_}'/ߐ_|ޟՐY޼}C?<{pÂ|ޝ;zEN};G 6m-d;0g'KWx }ۼ|nx) /Un^|w;e"] /<b"+mk# ;2\C5y1inH}mJo_H1~qmB(xw[ڳ3g;{y4|Hms$t1׉~vu#%=oUZy]{٬yvsT)-ȫ;ae}esia.kA(zXSފd'eqeq;w=<`_ x-/@`Nd̲_AL:Ɲr_ijS 709f֟ IqDB* n4uߡYG4cμiTPѽ3Õ/%@>"]؅q"&On.0|HĘ~ji`9Fix! %Pe֣6scӫ+pã|_kUB )OX" 2onq,o"V6K?Go?]o)L1< *E|@W]0'=; Izvv1~nX4^dO%q5+2 +?;Np/(UPm*mrr,߳JR5/=J9sɭԉ"s0:RMrAz"fߑ( Y3@*sj uQfL9dOs^d;Ik|( iOQ'=7{h?FwZ gFv,6b1m89b5P#&vFb0/Azo<{`^"F zYhmSLyo^1e]MHHgo|/3&cQo,&v;<4g7AGזpL oK_Ȏ{K1?3vm9sDhx L'FHmn btU&>Y5~}2] oyt__urȼ2:nb}E?D$tfOd<֘"}n85 a|V;:Q>?κaa&ECp|E27# :69vtAgoNm=o躾$rvqk]q}Wc4Yfǔ ,txnjW83?Y:8_ۘ5|37p\am/7E4n#]›PVSѭprn1\9LA!>r 0qS0ۙc8EhICTs[YC(1`~6]à'Oop>I~h>i`dç+ձs ɱ K:t#AExn_Ecwlv}+a`@&GI4us(F?8 p% Ɔe#M6L <d988 b-@\nߙWf.Ā !"jɜ2o`r㣼~ڋ}*_GGU}M~G*O|__|9ݼOi ͟͞Vën;TeT4tA٬W'Yy߯r NM;%Y̋bhZ)?|i1i*h'ɔLφyڲⱀm7zH9ɍClN᫊Im[o;%D A#JZ렗|<v3NtG0벎ځ 7V;i;Y^'==$#CUuA~k;:TOtq{0-vq;:Yj>_W Fdjk7τbvcHU?̽•ȏLk9pv)Rxsޗ SbtSu 85\$vɜSSx?G AyVS,1ԉa $7~^x.y&a5Z[_+N%|"nGy(SO"|F>yFWn?|ー=G-gyzzGky | hrQD\BڜgXDfS=VrcyOOrЎm퉚)Dqai*<׼ f:jOz_]A@~NܺkmxY6s:\`Ӄ;x @. i.YiLh7>xeֽmk ΍hrTO}fN: imfdȊBVڧ ^x|M;DLdLbU'ˇRVfm0 {Q)|=ybeS|J9>-ðY/h?Hx;F7T*f>,7:Br*1JyȶPQ|eݺ]Ԇv{~ 븣cw"Q^;}u&d'b3E:=Mbln:榾s-ĤkφWBWSo҂ug1蓙$}!8]zبq6܀Pz2]쒻Cݽ foB#%%Y|loԜȫ/ | EU\$L w%kTppN&d[ҭ=u#c Ż~B<"2yAv[:s(1 츒"dfWǷ^沜ΦNͷ-#44 ِr d~խ<(ܸaLvuՌ\(yۅ~o7nlt=M.x, }g6tLq?I{σj-b;o܀vF8Q`r=-cD:8(oɧ-ɩ'F!gp,<%9 bt i"i=M'|ɟ:GUAdy`!!d9~&Yq^D}q\ _il<,;t|Š=.Hv>d'Xy^?Jh6e\.\F/f&!R#{Y7˚38P"mN˕auupҲ+8=–XWx8G}AsY4;s b 5sEu_Co^>|C/c4M^ ^bH9C+>7u=h k{xBGG+SA8bNڭ}g08kbgVƇ!Rs1"^w sw*ǹfS 6thB8˝ rD6/Rnmbt.?^*b=^YkZ{Țw,K;s<.yIַ<s7'cP敬${Y1Xzn{7'8'sʇ͹LVďQ֐!6p{1'o;.\߸V'X}'ю-!"ѪS,NG@ӵ0?o mؒJMjl'vHl@ۊ %Ӱ$1]o3Hѡwk8N\2BoiC/tseAlXr|>=SdvѨ;+/`$z/K-+>ٟ,}tz_g9PW.Ζ@{8`Fl'#IcfÞ)6zk3-ՉbrT6ǀL.h'Cxg][ޠΗu$uzО1a Z \RȼxQ9KvM8u1; v<>zfM@GWtZ4spx|wCg]e&ʏ[d%A≲:_RL;߄W7Nژw#>]qns } w}.8NSf6JaLYPs:G2Wc' CKc.&ظq<Z\W)71d;.3hx<{4:F#zc>77{]ܠ&? pa yy8:[4(X_՞n!r|CkD[ۑ{Qy~?Ѷr\@69=vl۽`~< `]jakZbܛhOyGYgPR.qу.1ނCz U:m5#MݧR QKJYx5.q~:C0*o /Ohi|Qy`6~)JP.S*I35>Y\A#%(aZ׼#>\ ?P˥ꤥ+5BUNg1k2~_HI+62/0aMXcVؗ)4m!+>tL->rs=_(S \'kB ;oڲT697_y>?wZNc@JZ/kʑ&GO(Mb;جUrZ٣)Re.cs ~=Y"BdEt.qV~ ~?_wq0C.{!j|.[]'GZG_1p޸&: Kb>ƍ߆{0Ssp8:zP[~.A =M>@eM56Cvۡ;>m~F> W*=\'}`׍Mk3M3Ti 40] LfdaXJ`BasAm)'a QR4vn<`)8/27rR5y(-DԉCY[0/GvfsSuv$$:lUEq03{x\H1QR_}Og7`Iyb~!]auQyw'>M`ҋy̓=i]@o&3~;K&-65r #U&>aۨn65]s|&️秜lW@ζ|53P~+%Rvv7 ;0/AW5iQa H)3b'tE&!lц2Q\^BdKO1~ #mc4S:]2 3`|M1 )b.bhIiXut+rpb}*6=meQ.G>2HāMP$3)KF+~az1?GJsw[y)OڄFds:8B6ێ vK_L{Ư |jjK)ac=!j,_"W/;ឈ[|t >;Wrv8I KNA cx^k&ۧs:s;zaPЎ4q5Si}dzY|CFG'ry|VcW_ȳ+M_}~+E5m35RQ4ava8oΉle a˛|$H݃a?.~'sڻtQGבŹST6`=9(,WDy66r\3!_FR<"Rb:\`LM }-aqSnM1T˓vQz]:J~PXiG1}&_:߄(e1sx4tIG;?:A#lO(8K]m@߲}}-mu&fdT'0G42t|S^NMMဈ%XN! Zy@ٕ|U9k,@o~q:#sί6MՂCGOy, hW>L㝏(c$}9ZSBׂ#S P&܄6ʺny<}-ϣwfD{эO7sa_YG]n,b~X@ڏ6 \DD{d1?v]҃*NqGr&S^mY$[˞izUޝKnq~S۰3mOM, (/}rР{sD&?9JmM oӫkAJ!bxC].8N(y1{ Y.ySF|Ǭ9(uJO\\L z']֣: 39anI'T9T}]ƣE~tkO:#*&0٧՘F }Ħue!`Gg_l(ъ}>(sgr,w*9z^`~1OkNw9].l1'^a=['bxXF3 ETrO]r\W<} a9x%B1i ۮՂE !踵dN-= ż 7gޝ_$}?M~I'c %~N769FS/v'@@p~n(ob]tf}971)_K_g{*dpKH6Kɣ Pn'"XD7dĊ@o܃n)4XH-CG#Iڙ㱻֏6󷛄Уu%!n;Kpmh׎1`zt0Ry=vM[ xE}Hז1:L>Ȑ9l/`䚙tv&x >Н@%?xqY9"uq۬o'8L.]s:']Z_@8F>ά`K7-JD90}y'%{B]n⡚XXTPj"t:sX3A 8Ǎz\ S/kx9"nPǚ4gg> gpMH0~H^bxVzŐӳ_Ee1sm)y{ [؏ >t=s'!MsM Y j]k%ߍv:gs@>13ύ/]K\:?89L\oq6!idw[V6of4aE iN~IPnv4]ǺA &jqRuV}KOթуzQ^?pa,/MWoq|0Xvҹ5L/PH9W ǾPWϊ%bXY,4(2F`?o9]9ߔaVkѮsrWJ+r **-20;]&fuv }/:g9ŵn,le*[7JDnK:_۵ob!dn:0du}v)}85sSa]kgcKF6u>4uu6U6 \KP]0~+L,~c >?rwI6:}B.Qe8Ƈ{*~Άe!">޴^~9?8]mqO>!d 7XpD z!Oy^XK[Uj !<f)Ȱ-şD0p󣢫\(k1$rX_yqι['Wwnr׵v΍cSĚcV%;(6(/6}6 ۶s_D7J]LǸx(jJƜf :/UEEbkNt Ć-Xcy\֛mqn2~Ayc@>(P\\._F:+w)~R>9CA9TnyY] kLJCmOb}M_kRip4m9*"K^1ekDjmB2_cݫXq~(@0ߓ4w4"NܘH5kÛc6cs/:8'SL^9/mNҫLW1Y@`tC|YG HdX9867aJ@2Ԍc+8 o]`쎇O18'OC;/F'!$ʼn}wiӹ7^_GD8إ'$ /2&\,? ~CH)Qïp`qo\7=97:>/_x7qHWč&ǹp§I' 1vRԲY[Ya3_DM'd\O(fԙXԵE_ˍ~ƟxٵGvh4C$G;>si 3^#صCwSiys ȵcZqx%w18W;ljr5ouj1mG>6E㶚4mŘţ_MAYM&4%O>(Y[x9( 3Rm8-wk,>"C3:]^'\ ǩfs.?99fz ې~ZЁ4/7D{%?܎)^ +v> ܏^|ε7v.Ѯf'ۼui(r^%\uh7xh՞.d8tXC鴹G#:;=%Pcyy ~u[ dF}ibT;] JL sEn/֑v::ƦQ]fX̻Ь"XFn>OH0JG1#ZdI\jhy]Mo [ )1ڤH}T ob5'dž2[a<9΍fq]L }LifHcv趱ƣa-t*F,x^&OºIy1Wp~<<ϽAٷ$oR^5{ it(ºuo 4(u+zgk G'>F~~dPSuOf׮@aqE%Gw=4svZY'4J_욲 pfv6$pr td+&eLID87'oupT~5BGCXi9AǼk/ B'-$Hf$:!Ɩo,ZԇѺ߭X%~^86l[tݜٲhNº /χHf@ b&~ qY3> /񌤋_|ûpSﳆq7 c<LC'c7D+]3w/u׀Yik u [t3cril,3N''afaR[TϏ74>C.r7u^6ڱ|1NB 8ۀzu1=seG }HmxszJ3=kǯHo 9P8_@| 7ЅD\o#*2|U?`Wyk"ՓemxHE *`5&Vy,"%V_Fs:txUޢH*fnw|(?r#5Y*>Ζ(k~G l6 ae>,Zįmg\q ǁo"A?;sޣxScH(u%9x:r;a\<l9Tv}\ݟ,߁̑ԄWjjoA73|sOXecyF#]ǏG @'=Ͷ&Hh7Hk9NxMi.!lO:}/#=v(NҵUTF`zDm_,oz@C{ď};$\b y bnܔ+Aks&1s#ąs-J3aMs}~;J?$=h!o;^\`v q. ī4HuӚi귾xN98m`u©GJv+V2,S8;x'Rkb@Jq@}lc)>1o0O ܼnNnYfe80,1ZoF `bIJaFࢄ[.\}^:&a#v~?()l0#\#RMNp ⶐ->ܔ2b/X1s9?]'!?dqs't<ɹ6Ƨ6E?n !i_XVG` YXul-iF;pّⲛW; L!g3-9y2 x]T9C(AWڒq}iOt> ]HO.K4x|Da^{akksǺ1ξ>)ހ`]bvssРOu+ˮvm!ڀ)\N,GN`5b9֜ WFt3uN nv^~*{&`LS.6kyg6z2mӆ R'mv9n w ])& l"6j qMZ3s[I{D> Šx-tNu5;f{}7֒sOלpA _+;:-Ǯ}ɴnŇ6v#GDdҩmAi.S0qX/ja.J,bt}54IZz@Ttjco͛.$!Pz7뾱W [/6;I\fx'W{}JYo4&^M^i w e+k,6+3Cig6mL.[NlALx [6]&_I0YV`&1_7:d~c ES 0"˚M)]뗊HTx'9 ׎SD}wh}}lxת5p}]V笷@T6nD{1W#:q݀_qdD| ]^e_&11yWq<'ﻛ=Hܔ?%aΏ2O1D>i=4q|aG}e|p%rVl/[t|i[-kk@Ш1:blt@)rrדTQNע`1a!hq#G\j$r|%ˎ,=pNgC 2ȾM!oPBxl7wңmt>ArVrD]o|xy9[>lw]ԭ5Ldk<}c4xQ9nRMkb 5QO22VdyV|ݚumƮĘHs?bm'k sjK|jaI"1Zkۉ\$3p|Oq\O~杠0cb@nPĹk{}=- 9OFS@d6$'{os*֍MKv O(wUez|xb-708ݖ9GqFyLc= TrFr-l{?*;ܱ|r-{֫D՘5 `mRRI>'5:| xLp.] TUuߝ[$M4\LÖ P'(FF@й1hי :umhn%euӐk_pe5@ιmߡ@6,_x"Wcu>leҝ}ʃyH|n@YueNS]po4 Hܓ2촱/RP^dV>nI57~f[?y,|':1$F-><`#<\p$](Kt\VI:aOus[+uE1W0N5q`o<7@y*X.MgC]ߔq+gs xs'ݧ澕E7e6@v,qZ?v~g$3u)F t]ZQ_wcQ7YYl"h5=6>4yq<{~>+ѐ]I<װ{0qd]OHsWg?,sS+uR4tC( w<oYbY<*JFD^\>̤ _P|N U]Ȏ6"v0NNNVՉ9l(6ȣQmXe!S]M9 %i2Aq3icu]ͩqhgY=kQYsyq.)UV[N,+nn 4ֱ:l+Ǐ6d[`,g~;1l=4'$0`s6_nN` Hl+Xd%l;hR^$`GwisO܁&osb3Sw\3l;ˊ̮A3!"e9=#PHn^3C].䨺{.̏\ȾxC˞KYZwgڳp tg>VlAT8>=y9ϥLEcC$Dty B7#sXf+·}vc`uȼafC亃L= ׍Ȳcq AS xصC:PcpM`@t9'1k^>RM3p)UlY9L%į-s{Wk~n:Mml#<0yjy+%#kׁe1CYGׅxVb:8x1].Q ,j@j8syqn%wZ;f^ A -WZx"덇Z|Dy.h4Ltr'ˍ6ЅnZ36JYn9w,R<Rh lpR{ H'fSr}t6$^4Gi.v Uj 5ͩc8t8 Uh@b2^}p:&wN`?.xC9fļcWfѱO1i*QсeAJ5T'99 [7ܠu_xqY <ء/N;ʅըy;{۹on&ӣC?v>7bCmk?5;P,& ^";ٳq~s͋G+[S~;5DXO:'.\ǝn0EUPBC2~1ژn#ceʚP8X/ .Կz^3#:N:uwi++j XoTsN{wk?yv*o_:ϫ \W?)% a{|kc}&bwvyamoiVufGP8S?H=Ff;ӡ}7ZЂб4nNpgwAM(X´lMw}+_'/FYdƕ\3@s27SE"tDlIbcg'M ..8>T)u#fڀxmu؍.ݜ{ߡep 8wz'|! bFf=B:1e],4Od h6'3\?"??v}X/P& b&|Sh.r:;Nv_#P|CJDcCAդicB.|%o&? &⪍>.Ė%,CzNU2ţa7s?Ls6߻u! Ҫn^"9i}5!vKYst{26Fɉց s7c)mfF|!Cuk}vSc"ɎZC kgVmB^:9sl7:!_\Ըe";Źnf@x-Nc?`|=Rr|} j>d^O;r< b;p Av] Zy ˎŌ)+6Y'ޓb- 42~ʘ|Qv/#@˶Xg؉߀C^Q{ȯu]G]g}1HV H*˄{7րxB6@Y@s1,{jZq,`y@}S )"lΣfjthx>7iZ8 3kMc\~݂dش8Ny_liB:tsl/.̓2òtF~7ikhv# !,qڙysQ80;:y_A6Cc߃/wrZ"f̌ e779>i|Smf^οɧ ЖO#-ey6϶fx&Yhw!&,\%,Qxts|υcf#8>~ط{=kt6M9^Bx׭l+^TRtfny8ʦzq|Y6W_xEpnl>dnLmY?ƋXEօhֵ֫Ļ 3*dbG֑9рmY-돆7w{e'?Jܻkn3x1^+ -59k$8Ϥy3'n&`S@6!D^@0|7,pG?n|ƴ~p/'~Cr}U$›2e=KG.n[ěۼ'Ju C>ӯʶq[d+@d~cklҹmLB,؎Y?uNjKjјHzHP6/UwKЏ%uwxzn}MNsn)Nz"p;Z 7׽tb;*MDLY&74R4eg@#X*;Vqo-IψS;LçAh\[ HV{#1C׹l-([]6{19\X2\MnOA \F5[\o· ﮿ yYd^̜PdShP>:j}hԎL_:?:Ƙ.^cϩ٣Կ#}'٨ !M-NA2a\?|ju4Y\7]t׺mnwjKa7Orxm\%$hOff=5GoC|&vai]y?k&>(WgG1pޜ-P/][Rrx jD3]ēɎdK5bP\}@#6[hn8*/gRÈƪ'ya}bu \CաH$_w󱾘v wא Xbdx>(y6`.yAG>Mw%Q\yǜAȈ }+ŭ?W,j :Y-ī@GΓxAt-ur? -rn0-$s [F6F"o,R"9Lun'tvbn |v釬1]9d<抈}}Y>QbL%WӜ)bݜ֙O6 +<-]HgImc|#ґfwDtkŅa 5roF8kH,v g3h?aǜ!YS1's>W%?G/#l^Hb|燎bw>2WK҉c}N\ܴ2tH+r@|C\=69'WmѦgڹR]Э-,Nkt~ut;J[瘉ƄGMl נЏI^gA_A[6H51fbTRDžgf/ :+^Ab6RQOۛmE|tv`\vyOE J0ۓk=wAR[k`f8XwOs`\=lphk[{}NxW2l: ^"a}us0Wyqަƙ:^2\5Н^uM|P7E)lTbQ1huT %{8)I?uq(X7:7lvle8yULWG c.t:L'-w3Փ{E/\ٛl"67eH/i}6)FMNt}E?׶e\; z1 RNs\7.Rkf'#?8Jφ})ע%7`C3+EEz]>c OҚ]1pR I<6_X7#wꕗgW3S;r^ .[mN6nCb9 Ad-ߡ'<Ee=LbԘtċۼtrjsuv*0$nz:^'n_x=cpM@C7#'5񧋸"+&Nl;Kn_xB1,Mg 'R_]}ȥ;=KoMj"d<܈~˹>S1isixz)]nem04x$rk(~ᇙZg[E}sk&1`^WHI'V%tTѡiF6Y}az%s3SPpZ%?،5q UM`yzqQ͍<7띉S(>޵}M%PɗόIO(_eHg4t='SK`(x5w}_`nϢV;:_ugǂvkƬD6Vc(Mp0ՄCYh杏c;sj|W"E7GďI_=&ay <@Vbob=7Ȝdm UOC4ca#}*&^ 7 ߀Z*xt~TYWWW挆>Z^˚wףy'6׵wֈ{cC{=;;t β5TAl} Ĕ}@_ x U^&Pj-G"ɓ,GǻA/8a:>z*+:N9B1Ԏwk_}<^&Z\Fcc;xt_zh4&tցHQ[~ǵ (Ўݼ n>}9@V:RL$w;n΋p58GcJ;"wve1/~4s/eaN3 'ߡL;E8&Y1qK&ƿ\u(}ق;Rli甌<"Q{D:[|~ohќNz z^3pJӛ\v ;Tw;OAx}l]Nu>vog87ۛxoxA2.ߎ5g9I @&AݵWfffRHY[?;tgЗx51|w 3W{CLG9=26"0B3侷ba ^ cq`vtbu"ǼUJ >aDК{pţ$Zvvtc b;blDꢪ>)aDZɅHqLh0W8s,u]5> ,HZN»}x'HA_iZMmS> %n ǐ}Eo67/`( X_ʇnX`g@Svyҁt f`6nݍZr=8&"]=A]C ilrk|h|~拾 [./VJd4,Np<)\fT|!btw1yWtr˺0Mb߻[]|n&^+mCyrYzܖpchQ8Ï yxԁeqGu6 3.޻EkG1(1LEiNF~?ǗL_>ry' Ffx< rjmNTcR<%Gܰ 瀈U^6%Nbr|vtnU 7'EJ*  FKdVlidy秒O[߁sznuDoFVΗ>ʜΫU%*{2>|˧9vK)yZ~-~)lj~H=0fwZ\_b>o_KO`!/Is1caCGқwĥCw#^ }jdtQoQi4j 77rdgV xGn/V8?xl~^` %~Ds^c_3ˆ'c'Ưłk JvcmR 6culx#|Xiһ`?u$S1x&xnp|S0=ۑ@'+!z˼ycHq|ui*7>,ٯ~,Vu;^fاfPVk2ǝG>0vދnͳW}& mzpͪU!!ۺ[[ t3n@ozw`ɒۆccPAS$#VRbp8L_2nk%p}8b]]9?66zk`n,X6뻘=[lOv6R0e.^pEߘF i> N 3i_I֎`~l*1E`Soěg_I R:a[m[-̻,R54m]]k)~W`ۙ7~. y 7$l#\ 9=D}IVǷaV{)&}3 ?eBUO6dދK2pkp7[#"2 +73Jife[ݯK%Oyy6k]st| ug]w}2^Ϭ7N/G_LW<8/]nd7v1l}𢡄n:l`K?d3x\ y^Ogvc'3tB;vuEX;+ ;G  ik:yN wt1c#CW[9,.6v<ù]^]q×ػp hZZR?s7P7&$SWQ&f&!|TuL~7[7rW7oE] v:p%xwJV< cgPv=%۩y"/L_LzЅkGB|Ǭ#HFn{1h $TZ8gG҇>T?B  jY-Zl]tЍ߆fissQduC`'kbȑӧ~4|]]1,(1qEо`{pak3p']_Jb\o ǣT ҡlԓip&;^=9 ]4|^iXC_꾩cqlL(FXWE>/ rgR}}\vpN 50}_p~$C%o/b֮҃w{sӕyqn*&vtr6]~x XZ;7K.2vOf1s,KeW;vsv<=m`3ɺFAO6(J1$p;"ʇi6qD`L|}xU9q=`7lv߀sڍ6\׋bܸq<;[5R'9bͼWz0nn2osyMu5Ǎo^\ʫUggVo3p>ND+b_'=݇HIDATA A '"M|>>Řa?_]5 Aa(So2ޱ=yήt9$bݜ? Nyj@Ik@la5\f9$Evހ@@} }@й߂&QY4(QSٶm7W%9]rڸߔc&o^Jwl1hMҝmd;xH5El,1yJ~B{W`j lX'<yLkA~9Ũ)FMp5L0?iяjok}ǻZc~ӛfM~0K·bXE7ϽǏ-/ 1[ɛS05RM;AԤgͲU9ꀠ;IwgH4>FJVc, ܽZgpƤ .l<]5+qσFo'_D{7lmj֌_5/+Vdl Ky~7qo9o+N5/Ŋۨ/*kf}]N<`lAfukd]ak4Q92ӈq!7pcvx)AITlI$Z9Fr.r"EZKC+?~wnS;b@V< W'd+'Zl|^A^ `'{ 4m9{n_IjzM9Tn&ٞb֭~; tx\ۑ\''ox*yZ=(8]jcMz"9w1y34d)Rb-31F$8No]>#9{WmTV^uR] e oeu$ Y? 43~c]9v' iʹeFy;X(a&B,Ya{"׭/h6*6Zp8&91?x.tXBr~`?:_cY^֬.&Y`8ucv8n,Q`Y^Uԋm@ ?vtRn]^ 0z5Z=.p~ 5~q'2uD9:&]p>6[gIߞ8\G^ kGH1hDA`/{IB[bYĶfνvr#RafZ0-tҒlWv$3>ꀺu1.m͞9&MH`wuqݾ%񽤍ve1T/waXrA5\[>v-' qm3 7Z.R3ag&pcnF#9kgُ={r?gS|6Q4MHkȱ#goj8mvCN~9J6b]N`]+SXv~'=-o,NT O1HǴSkd=`({,/{ONܐ;յ[/E)=*vVWn鵋 /AǪWuk#ssnyIci^-.vVu;.trVws:1ԙX3yð{1j?6iwuk|~l.fF&:0d_zh#joniv_y܃_.nuEd|P?)F&$Ŕ *\SgaPU{o"uyT(b/Lu8OuktlhE$?-hOD(iEރxlǀOJ^h Gԭ7Gǒw?ap?w%ɯL~ ĠĠ mu,do|QobOk7Wkmٕ.18ĖZ_z5J&ۊSfy@qZ.>RV.'Lu ݊C?k.wtnۅ32vHHTfQocFȁ+']UU3$VǞJY'Dgnn[g`W\23WFA^saAC6Ӳ/D:Q#+=3COyhN"o[A7X>: 4C3Mn(fm^[ں üB}#Wqԅ(zAc^.fOa_koW-G22N s]>Mߜ"ʺvJ o3GH~Ԝ5^B ٱnsNž~4ّ|HuCsrC݆'?8'ZzT\,S 1PW@&'ug wϽ>~UţTu4zom0ĺki=  yB;TwSK05?Ův3_8<B[]ɹ\ro6E驽WJ8]:U:]BΟHD1b4RȪ_{z:;7oD|0d=[y)YAG3,燀Oen|o$z$[m*40jrXv k(~p7=[^N&^3&Yy#J>kPם3dۘqUTZ:ԂbO';<4M%߉d!臍. ?1WεuՀs;=;19] t$s1=] })*͵\Jm{3XOD;/̜w$ rě|c y1(<Lz7)DVݗ>ߣg~9{) ck=A)v'RGuiC7(28/±5l`\;cM}anF_N9 찼IKg{g NdaOf%/c_!2 l䲮gvs38dWg1>ksvR/۝.&9C/2@^<*G٢?֩"xX"U~];Eu rdخI!UctaC~rL@@D1`)967X>֞Tv`ai޶qk@(df{5k'̃uۿ#F|`)]E#qGcnJ~|i㗤;Ѡqvu/ ( ߻g$>;df-.U%ČSi-`^Ɏ6:stq:# # 7ec={.3 ǝ|r&whs0tBC#+5kQ/%4Ζ>\]mkI] ltpcī$#G`co2٧SY<{ZǸZ7׈}%Q};\`Fָ9(29nqBDP ͺ4$6̷YsVjGĪ\.|'>:ρ}l{yruM|c]t\a#{N}أ̿F>4qǽt://b}}ĐOI]sʉvz#"6'nd~7K7%~y:Ⱦ\r'/4Nʭwy1lN>Zio #%Hwah`s݁c8#l  1l.ܘFVnσM wh$j$] HT(\;hö́y]z&C$=ͻpH :. .tȇhW0}Q.jsѶk^6=huų^OHшs`1Y߄G6@ߋ71E( 8}2m\?O in,+bƃ}ZGt󋌥G+ǘtO54 TS'sSZ1P#~-9t?lM膌{aaڑk#+P̡5&0{2 :0SdֻX!U"_ 5&8FB9 %06 <۝CMgvqn 펶虧.릝ܑ}]M}|~alO!QWb|lZ"inܗs4"ŷ!; R .=vӖ&sSvEJs-ފ!!37ZTvb2iy 唸.2CR^sƐ$rMv!mArFR7O  ]Ugfˀ-3{\K,۝Xa0H0^;V~ 7M%6&u?H &.}4H0 `IvJgnؚP7~PMXbC>>3~Â"x#kP'mˍ0gYHu/?`?g7P(1hgK% ´`XoQ92Y1uG[8@;*5}-/Fyय़qQ ð"'ov_`Y7b!kEQ!ѶcyΥD e*Xkb>:s| t\223Xvl"N?]tÔ{t&b> e@=fˬ${CD쬄'{1C Hǘ>95<׹Ɗca?؞F#ധϹͅ<ƭ>U}ZG 3I ηQQa/&Pk"Oq&cZ{ujQ]O-_23^X}qrk< Yz؏qZ8r>j4-M3>ѡ9HCj)ޖ"`;!"H؀Tԙ;9g5{ᏊC|t(0Imݰ3 Vk{cnT1[pވkk 2}p8]w~T^+|={&gsap.py8GUwOŘ!ku\m܏ULebZ6 ] Ӭs-k.i)M̯~f)*pY,?ylS)gM&עvmGeG;mesf7pA_)[lA\ @Toý˜-4;r󵆭8^:<#8-)No/,QWcJ}C/@O+|hH_ҾJsX>T_1n!!' bT/Z gkǖ'u[8S5)Ԏh!aR 'GcNZT*mqU1NŸmQt}L\յSi˙眫6_!D$)?'bU4 e |x+c>q?B%`DC@u*S>zw`tU8!Z3/!P>_)q> ߢ={Ps5}oOfܞsV(Y n8\Q JWiAD7A`/מ 51-;#9`Z!c*t "'Io@1]Ň67q:_Ƨs98ߟ{9Y/3o>8`7ǩ=<`kҌS@uoƸy|+, DEó=ֵZ!wIűϡ̦^XYeFNJVm]_󀦈[)h79 |Z#v< iϑ4xRd͢'v1-] qKL1'`3U%_8ۗ9C%GtqENFcwq'޾tc$}O"4{qMO^^,1 1yH_D>?tC#0۱xcc !M.z(h:mm+O ][0'TkÐ'<*ޫIv[xpYcTLo//tcLR}Vm%p@6 ׮~gyz D\Ct@)xy=plM9pu@K\a׹{Vh 6m\9L73=̼:]}ж u;M!MM'i(L9ckcXH$*ݴ?H5 23|5Z_4`r;}gͤ[n_uznHϲ^q=j9/mQpNBM9i\X2kʖdK19dԑH ϴgc 7=1|ؓi\J?ls\$lp8< ~3)&f[S> 9$8-@\*U6!׳|G3X,F_u(I/\S8=CwM-k8nj3n wk{ŃN 3ݯ,sI5=)|}6ZlC=uǨHJ_8\}$;/wX9!W]>o@/3c̮GsMiO^C` $ۏG/kƀs֗H~_h7<$ ƶw,xN{|usvq瘐s%^dfX;T;9oz?>S-PXWppݗZ<8Ÿ8vN&E={R尴"&]عv ]H)YoqⅾZI3!Ǔk`> B*E,(]^G*Ko{4Csy9 hݴSW!),׵"n]D>e8G,| Pg+As"kb# ZxSk(-Q-2BBw.U^Qt74VgSy qOͥVH]\v{BXD;VjR5!}l6mdɦK_rHWA_~R@9Kf4TOJ[uT(XT|*U>>v~\`W!qou5/e jT>j;ocg̾"XsT4~a_+@UsL[mIyĜq@񁔳/|̑ϿGq9w1D?O.ހ@BcԝB/gxX)N@m@@S!FlƑdk\GVvP]l^9q|CwڐK>LYFhGoqռ a&o56AshI̋8{jA_\+1 >B^ҾL|E`73A2`>ґ0xe.[.yrvl~b!ۼ`o%^\@xC}0,po}õ@3q3dqrtȏhMn^oN]v`%'gGNpNZz Aխ?$:eˬ$CatO)P]#8v5v;e2Rܓ[Y&,8-ЅgKx$9q 5eg-4}=_*ZzQ׈v:^olۡM7__9\.ۓY*~w܌j^yŘ-*f>9}<: |pU|\eCb?@yYsuuӕU :ޟ2iךFk;^C<1gЌ~s1gt >aZm2yeO=5\ܵUѱl}7*_JbLIp{ |f^/bm~֔XG-^ڼ#M0CIJ/AӁsz ;پVؾ 8>>;#Fu1sYkx?i%K/ў;cqSΩހcKܨ7rڱWz:6/&Rp҂ovy ;ٳwl]ӧl (2bv?z|w:h:LRŀHD[> r'YpAٌ%}k17a;7 @x4h |V􀳯* ϫ%s)Y۫}[KQ^ϿP`FD_Bܤ⹖嵲1ƭ-3Tyao0qW%~p꺺|/=R'"/t1T>nT|;ѝ^%K%e9feKy5 Frp7Tw/C~Z?#qo)u8]-@eD|U1^GD}py ! Ap>x1 t4L!O`y׬.MaQE\ZIx=ӚQ\*W *uWMqL_n=8z /"NJNPoAU~wXAwŜBsޛ$ ,gPhe'Nse/0mgm} q_лk%.Sj}cz390e4en̨F{l3, .;V' Gh&Q68/#בw1$[kysMm9f3d&vUfW%r@}\|Ifhl )(s֐}={cgS9)38\,`s1-l7̺+Xwk1wYO;/q|1esnsx|/.ƉeYʔQ 0J <[Χ8(^~ l`q3 z#5 HcC|}pxi~t&IF~A]mJ_?q&0grH,;0"pܘ<7mHd~s= 濂\Q 5O&mmN na:z(2x2Ajsӆ8RYyxe9q~9a:|7Lϭl瑵Yv[Iyy&ōigeٌlQYn6Al0-mdIߡ1f ݞd9O^cya|,.Le]IidJ1K@`>QV43,7o7YɖqL AI!t~]kgՕdg1um~_=8/̃yO/eu׊;4}lQv)η4n 6i%m A%m}3^sQAx:P'Dt;.Y?4tӧƕ S=7:|J[^:(FHR?_:MAĽ G-_E5~-V= ElAP ._)&w: dU|ޢl7]r]?UOrQXqi|9_)PQWE Z=&P9_[/O=8 CɾT8t "nǚy%O7-1uk)}P\#bq'@ℳW!N-_܋6{d>rTfi>+:S`tDt3aU+#*k]@Zۊ- Ubӵ!{D=Ψ::g6!ק " hsDV97nvڏ$'/{kv:5z7{m7-ƻI҇{fO-Ǫ&4f :U^.z⤿ v?^A/$/u rµ]~wM(($4:)1/x#j>7`^qR&4['ug M*ϵ!PiB҂zϷ06WF4 Ā㇄`6px.vl'=5%/V{Dl=x}'8/R\BJZT{E_ 4Jn|L1d&TIJ"˻ᜤT*cm.5yvY7FuKl'cWIbټJ6a0wA+ )XG?P2IU <|"ݹYQ&^51hy hnH9ݶVe274!Ε1&}V{|+uFu>bt>Cw %EJ@dka(u(q3tЩ\k N'73zlsf}>+0'm ~j\;5b hPcZêsࡰ$L˶0&<NjiQ=_7G+vl^٦_" }cBŒ 1D s9A p33[S!b>"|@~LZF.\UqD5z1ˆtʧT[xf?<9a_%;\Z&9ZuOA}7ԤJɑ % [X 2Jg]H/El#dPz*}owYi޼׌%=#q6c=p!Ame;lB/"̈&.}" as^߸ tW}rS4zT^qIFD T8_m,h@x"X _Y/ Pef6EiScl&-. NfOk$z0 _78rʆ  ΁J5у FO\}r&~ zOR8>0ͱx^y*1ZNJЙ?ǧ3*'&b t:,KEud%}ՓOS q]X,\}NU&öl]Ϛif^AAslsKE9xXƼ7-BǦvm\faƉ! F 36ωdv@3O,!oJaٚ>6%j9a1. r~5bNGysNEY%=C#v3XWDȅܾG.`{ Ӓ/|e}?Kag?f9?Ȇx^fH,ō8AxS4.ƶ<~",e=@c8rl1o7@Psh5$ t"3vv5.>8@DE Es-q7Pq[ቸEjx?#JW7fNha/^4w}D:@#:+=+[fq=`;3 ۢ4Z1=Ȗ]ph`qN \;# ̌w 3 pv]~h9a+hARxH?a gc=z~g+8S,o66;6 J1Q xwgn؜/lje1o݇cŵYUF_εFI/ 㴡`D\8nk_:% nΗ88_0mݷL͎}Do8cU*U~.sX,Ѥ>Kă4?>tbx|Pnfw]n$Ce(?v$02'Ew ZdYN4ukk7;+0<=MNnȳY8s(o'Y,s}]9+ 3U܇`Id0 =}=Vy^3/U_fA7Z?sُ ǂ2 ҃.ip$[ˈ9oc'e:1nV3gY]L嗰 1˫skl;?w}`?1Ͽ{qS,+;uLJb,k ]p |;Px 5N}?$-zE_ʹf+\؟JԣHsx̨R-]^ƱNfp[s/۸Pr>αϫvL~ 0Bt~ !.:>xCkvFbIT@`٬S~ $=&#mͽևu#lK sM'?bsP-oK^h0ǘ6AUϒRaq~ʑ5wŅkI J q "Im  _١ltdykdwqNq_ bI'1B_Vi>^\?0dW@#QFixݖԼ]!X[.$]s NW* NGRkꚋi^bn=kmsv yi?U<7}h".8y&6sm1=^S6I)q?qw\M5EsOJf,a,bU9jn=zwIV+*9VzBL7"zsn.9K^!}=rDvs <`OvZ^UlMu=\Y>6(kU@ēk{22^,;ouPgPPSx¦&2DwkS1|S?oBvBcY0v'-n?U6fc620VǑWؐb!ŀk&~>ŧ X;S\s|8sA=&}vNDgAq Z6CI+fi0 \6̫=|wzǪ^|pT﹃ c"K.z\D}'St1,Yږ|*@'WoCI{vBL`GΧB^n$({ɲkm&LM5tqs 7꬀1aG"F}s{6Xmg U.C )gd _=d&.r.C=:'J&ɶB);B{IܿϏvLwC _f:٢c0ݞst"8gpXYG^ s>ҍ X$:q߫&˨ [~ BZ>b7 'GU,sǡ<' lVV \ 0ٴoI1s8{RF3ӌn>̢Wڿ#'׎8nre0ŵ?覀lczH/xBLIVzVDkjs\ >l]zr1"=,SZ~RgTUa P.\gT|V R 5}ulZƝaK/X'‘YC>ľr, 0]=K Avcێ,VXa[:.=nErqS<~뱇>4bh[m2(6|gĘAwZv>!|t7ż:?@.F5 y|\E>!ؓR'TdLJ,L ̷Zq~8Ozh^3 }?ė "KN Bw_g7vgNĔk1,޷џ0~`}=6KDžLŪ]s@ hICY8iǥ8,ʌ?!SU]6ntmF-AE+>U1"$@7m j%RU=!rz/N?x |4l*SwfzOX. ~ϯn}sSmG{{$P1[t+x\|rJ9YEs҂sql6]G(+1-l7ID[9߶&=8aK h A+ʂ}Gupɍ뱣5gڹ&5}5LKcN`l=頋Qy>OJ0ۅcyNlnZ/0?^f`=ݡh՞w1vL_KTG05ٮPky/̩"p]w@b4ڲgY qRlR^ z5M#:M~n?8i\}s+V+1LcjA= P_T.<*65*eݴ!Q\^G}8$_9eIC0ʱfh:41>AG #B}=Ԟu֟@{ˎ9|sF]]=p9JزsPzK4Jg5j.sK"36q?|oM4B5X2LQ\s ہg1IZ /RIUBLkt6!l2yV㊿${}ѧ}s-I=25ϔC+KD:xǶzg )Ҟ=pH ]"=ܡrp ϬKtg&obE6?®e3&_3yHqd7!ad]z+[,u!sӌ-wYKbU%@B/_ 룂7YSo:vБd\xiYՊ LMv~GX[!T/DJ[>+xofQ,4G8`؈M?f ?cG<ޓ*S?U-IXžhӅCׇS\m?<@Kǭn 9O>~ )F'먀to\ y8'γO> /L}{A<`F>>~^CȽ~c4 q\xYe1\g||(7cͅ790ٕ^=ڲcv|u*du2l_s'YL7}XՇʮ>>f$PɶluAQØ=2 $ Ib>\;ZW$ ?rLK$/L3w#C 7hQad\5 CaLZ_ W(?ֱiE7wHD / s P]$9"E&CV?2$Bj&Bmڅw s/SMē(\D`A*^rb!"W^T{EDuSt0P~⁋t5gM76 63}_ڻI-ԕa|JP*GAX2.s.ouܶ*f|V:|z|v8\_>+,7{sh>ek~$)n:Rl:[u+W\@-'G" qdmLkEv(1l=3܋JП #ki*VިN`c|QB;a~alGbuTWַ ,+Oׁ)Cv\ޣC0~w 6+9zKv<4;:3דȎmicp/2OܓafB0Tt6ŷDpPoCv!:Dk1oyO=yWcq?ϼ:t\TkDYӟo1BX$"G Qnu`QfMѹ-UCuٝzx# R30{MA"϶=IFOG>:yW` mr=K۶L姒~x[ԡq}񳐶ksc ZYτM;Cآ;4>4s-#mM  G8{Iszy li lgFBn}C\UB>vqwX olLf>1~Iv@n A#$"rX ]ѯ嬠͖7mjc`G?~hESm޺9bO!&8sG1 p+ C{aO vP8_Q-X*^lJPb(8@1-uǪX 5!4A A0|:*lꆸ}(kR,>(ky;udXP@VʉO}0: +T{_/Ym>\мRsr66-H= K"_*?o9}R]3g;T7T>7tXs 6CyJ /տF8s+ZcwKvX%Ɛpz*\" F4b6N\~(dP9 r|M׋ q-7xm_ H(*TVyƫ8ku8YVw@%GO yÈVk ϑ8߷*3* -oDvyBюȓNn(e C~+k}\9GNMZ䟁Sɋ%Vo9702vp>zڬGAWwPabN!]|VsP4W=i@+kg >`irgM[S"u1h& X,."N<sΜ@3Y6,܇8M(;sx|EOBydLg,c'آ;)`FqKxȰob}']/_p3t<p-+ V?G>1b lfI1HGѱZZzinv^0 CI_1T Ӌzf{%䠫-FX \n\+Hf)__utGK/OiM8CS9EDetq  =,?DtLo_턆ٴ>;IH;"b؎u^R<5l|?pim:8={.mB*7wK~/)U7JVۇ(ks OlųlX+(z`yQuv"|}9:<Lg⇍k7_Kˇ2~ʚzM6Ptg'w >U@/~ϽӪ\ʗu;YH7 a$Ki"hލ,wfຝvN̉!s aH2]uK7PpHxN} 鱼Pb? .HqBc, HTмBc9̀ 3ʾ0O{{ڶ?9=p$`va=rU.4() ^|ꆡlKkOEqBꙝ 8Gž9α_l8*&hRR )數~0Ɛ4+|6m(7s «xd8#Hw&k AٯP1đ@VحgKW.;Fp9jxl-n0B܅:e '.3 !O9\ Mnz!)= (/^4&wI~B'%:5F\ypHRhtOw79*:GՈsض`a˲xlTaO tsnQ1}Q!.O(o(I^]u/<ƨk۪A%V mUWo!8iGrcw)ZW4>1v4+(u}ŭ:qCqڋy-Z~ze~_z qYkDp8bx0F5#rv=<&lg_>@g njA3oP[ +8fF&=U`_jպI+8q|>s\䗟T'8ǎTk Y~-Eίkm&.7$}<&Z2=9ﾶ;\ y5 8F& @,c̸Ia.}(nAמXsSN8\ ֏`Y>#+>3|f ~<21?kbmbx\1tqŋv;Fcr@`X/!Oݣu]pVY}%_Qsn':t}?-ƊkHz縵&|#[iqGZ/` ]xY%|XJi;+˨P;6N,;Coobrel * eq`K JoX\H9?3F?;43Lk%%:@}-7ψ\qߠo|o8//L}(1}R4ȩ[ZF^.BWyj@װ>Hހ.5pc+p وTSd욞@_95,8r) v!YYtxΡsTz&dU>kUP_~מRÍ\sZнڗk=H=BN#ߎ%dG2Ns irO|DXA״ < ea"+eiĩPzNSIXSOME=-~rzk!Q|չS{5O@ mL{=|XRv-Z'K\ X ΢$73'Աd ރ6.loH9"Yquz^X,=(k>5i=y=+?^>18 /hb>r`U#"sY!P = hZQ'] 㲗'>Tø80śg~CA?nB=pHHz>`\Z}NL15bnN>6Q\I7Jq ]JAS4d p\xx8*NOy(s(W+t~e?|!~"2#՚@^r$ s/bͥڳ{T_B=MAo7;ͱ승A 7IT*lr>D9X"TM 8U@gg|e~6N{t\6!1^1 $BMHlTx /s!F0JɵRى*t2uFUN|b*k]QuO 4tM9# ۏُ2N'֧Pvcx~h}= 31s딙i\ #ѿX::9a5Ħxl)KQ0p0'( f63ӑjӁc@&8L< *u9/ދ_f@dHαIJ‘beY1 /0k1> aBeȍτ]sO~B8s 0}8fgb],.x 7~eћpPR~IMW/5qT2<'u7&H~8f7[iGDKӬ5Slq1ĕMNƶ(pLkve8iV{aiNĉ/)EQ7,L”,hbhO>vr}0/AcNqo4L'̟r8& 3F=nNlf3b=scpRlq7\+}Mז(oAy'k ~u'҃6H5P|utW{c2m>S?cnqAx&|?n/G"O׿N\ {9Eǁqqȷ]ss>3vubq,::8$3Nf7A`!x9fc2x`]&d]9g<|&&`ލ ;+L=Sعk]w߂c1ETy s|{w~`0n C~EN9)&T\q#lˑU>f-ox?Ap* _b! ZAoJJ)\ÊའkfhM8'R05K':Vo8ss#QI6ɲ#_IˏWN1/Kvx)Ex(, . )z+ĜdۺVe݄˲ ]f&Y2>VƼ#{oWmx$yN,|&[tIgy3_أbikr߼q}\e˰gaSGSGFi}Iη_D_^ Dl-B~o, ljFm(n.Oagg[e .Mʺ %bBfA؃ifv&δ]8Vݠt}R'%}hnNc|s|Nڟ9gdLEOO3K/t0ƞׇbrZfWjW8=dL'b(&> 38aӖlyG׋L난mw$ =gZ~p+:6 M6űVnq.ja'ud={ s\B?_ިbAdN{{@T>uCcU܃+:;ٿ ɲ_ڛx]Jt1b)q|vpY_'LS:%L+ ||l _8Yc+` nȹT9p\ckdu2jÆ SqￂBgMN*Fܟvј+ QgC Hlkw^8pDJ e%_`^~2S>@CŬʟ ?'7j[3OYo_eΖ~7X7D)k҇\o{x晿-lcEg֬Ŵ&Ԡ-:AٞJ31A&Odx]T^晴)/uԞ7ƈ't֯lgζT9&:UejB% rL/l55}^=P\-{Z Lmjᷡh&j?!7|yٌ ÐiC,a {F~Ee3Bv;Mq1o0! !3Y7Xd"p飷?iˍnYw~usIS CBfu3s909|)pLplO~ly_bҙic=F9Q_8syM2?cO K#!l C>n* sp1ljuq\}%|A0?t#IJ-!+lb 7wd^0dJf[PATmgY<~!e0/o Ll{M\_kK[Vr<^r;#bUg]8)x+/8YD_s 0NuN`zP`x,`ZH˴,}c =ӎѿpd/C~3#[1~(LcFg]b̓∱`ZI|y1;"/F{vźx?v7֬SYK\@ވ=oaDŽfMc?QRjNaZ/QiD-Ǒ凟zL϶v5fE)K4_ #x7-!+tA[>Ɏ7+}l><_lw- u1XWm@XPF7/gN/<+-6~N`m(}#`ӘDkZfyn,QyQ|zA>p'2Oz¸\e>i45/c6ă9=5tTX +A/շ~E7ܚ 3f! ʘK}Bj/J_/v^{Ec 9׃:>GǶ+< b ECeZ$ ev.W>5zs먰}u՟/ZGgI/ )?P %c%V6; p9c(i?'؏%&uZؘDUt 2RljҥGGzm]Ũx *67$ z4q9'"و>ȓʼ~6)%LHY0ֽxXbp5h#V.-df6ӾN""?qqaHy*+fÏr@~|(r~YB<5{mkBOw9+2nB crOe8pڋ=,-g%@]^X&nOp0o2H +j>H(jޡ%dPL f/hOm+mOʜ`NNd]y6"^տ6Cjċ;TpsqdV))4T31ύ%*8'ZGp 8߬s(H'Py+nBGSBTkIa؜nT*ZC/3Lr/&lO(r`وc^'3 P<٣{AǴvymA8up_ǑNS|"X꣚+J(%58clF~ b 70JdPg[Ϸ6ؗ !|/@T|אDƋ=&O|ExW{Ug%(̣Jgzs͗8Dܗ nXCC>!kq9pA^WB[ɪq&k%?7׎9]Uʉ xfinNre^O8s]}L;a1zMujhrE0\% ezޣIY{&R{eU G\a, YdAԙǂ̴\ *ߵowUF<>?Po:_()wotu\A>quWfoţcO89Us*F$^?f<\d>d~sd-2dqlv۪^f[.TciaT}5{l=_}P =#' ol 1f\ xqm+C |7]yNqP瞧dk›N/CtX^fFoxy `d't ցzDv;67AC8)4fwg XNٶDX[l׀3GJߨxZ;y>-Y%\F{ǝo7?#q|h1ܸ4硿вm85> HquBFߐ5P0-[J`_\9,lrsOa[ǫ}IAI'@?G$W`Zx7q ''|V׭=Sԃ;lgl%D|'>F;ȥM N&f_cO&ګٛ$!cD + YT9%#>@qQwPsovW(st}b9 #{yhBtg vaZ #\K۱SՈrK~HWxm1EFֽ_-v7"gv:!^4U?=}c>Vk(.!LE@%z8v@zKUށu3:ٯp-A69u ,K9sjE{a5-ܖ`rW_uW5 _m AL2t8xdj'0-爳A{[PIš-]k6?hxn3=?3')ppOqC0ɁrgxX.taCS.v6i#hRK姉q}T!Y Aܼ7[:NǾZos7"RZf"8FSg{!IJڽ=yν_W0-IGv2n6t1A9sMBT:Smbnq.f_qo%upAK[9 ǞSJУLOXq ^{ d*kp-ZL]bf"'njl"[>BCy~a*SSp z(o>T)D;k?-~֫4htsw[Qq@SBE/9cH15٠c3眢 }GB=0:Ծ+-Tg%w^Lb]Н5Pgcz/UH |rf=^,yށc|sCf gק-Ak/PCūm*6N8%̩'%n{ J9q;'c.T!*\_Ck6yzKOܥjQ+Zqd/9 d)12C70ctsB֌z]7ϡlU0۴ $x╅Wse\)_@?Esqrlcը+dxqzXzo/&}_&{q,BPP 7G ~3!lűڤi(TJ۳yE(]d5ضY,? GTX^:_~ /^~v6/&[u?1~a}F&i+|gOOxlSr8z}\"|0fٳ1.7(yjNv@_9WxaG1Tx. KT_je)^ȁzc`ͩl_ ?RX\s سYǃ,><|6]Q5-^8s:7恄ӝ՝äA<ȑvL{'c6''b1Qy#oqp:_ɵH sҧ@'Y>BX*^ 7K7Ծ= TIPfxOYu2] !GN9֣)To׵ 9yn bid=6alW>9'/LvpJRnjUK=.3x –_/l^+@XHӔKO~kiev#qrg(VBw3末Q )$dO HO@|ҭt=v\bh0"m8a1D*8cc-#Iu#r[l#泂qۆpy1sWA\s//3%X&:̶M^+@i"[_a|\!rlfeT;bCLȩ ;/=9F!/tuq╶Et{ }WW1pܫcɻ 98<96q|$q|<<AKlvds"ŞszqK<$%f6m3}<2rY$E~@%,(؍n׵NPN;8]/p4gT?xn+:8H\>'YW!=b0gDϽAڷl^$7T=0P_d#Sg's b$ -T\xMٟ#VG=ibRtܯfܛeА밃WEu''^=~cl9X2YN>&bb \^hR]rl]?D=Pt'לBpzA)86;d&(N5ypzox]Xo5v'j9RȘ"na{'|c6ggfډ9!܋xEW?3@Pb]HYLV8+|ɮQA" f uc/R _47V6ʜQf 3.Cc0Tq]pd;'yMmǗXP37@818#UA! D8/ht^Rٗ=`.1UԒ"*;rO Aֹ/CRݞ{\^XjO"cPh)M-7 7C Tq ?4 gJy]Vq/xW>}s;_ׅM+`_cZHB$sw.)[~UoP)v s3Nn2V<4ptl:Ϯ^wZBZ^(̯׹l( fO2omW A8w4_ީn~us`:Tk|Z|z,hF3 u ,ǕPyRA&g3 kojvEӊzCC:HѹkeGDCQC Vx'7obv)jUq0]%V&=^sd*Z񞑚#U:mꃈ&7͋ܭ'h -=.ϥ~pPrH͊s`&z{Əߢyw9߿1)+7 &l8),؈]B‘Bf/P^dqflO`1INʅ˄l8)FIJP7sr0ʵ݇u${ G]>f:%nȳ+y#%0l?+T>e`\)a|y ԘE!V&~=Yf">іvB< ?d*ȏwUTc ĭZ{ϵX+#NO&f_&>j~: x?.m)Cxq|&|nqO*}|5@&ː9D)ox?ر@P_DZXqrD )>a` Q"*jOS͡/O9w;'}3)">I, 4뫴ЯL=;r%>Gs/4< J7 T-@j}8@:ƜseL𾤲rVgS{0[q`;8G^m>ΏY 8Kƍ^ꗲɶ7qhuȈ獞6r>EZ?"-tf5BuDmI$-9܋ {2^؋g긾Ȼ*ݪw8Nk&v0)kg$oo##]pb,?{I:E시o9JF'8Ay@Ck73l_=4CudChФ\@L@QeCp3} L1Dyg)1L-j.\$,W]eqsQ["[0ʟ_^ 7<3uB%S]{Od˯DHqӛp=GUd]ǏMF+d Iͫ&8Pq7j秸*xp$1?I9 ̛B }tza_ا8:131~É50NMXϡ'(~&k{{tBQߘϽ~e&Fkǽ5?'DKDxmlBy^(i=eˬ*uAnum>W#8֐3~EY\ Yk/(;4_R<3=PËP'Z~A9Eu$ОcAnmgE e[C*BOAlM֔pK'&bOڛ8G} iY?Bƺ?nh&/Gx~ ʇ#l1kHqSE#v_/mx;kzdJurPBpbN?Sеc.3 ۾'Wdpdf!gfgL'mkd>Yc+e}ƋķIH m3-_s`]*Ȧ_CŀM:Kź' fjf \1U\M (ΑE׶L]%=MS;k?M^s^?mUbCi?TќCXy e~?uu]=tMg^-W3U{ɲIpC;5Dž q?-s]Z7kOc[f)>,O2^vuYz zU 6&~~LWNd-Q f =[ֿ. ec}߮t\ʷgUdcEqwt[apBY|Xv8bpWDT|&{ڳdQ7G-fz@Fu%^ՐBB3?V?4(_ O<]\_n.^ tIF, x;>"Z(v)gqCL˾ra/J p±SD3 {|0?gGsl`[Y̱ xF CEL{rn_#9 /6ѷ6ڤOq~POrc\wlW'zߤAr% 1J 6y[j\]iZᑻ%:-s@Sǩ^ط_ۙd:{(΅Ulj[ex{(;YkqyT|ƎoLs2Yd읤k>fz,gݡ7 SK /3_)}f汽y1C`,u'BxjyuQɁjIuM|;z3LMϊY+l/x?l%n ʟ ^yyg$}AtsTqE.Iώ{\w" E/+.c<=Vg3Ke^I!2X>q % ʃkË܂|OBpor^8se]i(f@1Iyl1s?u[cdɻ4:lK uE8WS-讧dy.Qܬ IENDB`timescale-pg_textsearch-b63fe58/images/pageviz.jpg000066400000000000000000010314721525762047700224050ustar00rootroot00000000000000ExifMM*V^(ifHH0221̠0100HJASCIIScreenshot 4ICC_PROFILE $applmntrRGB XYZ acspAPPLAPPL-appldescPbdscmcprt#wtptrXYZgXYZbXYZrTRC( aarg 4 vcgt T0ndin >mmod (vcgp 8bTRC( gTRC( aabg 4 aagg 4 descDisplaymluc' hrHRkoKRnbNOidhuHUcsCZslSIdaDKnlNLfiFIitITesESroROfrCAarukUAheILzhTWviVNskSKzhCNruRUenGBfrFRmshiINthTHcaESenAUesXLdeDEenUSptBRplPLelGRsvSEtrTRptPTjaJPStudio DisplaytextCopyright Apple Inc., 2026XYZ XYZ =XYZ J` XYZ (]ȼcurv #(-26;@EJOTY^chmrw| %+28>ELRY`gnu| &/8AKT]gqz !-8COZfr~ -;HUcq~ +:IXgw'7HYj{+=Oat 2FZn  % : O d y  ' = T j " 9 Q i  * C \ u & @ Z t .Id %A^z &Ca~1Om&Ed#Cc'Ij4Vx&IlAe@e Ek*Qw;c*R{Gp@j>i  A l !!H!u!!!"'"U"""# #8#f###$$M$|$$% %8%h%%%&'&W&&&''I'z''( (?(q(())8)k))**5*h**++6+i++,,9,n,,- -A-v--..L.../$/Z///050l0011J1112*2c223 3F3334+4e4455M555676r667$7`7788P8899B999:6:t::;-;k;;<' >`>>?!?a??@#@d@@A)AjAAB0BrBBC:C}CDDGDDEEUEEF"FgFFG5G{GHHKHHIIcIIJ7J}JK KSKKL*LrLMMJMMN%NnNOOIOOP'PqPQQPQQR1R|RSS_SSTBTTU(UuUVV\VVWDWWX/X}XYYiYZZVZZ[E[[\5\\]']x]^^l^__a_``W``aOaabIbbcCccd@dde=eef=ffg=ggh?hhiCiijHjjkOkklWlmm`mnnknooxop+ppq:qqrKrss]sttptu(uuv>vvwVwxxnxy*yyzFz{{c{|!||}A}~~b~#G k͂0WGrׇ;iΉ3dʋ0cʍ1fΏ6n֑?zM _ɖ4 uL$h՛BdҞ@iءG&vVǥ8nRĩ7u\ЭD-u`ֲK³8%yhYѹJº;.! zpg_XQKFAǿ=ȼ:ɹ8ʷ6˶5̵5͵6ζ7ϸ9к<Ѿ?DINU\dlvۀ܊ݖޢ)߯6DScs 2F[p(@Xr4Pm8Ww)Kmparaff Y [vcgtndin6RC&@PT@333333mmod:%,vcgpffffff333333AROT,CZN%/3y;v@HJ9M&RZev~dm‹U*Cr6.#2/=LFV(f{o]DZO1f/#9IXR:bko{„٭,aCx+55EjNL^gwԀ > <sO+b>w 2I`w//B///BZBBBBZqZZZZZqqqqqqq!##737^33^}JH" }!1AQa"q2#BR$3br %&'()*456789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz w!1AQaq"2B #3Rbr $4%&'()*56789:CDEFGHIJSTUVWXYZcdefghijstuvwxyz ?h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((pX d^S@#d_o(臇*Ta|oG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;ףooG;פt?z樮z^үPvd>Pn{ne~R ( ( ( ( ( ( ( (?h((((((7GDQ5v^T;1, P*3aA֠i_bj*CaN5Ղ(AEI QHM- (E#2 PtRVVSKLaEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPF skAWAEy1YQQEQEQEQEQEQEQEh((((((օCl6ƾSPb$JGY?f\#ҳgd}Z+Rrߺ~ҰmZvT-^M(8ʒtW7J8UzU~kEV&KB#0E,z ZCcںً̒F7=)QX㑢mVʰe :ík_ WƔBz(5 2֭_ 4J8\쒤,n 8j[6]yQ巯Ҁ7 * R4<+ۏi@1hMq1@[;r3K\bةY+oʺ lT=hj*Z MK*FT oM|cV} !~>$%o nEf9B81FzUu_QR^H&84V1lzPܦ#do>rRC+6'AVVIn1Y [԰ǰd߾=Tzt ZR2`AZH-=bg8ey"m|)j Eԑ8@=*ۢ}I*JlJrKu@x8OCұ'O3Qp8TOo݈"b`~:xqJH'xEk88 uE3b폥nȤ$8#U`\\ȌH^sodey~ }jX;⦴S@ ʽHJ™m<3\(Rinw`Ƞ 8#\]Չ =*i}Xv( A)ҔO%|'w}v8E X|?JZtyGlq[R-`CwOҀ/~V*$ⰡŨ?8&cpN @ 5s{| w?Jf\#'~9S3|m?€7!g҂yVU@YHU@h(V9s(֩dHX19?(ƷhQL51))ZU8ZhzsH +]"$"8HVݷA#8>kE6k.3&僌~%ݲ[ZmBNXuP[9lspe<+5›}._l͸w k`dѬMD1?¶&3$R:dVvw_lT}(Z4r1IV8̝v iMgYmwnnx@|jנ((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((( g-Go,6m)Z7}:`II.Tv_V´F:y.(!d}̿tVG Uԍ9үYOlX`6:r\VlUW{W{Sڵg-{_\}G@QEgyn}PҼ ( ( ( ( ( ( (?h((((((G( 7Ĩ9YxWEP2q\MVQgO޸<VUFjjBun(1*JNGCQVGF{W4)je1ƛE_1KWN.NW FPTs4 U'u#XEj[g5-fZ ;R,ɈuEfP=xh6D2r*qi,*"P^`Vwm4Eуڬ/ҮQ@/ yظm *0+Ft3'"ƓIjbmlm8PeWp7<:(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((([TP|6O̤ڥ3tm [*ChDѱݿb(>(Տd߿n16A_zբQ@ q#ڼڽ.Ҁ ( ( ( ( ( ( (?h((((((+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? j+'Zߡ ? jJ* ( ( ( ( ( ( (?h((((((zASCo,ԑd !_N>8.F8q"@$+V-%18"y2]T *I%`g=mq"6# rlQ*O\l<;!*;dRЌSQ@Tef6qQolJCHQV ]6ҙK vPTTp0 0HX{ s$RNXe` ;`uGEI$2vʥO=mnY7lW@QNTwm >$C5 EEES+m*tAcURH EX%[g;RUnC:( Vm5\[z[-@UahT18 %mKN6!g8qMeP}ڿ7@? )mBl-wp1)#6?Uϟv$c 3^"Ii`9F*K.e$N1Q&ɖl} Kysݻx:Z| ynAm7K {}? }ub@b0qCLi#ҫ\^y!GӮB}Uag3 B-I=^__{?5رkamXx-ϝ#Μn QE0 ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (U::@Q@Q@Q@Q@Q@Q@Q@h((((((zASCo,ԑd !_N>8.F8XFF z*yݱu+듚{F}q@#gtdS֥{%r tI`w68 GjEM ą6">m5Uh`eT HS㊎V/}5ktV U׊*\)B2@8 @U1"Үy#FNr8 *奺II#d;C-VMr=2Ҏ+DIVbG*au*1ߚWI :n?VӴ9:}(S7pJ)8LQRЀeBȥybBR)ΏlJCH-1\q@NtxkR[O0}q@QNDg`2}I%ȅGh`UUXv?TEwe='ޕRڨ&3tEh6[5Ű(c<})v$1¨8 Tmԧ'5qⴴ dPe</`i(.&Ȏ1Tm Y6ynszUHkEiIfmHc>,T7~1X讌s+=Ȝzv"Bue@. I=i*HLJ_AiG9Q(Qs @dQJqڒ$P7)lz I"U*} EMī8هpA9\f6Ԃi?ա=1)rxMf8K}GE9xdT*Uccljz#KzWgG#}(>r;lA*I-fD*=CES(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((EStQEQEQEQEQEQEQEh((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PPZVuZͫ0y0 b&^vIeq3ޮ$9T!e}lfA@C48_0>iwn% B C=(K?tVuY6~Pk)x<O=ܳA*c{ U:q{Go^9ϠǥQ_ Vn <[(dM֍Ɗz3?g]\A?̑lbrN/!){ A@G0g7QP nԨW˽Ilsħfe+jY?gv!AF+RKumՎ[@WB=Fj[ō! ӏJ9aesᱚmќ @ y9қEtvyϵJ.lAn}7qU'%2S@֦˧PZ{p\>ŽG!ʄ3؊dqWc6vL{- ~Xs„\Gg Wa$`ņJ<S/+ݱx Lb^y!g8zUVCg`zfpoآ99̌=}Ҭ wEsQ{6B=qQ ,?g7r9bq] Ŷ8becE= nʳn.DʱF#N֦d270d8B(²ǦjG"YHE>wg*aHvm> "N +ϠK$W Gֳ-M+ 2[oorW4Tky|lgonnHCK*}VBv#XܿL6]϶=(OܜYSGJvU{oyۅ~ı?SBNsNXSD V.D t{IrIp*'? )?#Iq2i/>[8ͫ77?hv(P)Z(еi -9ۊ xį9E-ME⋛?b"Hiz$YTp8D\I,|rr1fEqDQ\n&{ṉߜU&!CގJh> v9ޕޭ4#^K?/ӷJ1XBF>I:>ϼ1lzS"QO1+PSA%Dx$cJRTUF*F1TfPEEi۷1 :)z̜dJ}SLnݼL.x;Pat{*i. {-q@`QEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zASCo,ԑd !_N>8.F88.F88&RǠ3±po(:ܲoXدҢTwm > ǚs"KʶF(>s H.b=@h:3lX#( )YY#%0 *x"y2]T`JW` {{R:*ucclj0zbѳƷFA@ ܨMQ> F6ԱEK$CĨW)it˹bl}(Ib)' >,b?) -C@R0T> ǚs(**Hn"RE$'lTeQLm.1>9 $ EK$C5 QM cPTTWB2p8ۓAE9xdƥ6 a QI;S䷚ D*EE_y3+J:UI!,yW=21@WEDqT(heEBTt(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((QT*QEQEQEQEQEQEQEh((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PPZm| ȷ׃Y"w>?*pKw.7$w0${N6GZȺ :Jٓޛhcߎ@_e&YNW9 5^Y$~b0z`+V*(Xq=};UR5y?{dT {Xdv=J߼۳@O噞?,G1%}N)"snu~\wK~Eؑ*Z+KQ@GWhX?/ {p\u 1EHr7 "]vQZhx?>8[1;Uyo5_ F*]`X.#߳qR\^y!kyd}:\.gicP1`+V*+6*)b۟07ΖOܐO95{QXتP;qYud0uqz Z&'늧c!%yw+] V zf8x qLwd$s天>hҒ*b۟0LP١Hcx%_u-#ުr8"\EAzP:d%h Ԫ̌x#@.ng7.ۈ;bb7 QLk]deP[ P (իrw "?{& hx$TFE:Ƞqj5ˍ`Zx&-NITmMsQǨS\9YhJx8g{);니',[]s?)kRTUF*F1V)2GǾ+mT^]1b-i= ć*Ҵc|`=~oF'\[ +nOZpfL G;SeT4qv¡ET#nqڀ 2JV\qŒK,os՗Fe{P|ӎ$ o#1/zwqm }*QwƑE` ,XlKi,T.9'X$o:JXm 0{kMb-}({ikCj[vs^># b)mBl-w\bF0@j[y)F AEO3.;P.ϸ ҙa-B0Q%{hv0 lU dqSހ/ V "2<6vhlm#6X㹪5±C{>,90~\d`fGi w)418(VLAS7y_9 8ei9*;=vx#ӭR7m1.߭Hnl67?*%ܐp}⣷wd$q夨hҒ*b۟0LP92g3}N=j[km&`Q3cy[x31#ͫ4qYhy!QTrp:1Tmyx>F (Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@*\ES(((((((h((((((zASCo,ԑd !_N>8.F82Pit>^ )%uIЀeBȠci!lz kE"]JJeQL*żp9&wث9?J@W_A -)(v(9+K@pI/fQO.|BLkMdxw&n)C7)9R>Ծ`߈⋻8G\Vˢ.bӭr Q, r:U*oҌglz@QL ^UVKݬ_{VrWOC&+Ώ[vE:TֲNV9mG-W#T?| |WgB3Ek9kܡ[:^_߅)\S(@*,Q>y~b}* sjh^)^iG]xmk`PǍyTQ=F<ۿ8Tff;UۻxH;$bifѸ] qŒv *H|0yoZц+"Fq@TU[x9';#4{wJIҀ3袊`TmǵA1 *sir1۞TQE(h.ꅸF)EOk'HsҴ#yjy@U[ds#pq [&Â9E g($:|#X$|e,X ҳW t6/($Bp8EeZF Yr1QXlyMmJsF(T(QFтEIEDK]yV ""ȋҴ8be/:TRN @4%¼RD^ vml{?ҡ/&S\Pvے[<*ʯJѪ5x6"c pu(&͕=aWia(6,ݘf"u80cxDHѾ>S(*P"OVդ0Z+ˇǷNEMo N9cX(%Av(W-Rv7 UgEwz+? )%u(dh@2PzdQM cR*)u*#)((b*k!374Eh 6sp 8UeQgFWI :n?VӴ9:}(Usjh^(MlYHoq@tVݜPZw+ztIsm7chyoY@uJKxc?#cP (Z%HS? WtUM Ͳ6yRUI4(f?e<SZ- zim/?lkCm)Nk=핬6$i1O u/嫿 & (e'DP[UnBV^i1rx>E\1)nc pQEi797gTR[mva޽8 TV1X74o#p>_A^}0~FvE8lq@QF859mN( (Q@ F9wT- 1T-`DqR +^8l'F>F,V_A -4QE_It&vw'P-t!c8*;;%y;/Pm K ;M0Zo$ېnj͢lyMmQԣnqPh`n(h"U ڂNZh-Cɋҏ&/Jp 0,_tc3(bRN @Pj[XC>OR@PbɋҬJ$d"TTΕBQҙRU?l׼Šc py'*#ia*`6YӮ1Sa񼈑||9U(#CUa@C Ggkni=F<ۿ8SG< fta@4Sarc^›Lz E'p)A4 P=2),pM7-AMh@ Px@ )QEX$}'H U!374ᶆnnr q@UxV(-L*٭a-eVyEbNОqU@֪uk7Deڹ=*Ծ`߈⋻8Q;:f2QT*0 ( ( ( ( ( ( (?h((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PPES(KOUs\fջK7]gȩ m>s /['܈7< FOI>HMUΞbg#i7>~U0;P⻖+gU[*^ɴ{]ߞsӌtQGWQScȮ!:v"դ`IbdBjbI"\ÐpAVycY UcE؋T6 f{ 9OZwF<ѱu=U Yi u,ޜybY6z]=_ZoQجڱk?Y{ڣF%a g32-RiXjcmmL2JcqU,{ZI79gX`n;® BS<£[m_pE>DrszNVe@㰪rOfV;}N? @:2lpۨk>TnMSK#m۵U4w6aoq\P Ȭ|<S(SOvӨQ-C gip 8> F!hm,ۤGhlsKx%_mr@f?I1<` ;PV2 }%) ұ3'18TwQj\>>,݀S΁L*Trot1SMtL&4\}ƍ⼊3t#ui*dn.>ڭލ?jֿX/0PSl.I(a,?%[T.M@[i<ʳjјPr}XtQEkjج{_{Vqb>#|Hz9/ji]{FGN%=+8>TݷzvUgU6\n*g7`s҅k0^juUuRĉ#(H-Gw\V|q#d4f~_u?KgvGs @Ipoآ99̌=}Ҫ,q6dH= K$lc{{Me$w ߽+&-U{<,h#O[dq3kQ|^cq]n fHp953Y$h;IIm? &9 x.& #,{*Y<i:(-ª1P1Sn0AYjKp ()oM'~sVK}GǙ$KLv52F #Sg%:/U +KMɫ6mL2Jcq@X"m=r;ba{I$՜.V+<Ü{mN? 6PO?j|s,\,|E{TZͷh$f#xq>[oor)?AM85vnͷvF:HݟNh ݷHWKfKCtǩJZ}W6}i c6]JK3(kzCaSYKL;NqVDYqc /-TDUwZf 3^D"0NrqQKf~>AhqU$EQE0/TK^>yj;@:v·=X6<RK0p!MGADqzSPHW.M[{$z HQ1֜o{U T&Zk{hv8$J@MDq*i-.##Wy1X}y5dfZ۳zPJ/,r;QOƱjYnݨ k:±b+O+JZ$omo 1܉,( Z>/18Z˭'Il8^˜PGX%Ҟt `UڧqӵfZ-ª1P1ȭ[ſjqP8#|{=* b-g.=1Ҁ0&V̼sRs,dE*ESKMɪ [TmGlT&%vy8p's@hu8gH1ԣ=T`'nxSHąO0?\rZDw m@ ]w=;TV7cvm1QPq]naX19W?#ov~9/2ˌ9l ni9ecx3ߥAL.3 ʆNQyo,*"zxQ<${cg)6!sw?( ˏu@gSU $ESBVDj;@:vU ,H+KM٭6$RH8v5lA(5QY$Uj/ALvec>I4Iu$͸u$W;ov~9ֱZEY+&Xj*o:<4lYbβG2QQܙPAjE]A*ҴիBKY 0@*MB6c3hJ~yNדұJSۊ[x>{hv8$J įdq,?H%&̑s΁L*T F!hm4ۤGhlsE㴖6~oӊd90~\d`&F'o@X8V.-'?JƠ*X^$|̞`LPGX%{hv8$Je*On+GQj/΁L*T+O+JZ$omo hykuGor(e֦\p@r7 "vhlm#6X㹨1 Cm#qF@v߯qYx81<` ;T\((ZZKTx2y3Wg0UvxH "=?ϭ>Yeh+ȬnM~Zsj+n(\}(ečզ'7k|c9qUѷgW%-QM,$H{{SO=s&3VON(V-?Uŧ|/'𲡺62Ӑl^r;eL36һI( ,i"`j49y3dmeFAUZʡ Fs֠ظ{ݿXՇNjij<RjMkwdc(wc[W_g+)[#9V,d}w ddZ$_i iu`}@R7RvK̲;Nq@o\l0j8ª"*NǭWGf9qґQcho'98l7qc‹ ܸYKf~ 5PNMQE0 ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( ( (U::@Q@Q@Q@Q@Q@Q@Q@h((((((zASCo,ԑd !_N>8.F8qQ~Ss叵{}ߠQ>qQ~G${;L:cT4QT[ QE1Q@Q@Q@Q@Q@9rI@$1RȻdvaMEE (Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@*\ES(((((((h((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PPES(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((EStQEQEQEQEQEQEQEh((((((zASCo,ԑd !_N>8.F8nc]"Ft1**q\mmAT9%;bR eAS_Ivݸ[v?\Ἄe޼hK|eF} 9aF0?U*.JET[\21 )PzdPtT\J{ b#R[ e>,b?) -CTINԱ*Y W(H&|lBsҀ"ME*} JL&Ҁ+FqSK Q($M<+Q@QE*x_;JY?JvT.<*ʯJߎ;iˌ27oJœbR+jܶXƕ"r%A!zs#hx|U}\J{ @CEc*udޱ_AE9Qݶ $ |M<+G@ tc0v@ UlT$c([[mdv0*JJr#Bp3ǥ6Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@*\ES(((((((h((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PPZm| ȷ׃Y"w>?* j~[Lڏ$iKYRW(7G+J/?6x&XKݳkl HQV "2<RhKne=5±C{>8fM G;V{K#(Fb@ XET#nqڙ42=NM[ӝI87r:'kzU/-lNDQ:oͽA9hEU' pxG9I=ݽfŽhCT *KY I{ vN)W_c˷vW(w,zm:F2sM?zֵ^HaD+М=&ΐq8^O En 0^d%ur@l;@1B|~J*Mo# v9ޔ^bAq1baw'X$o:J 6T2? Zl+8'l@袊`Y5KHdqڪ ȉ#ZOفTawW:zF7ۇgQսhwfǢ}=p\8bj7TRŷ>`:Few$1ɭ{ց' ְ@0-7hfk{B"Zsr@*|$ mwk+y?:ъZ\\/F}2im]CgVfʆGeR=b~Xosqڀ?cou`FYt2V isW,Wr*)37ؼh,`8}k>XgIFx4w 3:H/L f*TV2d{yN|OUn>/ocMi$_ A;PٸyN³kKTKs~qǯz͠(EPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEP:WQT(((((((h((((((zASCo,ԑd !_N>8.F8Etc```~'mb/FIcV#$}01PN7&AEN}*ŭX(BTԎe *I!,yW=21S i$ 8 QS& QI;S䷚ D*EEM ą6">m"+U )[1qcCOI$"6+8!Td2;X`QVaY_*QRM'+Idjwfy[hP )8ܹWVx!bAǵgQO)%;bRSh֡_V N"&">E]8!ԟTx3E*}*,](~FǨQEJL؄EE=xdTz[\H㍈!v.c@q"@$qi,Fا@(puR2vM+]<)@Sd{T$vTc1=8bOSIm<#2!Q*(UlG(*Ta;RSd Ki qHh(͹U1A!R=U!`$SY =S}pa⡠*ampWp@R<2HAQEgfin#!=F;Pu1 tM<+mc9戅 ӊ ܜ)S) nc]7pJ)8@ ʅAQS,t*p@_(`SF>P19}iZ8&RǠ3BpX)ΏlpTNJ7u*}(Q@T\J{ @CE=cURHKK<3v S);Tp:Sh@2PzdPTU[32PUx&D+ddm0GcJ6(EPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEP:WQT(((((((h((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PPZZmiL\#Pzb+.8U`ImN@"LZm̝@ Q ! 4iwnXgIFx4w 3:H/L f*Tw+ӱnU{mRA$(O}< Şq*K-d2D$c|Q=8]}S.\c8 ܱNi{6ZG~wj15".>0n c2_"eztb̗+ܐ)úXH^FqfMj<CD٤<>٪o>dۍh{ǖO?Zռh,`8}j]ޡ8b[#QfEqFB+<6P]YBrm.d(>4I={zJהw$yhݼ>prn3jOa 0'hꪦ!C|w棊D_g<8#@M;=*V"GQXBwMq-a'?ZL(Vc<|AR[# !v>[\z8/J {hbگKI…p0p:Tw xP Y94dzV2@@Tz7* Oe4>xEAqr>{es@73d{(ӾA4F/lAٝOJϑ1ž6F=(?>c\yse86?JI ;X-}*+km&`Q@3}N=kGNv&U$Dvs<&?/לը/-lN7LE{ '%|ؐvVBG*MlY_|XFha!-vX,6X<匌 RCqC,[s3e-ƍԷV/#].0,H8)./e lzU/m-}(2 ޻Uqڪ#`®OsීF ~TF O/Ɩx!<[NAݎا\\A>Ybs1@)KTbTckbc${LfݼLrKx_jL{9,BT: ;V?%;zV42B|P@eo6,QYvKoob dU*+dw7ʡvGjc `\~T裼KEP8u?䵍4<3uoJ&4As(pW*>)(lYA$HO1SIumجAljNKҨ़t?ޏ?1s OޔgNv&U$Dvyo;`wo |[Lڏ$iΝBL? 7G+J/?6x$[H{4/j{Xdv=JYmI'AU>OxG-e}vjMnB JoCL396 VT~;!Ά \q@1Z̜8ƒ [K9 {Um|ldqZ-}㜓(fL G#v#K3NET#nqچ`nӞNI &_ߘ{sDym|1T|wNi| Ftgi #*&0tTqQ}oH zSd̸;oq*pb+.=b?,?9Th=9ؙTqڳ+F x%v@ tzt=ZǕb<5MK)'b#8VJۓ<+fd18[LW@t 8i׎X;N*WQ"=+(Y//jjhf7<\'x? ĒGC$V%w1E;Vysyvu4rQ  S޴ ?fxU>Z7Qo`1g@Rxy䜜cLY5Hg0Sh=3M8(mZF1='tG!ʄ3؊p9>T{- ~Xs€%ӝ(Vs;$⥶6oF=<&?/לַUU;,WʿAW[;ZAw$c#j(`QEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zASCo,ԑd !_N>8.F88.F8X~R;n@[Ҁ3*3>xFdBP4SINؔAS(6 aK vR**co8Rґ LJ{$m底>Uѧ_7c@TDpiU'%#%0 +J8'Y1tsԨz@CEkK ,+;`n`q$7g/(='ޕRP~Ќޘ8 VGdop˃r{UQZWvm춌tPmTڧ6 a RJvĥIBQ@QRBfxV;8_n EMī8هHPEN 6튅a;PQE\B$l[ Sob(%*7`gPEgfin#!=F;Pu;I>P19}hmc:H& EEEh >_ݍ튢HhQV-ͪn7Mx Vm5\[\Q#'85fW![gb 5,gۖl>ܞ;zPev9.'M=R$WpAZ-nYrxH *奺KI#d;C-VMr=2XmI;TGsHC.NǶ;P*+I#'85k dfZa<5e<02KcPj+Eᶚٮ-C2ϥgPEZ LJk64(Ċ(=)+N{{E6nO+5T $ Ժ Et9pګKcgn~Nu1sß.NǨHOk3h6:L 2VԲ,o9 (,pM7-AVgc`߁*dwCӊ(z~Ԃi?ա=1)rxDv3€!|VM cPTS6Jַ6 pG)q*;`''68e%-׍m)SxR,2P0:mg;P4R!Td2;X`J(v$1¨8 *e:A9-VfA@TUײu"R9ӣ& ~gQE(((((((((((((((((((((((((((((QT*QEQEQEQEQEQEQEh((((((z־ @ +8VAS7̀ @kJ 7.NtדS7"PP[b/$-p+ \}A&ݸP1 &YoA [0;HNAYӵUj OoA@62GA-sqsPZ(yG+=`+V*(Vvu4[$JF8QO1+TsKX"َɠ fKUU:G".\n]vMh-.n>4ֻ`` zPWim.#*E2_S1 ?,798I kz(IooRSʗ B*؞ @=*;/oc@u"R8>$ o#1/zuqݻxJ]wo0;xxw ]pN DXb8Z !B|T]G{x+['$is%HF9ܱNj͕2ݕ3#ov~91wl}*I ,| X`ImNVS7qQ6;Ϳ黏ʫ 臨(fGv2~_'nMxncT&3>l}V|aYSzn:tc&+ڔI'Ym>n:U[2QӠc8[,s%޾e!Qi^HCk'F[: etf5p[@Vu-t=GJ2hWB=Fj[ncPLL*( \ʅP h<6Ҭ3'с<~_JFܒͿp Zb$sj5Δq8v _ ҳ}j}>g8zUF)R9Ļ2.*2ͩ&+JX棠)KTbTcV.Lfݼlѵv |T;jB1qק!cޛ=4IB'۰I54~a}*Kx@zc1R}5j;466,qԗ,Yڙ$V+oq^WR\݋ AqJѼh,`8}k>XgIFx4w 3:H/L f*S- ZAKNp=je6 }*H@úYv"O?JVm~v@ {0~~Ehijxt?S= ć*͆WU>W㿂|`=~oS^ +nOZĮZTlym2h>_83A#<#(Vb@U.Q%O260}I(X#َ$eU LL*$mzArJюq`2&%ؽs@?䵒%ɌN6&Yݥ,1$tQFU`82F${6B=qR=;hkp zb(BmzT7^xXBFuE'qiU\|ۉ %SZ? ҳ`Xd+O־Oo9m&E%$v"@-㴸XcVlR@1=z $J}OrhQL }NyUUYH@{⣾ <[fu=)fg]Hs;bX%F%A #c40@ !>zyfx%c͋mqL'9nfE!qYqztA6r1@6_N)(sCcn>[+Vw Z+oqS7b(GcҐ5ˍ`4FaÅ$Uo&׹{sUYne9펔5qXlT;) GN;R5ݬK_[oϧ(fGv2~_'ne-ƍMgi}ٌcґ.0,H8(ƳՋ.Sga85'qiU\|ۉx}&@v_\"8X/`k#>@Yv[’ujx!<[NAݎؠ:իldLK[s?*S{Yb *:dP'hpF~Urqבdw٬I.>2;vm2#Ҁ,Y 8.F8:PW1@RtgRM/1KI'CK8XZg {zRH%X51Pp3 b6ҟ4k8FRTMqLr8 >LXޫ`2F)K?*cJB F( [*I_v1,y$g4 .@9b}jul&,u@J#bYT1@XcJ|# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$Mq4[}׽W*dRUgQSW< R F)cm)KcPaK}џ%MI'D,O[JŎ)V$b *ijb8|t bҤ_5cR7Is@ld(c'֧Yb^^1IV%ELc\ )H+8|t/(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/1KI'CK8XZg {zRH%X51Pp3 b6ҟ4k8FRTMqLr8 >LXޫ`2F)K?*cJB F( [*I_v1,y$g4 .@9b}jul&,u@J#bYT1@XcJ|# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$Mq4[}׽W*dRUgQSW< R F)cm)KcPaK}џ%MI'D,O[JŎ)V$b *ijb8|t bҤ_5cR7Is@ld(c'֧Yb^^1IV%ELc\ )H+8|t/(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/1KI'CK8XZg {zRH%X51Pp3 b6ҟ4k8FRTMqLr8 >LXޫ`2F)K?*cJB F( [*I_v1,y$g4 .@9b}jul&,u@J#bYT1@XcJ|# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$Mq4[}׽W*dRUgQSW< R F)cm)KcPaK}џ%MI'D,O[JŎ)V$b *ijb8|t bҤ_5cR7Is@ld(c'֧Yb^^1IV%ELc\ )H+8|t/(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/1KI'CK8XZg {zRH%X51Pp3 b6ҟ4k8FRTMqLr8 >LXޫ`2F)*ijbES((((EStQEQEQEQEQEQEQEh((((((:~)?/?¨RU ((EPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEPEP:WQT(((((((h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#bYT1U)QEQEQEQEU:(((((((h((((((zAEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#bYT1U)QEQEQEQEU:(((((((h((((((zAEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#bYT1U)QEQEQEQEU:(((((((h((((((zAEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#bYT1U)QEQEQEQEU:(((((((h((((((zAEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#bYT1U)QEQEQEQEU:(((((((h((((((zAEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEupuN ( ( ( ( ( ( (?h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#bYT1U)QEQEQEQEU:(((((((h((((((zAEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEoo ¬fV)t֕->oY\Tjtf9{~oK0NsQܮ7-$ةK[w5zuT)B|/{\-^->oՠ(oKܮRR08+Gk~oK"$^ B^]U%SG-X#UpJ=dh%SWć'9nWpSpJ=dh%SWUIJ'Q"iwѵgMZ IŠco`br}A->oեFo3R0NsBrvCMdj؍ qOU%*/+^vGMdkB(OE!I8QEmU%SG-X#UpJ=dh%SWć'9nWpSةK[w5zuT)B|/{\-^->oՠ(oKܮRR08+Gk~oK"$^ B^]U%SG-X#UpJ=dh%SWć'9nWpSpJ=dh%SWUIJ'Q"iwѵgMZ IŠco`br}A->oեFo3R0NsBrvCMdj؍ qOU%*/+^vGMdkB(OE!I8QEmU%SG-X#UpJ=dh%SWć'9nWpSةK[w5zuT)B|/{\-^->oՠ(oKܮRR08+Gk~o],sA1(x5^~ʷ}LBQ]G8QEQEQEQEU:(((((((h((((((zxcJB F( [*I_v1,y$g4kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ U< ,j*ccm(+)o3&}o$!P,ONŎ)V$bK?(RW1KlpO_5cb# [)*h&I8j&9b}hRUl&,uUJ#gV%ELc m!x#-F~$ N3kc qC>:϶: X J,j*c gJA^,m>i|݌q@-F~4y$g5>mld)*϶:X @3梦16Ҁ?Ji|݌qX&I8h] r>LXހ+ҕ`2F)*ijb+)x#_v1(0Ғo$c'ր %XYb^^1HxbYT1P>:PW1@RtgRM/)`$9iv21Sa1czJU梦1xR>:S}RtgISA7IsQ1@@⒬,a1cz@ UN;l >Xr梦1ֲmYUvF٣GB7%!/I6vlu;W/6:9ld%u$6z9N,;r] JG3 LLjStF܉Ɵ2]nxm*(ِFT9Bp J%!/!/[%vlu;Q$T mX9|&vYJ@7A*9N,;ZbeTb n9dF^ tF܉ƣMd2VW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+yQ=gDmȘi+HׁJ@{bMd(C.RCq1<Ӥ%uD.͎j$zs+/$6H?A)2zkLLjSB{[%f=i#b"cfASPJ@{bHc yƋiEٱ tJkd%u$6z9N,;r] JG3 LLjStF܉Ɵ2]nxm*(ِFT9Bp J%!/!/[%vlu;Q$T mX9|&vYJ@7A*9N,;ZbeTb n9dF^ tF܉ƣMd2VW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+yQ=gDmȘi+HׁJ@{bMd(C.RCq1<Ӥ%uD.͎j$zs+/$6H?A)2zkLLjSB{[%f=i#b"cfASPJ@{bHc yƋiEٱ tJkd%u$6z9N,;r] JG3 LLjStF܉Ɵ2]nxm*(ِFT9Bp J%!/!/[%vlu;Q$T mX9|&vYJ@7A*9N,;ZbeTb n9dr>[U,荹s߇(mt3حEWQQEQEQEQEU:(((((((h((((((oKRXFks=dh%SZ #}c 7y%(*M_t `]N(n](*}M^U%*tP1Dy FױK[w5h)' *e1Idh%SVJ$89 ێ[w4}b7+)>uT({[w4}>bP$E GWMdkAco`bTf4^}(*M_t `]N(n[\b->oRR08E Hwpm{dh%SVpX#Pj+K[w5iQԮbCМP%SG-#rSUIJ׸Q%SG-Њ(x5RNQyw GWMdkAco`bTf4^}(*M_t `]N(n](*}M^U%*tP1Dy FױK[w5h)' *e1Idh%SVJ$89 ێ[w4}b7+)>uT({[w4}>bP$E GWMdkAco`bTf4^}(*M_t `]N(n[\b->oRR08E Hwpm{dh%SVpX#Pj+K[w5iQԮbCМP%SG-#rSUIJ׸Q%SG-Њ(x5RNQyw GWMdkAco`bTf4^}(*M_t `]N(n](*}M^U%*tP1Dy FױK[w5h)' *e1Idh%SVJ$89 ێ[w4}b7+)>uT({[w4}>bP$E GWMdkAco`bTf4^}(*M_t `]N(n[\b->oRR08E Hwpm{dh%SVpX#Pj+K[w5iQԮbCМP%SG-#rSUIJ׸Q%SG-Њ(x5RNQyw GWMdkAco`bTf4^}(*M_t `]N(n](*}M^U%*tP1Dy FױK[w5h)' *e1Idh%SVJ$89 ێ[w4}b7+)>uT({[w4}>bP$E GWMdkAco`bTf4^}(*M_t `]N(n[\b->oRR08E Hwpm{dh%SVpX#Pj+K[w5iQԮbCМP%SG-#rSUIJ׸Q%SG-Њ(x5RNQyw GWMdkAco`bTf4^}(*M_t `]N(n](*}M^U%*tP1Dy FױK[w5h)' *e1Idh%SVJ$89 ێ[w4}b7+)>uT({[w4}>bP$E GWMdkAco`bTf4^}(*M_t `]N(n[\b->oRR08E Hwpm{dh%SVpX#Pj+K[w5iQԮbCМP%SG-#rSUIJ׸Q%SG-Њ(x5RNQyw GWMdkAco`bTf4^}(*M_t `]N(n](*}M^U%*tP1Dy FױK[w5h)' *e1Idh%SVJ$89 ێ[w4}b7+)>uT({[w52"ơ` 'Q"j(/AU1QcM t `]N)8he4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEQ&S0NsQܮFEM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ|*LjxBFhbM t `]N)8he4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEC0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAE[UB6f+QEsQ@Q@Q@Q@*\ES(((((((h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[aֹoWD6L~5~ʴΪbQ]FEPEPEPEP:WQT(((((((h((((((먢 3^QEXX#UW))TbCb7+).QQSNQ:(OEwW2Ljx%SJ$89 :SnWpS|)UQ˥b*P1DBp.t670F1P3}њ9XChć'9nWpSN=EM:JUF>bOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/ IŠ9Bmaco`bTf4DEN$89F#rRhe4)US$^ >G{ Zz)I8QS,xRQHE9QԮbCУ]lAEbOV^pRNT#}cn6+NTf5+X4([QO]N)󪤥T`qG. hB|/=h ERQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEV-|7jn]¿W+ ( ( ( (U::@Q@Q@Q@Q@Q@Q@Q@h((((((_,.ׁUHِFA+̋;&Fco =H1<|cE4":v[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֒?)z(Yr&?96dz\Y\|Dm)0go hd]rNԮ[ȖB^(RLax IwNrý7%߰(ۡ9s7'F/"c̮.Wk[G#/^*3y튊96dzw hI sN-Ȼ6:]6̬D;,^  ˖121O n9dF^ tF܉ƣMeOQBVW/*3yG%!/(C.Yf\ӵ+%-ԓ;,^]6Ӽ8f\Mw 6N@7A*+y2zm121OYr&?|v,R2R1بfASScE5*Hn>C0gtQnEٱ tDUP1e`$f)<8f\MiQd.W[a֫agDmȘjg'EkQEQ@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@b~ʶ>Uч+%z(NP((((EStQEQEQEQEQEQEQEh((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먢D(((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((+>UX߇?a_+ErQ@Q@Q@Q@*\ES(((((((h((((((먤-yGQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEQEw *ڬ[oWF0袊9B((((uN @Q@Q@Q@Q@Q@Q@Q@h((((((SoQ~*> 2OTrG\Ϲ?? /svȥOHwܧwvt:]A[7Q>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]AG? Ecwk*rGs叵{}ߠQ>qQ~G${;X]APM[!97QEB ( ( ( ( V ~V ( ( ( ( ( ( (?h((((((#JYGTAȫJD @\Is/ q +\nݼck;竡W?z (Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@Q@SYh )994QEQEQEQEQEQEQEh(((((()G)( DiEh|?«OZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞoOZEZޞ֫Q@w'^ъZJ(((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((h((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((((timescale-pg_textsearch-b63fe58/pg_textsearch.control000066400000000000000000000002411525762047700232200ustar00rootroot00000000000000# pg_textsearch extension comment = 'Full-text search with BM25 ranking' default_version = '1.5.1' module_pathname = '$libdir/pg_textsearch' relocatable = false timescale-pg_textsearch-b63fe58/scripts/000077500000000000000000000000001525762047700204505ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/scripts/README.md000066400000000000000000000041211525762047700217250ustar00rootroot00000000000000# Scripts ## `package-deb.sh` Builds a Debian package for pg_textsearch. ## `msvc-build.ps1` Builds `pg_textsearch.dll` with MSVC against a PostgreSQL 17 or 18 x64 tree, on Windows. PGXS cannot drive MSVC, so this script is the Windows build recipe; it reads `OBJS` and `DATA` from the `Makefile` instead of keeping a second source list. Requires Visual Studio 2022 Build Tools with the "Desktop development with C++" workload. ```powershell .\scripts\msvc-build.ps1 -PgRoot 'C:\Program Files\PostgreSQL\18' ``` `-PgRoot` is any installation or extracted EDB binaries tree containing `include\server` and `lib\postgres.lib`. Output goes to `-WorkDir`, by default `target\msvc`; nothing is written into `-PgRoot` and nothing is installed. `-ExpectedMajor 17|18` refuses a tree of another major version. The build compiles with `/W3 /WX` and suppresses three conversion warnings (C4244, C4267, C4305) that already appear on `main`; every other warning fails the build. After compilation two gates run: `src/layout_check.c` stops the compile if MSVC lays out a packed or aligned on-disk struct differently from the reference GCC x86-64 build, and the export gate stops the build if the DLL does not export a symbol that the `DATA` SQL files use through `MODULE_PATHNAME`. Only `OBJS` and `DATA` are read from the `Makefile`; the version comes from `pg_textsearch.control`. Everything else that affects the build lives in two places: a semantic `-D` or `-I` added to `PG_CPPFLAGS`, or a link dependency added via `SHLIB_LINK`, must be mirrored in `scripts/msvc-build.ps1` in the same change, or the Windows DLL diverges from the PGXS build. The `build-msvc` job in `.github/workflows/ci.yml` runs the same script against EDB binary zips pinned by URL and SHA-256. When EDB replaces the release and the URL stops working, pick the new URL, compute its hash, and update URL and hash together in one commit; never change one side of a pin alone. If the hash does not match while the URL is unchanged, treat it as an incident and do not update the pin until the replacement archive's provenance has been independently verified. timescale-pg_textsearch-b63fe58/scripts/bump-version.sh000077500000000000000000000157071525762047700234470ustar00rootroot00000000000000#!/usr/bin/env bash # # bump-version.sh OLD NEW # # Bumps the pg_textsearch version from OLD to NEW. Run from the repo # root with a clean working tree. Auto-detects mode from the version # strings: # # OLD=A.B.C NEW=X.Y.Z-dev → dev bump (post-release) # OLD=X.Y.Z-dev NEW=X.Y.Z → release (same triple) # # Updates the SQL files, Makefile DATA list, control file, mod.c, # CLAUDE.md, test scripts, and the msmarco benchmark check. # Does NOT edit the upgrade-tests matrix, author the upgrade SQL, run # tests, or open a PR — those are listed in the "next steps" output. set -euo pipefail if [[ $# -ne 2 ]]; then echo "usage: $0 OLD NEW" >&2 exit 2 fi OLD=$1 NEW=$2 # --- mode detection ------------------------------------------------ is_release() { # X.Y.Z (no -dev suffix) [[ $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] } is_dev() { # X.Y.Z-dev [[ $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+-dev$ ]] } if is_release "$OLD" && is_dev "$NEW"; then MODE=dev elif is_dev "$OLD" && is_release "$NEW" && [[ ${OLD%-dev} == "$NEW" ]]; then MODE=release else echo "Error: invalid OLD/NEW combination." >&2 echo " Dev bump: OLD=A.B.C, NEW=X.Y.Z-dev" >&2 echo " Release: OLD=X.Y.Z-dev, NEW=X.Y.Z (same triple)" >&2 exit 2 fi # --- safety checks ------------------------------------------------- if [[ ! -f pg_textsearch.control ]]; then echo "Error: run from the repo root." >&2 exit 2 fi if [[ -n "$(git status --porcelain)" ]]; then echo "Error: working tree is not clean. Commit or stash first." >&2 exit 2 fi SRC_MAIN="sql/pg_textsearch--$OLD.sql" DST_MAIN="sql/pg_textsearch--$NEW.sql" if [[ ! -f $SRC_MAIN ]]; then echo "Error: $SRC_MAIN does not exist." >&2 exit 2 fi if [[ -f $DST_MAIN ]]; then echo "Error: $DST_MAIN already exists." >&2 exit 2 fi # --- mode-specific actions ----------------------------------------- if [[ $MODE == dev ]]; then NEW_UPGRADE="sql/pg_textsearch--$OLD--$NEW.sql" if [[ -f $NEW_UPGRADE ]]; then echo "Error: $NEW_UPGRADE already exists." >&2 exit 2 fi # Rename the base SQL file. Body substitution happens below. git mv "$SRC_MAIN" "$DST_MAIN" # Create the upgrade-script stub. Authors append feature DDL as # work lands; release-time audit verifies completeness. cat > "$NEW_UPGRADE" <.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO \$\$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END \$\$; EOF git add "$NEW_UPGRADE" # Makefile: rename the base-file DATA entry, then append the new # upgrade entry. Use a targeted regex (`pg_textsearch--OLD.sql`) # rather than a broad `OLD.sql` to avoid corrupting legacy # upgrade-chain entries like `pg_textsearch--A--OLD.sql`. perl -i -pe "s|pg_textsearch--\Q$OLD\E\.sql|pg_textsearch--$NEW.sql|g" Makefile perl -i -pe ' if (/^(\s+sql\/pg_textsearch--[0-9].*\.sql)$/) { $_ = "$1 \\\n sql/pg_textsearch--'"$OLD"'--'"$NEW"'.sql\n"; } ' Makefile elif [[ $MODE == release ]]; then # Find and rename the upgrade file `sql/pg_textsearch--PREV--OLD.sql`. # Globbing here uses bash's nullglob so we can count safely. shopt -s nullglob upgrades=( sql/pg_textsearch--*--"$OLD".sql ) shopt -u nullglob if [[ ${#upgrades[@]} -ne 1 ]]; then echo "Error: expected exactly one sql/pg_textsearch--*--$OLD.sql," \ "found ${#upgrades[@]}." >&2 exit 2 fi SRC_UPGRADE=${upgrades[0]} DST_UPGRADE=${SRC_UPGRADE//$OLD.sql/$NEW.sql} # Extract PREV (last released version) from the upgrade filename # `sql/pg_textsearch--PREV--OLD.sql`. Used for the upgrade-SQL # audit diff in the release next-steps output. PREV=${SRC_UPGRADE#sql/pg_textsearch--} PREV=${PREV%%--*} git mv "$SRC_MAIN" "$DST_MAIN" git mv "$SRC_UPGRADE" "$DST_UPGRADE" # Makefile: update both DATA entries (main install + current upgrade # file). Safe to use the broader `--OLD.sql` regex here because in # release mode OLD=X.Y.Z-dev only appears in the current cycle's # entries, never in legacy upgrade-chain filenames. perl -i -pe "s|--\Q$OLD\E\.sql|--$NEW.sql|g" Makefile fi # Body substitutions for the renamed SQL files (both modes need these: # version-equality block in the main install file, version comments in # the upgrade file). perl -i -pe "s/\Q$OLD\E/$NEW/g" "$DST_MAIN" [[ $MODE == release ]] && perl -i -pe "s/\Q$OLD\E/$NEW/g" "$DST_UPGRADE" # --- common text substitutions ------------------------------------- # Files where every literal occurrence of OLD becomes NEW. common_files=( pg_textsearch.control src/mod.c CLAUDE.md test/scripts/memory_accounting.sh test/scripts/multi_index.sh test/scripts/segment_wal_recovery.sh benchmarks/datasets/msmarco-v2/run_v2_benchmark.sh ) for f in "${common_files[@]}"; do if [[ -f $f ]]; then perl -i -pe "s/\Q$OLD\E/$NEW/g" "$f" fi done # --- straggler check ----------------------------------------------- # Find any remaining references to OLD outside known-safe locations. # Excluded: this script, expected outputs, older upgrade SQL files, # RELEASING.md (carries the historical upgrade chain), the # upgrade-tests workflow (carries phase-history comments). stragglers=$(git grep --fixed-strings "$OLD" -- \ ':!scripts/bump-version.sh' \ ':!test/expected/' \ ':!sql/pg_textsearch--*--*.sql' \ ':!RELEASING.md' \ ':!.github/workflows/upgrade-tests.yml' \ ':!docs/' \ | cut -d: -f1 | sort -u \ || true) if [[ -n $stragglers ]]; then echo "Warning: literal '$OLD' still appears in:" >&2 echo "$stragglers" | sed 's/^/ /' >&2 echo "Review and update by hand if these need bumping." >&2 fi # --- next steps ---------------------------------------------------- echo echo "Version bump $OLD → $NEW complete." echo if [[ $MODE == release ]]; then cat <\target\msvc. .PARAMETER ExpectedMajor Refuse to build unless the PgRoot headers report this major version. Omitted, any of 17 or 18 is accepted. .EXAMPLE .\scripts\msvc-build.ps1 -PgRoot 'C:\Program Files\PostgreSQL\18' .EXAMPLE .\scripts\msvc-build.ps1 -PgRoot D:\pg17\pgsql -ExpectedMajor 17 #> [CmdletBinding()] param( [string]$PgRoot, [string]$WorkDir, [string]$ExpectedMajor ) Set-StrictMode -Version 2.0 $ErrorActionPreference = 'Stop' $global:LASTEXITCODE = 0 $SupportedMajors = @('17', '18') # MSVC version of the flag block in the Makefile. Use /MD: postgres.exe uses # the dynamic CRT, and /MT would give the extension its own second CRT heap. # /WX works like -Werror on Unix. $CompileFlags = @( '/nologo' '/c' '/MD' '/O2' '/std:c11' '/utf-8' '/W3' '/WX' # Conversion warnings that already occur on main; gcc/clang -Wall -Wextra # does not report them. Nothing else is suppressed, so any new warning # still fails the build. '/wd4244' '/wd4267' '/wd4305' ) $CompileDefines = @( '/DWIN32' '/D_WINDOWS' '/D_CRT_SECURE_NO_WARNINGS' '/D_CRT_NONSTDC_NO_DEPRECATE' ) # --------------------------------------------------------------------------- # Helpers # --------------------------------------------------------------------------- function Write-Step { param([string]$Message) Write-Host '' Write-Host "==> $Message" -ForegroundColor Cyan } function Write-Detail { param([string]$Message) Write-Host " $Message" } function Get-FullPath { param([string]$Path, [string]$BaseDir) return [System.IO.Path]::GetFullPath([System.IO.Path]::Combine($BaseDir, $Path)) } function New-CleanDirectory { param([string]$Path) if (Test-Path -LiteralPath $Path) { Remove-Item -LiteralPath $Path -Recurse -Force } New-Item -ItemType Directory -Path $Path -Force | Out-Null } function Write-TextFile { param([string]$Path, [string[]]$Lines) $content = ($Lines -join "`r`n") + "`r`n" $utf8NoBom = New-Object System.Text.UTF8Encoding($false) [System.IO.File]::WriteAllText($Path, $content, $utf8NoBom) } function Find-VcVars { $candidates = New-Object System.Collections.Generic.List[string] $pf86 = ${env:ProgramFiles(x86)} if ($pf86) { $vswhere = Join-Path $pf86 'Microsoft Visual Studio\Installer\vswhere.exe' if (Test-Path -LiteralPath $vswhere) { $found = & $vswhere -products '*' -latest ` -requires Microsoft.VisualStudio.Component.VC.Tools.x86.x64 ` -property installationPath $global:LASTEXITCODE = 0 foreach ($install in $found) { if ($install) { $candidates.Add((Join-Path $install.Trim() 'VC\Auxiliary\Build\vcvars64.bat')) } } } $candidates.Add((Join-Path $pf86 'Microsoft Visual Studio\2022\BuildTools\VC\Auxiliary\Build\vcvars64.bat')) } $pf = ${env:ProgramFiles} if ($pf) { foreach ($edition in @('Enterprise', 'Professional', 'Community', 'BuildTools')) { $candidates.Add((Join-Path $pf "Microsoft Visual Studio\2022\$edition\VC\Auxiliary\Build\vcvars64.bat")) } } foreach ($candidate in $candidates) { if (Test-Path -LiteralPath $candidate) { return $candidate } } throw 'vcvars64.bat not found. Install Visual Studio 2022 Build Tools with the "Desktop development with C++" workload.' } function Import-VcVarsEnvironment { param([string]$VcVarsPath) # vcvars64.bat changes the environment of its own cmd.exe only, so run # it in a child shell and copy the result into this process. The call # goes through a temporary .cmd file because pwsh and cmd.exe escape # embedded quotes differently. $marker = 'VCVARS_ENVIRONMENT_FOLLOWS' $shim = Join-Path ([System.IO.Path]::GetTempPath()) ('vcvars-' + [System.Guid]::NewGuid().ToString('N') + '.cmd') Write-TextFile -Path $shim -Lines @( '@echo off' ('call "' + $VcVarsPath + '" >nul 2>&1') 'if errorlevel 1 exit /b 1' "echo $marker" 'set' ) try { $lines = & cmd.exe /c $shim if ($LASTEXITCODE -ne 0) { throw "Failed to run $VcVarsPath (exit code $LASTEXITCODE)" } } finally { Remove-Item -LiteralPath $shim -Force -ErrorAction SilentlyContinue } $started = $false foreach ($line in $lines) { if (-not $started) { if ($line -eq $marker) { $started = $true } continue } $split = $line.IndexOf('=') if ($split -lt 1) { continue } [System.Environment]::SetEnvironmentVariable( $line.Substring(0, $split), $line.Substring($split + 1)) } if (-not $started) { throw "Could not capture the environment from $VcVarsPath" } } function Get-MakefileVariable { param([string]$MakefilePath, [string]$Name) $lines = Get-Content -LiteralPath $MakefilePath # Exactly one plain assignment may define the variable. A second one # (+=, :=, ?=, a conditional block) means a Makefile shape this small # parser cannot evaluate; silently ignoring it would drop objects or SQL # files from the Windows build, so it fails instead. $assigns = @($lines | Select-String -Pattern ('^\s*' + $Name + '\s*[:+?]?=')) if ($assigns.Count -eq 0) { throw "Variable $Name not found in $MakefilePath" } if ($assigns.Count -gt 1) { $at = ($assigns | ForEach-Object LineNumber) -join ', ' throw "$Name has $($assigns.Count) assignments in $MakefilePath (line $at);" + ' this parser supports exactly one' } $buffer = New-Object System.Collections.Generic.List[string] foreach ($line in $lines[$($assigns[0].LineNumber - 1)..($lines.Count - 1)]) { $continues = ($line.TrimEnd() -match '\\$') $buffer.Add(($line -replace '\\\s*$', '')) if (-not $continues) { break } } $joined = ($buffer -join ' ') -replace ('^\s*' + $Name + '\s*=\s*'), '' $values = @($joined -split '\s+' | Where-Object { $_ -ne '' }) if ($values.Count -eq 0) { throw "Variable $Name is empty in $MakefilePath" } return $values } function Invoke-Cl { param([string]$Tag, [string]$RspDir, [string]$ObjDir, [string[]]$Flags, [string[]]$Sources) New-Item -ItemType Directory -Path $ObjDir -Force | Out-Null $lines = @($Flags) # The doubled backslash survives cl's argument unescaping as one trailing # separator, which is what makes /Fo name a directory instead of a file. $lines += ('/Fo"' + $ObjDir + '\\"') foreach ($source in $Sources) { $lines += ('"' + $source + '"') } $rspName = "cl-$Tag.rsp" Write-TextFile -Path (Join-Path $RspDir $rspName) -Lines $lines # Run from the response directory so the @ argument needs no quoting. Push-Location -LiteralPath $RspDir try { & cl.exe "@$rspName" if ($LASTEXITCODE -ne 0) { throw "cl.exe failed for $Tag (exit code $LASTEXITCODE)" } } finally { Pop-Location } } function Get-DllExport { param([string]$DllPath) # link /dump prints what dumpbin prints. /dump must come first, or # link.exe treats the DLL as an input object. $output = & link.exe /dump /nologo /exports $DllPath if ($LASTEXITCODE -ne 0) { throw "link /dump /exports failed for $DllPath" } $names = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::Ordinal) foreach ($line in $output) { $match = [regex]::Match($line, '^\s+\d+\s+[0-9A-Fa-f]+\s+[0-9A-Fa-f]{8}\s+(\S+)') if ($match.Success) { $names.Add($match.Groups[1].Value) | Out-Null } } # Leading comma keeps PowerShell from turning the set into an array, # which would make Contains a slow linear search. return ,$names } # --------------------------------------------------------------------------- # Preflight # --------------------------------------------------------------------------- Write-Step 'Preflight' if (-not $PSScriptRoot) { throw 'This script must be run from a file, not pasted into a shell.' } if ($ExpectedMajor -and $SupportedMajors -notcontains $ExpectedMajor) { throw "-ExpectedMajor must be one of $($SupportedMajors -join ', ')." } $RepoRoot = (Split-Path -Parent $PSScriptRoot).TrimEnd('\') $makefile = Join-Path $RepoRoot 'Makefile' $controlFile = Join-Path $RepoRoot 'pg_textsearch.control' $srcDir = Join-Path $RepoRoot 'src' foreach ($required in @($makefile, $controlFile, $srcDir)) { if (-not (Test-Path -LiteralPath $required)) { throw "$RepoRoot does not look like a pg_textsearch checkout: missing $required" } } $version = ([regex]::Match((Get-Content -LiteralPath $controlFile -Raw), "default_version\s*=\s*'([^']+)'")).Groups[1].Value if (-not $version) { throw "Could not parse default_version from $controlFile" } $installSql = Join-Path $RepoRoot "sql\pg_textsearch--$version.sql" if (-not (Test-Path -LiteralPath $installSql)) { throw "Missing install script for version ${version}: $installSql" } Write-Detail "repository root: $RepoRoot" Write-Detail "pg_textsearch version: $version" if (-not $PgRoot) { throw 'Specify -PgRoot: the root of a PostgreSQL 17 or 18 x64 tree (containing include\server and lib\postgres.lib).' } # Relative -PgRoot/-WorkDir follow the caller's location; the repository root # never does, it comes from $PSScriptRoot alone. $callerDir = (Get-Location).ProviderPath $PgRoot = (Get-FullPath -Path $PgRoot -BaseDir $callerDir).TrimEnd('\') $pgInclude = Join-Path $PgRoot 'include\server' $pgLib = Join-Path $PgRoot 'lib\postgres.lib' foreach ($required in @($pgInclude, $pgLib)) { if (-not (Test-Path -LiteralPath $required)) { throw "PgRoot does not look like a PostgreSQL tree: missing $required" } } if ($WorkDir) { $WorkDir = (Get-FullPath -Path $WorkDir -BaseDir $callerDir).TrimEnd('\') } else { $WorkDir = (Join-Path $RepoRoot 'target\msvc').TrimEnd('\') } # Keep PostgreSQL inputs and build outputs disjoint. This makes cleanup safe # even if the scratch layout changes later. $pgRootLower = $PgRoot.ToLowerInvariant() $workDirLower = $WorkDir.ToLowerInvariant() if ($workDirLower -eq $pgRootLower -or $workDirLower.StartsWith($pgRootLower + '\')) { throw 'WorkDir must not be inside PgRoot. This script never writes into the PostgreSQL tree.' } if ($pgRootLower.StartsWith($workDirLower + '\')) { throw 'PgRoot must not be inside WorkDir. The PostgreSQL tree and build output must not overlap.' } $pgConfigH = Join-Path $pgInclude 'pg_config.h' $versionLine = Select-String -LiteralPath $pgConfigH -Pattern '^#define\s+PG_VERSION\s+"' | Select-Object -First 1 if (-not $versionLine) { throw "Could not read PG_VERSION from $pgConfigH" } $pgVersion = ([regex]::Match($versionLine.Line, '"([^"]+)"')).Groups[1].Value $pgMajor = ([regex]::Match($pgVersion, '^(\d+)')).Groups[1].Value Write-Detail "PostgreSQL headers report version $pgVersion" if ($ExpectedMajor) { if ($pgMajor -ne $ExpectedMajor) { throw "PgRoot is PostgreSQL $pgVersion but -ExpectedMajor $ExpectedMajor was requested." } } elseif ($SupportedMajors -notcontains $pgMajor) { throw "PgRoot is PostgreSQL $pgVersion; this script supports $($SupportedMajors -join ' and ')." } $vcvars = Find-VcVars Write-Detail "vcvars64.bat: $vcvars" Import-VcVarsEnvironment $vcvars if ($env:VCToolsVersion) { Write-Detail "MSVC toolset: $env:VCToolsVersion" } if ($env:VSCMD_VER) { Write-Detail "Visual Studio: $env:VSCMD_VER" } if (-not (Get-Command cl.exe -ErrorAction SilentlyContinue)) { throw 'cl.exe is not on PATH after running vcvars64.bat' } if (-not (Get-Command link.exe -ErrorAction SilentlyContinue)) { throw 'link.exe is not on PATH after running vcvars64.bat' } # --------------------------------------------------------------------------- # Sources # --------------------------------------------------------------------------- Write-Step 'Sources' # Group by directory: three different directories contain a scan.c, so one # flat object directory would silently drop objects. $sourceGroups = [ordered]@{} $layoutCheckRelative = 'src\layout_check.c' $haveLayoutCheck = $false $sourceCount = 0 foreach ($obj in (Get-MakefileVariable -MakefilePath $makefile -Name 'OBJS')) { if ($obj -notmatch '\.o$') { throw "Unexpected OBJS entry in ${makefile}: $obj" } $relative = ($obj -replace '\.o$', '.c') -replace '/', '\' if (-not (Test-Path -LiteralPath (Join-Path $RepoRoot $relative))) { throw "Makefile OBJS lists $obj but $relative does not exist" } if ($relative -eq $layoutCheckRelative) { $haveLayoutCheck = $true } $dir = Split-Path -Parent $relative if (-not $sourceGroups.Contains($dir)) { $sourceGroups[$dir] = New-Object System.Collections.Generic.List[string] } $sourceGroups[$dir].Add($relative) $sourceCount++ } if (-not $haveLayoutCheck) { throw "Makefile OBJS no longer lists $layoutCheckRelative; the struct layout gate would be silently gone." } Write-Detail "$sourceCount sources in $($sourceGroups.Count) directories, from the Makefile OBJS list" $dataFiles = Get-MakefileVariable -MakefilePath $makefile -Name 'DATA' Write-Detail "$($dataFiles.Count) SQL files from the Makefile DATA list" # --------------------------------------------------------------------------- # Compile # --------------------------------------------------------------------------- Write-Step 'Compile' $dll = Join-Path $WorkDir 'pg_textsearch.dll' if (Test-Path -LiteralPath $dll) { Remove-Item -LiteralPath $dll -Force } $objRoot = Join-Path $WorkDir 'obj' $rspDir = Join-Path $WorkDir 'rsp' New-CleanDirectory $objRoot New-CleanDirectory $rspDir # Same include paths as the Makefile (-I$(srcdir)/src) plus the PostgreSQL # tree; no source includes "src/...", so no repository-root /I is needed. $flags = @($CompileFlags) $flags += $CompileDefines # \" puts quotes into the /D value so PG_TEXTSEARCH_VERSION becomes a quoted # string, same as -DPG_TEXTSEARCH_VERSION=\"...\" in the GCC build. $flags += ('/DPG_TEXTSEARCH_VERSION=\"' + $version + '\"') $flags += ('/I"' + $srcDir + '"') foreach ($include in @('include\server\port\win32_msvc', 'include\server\port\win32', 'include\server', 'include')) { $flags += ('/I"' + (Join-Path $PgRoot $include) + '"') } $objectFiles = New-Object System.Collections.Generic.List[string] # Compile layout_check.c first and alone: a layout failure then stands alone # instead of mixing with errors from every other translation unit. Write-Detail "layout gate: compiling $layoutCheckRelative" try { Invoke-Cl -Tag 'layout-check' -RspDir $rspDir -ObjDir (Join-Path $objRoot 'src') -Flags $flags ` -Sources @((Join-Path $RepoRoot $layoutCheckRelative)) } catch { Write-Host '' Write-Host 'LAYOUT GATE: FAIL' -ForegroundColor Red Write-Host 'The layout-check translation unit did not compile; inspect the compiler diagnostics above.' -ForegroundColor Red Write-Host 'If a static layout assertion failed, this build cannot safely read existing index data.' -ForegroundColor Red throw } Write-Host ' LAYOUT GATE: PASS' -ForegroundColor Green $objectFiles.Add((Join-Path $objRoot ($layoutCheckRelative -replace '\.c$', '.obj'))) foreach ($dir in $sourceGroups.Keys) { $relatives = @($sourceGroups[$dir] | Where-Object { $_ -ne $layoutCheckRelative }) if ($relatives.Count -eq 0) { continue } Write-Detail "compiling $($dir -replace '\\', '/') ($($relatives.Count) files)" Invoke-Cl -Tag ($dir -replace '\\', '_') -RspDir $rspDir -ObjDir (Join-Path $objRoot $dir) ` -Flags $flags -Sources @($relatives | ForEach-Object { Join-Path $RepoRoot $_ }) foreach ($relative in $relatives) { $objectFiles.Add((Join-Path $objRoot ($relative -replace '\.c$', '.obj'))) } } # --------------------------------------------------------------------------- # Link # --------------------------------------------------------------------------- Write-Step 'Link' $linkLines = @( '/nologo' '/DLL' '/MACHINE:X64' ('/OUT:"' + $dll + '"') ('/IMPLIB:"' + (Join-Path $objRoot 'pg_textsearch.lib') + '"') ('"' + $pgLib + '"') ) foreach ($obj in $objectFiles) { if (-not (Test-Path -LiteralPath $obj)) { throw "Missing object file: $obj" } $linkLines += ('"' + $obj + '"') } Write-TextFile -Path (Join-Path $rspDir 'link.rsp') -Lines $linkLines Push-Location -LiteralPath $rspDir try { & link.exe '@link.rsp' if ($LASTEXITCODE -ne 0) { throw "link.exe failed (exit code $LASTEXITCODE)" } } finally { Pop-Location } Write-Detail "linked $($objectFiles.Count) objects into $dll" # --------------------------------------------------------------------------- # Export gate # --------------------------------------------------------------------------- Write-Step 'Export gate' # Every DATA file, not just the install script: tp_memory_usage is only # bound by an upgrade script and would otherwise go unchecked. $wanted = New-Object 'System.Collections.Generic.HashSet[string]' ([System.StringComparer]::Ordinal) foreach ($data in $dataFiles) { $path = Join-Path $RepoRoot ($data -replace '/', '\') if (-not (Test-Path -LiteralPath $path)) { throw "Makefile DATA lists $data but it does not exist" } $text = Get-Content -LiteralPath $path -Raw $matched = [regex]::Matches($text, "MODULE_PATHNAME'\s*,\s*'([A-Za-z0-9_]+)'") # The one-argument AS 'MODULE_PATHNAME' form would name the symbol after # the SQL function and escape the regex, so count the mentions too. $mentions = ([regex]::Matches($text, 'MODULE_PATHNAME')).Count if ($mentions -ne $matched.Count) { throw "$data has $mentions MODULE_PATHNAME mentions but $($matched.Count) parse as explicit symbol references; the export gate would miss $($mentions - $matched.Count) of them." } foreach ($match in $matched) { $wanted.Add($match.Groups[1].Value) | Out-Null } } if ($wanted.Count -eq 0) { throw 'Found no MODULE_PATHNAME symbol references in the DATA SQL files' } # Case-sensitive comparison: GetProcAddress matches case exactly, while # PowerShell string comparison ignores case by default. $exported = Get-DllExport -DllPath $dll $missing = @($wanted | Where-Object { -not $exported.Contains($_) } | Sort-Object) if ($missing.Count -gt 0) { Write-Host '' Write-Host 'EXPORT GATE: FAIL' -ForegroundColor Red Write-Host "Not exported by the DLL: $($missing -join ', ')" -ForegroundColor Red throw 'The built DLL does not export every symbol the SQL scripts bind. A PGDLLEXPORT is probably missing.' } Write-Host " EXPORT GATE: PASS ($($wanted.Count) SQL symbols, $($exported.Count) exports in the DLL)" -ForegroundColor Green Write-Step 'Done' Write-Detail ("{0} ({1:N0} bytes)" -f $dll, (Get-Item -LiteralPath $dll).Length) Write-Detail 'Nothing was written into the PostgreSQL tree.' timescale-pg_textsearch-b63fe58/scripts/package-deb.sh000077500000000000000000000057351525762047700231440ustar00rootroot00000000000000#!/bin/bash set -eux VERSION="${1}" BASEDIR="${2}" ARCH="${3}" PG_VERSION="${4}" # Clean version without 'v' prefix CLEAN_VERSION=${VERSION#v} # Debian package version format DEB_VERSION="${CLEAN_VERSION}-1" PACKAGE_NAME="pg-textsearch-postgresql-${PG_VERSION}" # Setup directories DEBDIR="${BASEDIR}/dist" BUILDDIR="${BASEDIR}/debian-build" rm -rf "${BUILDDIR}" "${DEBDIR}" mkdir -p "${BUILDDIR}/DEBIAN" mkdir -p "${DEBDIR}" # Get PostgreSQL directories # Support both Docker (TimescaleDB) and regular PostgreSQL installations if [ -x "/usr/local/bin/pg_config" ]; then PG_CONFIG="/usr/local/bin/pg_config" elif [ -x "/usr/lib/postgresql/${PG_VERSION}/bin/pg_config" ]; then PG_CONFIG="/usr/lib/postgresql/${PG_VERSION}/bin/pg_config" else echo "Error: pg_config not found in expected locations" exit 1 fi LIBDIR=$($PG_CONFIG --pkglibdir) SHAREDIR=$($PG_CONFIG --sharedir) # Create package directory structure mkdir -p "${BUILDDIR}${LIBDIR}" mkdir -p "${BUILDDIR}${SHAREDIR}/extension" # Copy extension files cp "${BASEDIR}/pg_textsearch.so" "${BUILDDIR}${LIBDIR}/" || \ cp "${BASEDIR}/pg_textsearch.dylib" "${BUILDDIR}${LIBDIR}/" || \ { echo "Error: Could not find pg_textsearch library"; exit 1; } cp "${BASEDIR}/pg_textsearch.control" "${BUILDDIR}${SHAREDIR}/extension/" # Copy all SQL files for the extension cp "${BASEDIR}"/sql/pg_textsearch*.sql "${BUILDDIR}${SHAREDIR}/extension/" # Determine architecture if [ "$ARCH" = "arm64" ]; then DEB_ARCH="arm64" else DEB_ARCH="amd64" fi # Create control file cat > "${BUILDDIR}/DEBIAN/control" << EOF Package: ${PACKAGE_NAME} Version: ${DEB_VERSION} Architecture: ${DEB_ARCH} Maintainer: Timescale Depends: timescaledb-2-postgresql-${PG_VERSION} | postgresql-${PG_VERSION} Section: database Priority: optional Homepage: https://github.com/timescale/pg_textsearch Description: pg_textsearch - PostgreSQL extension for full-text search with BM25 pg_textsearch provides full-text search capabilities with BM25 ranking for PostgreSQL. It implements a memtable-based architecture similar to LSM trees, with in-memory structures that spill to disk segments for scalability. EOF # Create postinst script cat > "${BUILDDIR}/DEBIAN/postinst" << 'EOF' #!/bin/sh set -e if [ "$1" = "configure" ]; then echo "pg_textsearch extension installed. Use 'CREATE EXTENSION pg_textsearch;' in PostgreSQL to enable." fi exit 0 EOF chmod 755 "${BUILDDIR}/DEBIAN/postinst" # Create prerm script cat > "${BUILDDIR}/DEBIAN/prerm" << 'EOF' #!/bin/sh set -e if [ "$1" = "remove" ]; then echo "Removing pg_textsearch extension. Please DROP EXTENSION pg_textsearch in all databases first." fi exit 0 EOF chmod 755 "${BUILDDIR}/DEBIAN/prerm" # Build the package cd "${BASEDIR}" dpkg-deb --build debian-build # Move to dist with proper name PACKAGE_FILE="${PACKAGE_NAME}_${DEB_VERSION}_${DEB_ARCH}.deb" mv debian-build.deb "${DEBDIR}/${PACKAGE_FILE}" echo "Package created: ${DEBDIR}/${PACKAGE_FILE}" # Cleanup rm -rf "${BUILDDIR}" timescale-pg_textsearch-b63fe58/sql/000077500000000000000000000000001525762047700175605ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.0.1--0.0.2.sql000066400000000000000000000001641525762047700243610ustar00rootroot00000000000000-- pg_textsearch extension upgrade from version 0.0.1 to 0.0.2 -- No schema changes required between these versions timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.0.2--0.0.3.sql000066400000000000000000000001641525762047700243630ustar00rootroot00000000000000-- pg_textsearch extension upgrade from version 0.0.2 to 0.0.3 -- No schema changes required between these versions timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.0.3--0.0.4.sql000066400000000000000000000002371525762047700243660ustar00rootroot00000000000000-- Upgrade from pg_textsearch 0.0.3 to 0.0.4 -- -- This release includes bug fixes and improvements but no schema changes. -- No changes required for upgrade timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.0.4--0.0.5.sql000066400000000000000000000013261525762047700243700ustar00rootroot00000000000000-- Upgrade from 0.0.4 to 0.0.5 -- Function to force segment write (spill memtable to disk) CREATE FUNCTION bm25_spill_index(index_name text) RETURNS int4 AS 'MODULE_PATHNAME', 'tp_spill_memtable' LANGUAGE C VOLATILE STRICT; -- Drop old debug function and create new one with simpler name DROP FUNCTION IF EXISTS bm25_debug_dump_index(text, boolean); -- Single argument version returns truncated output as text CREATE FUNCTION bm25_dump_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Two argument version writes full dump (with hex) to file CREATE FUNCTION bm25_dump_index(text, text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.0.5--0.1.0.sql000066400000000000000000000026601525762047700243670ustar00rootroot00000000000000-- Upgrade from 0.0.5 to 0.1.0 -- Create the new scoring functions with bm25_ prefix CREATE FUNCTION bm25_text_bm25query_score(left_text text, right_query bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_bm25query_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; CREATE FUNCTION bm25_text_text_score(text, text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- Drop the operator class first (it depends on the operator) -- Note: Users must rebuild their indexes after this upgrade DROP OPERATOR CLASS IF EXISTS text_bm25_ops USING bm25 CASCADE; -- Drop the old operators DROP OPERATOR IF EXISTS <@> (text, bm25query); DROP OPERATOR IF EXISTS <@> (text, text); -- Drop the old functions DROP FUNCTION IF EXISTS text_bm25query_score(text, bm25query); DROP FUNCTION IF EXISTS text_text_score(text, text); -- Create new operators with the bm25_ prefixed functions CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = bm25query, PROCEDURE = bm25_text_bm25query_score ); CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = text, PROCEDURE = bm25_text_text_score ); -- Recreate the operator class with the new functions CREATE OPERATOR CLASS text_bm25_ops DEFAULT FOR TYPE text USING bm25 AS OPERATOR 1 <@> (text, bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text, bm25query) bm25_text_bm25query_score(text, bm25query); timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.1.0--0.2.0.sql000066400000000000000000000005001525762047700243530ustar00rootroot00000000000000-- Upgrade from 0.1.0 to 0.2.0 -- No schema changes - internal storage format improvements only -- IMPORTANT: Requires REINDEX for existing indexes to use V2 segment format DO $$ BEGIN RAISE INFO 'pg_textsearch upgraded to v0.2.0'; RAISE INFO 'Run REINDEX on bm25 indexes to use new V2 segment format'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.2.0--0.3.0.sql000066400000000000000000000001741525762047700243640ustar00rootroot00000000000000-- Upgrade from 0.2.0 to 0.3.0 -- No schema changes DO $$ BEGIN RAISE INFO 'pg_textsearch upgraded to v0.3.0'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.3.0--0.4.0.sql000066400000000000000000000001741525762047700243660ustar00rootroot00000000000000-- Upgrade from 0.3.0 to 0.4.0 -- No schema changes DO $$ BEGIN RAISE INFO 'pg_textsearch upgraded to v0.4.0'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.3.0.sql000066400000000000000000000136501525762047700236570ustar00rootroot00000000000000-- pg_textsearch extension version 0.3.0 -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_textsearch" to load this file. \quit -- Access method CREATE FUNCTION tp_handler(internal) RETURNS index_am_handler AS 'MODULE_PATHNAME', 'tp_handler' LANGUAGE C; CREATE ACCESS METHOD bm25 TYPE INDEX HANDLER tp_handler; -- bm25vector type CREATE FUNCTION bm25vector_in(cstring) RETURNS bm25vector AS 'MODULE_PATHNAME', 'tpvector_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_out(bm25vector) RETURNS cstring AS 'MODULE_PATHNAME', 'tpvector_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_recv(internal) RETURNS bm25vector AS 'MODULE_PATHNAME', 'tpvector_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_send(bm25vector) RETURNS bytea AS 'MODULE_PATHNAME', 'tpvector_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE bm25vector ( INPUT = bm25vector_in, OUTPUT = bm25vector_out, RECEIVE = bm25vector_recv, SEND = bm25vector_send, STORAGE = extended, ALIGNMENT = int4 ); -- bm25query type CREATE FUNCTION bm25query_in(cstring) RETURNS bm25query AS 'MODULE_PATHNAME', 'tpquery_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_out(bm25query) RETURNS cstring AS 'MODULE_PATHNAME', 'tpquery_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_recv(internal) RETURNS bm25query AS 'MODULE_PATHNAME', 'tpquery_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_send(bm25query) RETURNS bytea AS 'MODULE_PATHNAME', 'tpquery_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE bm25query ( INPUT = bm25query_in, OUTPUT = bm25query_out, RECEIVE = bm25query_recv, SEND = bm25query_send, STORAGE = extended, ALIGNMENT = int4 ); -- Convert text to bm25query CREATE FUNCTION to_bm25query(input_text text) RETURNS bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION to_bm25query(input_text text, index_name text) RETURNS bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text_index' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Equality function: bm25vector = bm25vector → boolean CREATE FUNCTION bm25vector_eq(bm25vector, bm25vector) RETURNS boolean AS 'MODULE_PATHNAME', 'tpvector_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Create the = operator for equality CREATE OPERATOR = ( LEFTARG = bm25vector, RIGHTARG = bm25vector, FUNCTION = bm25vector_eq, COMMUTATOR = =, HASHES ); -- BM25 scoring function for text <@> bm25query operations -- -- COST 1000: Standalone scoring is expensive. Each call parses document text -- with to_tsvector (~14μs per doc), opens the index, looks up IDF values, and -- calculates BM25 scores. High cost helps planner prefer index scans. CREATE FUNCTION bm25_text_bm25query_score(left_text text, right_query bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_bm25query_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- bm25query equality function CREATE FUNCTION bm25query_eq(bm25query, bm25query) RETURNS boolean AS 'MODULE_PATHNAME', 'tpquery_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- <@> operator for text <@> bm25query operations CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = bm25query, PROCEDURE = bm25_text_bm25query_score ); -- Function for text <@> text operator (planner hook rewrites to text <@> bm25query) -- COST 1000: High cost makes planner prefer index scans over seq scan + sort. -- In practice, this function errors without index scan context, but the cost -- helps the planner choose the right path before execution. CREATE FUNCTION bm25_text_text_score(text, text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- Stub function returning cached score from index scan. -- The planner hook replaces resjunk ORDER BY expressions with calls to this -- function, avoiding expensive re-computation of BM25 scores. CREATE FUNCTION bm25_get_current_score() RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_get_current_score' LANGUAGE C VOLATILE STRICT PARALLEL SAFE; -- <@> operator for text <@> text operations (implicit index resolution) -- The planner hook transforms this to text <@> bm25query when a BM25 index exists CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = text, PROCEDURE = bm25_text_text_score ); -- = operator for bm25query equality CREATE OPERATOR = ( LEFTARG = bm25query, RIGHTARG = bm25query, FUNCTION = bm25query_eq, COMMUTATOR = =, HASHES ); -- bm25 operator class for text columns -- The planner hook rewrites text <@> text to text <@> bm25query, so we only -- need to register the bm25query operator and support function here. CREATE OPERATOR CLASS text_bm25_ops DEFAULT FOR TYPE text USING bm25 AS OPERATOR 1 <@> (text, bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text, bm25query) bm25_text_bm25query_score(text, bm25query); -- Debug function to dump index contents (memtable and segments) -- Single argument version returns truncated output as text CREATE FUNCTION bm25_dump_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Two argument version writes full dump (with hex) to file CREATE FUNCTION bm25_dump_index(text, text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Display version info DO $$ BEGIN RAISE INFO 'pg_textsearch v0.3.0 installed'; END $$; -- Function to force segment write (spill memtable to disk) CREATE FUNCTION bm25_spill_index(index_name text) RETURNS int4 AS 'MODULE_PATHNAME', 'tp_spill_memtable' LANGUAGE C VOLATILE STRICT; -- Fast summary function showing only statistics (no content dump) CREATE FUNCTION bm25_summarize_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_summarize_index' LANGUAGE C STRICT STABLE; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.4.0--0.4.1.sql000066400000000000000000000001741525762047700243700ustar00rootroot00000000000000-- Upgrade from 0.4.0 to 0.4.1 -- No schema changes DO $$ BEGIN RAISE INFO 'pg_textsearch upgraded to v0.4.1'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.4.0.sql000066400000000000000000000136501525762047700236600ustar00rootroot00000000000000-- pg_textsearch extension version 0.4.0 -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_textsearch" to load this file. \quit -- Access method CREATE FUNCTION tp_handler(internal) RETURNS index_am_handler AS 'MODULE_PATHNAME', 'tp_handler' LANGUAGE C; CREATE ACCESS METHOD bm25 TYPE INDEX HANDLER tp_handler; -- bm25vector type CREATE FUNCTION bm25vector_in(cstring) RETURNS bm25vector AS 'MODULE_PATHNAME', 'tpvector_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_out(bm25vector) RETURNS cstring AS 'MODULE_PATHNAME', 'tpvector_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_recv(internal) RETURNS bm25vector AS 'MODULE_PATHNAME', 'tpvector_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_send(bm25vector) RETURNS bytea AS 'MODULE_PATHNAME', 'tpvector_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE bm25vector ( INPUT = bm25vector_in, OUTPUT = bm25vector_out, RECEIVE = bm25vector_recv, SEND = bm25vector_send, STORAGE = extended, ALIGNMENT = int4 ); -- bm25query type CREATE FUNCTION bm25query_in(cstring) RETURNS bm25query AS 'MODULE_PATHNAME', 'tpquery_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_out(bm25query) RETURNS cstring AS 'MODULE_PATHNAME', 'tpquery_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_recv(internal) RETURNS bm25query AS 'MODULE_PATHNAME', 'tpquery_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_send(bm25query) RETURNS bytea AS 'MODULE_PATHNAME', 'tpquery_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE bm25query ( INPUT = bm25query_in, OUTPUT = bm25query_out, RECEIVE = bm25query_recv, SEND = bm25query_send, STORAGE = extended, ALIGNMENT = int4 ); -- Convert text to bm25query CREATE FUNCTION to_bm25query(input_text text) RETURNS bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION to_bm25query(input_text text, index_name text) RETURNS bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text_index' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Equality function: bm25vector = bm25vector → boolean CREATE FUNCTION bm25vector_eq(bm25vector, bm25vector) RETURNS boolean AS 'MODULE_PATHNAME', 'tpvector_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Create the = operator for equality CREATE OPERATOR = ( LEFTARG = bm25vector, RIGHTARG = bm25vector, FUNCTION = bm25vector_eq, COMMUTATOR = =, HASHES ); -- BM25 scoring function for text <@> bm25query operations -- -- COST 1000: Standalone scoring is expensive. Each call parses document text -- with to_tsvector (~14μs per doc), opens the index, looks up IDF values, and -- calculates BM25 scores. High cost helps planner prefer index scans. CREATE FUNCTION bm25_text_bm25query_score(left_text text, right_query bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_bm25query_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- bm25query equality function CREATE FUNCTION bm25query_eq(bm25query, bm25query) RETURNS boolean AS 'MODULE_PATHNAME', 'tpquery_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- <@> operator for text <@> bm25query operations CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = bm25query, PROCEDURE = bm25_text_bm25query_score ); -- Function for text <@> text operator (planner hook rewrites to text <@> bm25query) -- COST 1000: High cost makes planner prefer index scans over seq scan + sort. -- In practice, this function errors without index scan context, but the cost -- helps the planner choose the right path before execution. CREATE FUNCTION bm25_text_text_score(text, text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- Stub function returning cached score from index scan. -- The planner hook replaces resjunk ORDER BY expressions with calls to this -- function, avoiding expensive re-computation of BM25 scores. CREATE FUNCTION bm25_get_current_score() RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_get_current_score' LANGUAGE C VOLATILE STRICT PARALLEL SAFE; -- <@> operator for text <@> text operations (implicit index resolution) -- The planner hook transforms this to text <@> bm25query when a BM25 index exists CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = text, PROCEDURE = bm25_text_text_score ); -- = operator for bm25query equality CREATE OPERATOR = ( LEFTARG = bm25query, RIGHTARG = bm25query, FUNCTION = bm25query_eq, COMMUTATOR = =, HASHES ); -- bm25 operator class for text columns -- The planner hook rewrites text <@> text to text <@> bm25query, so we only -- need to register the bm25query operator and support function here. CREATE OPERATOR CLASS text_bm25_ops DEFAULT FOR TYPE text USING bm25 AS OPERATOR 1 <@> (text, bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text, bm25query) bm25_text_bm25query_score(text, bm25query); -- Debug function to dump index contents (memtable and segments) -- Single argument version returns truncated output as text CREATE FUNCTION bm25_dump_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Two argument version writes full dump (with hex) to file CREATE FUNCTION bm25_dump_index(text, text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Display version info DO $$ BEGIN RAISE INFO 'pg_textsearch v0.4.0 installed'; END $$; -- Function to force segment write (spill memtable to disk) CREATE FUNCTION bm25_spill_index(index_name text) RETURNS int4 AS 'MODULE_PATHNAME', 'tp_spill_memtable' LANGUAGE C VOLATILE STRICT; -- Fast summary function showing only statistics (no content dump) CREATE FUNCTION bm25_summarize_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_summarize_index' LANGUAGE C STRICT STABLE; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.4.1--0.4.2.sql000066400000000000000000000001741525762047700243720ustar00rootroot00000000000000-- Upgrade from 0.4.1 to 0.4.2 -- No schema changes DO $$ BEGIN RAISE INFO 'pg_textsearch upgraded to v0.4.2'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.4.2--0.5.0.sql000066400000000000000000000001741525762047700243720ustar00rootroot00000000000000-- Upgrade from 0.4.2 to 0.5.0 -- No schema changes DO $$ BEGIN RAISE INFO 'pg_textsearch upgraded to v0.5.0'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.5.0--0.5.1.sql000066400000000000000000000006741525762047700243770ustar00rootroot00000000000000-- Upgrade from 0.5.0 to 0.5.1 -- No schema changes -- Verify pg_textsearch library is loaded DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; DO $$ BEGIN RAISE INFO 'pg_textsearch: upgraded to v0.5.1'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.5.0.sql000066400000000000000000000144541525762047700236640ustar00rootroot00000000000000-- pg_textsearch extension version 0.5.0 -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_textsearch" to load this file. \quit -- Access method CREATE FUNCTION tp_handler(internal) RETURNS index_am_handler AS 'MODULE_PATHNAME', 'tp_handler' LANGUAGE C; CREATE ACCESS METHOD bm25 TYPE INDEX HANDLER tp_handler; -- bm25vector type CREATE FUNCTION bm25vector_in(cstring) RETURNS bm25vector AS 'MODULE_PATHNAME', 'tpvector_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_out(bm25vector) RETURNS cstring AS 'MODULE_PATHNAME', 'tpvector_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_recv(internal) RETURNS bm25vector AS 'MODULE_PATHNAME', 'tpvector_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25vector_send(bm25vector) RETURNS bytea AS 'MODULE_PATHNAME', 'tpvector_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE bm25vector ( INPUT = bm25vector_in, OUTPUT = bm25vector_out, RECEIVE = bm25vector_recv, SEND = bm25vector_send, STORAGE = extended, ALIGNMENT = int4 ); -- bm25query type CREATE FUNCTION bm25query_in(cstring) RETURNS bm25query AS 'MODULE_PATHNAME', 'tpquery_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_out(bm25query) RETURNS cstring AS 'MODULE_PATHNAME', 'tpquery_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_recv(internal) RETURNS bm25query AS 'MODULE_PATHNAME', 'tpquery_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION bm25query_send(bm25query) RETURNS bytea AS 'MODULE_PATHNAME', 'tpquery_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE bm25query ( INPUT = bm25query_in, OUTPUT = bm25query_out, RECEIVE = bm25query_recv, SEND = bm25query_send, STORAGE = extended, ALIGNMENT = int4 ); -- Convert text to bm25query CREATE FUNCTION to_bm25query(input_text text) RETURNS bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION to_bm25query(input_text text, index_name text) RETURNS bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text_index' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Equality function: bm25vector = bm25vector → boolean CREATE FUNCTION bm25vector_eq(bm25vector, bm25vector) RETURNS boolean AS 'MODULE_PATHNAME', 'tpvector_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Create the = operator for equality CREATE OPERATOR = ( LEFTARG = bm25vector, RIGHTARG = bm25vector, FUNCTION = bm25vector_eq, COMMUTATOR = =, HASHES ); -- BM25 scoring function for text <@> bm25query operations -- -- COST 1000: Standalone scoring is expensive. Each call parses document text -- with to_tsvector (~14μs per doc), opens the index, looks up IDF values, and -- calculates BM25 scores. High cost helps planner prefer index scans. CREATE FUNCTION bm25_text_bm25query_score(left_text text, right_query bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_bm25query_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- bm25query equality function CREATE FUNCTION bm25query_eq(bm25query, bm25query) RETURNS boolean AS 'MODULE_PATHNAME', 'tpquery_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- <@> operator for text <@> bm25query operations CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = bm25query, PROCEDURE = bm25_text_bm25query_score ); -- Function for text <@> text operator (planner hook rewrites to text <@> bm25query) -- COST 1000: High cost makes planner prefer index scans over seq scan + sort. -- In practice, this function errors without index scan context, but the cost -- helps the planner choose the right path before execution. CREATE FUNCTION bm25_text_text_score(text, text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- Stub function returning cached score from index scan. -- The planner hook replaces resjunk ORDER BY expressions with calls to this -- function, avoiding expensive re-computation of BM25 scores. CREATE FUNCTION bm25_get_current_score() RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_get_current_score' LANGUAGE C VOLATILE STRICT PARALLEL SAFE; -- <@> operator for text <@> text operations (implicit index resolution) -- The planner hook transforms this to text <@> bm25query when a BM25 index exists CREATE OPERATOR <@> ( LEFTARG = text, RIGHTARG = text, PROCEDURE = bm25_text_text_score ); -- = operator for bm25query equality CREATE OPERATOR = ( LEFTARG = bm25query, RIGHTARG = bm25query, FUNCTION = bm25query_eq, COMMUTATOR = =, HASHES ); -- bm25 operator class for text columns -- The planner hook rewrites text <@> text to text <@> bm25query, so we only -- need to register the bm25query operator and support function here. CREATE OPERATOR CLASS text_bm25_ops DEFAULT FOR TYPE text USING bm25 AS OPERATOR 1 <@> (text, bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text, bm25query) bm25_text_bm25query_score(text, bm25query); -- Debug function to dump index contents (memtable and segments) -- Single argument version returns truncated output as text CREATE FUNCTION bm25_dump_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Two argument version writes full dump (with hex) to file CREATE FUNCTION bm25_dump_index(text, text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Display version info DO $$ BEGIN RAISE INFO 'pg_textsearch v0.5.0 installed'; END $$; -- Function to force segment write (spill memtable to disk) CREATE FUNCTION bm25_spill_index(index_name text) RETURNS int4 AS 'MODULE_PATHNAME', 'tp_spill_memtable' LANGUAGE C VOLATILE STRICT; -- Fast summary function showing only statistics (no content dump) CREATE FUNCTION bm25_summarize_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_summarize_index' LANGUAGE C STRICT STABLE; -- Page visualization - dumps ANSI-colored page layout to file -- Shows page types: M=meta R=recovery H=header d=dict p=post s=skip m=docmap i=idx .=empty -- Colors indicate segment level (cyan=L0, yellow=L1, green=L2, magenta=L3+) CREATE FUNCTION bm25_debug_pageviz(index_name text, filepath text) RETURNS text AS 'MODULE_PATHNAME', 'tp_debug_pageviz' LANGUAGE C STRICT STABLE; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.5.1--0.6.0.sql000066400000000000000000000011351525762047700243710ustar00rootroot00000000000000-- Upgrade from 0.5.1 to 0.6.0 -- Verify pg_textsearch library is loaded DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; -- New function: force-merge all segments into one CREATE FUNCTION bm25_force_merge(index_name text) RETURNS void AS 'MODULE_PATHNAME', 'tp_force_merge' LANGUAGE C VOLATILE STRICT; DO $$ BEGIN RAISE INFO 'pg_textsearch: upgraded to v0.6.0'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.6.0--0.6.1.sql000066400000000000000000000006741525762047700244010ustar00rootroot00000000000000-- Upgrade from 0.6.0 to 0.6.1 -- No schema changes -- Verify pg_textsearch library is loaded DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; DO $$ BEGIN RAISE INFO 'pg_textsearch: upgraded to v0.6.1'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--0.6.1--1.0.0.sql000066400000000000000000000014441525762047700243700ustar00rootroot00000000000000-- Upgrade from 0.6.1 to 1.0.0 -- No schema changes -- Verify pg_textsearch library is loaded DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; -- Revoke public execute on debug functions (superuser-only). REVOKE EXECUTE ON FUNCTION bm25_dump_index(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_summarize_index(text) FROM PUBLIC; -- Drop file-writing debug functions (moved behind compile-time flag). DROP FUNCTION IF EXISTS bm25_dump_index(text, text); DROP FUNCTION IF EXISTS bm25_debug_pageviz(text, text); DO $$ BEGIN RAISE INFO 'pg_textsearch v1.0.0'; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.0.0--1.1.0.sql000066400000000000000000000045341525762047700243660ustar00rootroot00000000000000-- Upgrade from 1.0.0 to 1.1.0 -- Verify the library is loaded. The version-equality check belongs in -- the *latest* upgrade script only; intermediate scripts (this one -- becomes intermediate after a future bump) must accept any loaded -- version because the chain ends at default_version, not here. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; -- Native text[] support: scoring functions, operators, operator class. -- Flattens array elements with spaces, then scores as a single document. CREATE FUNCTION @extschema@.bm25_textarray_bm25query_score( left_arr text[], right_query @extschema@.bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_textarray_bm25query_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- Error stub for text[] <@> text (planner should rewrite to bm25query) CREATE FUNCTION @extschema@.bm25_textarray_text_score(text[], text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_textarray_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; CREATE OPERATOR @extschema@.<@> ( LEFTARG = text[], RIGHTARG = @extschema@.bm25query, PROCEDURE = @extschema@.bm25_textarray_bm25query_score ); CREATE OPERATOR @extschema@.<@> ( LEFTARG = text[], RIGHTARG = text, PROCEDURE = @extschema@.bm25_textarray_text_score ); CREATE OPERATOR CLASS @extschema@.text_array_bm25_ops DEFAULT FOR TYPE text[] USING bm25 AS OPERATOR 1 @extschema@.<@> (text[], @extschema@.bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text[], @extschema@.bm25query) @extschema@.bm25_textarray_bm25query_score( text[], @extschema@.bm25query); -- Memory usage visibility function. -- Intentionally accessible to all users (monitoring/ops use case). -- Only exposes aggregate DSA byte counts and configured limits, -- not per-index or per-user data. CREATE FUNCTION bm25_memory_usage( OUT dsa_total_bytes int8, OUT dsa_total_mb float4, OUT estimated_bytes int8, OUT estimated_mb float4, OUT counter_bytes int8, OUT memory_limit_mb float4, OUT usage_pct float4 ) AS 'MODULE_PATHNAME', 'tp_memory_usage' LANGUAGE C VOLATILE; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.1.0--1.2.0.sql000066400000000000000000000011551525762047700243640ustar00rootroot00000000000000-- Upgrade from 1.1.0 to 1.2.0 -- Verify the library is loaded. The version-equality check lives in -- the main install file (pg_textsearch--.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.2.0--1.3.0.sql000066400000000000000000000112351525762047700243660ustar00rootroot00000000000000-- Upgrade from 1.2.0 to 1.3.0 -- Verify the library is loaded. The version-equality check lives in -- the main install file (pg_textsearch--.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; -- The bm25_test_memtable_page / bm25_test_memtable_append / -- bm25_test_chain_source / bm25_memtable_chain / -- bm25_memtable_dead_pages functions are -- INTERNAL-ONLY test scaffolds for the on-disk memtable redesign -- (issue #374). Not part of the supported public API. -- See pg_textsearch--1.3.0.sql for the full disclaimer. CREATE FUNCTION bm25_test_memtable_page(case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_memtable_page' LANGUAGE C STRICT; CREATE FUNCTION bm25_test_memtable_append( index_name text, case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_memtable_append' LANGUAGE C STRICT; CREATE FUNCTION bm25_memtable_chain( index_name text, OUT blkno bigint, OUT n_records integer, OUT free_offset integer, OUT next_block bigint, OUT flags integer) RETURNS SETOF record AS 'MODULE_PATHNAME', 'bm25_memtable_chain' LANGUAGE C STRICT; CREATE FUNCTION bm25_memtable_dead_pages( index_name text, OUT blkno bigint, OUT flags integer, OUT dead_fxid bigint, OUT n_records integer) RETURNS SETOF record AS 'MODULE_PATHNAME', 'bm25_memtable_dead_pages' LANGUAGE C STRICT; CREATE FUNCTION bm25_test_chain_source( index_name text, case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_chain_source' LANGUAGE C STRICT; CREATE FUNCTION bm25_cache_cold_build( index_name text, OUT result text, OUT records_applied bigint, OUT cursor_seq bigint, OUT estimated_bytes bigint) RETURNS record AS 'MODULE_PATHNAME', 'bm25_cache_cold_build' LANGUAGE C STRICT; CREATE FUNCTION bm25_cache_apply_to_tail( index_name text, OUT result text, OUT records_applied bigint, OUT cursor_seq bigint, OUT estimated_bytes bigint) RETURNS record AS 'MODULE_PATHNAME', 'bm25_cache_apply_to_tail' LANGUAGE C STRICT; CREATE FUNCTION bm25_cache_bump_spill_generation(index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'bm25_cache_bump_spill_generation' LANGUAGE C STRICT; -- Cache memory-cap scaffolds. CREATE FUNCTION bm25_cache_global_estimated_bytes() RETURNS bigint AS 'MODULE_PATHNAME', 'bm25_cache_global_estimated_bytes' LANGUAGE C STRICT; CREATE FUNCTION bm25_cache_evict_largest(index_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_cache_evict_largest' LANGUAGE C STRICT; -- Cache source scaffold. CREATE FUNCTION bm25_test_cache_source( index_name text, case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_cache_source' LANGUAGE C STRICT; REVOKE EXECUTE ON FUNCTION bm25_test_memtable_page(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_test_memtable_append(text, text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_memtable_chain(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_memtable_dead_pages(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_test_chain_source(text, text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_cache_cold_build(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_cache_apply_to_tail(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_cache_bump_spill_generation(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_test_cache_source(text, text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_cache_global_estimated_bytes() FROM PUBLIC; REVOKE EXECUTE ON FUNCTION bm25_cache_evict_largest(text) FROM PUBLIC; -- The on-disk memtable redesign (issue #374) makes the soft-limit -- memory_usage SRF obsolete along with the underlying soft-limit -- machinery; the chain_page_count-based auto-spill heuristic replaces -- the old global byte accounting. DROP FUNCTION IF EXISTS bm25_memory_usage(); -- Standalone scoring functions are not parallel-safe: they open the index -- relation by name, attach per-backend state, and walk the on-disk memtable -- chain under shared latches. Parallel workers attempting the same setup -- do not survive worker startup reliably. Ranked queries should use -- ORDER BY <@> ... LIMIT n, which uses an index scan and does not exercise -- these functions in workers. ALTER FUNCTION bm25_text_bm25query_score(text, bm25query) PARALLEL UNSAFE; ALTER FUNCTION bm25_textarray_bm25query_score(text[], bm25query) PARALLEL UNSAFE; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.3.0--1.3.1.sql000066400000000000000000000020721525762047700243670ustar00rootroot00000000000000-- Upgrade from 1.3.0 to 1.3.1 -- Verify the library is loaded. The version-equality check lives in -- the main install file (pg_textsearch--.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; -- Deferred, standby-safe reclaim of displaced segment pages (#380): -- count the pages currently parked in the tombstone chain awaiting -- horizon-gated FSM reclaim. Superuser-only, like the other debug -- introspection functions. CREATE FUNCTION bm25_pending_free_pages(index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_pending_free_pages' LANGUAGE C STRICT STABLE; REVOKE EXECUTE ON FUNCTION bm25_pending_free_pages(text) FROM PUBLIC; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.3.1--1.4.0.sql000066400000000000000000000042561525762047700243760ustar00rootroot00000000000000-- Upgrade from 1.3.1 to 1.4.0 -- Verify the library is loaded. The version-equality check lives in -- the main install file (pg_textsearch--.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; -- Re-declare the standalone scoring functions as STABLE (previously -- IMMUTABLE). Their result depends on corpus statistics that change with -- the indexed table, and STABLE prevents plan-time constant folding so the -- runtime privilege check on the named index always runs under the invoking -- role. ALTER FUNCTION @extschema@.bm25_text_bm25query_score(text, @extschema@.bm25query) STABLE; ALTER FUNCTION @extschema@.bm25_textarray_bm25query_score(text[], @extschema@.bm25query) STABLE; -- INTERNAL-ONLY test scaffold (issues #426, #427): return the live -- head tombstone page to the index FSM so the next allocator can pick -- it up, reproducing the stale-FSM / non-atomic-claim page-reuse -- hazard without an actual crash. Superuser-only; not a supported API. CREATE FUNCTION @extschema@.bm25_test_recycle_tombstone_head( index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_test_recycle_tombstone_head' LANGUAGE C VOLATILE STRICT; -- INTERNAL-ONLY test scaffold (issue #427): overwrite the head -- tombstone page's magic so the chain node is corrupt, simulating a -- page-reuse clobber, to exercise the drain's self-healing recovery. -- Superuser-only; not a supported API. CREATE FUNCTION @extschema@.bm25_test_corrupt_tombstone_head( index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_test_corrupt_tombstone_head' LANGUAGE C VOLATILE STRICT; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_recycle_tombstone_head(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_corrupt_tombstone_head(text) FROM PUBLIC; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.4.0--1.5.0.sql000066400000000000000000000075361525762047700244030ustar00rootroot00000000000000-- Upgrade from 1.4.0 to 1.5.0 -- Verify the library is loaded. The version-equality check lives in -- the main install file (pg_textsearch--.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; COMMENT ON FUNCTION @extschema@.bm25_force_merge(text) IS 'Run one-shot copy-on-write compaction into the fewest conservatively size-bounded segments; existing over-budget singletons remain uncombinable, and displaced pages enter deferred reclaim.'; -- PARALLEL RESTRICTED: opens the index relation, which may be a local -- temporary index whose buffers a parallel worker cannot reach. CREATE FUNCTION @extschema@.bm25_level_counts(idx regclass) RETURNS int[] AS 'MODULE_PATHNAME', 'tp_level_counts' LANGUAGE C VOLATILE STRICT PARALLEL RESTRICTED; CREATE FUNCTION @extschema@.bm25_compact(idx regclass) RETURNS void AS 'MODULE_PATHNAME', 'tp_compact_index' LANGUAGE C VOLATILE STRICT; COMMENT ON FUNCTION @extschema@.bm25_compact(regclass) IS 'Run threshold compaction to completion, yielding per-index maintenance admission between passes. Published passes are not undone by ROLLBACK, so a cascade that errors partway leaves its earlier passes applied.'; CREATE FUNCTION @extschema@.bm25_compact_step(idx regclass) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_compact_index_step' LANGUAGE C VOLATILE STRICT; COMMENT ON FUNCTION @extschema@.bm25_compact_step(regclass) IS 'Run at most one compaction pass and report whether one ran, letting a caller spread a cascade over several transactions. A published pass is not undone by ROLLBACK.'; CREATE FUNCTION @extschema@.bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION @extschema@.bm25_background_target_is_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_background_target_is_current' LANGUAGE C VOLATILE STRICT; REVOKE ALL ON FUNCTION @extschema@.bm25_compact_step_if_current(oid, oid, oid, oid, oid) FROM PUBLIC; REVOKE ALL ON FUNCTION @extschema@.bm25_background_target_is_current(oid, oid, oid, oid, oid) FROM PUBLIC; -- VOLATILE because it runs the compaction planner against live -- metapage state, and PARALLEL RESTRICTED to match bm25_level_counts. CREATE FUNCTION @extschema@.bm25_needs_compaction(idx regclass) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_needs_compaction' LANGUAGE C VOLATILE STRICT PARALLEL RESTRICTED; COMMENT ON FUNCTION @extschema@.bm25_needs_compaction(regclass) IS 'Report whether bm25_compact_step would run a pass on this index. A level that holds segments_per_level segments but cannot be reduced -- every candidate group exceeds max_segment_size -- reports false, so this is safe as a retry condition.'; -- Session-local count of BM25 scoring passes. Scan depth is invisible -- in query results (Filter + Limit + backoff produce the exact top-k -- either way), so this is the signal that a filtered top-k scan was -- seeded from its own filter rather than another scan's (issue #435). CREATE FUNCTION @extschema@.bm25_debug_scoring_passes( reset boolean DEFAULT false) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_debug_scoring_passes' LANGUAGE C VOLATILE STRICT; ALTER OPERATOR FAMILY @extschema@.text_bm25_ops USING bm25 ADD OPERATOR 1 pg_catalog.@@ (text, pg_catalog.tsquery); timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.5.0--1.5.1.sql000066400000000000000000000011551525762047700243740ustar00rootroot00000000000000-- Upgrade from 1.5.0 to 1.5.1 -- Verify the library is loaded. The version-equality check lives in -- the main install file (pg_textsearch--.sql); upgrade -- scripts must accept any loaded version because they may run as -- intermediate steps in a chain that ends at default_version, not at -- the version named in this filename. DO $$ BEGIN IF pg_catalog.current_setting('pg_textsearch.library_version', true) IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; END $$; timescale-pg_textsearch-b63fe58/sql/pg_textsearch--1.5.1.sql000066400000000000000000000451321525762047700236630ustar00rootroot00000000000000-- pg_textsearch extension version 1.5.1 -- complain if script is sourced in psql, rather than via CREATE EXTENSION \echo Use "CREATE EXTENSION pg_textsearch" to load this file. \quit -- Verify loaded library matches this SQL script version DO $$ DECLARE lib_ver text; BEGIN lib_ver := pg_catalog.current_setting('pg_textsearch.library_version', true); IF lib_ver IS NULL THEN RAISE EXCEPTION 'pg_textsearch library not loaded. ' 'Add pg_textsearch to shared_preload_libraries and restart.'; END IF; IF lib_ver OPERATOR(pg_catalog.<>) '1.5.1' THEN RAISE EXCEPTION 'pg_textsearch library version mismatch: loaded=%, expected=%. ' 'Restart the server after installing the new binary.', lib_ver, '1.5.1'; END IF; END $$; -- Access method CREATE FUNCTION @extschema@.tp_handler(internal) RETURNS index_am_handler AS 'MODULE_PATHNAME', 'tp_handler' LANGUAGE C; CREATE ACCESS METHOD bm25 TYPE INDEX HANDLER @extschema@.tp_handler; -- bm25vector type CREATE FUNCTION @extschema@.bm25vector_in(cstring) RETURNS @extschema@.bm25vector AS 'MODULE_PATHNAME', 'tpvector_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.bm25vector_out(@extschema@.bm25vector) RETURNS cstring AS 'MODULE_PATHNAME', 'tpvector_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.bm25vector_recv(internal) RETURNS @extschema@.bm25vector AS 'MODULE_PATHNAME', 'tpvector_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.bm25vector_send(@extschema@.bm25vector) RETURNS bytea AS 'MODULE_PATHNAME', 'tpvector_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE @extschema@.bm25vector ( INPUT = @extschema@.bm25vector_in, OUTPUT = @extschema@.bm25vector_out, RECEIVE = @extschema@.bm25vector_recv, SEND = @extschema@.bm25vector_send, STORAGE = extended, ALIGNMENT = int4 ); -- bm25query type CREATE FUNCTION @extschema@.bm25query_in(cstring) RETURNS @extschema@.bm25query AS 'MODULE_PATHNAME', 'tpquery_in' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.bm25query_out(@extschema@.bm25query) RETURNS cstring AS 'MODULE_PATHNAME', 'tpquery_out' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.bm25query_recv(internal) RETURNS @extschema@.bm25query AS 'MODULE_PATHNAME', 'tpquery_recv' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.bm25query_send(@extschema@.bm25query) RETURNS bytea AS 'MODULE_PATHNAME', 'tpquery_send' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE TYPE @extschema@.bm25query ( INPUT = @extschema@.bm25query_in, OUTPUT = @extschema@.bm25query_out, RECEIVE = @extschema@.bm25query_recv, SEND = @extschema@.bm25query_send, STORAGE = extended, ALIGNMENT = int4 ); -- Convert text to bm25query CREATE FUNCTION @extschema@.to_bm25query(input_text text) RETURNS @extschema@.bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; CREATE FUNCTION @extschema@.to_bm25query(input_text text, index_name text) RETURNS @extschema@.bm25query AS 'MODULE_PATHNAME', 'to_tpquery_text_index' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Equality function: bm25vector = bm25vector → boolean CREATE FUNCTION @extschema@.bm25vector_eq(@extschema@.bm25vector, @extschema@.bm25vector) RETURNS boolean AS 'MODULE_PATHNAME', 'tpvector_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- Create the = operator for equality CREATE OPERATOR @extschema@.= ( LEFTARG = @extschema@.bm25vector, RIGHTARG = @extschema@.bm25vector, FUNCTION = @extschema@.bm25vector_eq, COMMUTATOR = OPERATOR(@extschema@.=), HASHES ); -- BM25 scoring function for text <@> bm25query operations -- -- COST 1000: Standalone scoring is expensive. Each call parses document text -- with to_tsvector (~14μs per doc), opens the index, looks up IDF values, and -- calculates BM25 scores. High cost helps planner prefer index scans. -- -- PARALLEL UNSAFE: standalone scoring opens the index relation by name, -- attaches per-backend state (registry, DSA, LWLocks), and walks the -- on-disk memtable chain under shared latches. Parallel workers attempting -- the same setup do not survive worker startup reliably (see follow-up -- issue). Ranked queries should use ORDER BY <@> ... LIMIT n, which is an -- index scan and does not exercise this function in workers. -- STABLE, not IMMUTABLE: the returned score depends on corpus statistics -- that change as the indexed table is modified, so the result is only -- stable within a single scan. STABLE also keeps the planner from -- constant-folding a fully-constant scoring expression at plan time, so the -- runtime privilege check on the named index always runs under the invoking -- role. CREATE FUNCTION @extschema@.bm25_text_bm25query_score(left_text text, right_query @extschema@.bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_bm25query_score' LANGUAGE C STABLE STRICT PARALLEL UNSAFE COST 1000; -- bm25query equality function CREATE FUNCTION @extschema@.bm25query_eq(@extschema@.bm25query, @extschema@.bm25query) RETURNS boolean AS 'MODULE_PATHNAME', 'tpquery_eq' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE; -- <@> operator for text <@> bm25query operations CREATE OPERATOR @extschema@.<@> ( LEFTARG = text, RIGHTARG = @extschema@.bm25query, PROCEDURE = @extschema@.bm25_text_bm25query_score ); -- Function for text <@> text operator (planner hook rewrites to text <@> bm25query) -- COST 1000: High cost makes planner prefer index scans over seq scan + sort. -- In practice, this function errors without index scan context, but the cost -- helps the planner choose the right path before execution. CREATE FUNCTION @extschema@.bm25_text_text_score(text, text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_text_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- Stub function returning cached score from index scan. -- The planner hook replaces resjunk ORDER BY expressions with calls to this -- function, avoiding expensive re-computation of BM25 scores. CREATE FUNCTION @extschema@.bm25_get_current_score() RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_get_current_score' LANGUAGE C VOLATILE STRICT PARALLEL SAFE; -- <@> operator for text <@> text operations (implicit index resolution) -- The planner hook transforms this to text <@> bm25query when a BM25 index exists CREATE OPERATOR @extschema@.<@> ( LEFTARG = text, RIGHTARG = text, PROCEDURE = @extschema@.bm25_text_text_score ); -- = operator for bm25query equality CREATE OPERATOR @extschema@.= ( LEFTARG = @extschema@.bm25query, RIGHTARG = @extschema@.bm25query, FUNCTION = @extschema@.bm25query_eq, COMMUTATOR = OPERATOR(@extschema@.=), HASHES ); -- bm25 operator class for text columns -- The planner hook rewrites text <@> text to text <@> bm25query, so we only -- need to register the bm25query operator and support function here. CREATE OPERATOR CLASS @extschema@.text_bm25_ops DEFAULT FOR TYPE text USING bm25 AS OPERATOR 1 pg_catalog.@@ (text, pg_catalog.tsquery), OPERATOR 1 @extschema@.<@> (text, @extschema@.bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text, @extschema@.bm25query) @extschema@.bm25_text_bm25query_score(text, @extschema@.bm25query); -- BM25 scoring function for text[] <@> bm25query operations -- Flattens array elements with spaces, then scores as a single document. -- PARALLEL UNSAFE: delegates to bm25_text_bm25query_score; see that -- function's declaration for the rationale. CREATE FUNCTION @extschema@.bm25_textarray_bm25query_score( left_arr text[], right_query @extschema@.bm25query) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_textarray_bm25query_score' LANGUAGE C STABLE STRICT PARALLEL UNSAFE COST 1000; -- Error stub for text[] <@> text (planner should rewrite to bm25query) CREATE FUNCTION @extschema@.bm25_textarray_text_score(text[], text) RETURNS float8 AS 'MODULE_PATHNAME', 'bm25_textarray_text_score' LANGUAGE C IMMUTABLE STRICT PARALLEL SAFE COST 1000; -- <@> operator for text[] <@> bm25query operations CREATE OPERATOR @extschema@.<@> ( LEFTARG = text[], RIGHTARG = @extschema@.bm25query, PROCEDURE = @extschema@.bm25_textarray_bm25query_score ); -- <@> operator for text[] <@> text operations (implicit index resolution) CREATE OPERATOR @extschema@.<@> ( LEFTARG = text[], RIGHTARG = text, PROCEDURE = @extschema@.bm25_textarray_text_score ); -- bm25 operator class for text[] columns CREATE OPERATOR CLASS @extschema@.text_array_bm25_ops DEFAULT FOR TYPE text[] USING bm25 AS OPERATOR 1 @extschema@.<@> (text[], @extschema@.bm25query) FOR ORDER BY float_ops, FUNCTION 8 (text[], @extschema@.bm25query) @extschema@.bm25_textarray_bm25query_score( text[], @extschema@.bm25query); -- Debug function to dump index contents (memtable and segments) CREATE FUNCTION @extschema@.bm25_dump_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_dump_index' LANGUAGE C STRICT STABLE; -- Function to force segment write (spill memtable to disk) CREATE FUNCTION @extschema@.bm25_spill_index(index_name text) RETURNS int4 AS 'MODULE_PATHNAME', 'tp_spill_memtable' LANGUAGE C VOLATILE STRICT; -- One-shot, size-bounded copy-on-write segment compaction CREATE FUNCTION @extschema@.bm25_force_merge(index_name text) RETURNS void AS 'MODULE_PATHNAME', 'tp_force_merge' LANGUAGE C VOLATILE STRICT; COMMENT ON FUNCTION @extschema@.bm25_force_merge(text) IS 'Run one-shot copy-on-write compaction into the fewest conservatively size-bounded segments; existing over-budget singletons remain uncombinable, and displaced pages enter deferred reclaim.'; -- PARALLEL RESTRICTED: opens the index relation, which may be a local -- temporary index whose buffers a parallel worker cannot reach. CREATE FUNCTION @extschema@.bm25_level_counts(idx regclass) RETURNS int[] AS 'MODULE_PATHNAME', 'tp_level_counts' LANGUAGE C VOLATILE STRICT PARALLEL RESTRICTED; CREATE FUNCTION @extschema@.bm25_compact(idx regclass) RETURNS void AS 'MODULE_PATHNAME', 'tp_compact_index' LANGUAGE C VOLATILE STRICT; COMMENT ON FUNCTION @extschema@.bm25_compact(regclass) IS 'Run threshold compaction to completion, yielding per-index maintenance admission between passes. Published passes are not undone by ROLLBACK, so a cascade that errors partway leaves its earlier passes applied.'; CREATE FUNCTION @extschema@.bm25_compact_step(idx regclass) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_compact_index_step' LANGUAGE C VOLATILE STRICT; COMMENT ON FUNCTION @extschema@.bm25_compact_step(regclass) IS 'Run at most one compaction pass and report whether one ran, letting a caller spread a cascade over several transactions. A published pass is not undone by ROLLBACK.'; CREATE FUNCTION @extschema@.bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; CREATE FUNCTION @extschema@.bm25_background_target_is_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_background_target_is_current' LANGUAGE C VOLATILE STRICT; REVOKE ALL ON FUNCTION @extschema@.bm25_compact_step_if_current(oid, oid, oid, oid, oid) FROM PUBLIC; REVOKE ALL ON FUNCTION @extschema@.bm25_background_target_is_current(oid, oid, oid, oid, oid) FROM PUBLIC; -- VOLATILE because it runs the compaction planner against live -- metapage state, and PARALLEL RESTRICTED to match bm25_level_counts. CREATE FUNCTION @extschema@.bm25_needs_compaction(idx regclass) RETURNS boolean AS 'MODULE_PATHNAME', 'tp_needs_compaction' LANGUAGE C VOLATILE STRICT PARALLEL RESTRICTED; COMMENT ON FUNCTION @extschema@.bm25_needs_compaction(regclass) IS 'Report whether bm25_compact_step would run a pass on this index. A level that holds segments_per_level segments but cannot be reduced -- every candidate group exceeds max_segment_size -- reports false, so this is safe as a retry condition.'; -- Fast summary function showing only statistics (no content dump) CREATE FUNCTION @extschema@.bm25_summarize_index(text) RETURNS text AS 'MODULE_PATHNAME', 'tp_summarize_index' LANGUAGE C STRICT STABLE; CREATE FUNCTION @extschema@.bm25_pending_free_pages(index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_pending_free_pages' LANGUAGE C STRICT STABLE; -- Session-local count of BM25 scoring passes. Scan depth is invisible -- in query results (Filter + Limit + backoff produce the exact top-k -- either way), so this is the signal that a filtered top-k scan was -- seeded from its own filter rather than another scan's (issue #435). CREATE FUNCTION @extschema@.bm25_debug_scoring_passes( reset boolean DEFAULT false) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_debug_scoring_passes' LANGUAGE C VOLATILE STRICT; -- INTERNAL-ONLY test scaffold (issues #426, #427): return the live -- head tombstone page to the index FSM so the next allocator can pick -- it up, reproducing the stale-FSM / non-atomic-claim page-reuse -- hazard without an actual crash. Superuser-only; not a supported API. CREATE FUNCTION @extschema@.bm25_test_recycle_tombstone_head( index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_test_recycle_tombstone_head' LANGUAGE C VOLATILE STRICT; -- INTERNAL-ONLY test scaffold (issue #427): overwrite the head -- tombstone page's magic so the chain node is corrupt, simulating a -- page-reuse clobber, to exercise the drain's self-healing recovery. -- Superuser-only; not a supported API. CREATE FUNCTION @extschema@.bm25_test_corrupt_tombstone_head( index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'tp_test_corrupt_tombstone_head' LANGUAGE C VOLATILE STRICT; -- Revoke public execute on debug functions (superuser-only). REVOKE EXECUTE ON FUNCTION @extschema@.bm25_dump_index(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_summarize_index(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_pending_free_pages(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_recycle_tombstone_head(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_corrupt_tombstone_head(text) FROM PUBLIC; -- The bm25_test_memtable_page / bm25_test_memtable_append / -- bm25_test_chain_source / bm25_memtable_chain / -- bm25_memtable_dead_pages functions are -- INTERNAL-ONLY test scaffolds for the on-disk memtable redesign -- (issue #374). They are not part of the supported -- public API: their signatures, return values, and existence are -- subject to change or removal in ANY release (including patch -- releases) without notice or upgrade paths. Do not depend on -- them from application code. CREATE FUNCTION @extschema@.bm25_test_memtable_page(case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_memtable_page' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_test_memtable_append( index_name text, case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_memtable_append' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_memtable_chain( index_name text, OUT blkno bigint, OUT n_records integer, OUT free_offset integer, OUT next_block bigint, OUT flags integer) RETURNS SETOF record AS 'MODULE_PATHNAME', 'bm25_memtable_chain' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_memtable_dead_pages( index_name text, OUT blkno bigint, OUT flags integer, OUT dead_fxid bigint, OUT n_records integer) RETURNS SETOF record AS 'MODULE_PATHNAME', 'bm25_memtable_dead_pages' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_test_chain_source( index_name text, case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_chain_source' LANGUAGE C STRICT; -- Cache apply protocol scaffolds (in-memory memtable cache). -- Same INTERNAL-ONLY disclaimer as the v2 chain scaffolds above. -- Each returns a (result, records_applied, cursor_seq, -- estimated_bytes) tuple. CREATE FUNCTION @extschema@.bm25_cache_cold_build( index_name text, OUT result text, OUT records_applied bigint, OUT cursor_seq bigint, OUT estimated_bytes bigint) RETURNS record AS 'MODULE_PATHNAME', 'bm25_cache_cold_build' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_cache_apply_to_tail( index_name text, OUT result text, OUT records_applied bigint, OUT cursor_seq bigint, OUT estimated_bytes bigint) RETURNS record AS 'MODULE_PATHNAME', 'bm25_cache_apply_to_tail' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_cache_bump_spill_generation( index_name text) RETURNS bigint AS 'MODULE_PATHNAME', 'bm25_cache_bump_spill_generation' LANGUAGE C STRICT; -- Cache memory-cap scaffolds. Same INTERNAL-ONLY disclaimer as -- above. The per-index limit/8 per-record growth guard rejects -- a record whose estimated growth would cross it; the -- global limit/2 cap attempts best-effort eviction, and limit is an -- approximate admission threshold: catch-up or cold build falls back -- when the entry-time estimate is already at the limit, while admitted -- or concurrent work may exceed it. Zero means unlimited. CREATE FUNCTION @extschema@.bm25_cache_global_estimated_bytes() RETURNS bigint AS 'MODULE_PATHNAME', 'bm25_cache_global_estimated_bytes' LANGUAGE C STRICT; CREATE FUNCTION @extschema@.bm25_cache_evict_largest(index_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_cache_evict_largest' LANGUAGE C STRICT; -- Cache source scaffold (in-memory memtable cache). -- Same INTERNAL-ONLY disclaimer as above. CREATE FUNCTION @extschema@.bm25_test_cache_source( index_name text, case_name text) RETURNS text AS 'MODULE_PATHNAME', 'bm25_test_cache_source' LANGUAGE C STRICT; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_memtable_page(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_memtable_append(text, text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_memtable_chain(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_memtable_dead_pages(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_chain_source(text, text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_cache_cold_build(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_cache_apply_to_tail(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_cache_bump_spill_generation(text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_test_cache_source(text, text) FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_cache_global_estimated_bytes() FROM PUBLIC; REVOKE EXECUTE ON FUNCTION @extschema@.bm25_cache_evict_largest(text) FROM PUBLIC; timescale-pg_textsearch-b63fe58/src/000077500000000000000000000000001525762047700175505ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/access/000077500000000000000000000000001525762047700210115ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/access/am.h000066400000000000000000000162131525762047700215620ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * am.h - BM25 access method shared definitions */ #pragma once #include #include #include #include #include #include #include #include #include "index/state.h" #include "types/vector.h" /* * BM25 scan opaque data - internal state for index scans */ typedef struct TpScanOpaqueData { MemoryContext scan_context; /* Memory context for scan */ /* Query processing state */ char *query_text; /* Search query text */ TpVector *query_vector; /* Original query vector from ORDER BY */ TSQuery boolean_query; /* Query from an ordinary @@ scan key */ MemoryContext boolean_context; bool is_boolean_scan; bool boolean_recheck; BufFile *boolean_results; Oid index_oid; /* Index OID */ /* Scan results state */ ItemPointer result_ctids; /* Array of matching CTIDs */ float4 *result_scores; /* Array of BM25 scores */ int result_count; /* Number of results */ int current_pos; /* Current position in results */ bool eof_reached; /* End of scan flag */ /* LIMIT optimization */ int limit; /* Query LIMIT value, -1 if none */ int max_results_used; /* Internal limit used for current batch */ /* CTIDs already emitted; used across limit-doubling re-execs. */ struct HTAB *returned_ctids; } TpScanOpaqueData; typedef TpScanOpaqueData *TpScanOpaque; /* Index options structure */ typedef struct TpOptions { int32 vl_len_; /* varlena header (do not touch directly!) */ int32 text_config_offset; /* offset to text config string */ double k1; /* BM25 k1 parameter */ double b; /* BM25 b parameter */ int32 compaction_schedule_offset; int32 compaction_lineage_offset; int compaction; /* TpCompactionMode for this index */ } TpOptions; /* Tapir-specific build phases for progress reporting */ #define TP_PHASE_LOADING 2 #define TP_PHASE_WRITING 3 #define TP_PHASE_COMPACTING 4 /* Progress reporting interval (tuples) */ #define TP_PROGRESS_REPORT_INTERVAL 1000 /* Forward declarations */ struct IndexInfo; struct PlannerInfo; struct IndexPath; struct IndexVacuumInfo; struct IndexBulkDeleteResult; struct IndexBuildResult; /* * Shared utility functions */ /* Cached score for ORDER BY optimization */ float8 tp_get_cached_score(void); /* * Access method handler */ PGDLLEXPORT Datum tp_handler(PG_FUNCTION_ARGS); /* * Build utilities (am/build.c) */ /* Link a segment as the new L0 chain head in the metapage */ void tp_link_l0_chain_head(Relation index, BlockNumber segment_root); /* Truncate a contiguous EOF suffix already stamped recyclable */ void tp_truncate_dead_pages(Relation index); /* * Build functions (am/build.c) */ struct IndexBuildResult * tp_build(Relation heap, Relation index, struct IndexInfo *indexInfo); void tp_buildempty(Relation index); bool tp_insert( Relation index, Datum *values, bool *isnull, ItemPointer ht_ctid, Relation heapRel, IndexUniqueCheck checkUnique, bool indexUnchanged, struct IndexInfo *indexInfo); /* Shared document processing function */ bool tp_process_document_text( text *document_text, ItemPointer ctid, Oid text_config_oid, TpLocalIndexState *index_state, Relation index_rel, int32 *doc_length_out); /* Extract terms and frequencies from a TSVector */ int tp_extract_terms_from_tsvector( TSVector tsvector, char ***terms_out, int32 **frequencies_out, int *term_count_out); /* * Tokenize a document into terms and frequencies. Handles documents whose * lexeme volume would otherwise exceed Postgres's tsvector 1 MB cap by * splitting on whitespace and merging per-chunk results. * * Output arrays are palloc'd in the current memory context. Returns * doc_length (sum of frequencies). */ int tp_tokenize_text( text *document_text, Oid text_config_oid, char ***terms_out, int32 **frequencies_out, int *term_count_out); /* Build progress tracking for partitioned tables */ void tp_build_progress_set_owner(const void *owner); void tp_build_progress_begin(void); void tp_build_progress_end(void); void tp_build_progress_abort(void); /* * Scan functions (am/scan.c) */ IndexScanDesc tp_beginscan(Relation index, int nkeys, int norderbys); void tp_rescan( IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys); void tp_endscan(IndexScanDesc scan); bool tp_gettuple(IndexScanDesc scan, ScanDirection dir); /* * Vacuum functions (am/vacuum.c) */ struct IndexBulkDeleteResult *tp_bulkdelete( struct IndexVacuumInfo *info, struct IndexBulkDeleteResult *stats, IndexBulkDeleteCallback callback, void *callback_state); struct IndexBulkDeleteResult *tp_vacuumcleanup( struct IndexVacuumInfo *info, struct IndexBulkDeleteResult *stats); char *tp_buildphasename(int64 phase); /* * Spill a memtable to an L0 segment. Skips when chain_page_count < * min_pages. Acquires LW_EXCLUSIVE for spill durability, releases it, * then applies the index's compaction policy. */ void tp_spill_memtable_if_needed( Relation index, TpLocalIndexState *index_state, uint32 min_pages); /* * Spill now but let the caller apply compaction after prerequisite work. * An empty chain is the postcondition, for callers whose correctness * depends on it: a level 0 with no room reports its capacity limit * instead of leaving records in the chain. Returns whether a spill ran. */ bool tp_spill_memtable_if_needed_deferred( Relation index, TpLocalIndexState *index_state, uint32 min_pages); /* Apply the index policy at a spill or VACUUM maintenance opportunity. */ void tp_apply_compaction_policy( TpLocalIndexState *index_state, Relation index_rel, bool maintenance_opportunity); /* Shutdown cleanup spills durable state without starting maintenance. */ void tp_spill_memtable_without_compaction_if_needed( Relation index, TpLocalIndexState *index_state, uint32 min_pages); /* * Recycle memtable pages stamped DEAD during spill: scan the index * main fork, tp_record_free_index_page when dead_fxid is older than * the global visibility horizon for heaprel. Returns the number of * blocks freed. Caller must hold the per-index maintenance object * lock to exclude force-merge truncation, but must not retain the * per-index LWLock across this full-fork scan. Each freed page is * WAL-stamped recyclable before it returns to the FSM (same as * segment page free). */ int tp_reclaim_dead_memtable_pages(Relation indexrel, Relation heaprel); /* * Compute the deferred-reclaim horizon as a FullTransactionId. * Pass the heap relation when available (vacuum) for a precise * horizon, or NULL (merge) for the most-conservative shared * horizon — NULL can only over-retain, never free too early. */ extern FullTransactionId tp_reclaim_horizon(Relation heaprel); /* * Handler functions (am/handler.c) */ bytea *tp_options(Datum reloptions, bool validate); bool tp_validate(Oid opclassoid); /* Relation options kind - initialized in mod.c */ extern relopt_kind tp_relopt_kind; timescale-pg_textsearch-b63fe58/src/access/boolean.c000066400000000000000000000611721525762047700226030ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * Boolean query execution over BM25 posting lists. */ #include #include #include #include #include #include #include #include "access/am.h" #include "access/boolean.h" #include "memtable/chain_walker.h" #include "segment/alive_bitset.h" #include "segment/graph_snapshot.h" #include "segment/io.h" #include "types/vector.h" #define TP_BOOLEAN_MAX_PREFIX_ITERATORS 256 typedef struct TpBooleanTerm { char *lexeme; int length; } TpBooleanTerm; typedef struct TpBooleanEvalState { TSQuery query; TpBooleanTerm *terms; int term_count; int exact_operand_count; bool requires_recheck; } TpBooleanEvalState; typedef enum TpBooleanCandidateKind { TP_BOOLEAN_CANDIDATE_ALL, TP_BOOLEAN_CANDIDATE_TERM, TP_BOOLEAN_CANDIDATE_UNION, TP_BOOLEAN_CANDIDATE_PREFIX, TP_BOOLEAN_CANDIDATE_PREFIX_BITMAP, } TpBooleanCandidateKind; typedef struct TpBooleanPrefixHeapNode { uint32 iterator_index; uint32 doc_id; } TpBooleanPrefixHeapNode; typedef struct TpBooleanCandidateStream { TpBooleanCandidateKind kind; uint64 estimate; union { struct { uint32 next_doc_id; uint32 num_docs; } all; struct { TpSegmentPostingIterator iterator; bool initialized; } term; struct { struct TpBooleanCandidateStream *left; struct TpBooleanCandidateStream *right; uint32 left_doc_id; uint32 right_doc_id; bool left_loaded; bool right_loaded; bool left_valid; bool right_valid; } union_stream; struct { TpSegmentPostingIterator *iterators; TpBooleanPrefixHeapNode *heap; uint32 iterator_count; uint32 heap_size; bool initialized; } prefix; struct { uint8 *bits; uint32 next_doc_id; uint32 num_docs; } prefix_bitmap; } state; } TpBooleanCandidateStream; typedef struct TpBooleanTermCursor { TpSegmentPostingIterator iterator; TpSegmentPosting *posting; bool initialized; bool positioned; } TpBooleanTermCursor; typedef struct TpBooleanSegmentEval { TpBooleanEvalState *query; TpBooleanTermCursor *cursors; uint32 doc_id; } TpBooleanSegmentEval; static List *tp_boolean_incomplete_warning_seen = NIL; static void tp_boolean_warn_if_incomplete( Relation index, TpIndexMetaPage metap, TSQuery query) { Oid index_oid = RelationGetRelid(index); if ((metap->capabilities & TP_METAPAGE_ALL_DOCUMENTS_INDEXED) != 0 || query->size == 0 || tsquery_requires_match(GETQUERY(query)) || list_member_oid(tp_boolean_incomplete_warning_seen, index_oid)) return; { MemoryContext old = MemoryContextSwitchTo(TopMemoryContext); tp_boolean_incomplete_warning_seen = lappend_oid(tp_boolean_incomplete_warning_seen, index_oid); MemoryContextSwitchTo(old); } ereport(WARNING, (errmsg("BM25 index \"%s\" may omit empty or stopword-only " "documents from this query", RelationGetRelationName(index)), errhint("Run \"REINDEX INDEX %s;\" for complete results.", RelationGetRelationName(index)))); } static void tp_boolean_check_config(Relation index, TpIndexMetaPage metap) { Oid current_config = getTSCurrentConfig(true); if (metap->text_config_oid != current_config) { char *current_config_name = DatumGetCString(DirectFunctionCall1( regconfigout, ObjectIdGetDatum(current_config))); char *index_config_name = DatumGetCString(DirectFunctionCall1( regconfigout, ObjectIdGetDatum(metap->text_config_oid))); ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("BM25 index \"%s\" cannot evaluate text search " "configuration \"%s\"", RelationGetRelationName(index), current_config_name), errdetail( "The index uses text search configuration \"%s\".", index_config_name), errhint("Set default_text_search_config to match the index " "or use a different index."))); } } static TpBooleanTerm * tp_boolean_find_term(TpBooleanEvalState *state, const char *lexeme, int length) { for (int i = 0; i < state->term_count; i++) { TpBooleanTerm *term = &state->terms[i]; if (term->length == length && memcmp(term->lexeme, lexeme, length) == 0) return term; } return NULL; } static TpBooleanEvalState tp_boolean_extract_terms(TSQuery query) { TpBooleanEvalState state; QueryItem *items = GETQUERY(query); char *operands = GETOPERAND(query); memset(&state, 0, sizeof(state)); state.query = query; state.terms = palloc0(Max(query->size, 1) * sizeof(TpBooleanTerm)); state.exact_operand_count = tp_boolean_query_exact_operand_count(query); for (int i = 0; i < query->size; i++) { QueryItem *item = &items[i]; if (item->type == QI_OPR) { if (item->qoperator.oper == OP_PHRASE) state.requires_recheck = true; continue; } if (item->type == QI_VAL) { QueryOperand *operand = &item->qoperand; const char *lexeme = operands + operand->distance; TpBooleanTerm *term; /* * Segment postings do not retain lexeme weights, and prefix * operands may expand to many terms. Treat these operands as * unknown and let the heap recheck evaluate their exact * semantics. */ if (operand->prefix || operand->weight != 0) { state.requires_recheck = true; continue; } if (tp_boolean_find_term(&state, lexeme, operand->length) != NULL) continue; term = &state.terms[state.term_count++]; term->length = operand->length; term->lexeme = pnstrdup(lexeme, operand->length); } } return state; } int tp_boolean_query_exact_operand_count(TSQuery query) { QueryItem *items = GETQUERY(query); int count = 0; for (int i = 0; i < query->size; i++) { if (items[i].type == QI_VAL && !items[i].qoperand.prefix && items[i].qoperand.weight == 0) count++; } return count; } typedef struct TpBooleanResultWriter { TpBooleanEvalState *query; BufFile *file; int count; } TpBooleanResultWriter; typedef struct TpBooleanMemtableEval { TpBooleanEvalState *query; bool *term_present; } TpBooleanMemtableEval; static void tp_boolean_write_result(TpBooleanResultWriter *writer, ItemPointer candidate) { BufFileWrite(writer->file, candidate, sizeof(ItemPointerData)); writer->count++; } static TSTernaryValue tp_boolean_memtable_has_term( void *arg, QueryOperand *operand, ExecPhraseData *data) { TpBooleanMemtableEval *eval = arg; const char *lexeme = GETOPERAND(eval->query->query) + operand->distance; TpBooleanTerm *term = tp_boolean_find_term(eval->query, lexeme, operand->length); if (operand->prefix || operand->weight != 0) return TS_MAYBE; Assert(term != NULL); if (!eval->term_present[term - eval->query->terms]) return TS_NO; return data == NULL ? TS_YES : TS_MAYBE; } static void tp_boolean_write_memtable_snapshot( Relation index, const TpMemtableChainSnapshot *snapshot, TpBooleanResultWriter *writer) { TpBooleanEvalState *state = writer->query; TpBooleanMemtableEval eval = { .query = state, }; TpChainWalker *walker; TpChainWalkerRecord rec; if (!BlockNumberIsValid(snapshot->head_blkno)) return; eval.term_present = palloc0(Max(state->term_count, 1) * sizeof(bool)); walker = tp_chain_walker_open_bounded( index, snapshot, CurrentMemoryContext, true); while (tp_chain_walker_next(walker, &rec)) { TpVector *vec = (TpVector *)rec.vector_bytes; TpVectorEntry *entry; CHECK_FOR_INTERRUPTS(); memset(eval.term_present, 0, state->term_count * sizeof(bool)); #ifdef USE_ASSERT_CHECKING if (!rec.is_fragment) tpvector_validate_v2_buffer(rec.vector_bytes, rec.vector_len); #endif entry = get_tpvector_first_entry(vec); for (int i = 0; i < vec->entry_count; i++) { TpVectorEntryView view; TpBooleanTerm *term; entry = tpvector_entry_decode_advance(entry, &view); term = tp_boolean_find_term( state, view.lexeme, (int)view.lexeme_len); if (term != NULL) eval.term_present[term - state->terms] = true; } if (TS_execute_ternary( GETQUERY(state->query), &eval, TS_EXEC_PHRASE_NO_POS, tp_boolean_memtable_has_term) != TS_NO) tp_boolean_write_result(writer, &rec.ctid); if (rec.owns_vector) pfree((void *)rec.vector_bytes); } tp_chain_walker_close(walker); pfree(eval.term_present); } static TpBooleanCandidateStream * tp_boolean_create_all_stream(TpSegmentReader *reader) { TpBooleanCandidateStream *stream = palloc0(sizeof(*stream)); stream->kind = TP_BOOLEAN_CANDIDATE_ALL; stream->estimate = reader->header->num_docs; stream->state.all.num_docs = reader->header->num_docs; return stream; } static void tp_boolean_free_candidate_stream(TpBooleanCandidateStream *stream); static TpBooleanCandidateStream * tp_boolean_create_union_stream( TpSegmentReader *reader, TpBooleanCandidateStream *left, TpBooleanCandidateStream *right) { TpBooleanCandidateStream *stream = palloc0(sizeof(*stream)); stream->kind = TP_BOOLEAN_CANDIDATE_UNION; stream->estimate = Min((uint64)reader->header->num_docs, left->estimate + right->estimate); stream->state.union_stream.left = left; stream->state.union_stream.right = right; return stream; } static bool tp_boolean_prefix_heap_less( const TpBooleanPrefixHeapNode *left, const TpBooleanPrefixHeapNode *right) { return left->doc_id < right->doc_id || (left->doc_id == right->doc_id && left->iterator_index < right->iterator_index); } static void tp_boolean_prefix_heap_sift_down( TpBooleanPrefixHeapNode *heap, uint32 heap_size, uint32 parent) { TpBooleanPrefixHeapNode node = heap[parent]; while (parent < heap_size / 2) { uint32 child = parent * 2 + 1; if (child + 1 < heap_size && tp_boolean_prefix_heap_less(&heap[child + 1], &heap[child])) child++; if (!tp_boolean_prefix_heap_less(&heap[child], &node)) break; heap[parent] = heap[child]; parent = child; } heap[parent] = node; } static void tp_boolean_prefix_stream_init(TpBooleanCandidateStream *stream) { for (uint32 i = 0; i < stream->state.prefix.iterator_count; i++) { TpSegmentPosting *posting; if (tp_segment_posting_iterator_next( &stream->state.prefix.iterators[i], &posting)) stream->state.prefix.heap[stream->state.prefix.heap_size++] = (TpBooleanPrefixHeapNode){ .iterator_index = i, .doc_id = posting->doc_id, }; } for (uint32 i = stream->state.prefix.heap_size / 2; i > 0; i--) tp_boolean_prefix_heap_sift_down( stream->state.prefix.heap, stream->state.prefix.heap_size, i - 1); stream->state.prefix.initialized = true; } static void tp_boolean_prefix_stream_advance_root(TpBooleanCandidateStream *stream) { TpBooleanPrefixHeapNode *heap = stream->state.prefix.heap; TpSegmentPosting *posting; uint32 iterator_index = heap[0].iterator_index; if (tp_segment_posting_iterator_next( &stream->state.prefix.iterators[iterator_index], &posting)) heap[0].doc_id = posting->doc_id; else heap[0] = heap[--stream->state.prefix.heap_size]; if (stream->state.prefix.heap_size > 0) tp_boolean_prefix_heap_sift_down( heap, stream->state.prefix.heap_size, 0); } static TpBooleanCandidateStream * tp_boolean_create_prefix_stream( TpSegmentReader *reader, const char *prefix, int prefix_length) { TpSegmentPrefixCandidates candidates; TpBooleanCandidateStream *stream = palloc0(sizeof(*stream)); tp_segment_prefix_candidates_init( reader, prefix, prefix_length, TP_BOOLEAN_MAX_PREFIX_ITERATORS, &candidates); stream->estimate = candidates.estimate; if (candidates.doc_bitmap != NULL) { stream->kind = TP_BOOLEAN_CANDIDATE_PREFIX_BITMAP; stream->state.prefix_bitmap.bits = candidates.doc_bitmap; stream->state.prefix_bitmap.num_docs = reader->header->num_docs; return stream; } if (candidates.iterator_count == 0) { stream->kind = TP_BOOLEAN_CANDIDATE_TERM; return stream; } stream->kind = TP_BOOLEAN_CANDIDATE_PREFIX; stream->estimate = candidates.estimate; stream->state.prefix.iterators = candidates.iterators; stream->state.prefix.iterator_count = candidates.iterator_count; stream->state.prefix.heap = palloc_array(TpBooleanPrefixHeapNode, candidates.iterator_count); return stream; } static TpBooleanCandidateStream * tp_boolean_create_candidate_stream( TpBooleanEvalState *state, TpSegmentReader *reader, QueryItem *item) { TpBooleanCandidateStream *stream; if (item->type == QI_VAL) { QueryOperand *operand = &item->qoperand; const char *lexeme = GETOPERAND(state->query) + operand->distance; if (operand->prefix) return tp_boolean_create_prefix_stream( reader, lexeme, operand->length); if (operand->weight != 0) return tp_boolean_create_all_stream(reader); stream = palloc0(sizeof(*stream)); stream->kind = TP_BOOLEAN_CANDIDATE_TERM; stream->state.term.initialized = tp_segment_posting_iterator_init( &stream->state.term.iterator, reader, lexeme); stream->state.term.iterator.force_copy = true; stream->estimate = stream->state.term.initialized ? stream->state.term.iterator.dict_entry.doc_freq : 0; return stream; } if (item->type != QI_OPR) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid item in BM25 Boolean query"))); if (item->qoperator.oper == OP_NOT) return tp_boolean_create_all_stream(reader); { TpBooleanCandidateStream *right = tp_boolean_create_candidate_stream(state, reader, item + 1); TpBooleanCandidateStream *left = tp_boolean_create_candidate_stream( state, reader, item + item->qoperator.left); if (item->qoperator.oper == OP_AND || item->qoperator.oper == OP_PHRASE) { if (left->estimate <= right->estimate) { tp_boolean_free_candidate_stream(right); return left; } tp_boolean_free_candidate_stream(left); return right; } if (item->qoperator.oper != OP_OR) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid operator in BM25 Boolean query"))); return tp_boolean_create_union_stream(reader, left, right); } } static bool tp_boolean_candidate_stream_next( TpBooleanCandidateStream *stream, uint32 *doc_id) { switch (stream->kind) { case TP_BOOLEAN_CANDIDATE_ALL: if (stream->state.all.next_doc_id >= stream->state.all.num_docs) return false; *doc_id = stream->state.all.next_doc_id++; return true; case TP_BOOLEAN_CANDIDATE_TERM: { TpSegmentPosting *posting; if (!stream->state.term.initialized || !tp_segment_posting_iterator_next( &stream->state.term.iterator, &posting)) return false; *doc_id = posting->doc_id; return true; } case TP_BOOLEAN_CANDIDATE_UNION: { TpBooleanCandidateStream *left = stream->state.union_stream.left; TpBooleanCandidateStream *right = stream->state.union_stream.right; if (!stream->state.union_stream.left_loaded) { stream->state.union_stream.left_valid = tp_boolean_candidate_stream_next( left, &stream->state.union_stream.left_doc_id); stream->state.union_stream.left_loaded = true; } if (!stream->state.union_stream.right_loaded) { stream->state.union_stream.right_valid = tp_boolean_candidate_stream_next( right, &stream->state.union_stream.right_doc_id); stream->state.union_stream.right_loaded = true; } if (!stream->state.union_stream.left_valid && !stream->state.union_stream.right_valid) return false; if (!stream->state.union_stream.right_valid || (stream->state.union_stream.left_valid && stream->state.union_stream.left_doc_id <= stream->state.union_stream.right_doc_id)) *doc_id = stream->state.union_stream.left_doc_id; else *doc_id = stream->state.union_stream.right_doc_id; if (stream->state.union_stream.left_valid && stream->state.union_stream.left_doc_id == *doc_id) stream->state.union_stream.left_loaded = false; if (stream->state.union_stream.right_valid && stream->state.union_stream.right_doc_id == *doc_id) stream->state.union_stream.right_loaded = false; return true; } case TP_BOOLEAN_CANDIDATE_PREFIX: if (!stream->state.prefix.initialized) tp_boolean_prefix_stream_init(stream); if (stream->state.prefix.heap_size == 0) return false; *doc_id = stream->state.prefix.heap[0].doc_id; do { tp_boolean_prefix_stream_advance_root(stream); } while (stream->state.prefix.heap_size > 0 && stream->state.prefix.heap[0].doc_id == *doc_id); return true; case TP_BOOLEAN_CANDIDATE_PREFIX_BITMAP: while (stream->state.prefix_bitmap.next_doc_id < stream->state.prefix_bitmap.num_docs) { uint32 candidate = stream->state.prefix_bitmap.next_doc_id; uint8 byte = stream->state.prefix_bitmap.bits[candidate >> 3]; if ((candidate & 7) == 0 && byte == 0) { stream->state.prefix_bitmap.next_doc_id += 8; continue; } stream->state.prefix_bitmap.next_doc_id++; if (byte & (1 << (candidate & 7))) { *doc_id = candidate; return true; } } return false; } pg_unreachable(); } static void tp_boolean_free_candidate_stream(TpBooleanCandidateStream *stream) { if (stream == NULL) return; if (stream->kind == TP_BOOLEAN_CANDIDATE_TERM && stream->state.term.initialized) tp_segment_posting_iterator_free(&stream->state.term.iterator); else if (stream->kind == TP_BOOLEAN_CANDIDATE_UNION) { tp_boolean_free_candidate_stream(stream->state.union_stream.left); tp_boolean_free_candidate_stream(stream->state.union_stream.right); } else if (stream->kind == TP_BOOLEAN_CANDIDATE_PREFIX) { for (uint32 i = 0; i < stream->state.prefix.iterator_count; i++) tp_segment_posting_iterator_free( &stream->state.prefix.iterators[i]); pfree(stream->state.prefix.iterators); pfree(stream->state.prefix.heap); } else if (stream->kind == TP_BOOLEAN_CANDIDATE_PREFIX_BITMAP) pfree(stream->state.prefix_bitmap.bits); pfree(stream); } static TSTernaryValue tp_boolean_segment_has_term( void *arg, QueryOperand *operand, ExecPhraseData *data) { TpBooleanSegmentEval *eval = arg; const char *lexeme = GETOPERAND(eval->query->query) + operand->distance; TpBooleanTerm *term = tp_boolean_find_term(eval->query, lexeme, operand->length); TpBooleanTermCursor *cursor; if (operand->prefix || operand->weight != 0) return TS_MAYBE; Assert(term != NULL); cursor = &eval->cursors[term - eval->query->terms]; if (!cursor->initialized) return TS_NO; if (!cursor->positioned) { cursor->positioned = tp_segment_posting_iterator_seek( &cursor->iterator, eval->doc_id, &cursor->posting); if (!cursor->positioned) return TS_NO; } else if (cursor->posting->doc_id < eval->doc_id) { if (eval->doc_id > cursor->iterator.skip_entry.last_doc_id) cursor->positioned = tp_segment_posting_iterator_seek( &cursor->iterator, eval->doc_id, &cursor->posting); else { do { cursor->positioned = tp_segment_posting_iterator_next( &cursor->iterator, &cursor->posting); } while (cursor->positioned && cursor->posting->doc_id < eval->doc_id); } if (!cursor->positioned) return TS_NO; } if (cursor->posting->doc_id != eval->doc_id) return TS_NO; return data == NULL ? TS_YES : TS_MAYBE; } static bool tp_boolean_segment_candidate_matches( TpBooleanEvalState *state, TpBooleanTermCursor *cursors, uint32 doc_id) { TpBooleanSegmentEval eval = { .query = state, .cursors = cursors, .doc_id = doc_id, }; return TS_execute_ternary( GETQUERY(state->query), &eval, TS_EXEC_PHRASE_NO_POS, tp_boolean_segment_has_term) != TS_NO; } static void tp_boolean_write_segment( TpSegmentReader *reader, TpBooleanResultWriter *writer) { TpBooleanCandidateStream *stream = tp_boolean_create_candidate_stream( writer->query, reader, GETQUERY(writer->query->query)); TpBooleanTermCursor *cursors = palloc0( writer->query->term_count * sizeof(*cursors)); uint32 cache_threshold = reader->header->num_docs / 100 + (reader->header->num_docs % 100 != 0); uint32 doc_id; if (stream->estimate >= cache_threshold) tp_segment_enable_ctid_lookup_cache(reader); for (int i = 0; i < writer->query->term_count; i++) { cursors[i].initialized = tp_segment_posting_iterator_init( &cursors[i].iterator, reader, writer->query->terms[i].lexeme); cursors[i].iterator.force_copy = true; } while (tp_boolean_candidate_stream_next(stream, &doc_id)) { ItemPointerData ctid; CHECK_FOR_INTERRUPTS(); if (!tp_segment_is_alive(reader, doc_id)) continue; if (!tp_boolean_segment_candidate_matches( writer->query, cursors, doc_id)) continue; tp_segment_lookup_ctid(reader, doc_id, &ctid); if (ItemPointerIsValid(&ctid)) tp_boolean_write_result(writer, &ctid); } for (int i = 0; i < writer->query->term_count; i++) { if (cursors[i].initialized) tp_segment_posting_iterator_free(&cursors[i].iterator); } pfree(cursors); tp_boolean_free_candidate_stream(stream); } void tp_boolean_rescan( IndexScanDesc scan, ScanKey keys, int nkeys, TpIndexMetaPage metap) { TpScanOpaque so = (TpScanOpaque)scan->opaque; MemoryContext old_context; if (nkeys != 1 || keys == NULL || keys[0].sk_strategy != TSearchStrategyNumber) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("BM25 Boolean scans require exactly one @@ " "condition"))); so->boolean_query = NULL; so->is_boolean_scan = true; so->boolean_recheck = false; if (keys[0].sk_flags & SK_ISNULL) return; tp_boolean_check_config(scan->indexRelation, metap); old_context = MemoryContextSwitchTo(so->scan_context); so->boolean_query = DatumGetTSQueryCopy(keys[0].sk_argument); MemoryContextSwitchTo(old_context); tp_boolean_warn_if_incomplete( scan->indexRelation, metap, so->boolean_query); } bool tp_boolean_execute(IndexScanDesc scan, TpLocalIndexState *index_state) { TpScanOpaque so = (TpScanOpaque)scan->opaque; MemoryContext old_context; TpBooleanEvalState state; TpIndexMetaPage metap; TpSegmentGraphSnapshot *snapshot; TpBooleanResultWriter writer; if (so->boolean_query == NULL || so->boolean_query->size == 0) return false; if (index_state->lock_held) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("BM25 Boolean execution must acquire its own index " "lock"))); old_context = MemoryContextSwitchTo(so->boolean_context); state = tp_boolean_extract_terms(so->boolean_query); if (state.exact_operand_count > TP_BOOLEAN_MAX_EXACT_OPERANDS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("BM25 Boolean queries support at most %d exact term " "operands", TP_BOOLEAN_MAX_EXACT_OPERANDS), errdetail( "Query contains %d exact term operands.", state.exact_operand_count))); so->boolean_recheck = state.requires_recheck; tp_acquire_index_lock(index_state, LW_SHARED); snapshot = tp_segment_graph_snapshot_create(scan->indexRelation); metap = &snapshot->metapage; tp_boolean_check_config(scan->indexRelation, metap); /* * Spill or compaction can replace the captured chain and segment roots * after the lock is released. Retired pages carry the publishing * transaction's horizon, so VACUUM cannot recycle them while this scan's * transaction snapshot can still reference them. */ tp_release_index_lock(index_state); writer.query = &state; writer.file = BufFileCreateTemp(false); writer.count = 0; tp_boolean_write_memtable_snapshot( scan->indexRelation, &snapshot->memtable, &writer); for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { uint32 root_count; const BlockNumber *roots = tp_segment_graph_snapshot_level(snapshot, level, &root_count); for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentReader *reader = tp_segment_open_ex( scan->indexRelation, roots[root_idx], false); if (reader == NULL) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("could not open BM25 segment %u", roots[root_idx]))); tp_boolean_write_segment(reader, &writer); tp_segment_close(reader); } } tp_segment_graph_snapshot_free(snapshot); so->boolean_results = writer.file; so->result_count = writer.count; so->current_pos = 0; if (BufFileSeek(writer.file, 0, 0, SEEK_SET) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not rewind BM25 Boolean result file"))); MemoryContextSwitchTo(old_context); return writer.count > 0; } bool tp_boolean_next(IndexScanDesc scan) { TpScanOpaque so = (TpScanOpaque)scan->opaque; ItemPointerData ctid; size_t bytes_read; Assert(so->boolean_results != NULL); bytes_read = BufFileRead(so->boolean_results, &ctid, sizeof(ItemPointerData)); if (bytes_read == 0) return false; if (bytes_read != sizeof(ItemPointerData)) ereport(ERROR, (errcode_for_file_access(), errmsg("could not read BM25 Boolean result file"))); scan->xs_heaptid = ctid; so->current_pos++; return true; } timescale-pg_textsearch-b63fe58/src/access/boolean.h000066400000000000000000000015701525762047700226040ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #pragma once #include #include #include #include "index/metapage.h" #include "index/state.h" /* * Each exact operand can retain a candidate iterator, and each distinct exact * term can retain an evaluator cursor. This keeps the combined upper bound at * 128, below PostgreSQL's 200 simultaneously held LWLocks. * * ponytail: force-copy segment iterators if larger Boolean queries matter. */ #define TP_BOOLEAN_MAX_EXACT_OPERANDS 64 int tp_boolean_query_exact_operand_count(TSQuery query); void tp_boolean_rescan( IndexScanDesc scan, ScanKey keys, int nkeys, TpIndexMetaPage metap); bool tp_boolean_execute(IndexScanDesc scan, TpLocalIndexState *index_state); bool tp_boolean_next(IndexScanDesc scan); timescale-pg_textsearch-b63fe58/src/access/build.c000066400000000000000000001570741525762047700222720ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build.c - BM25 index build, insert, and spill operations */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "access/build_context.h" #include "access/build_parallel.h" #include "access/rls.h" #include "constants.h" #include "debug/injection.h" #include "index/compaction_request.h" #include "index/freepage.h" #include "index/metapage.h" #include "index/registry.h" #include "index/state.h" #include "memtable/chain_source.h" #include "memtable/log.h" #include "memtable/page.h" #include "segment/compaction.h" #include "segment/dictionary.h" #include "segment/docmap.h" #include "segment/io.h" #include "segment/merge.h" #include "segment/segment.h" #include "types/array.h" #include "types/vector.h" /* * Build progress tracking for partitioned tables. * * When creating a BM25 index on a partitioned table, tp_build() * is called once per partition. Without tracking, each call emits * repeated NOTICE messages, producing many lines of noise. This * state aggregates statistics across partitions and emits a * single summary. * * Activated by the object-access hook after PostgreSQL creates the actual * BM25 index catalog object. */ typedef struct TpBuildProgress { struct TpBuildProgress *previous; const void *owner; int partition_count; uint64 total_docs; uint64 total_len; } TpBuildProgress; static TpBuildProgress *build_progress = NULL; static const void *build_progress_owner = NULL; typedef struct TpPreparedSpill { TpDataSource *source; TermInfo *terms; uint32 num_terms; TpDocMapBuilder *docmap; uint64 docs_delta; uint64 len_delta; BlockNumber root; bool published; } TpPreparedSpill; void tp_build_progress_set_owner(const void *owner) { build_progress_owner = owner; } void tp_build_progress_begin(void) { MemoryContext old_context; TpBuildProgress *progress; old_context = MemoryContextSwitchTo(TopMemoryContext); progress = palloc0(sizeof(*progress)); MemoryContextSwitchTo(old_context); progress->previous = build_progress; progress->owner = build_progress_owner; build_progress = progress; } void tp_build_progress_end(void) { TpBuildProgress *progress = build_progress; double avg_len = 0.0; if (progress == NULL) return; Assert(progress->owner == build_progress_owner); build_progress = progress->previous; if (progress->partition_count == 0) { pfree(progress); return; } if (progress->total_docs > 0) avg_len = (double)progress->total_len / (double)progress->total_docs; if (progress->partition_count > 1) elog(NOTICE, "BM25 index build completed: " UINT64_FORMAT " documents across %d partitions," " avg_length=%.2f", progress->total_docs, progress->partition_count, avg_len); else elog(NOTICE, "BM25 index build completed: " UINT64_FORMAT " documents, avg_length=%.2f", progress->total_docs, avg_len); pfree(progress); } void tp_build_progress_abort(void) { TpBuildProgress *progress = build_progress; if (progress == NULL) return; Assert(progress->owner == build_progress_owner); build_progress = progress->previous; pfree(progress); } static TpBuildProgress * current_build_progress(void) { if (build_progress != NULL && build_progress->owner == build_progress_owner) return build_progress; return NULL; } static void record_or_report_build_completion( TpBuildProgress *progress, uint64 total_docs, uint64 total_len) { if (progress != NULL) { progress->total_docs += total_docs; progress->total_len += total_len; progress->partition_count++; } else { elog(NOTICE, "BM25 index build completed: " UINT64_FORMAT " documents, avg_length=%.2f", total_docs, total_docs > 0 ? (float4)(total_len / (double)total_docs) : 0.0); } } /* * Build phase name for progress reporting */ char * tp_buildphasename(int64 phase) { switch (phase) { case PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE: return "initializing"; case TP_PHASE_LOADING: return "loading tuples"; case TP_PHASE_WRITING: return "writing index"; case TP_PHASE_COMPACTING: return "compacting segments"; default: return NULL; } } static void tp_check_spill_capacity(Relation index_rel, uint32 segment_capacity) { TpIndexMetaPage metap = tp_get_metapage(index_rel); bool at_capacity = metap->level_counts[0] >= segment_capacity; pfree(metap); if (at_capacity) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("bm25 segment count limit reached at level 0"))); } static bool tp_prepare_spill( TpLocalIndexState *index_state, Relation index_rel, TpPreparedSpill *spill, uint32 segment_capacity) { memset(spill, 0, sizeof(*spill)); spill->root = InvalidBlockNumber; /* * Open a chain source. This idempotently re-acquires the * per-index LWLock in SHARED mode, which is a no-op since * the caller holds it EXCLUSIVE; the constructor returns * NULL for an empty chain. */ spill->source = tp_memtable_chain_source_create(index_state, index_rel, NULL, 0); if (spill->source == NULL) return false; tp_check_spill_capacity(index_rel, segment_capacity); spill->docs_delta = (uint64)spill->source->total_docs; spill->len_delta = (uint64)spill->source->total_len; tp_memtable_chain_source_extract( spill->source, CurrentMemoryContext, &spill->terms, &spill->num_terms, &spill->docmap); return true; } static void tp_build_prepared_spill(Relation index_rel, TpPreparedSpill *spill) { Assert(!BlockNumberIsValid(spill->root)); spill->root = tp_write_segment( index_rel, spill->terms, spill->num_terms, spill->docmap); } static void tp_publish_prepared_spill( TpLocalIndexState *index_state, Relation index_rel, TpPreparedSpill *spill, BlockNumber *out_segment_root, uint32 segment_capacity) { tp_check_spill_capacity(index_rel, segment_capacity); /* * Finalize first, then mark dead - crash-safe ordering. * * We must disconnect the chain (clear metap.head) BEFORE stamping * pages DEAD. If we crash between mark and finalize, the chain * is still reachable via metap.head but pages are stamped DEAD; * once xmin advances, tp_reclaim_dead_memtable_pages would free * pages still in the live chain, causing corruption on FSM reuse. * * By finalizing first: a crash after finalize but before marking * leaves pages unreachable but un-stamped, so reclaim won't touch * them. Worst case: pages leak until REINDEX. */ { BlockNumber chain_head; FullTransactionId horizon; TpIndexMetaPage metap; metap = tp_get_metapage(index_rel); chain_head = metap->memtable_head_blkno; pfree(metap); tp_spill_finalize( index_state, index_rel, spill->root, spill->docs_delta, spill->len_delta, segment_capacity, &spill->published); /* * Sample only after the unpublication WAL is inserted. Every * standby snapshot that can still discover the old chain then has * xmin at or below this reuse-conflict horizon. */ horizon = ReadNextFullTransactionId(); if (BlockNumberIsValid(chain_head)) tp_memtable_mark_chain_dead(index_rel, chain_head, horizon); } if (out_segment_root != NULL) *out_segment_root = spill->root; } static void tp_cleanup_prepared_spill(Relation index_rel, TpPreparedSpill *spill) { if (BlockNumberIsValid(spill->root) && !spill->published) tp_discard_unpublished_segment(index_rel, spill->root); if (spill->terms != NULL) tp_free_dictionary(spill->terms, spill->num_terms); if (spill->docmap != NULL) tp_docmap_destroy(spill->docmap); if (spill->source != NULL) tp_source_close(spill->source); spill->source = NULL; spill->terms = NULL; spill->docmap = NULL; spill->root = InvalidBlockNumber; } static bool tp_acquire_spill_index_lock(TpLocalIndexState *index_state, bool nowait) { if (nowait) return tp_try_acquire_index_lock(index_state, LW_EXCLUSIVE); tp_acquire_index_lock(index_state, LW_EXCLUSIVE); return true; } /* * Caller holds publication ShareLock and the exclusive writer/spill gate, * but not the per-index lock. Return the published root before compaction. */ static bool tp_do_spill( TpLocalIndexState *index_state, Relation index_rel, BlockNumber *out_segment_root, uint32 segment_capacity, bool nowait) { TpPreparedSpill *spill; volatile bool success = false; Assert(!index_state->lock_held); Assert(!LWLockHeldByMe(&index_state->shared->lock)); Assert(LWLockHeldByMeInMode( &index_state->shared->memtable_write_lock, LW_EXCLUSIVE)); if (out_segment_root != NULL) *out_segment_root = InvalidBlockNumber; /* * Standby is read-only; spill is primary-only. All durable * state lives on disk; redo applies GenericXLog records and * never invokes spill itself. */ if (RecoveryInProgress()) return false; spill = palloc0(sizeof(*spill)); spill->root = InvalidBlockNumber; PG_TRY(); { if (tp_acquire_spill_index_lock(index_state, nowait)) { bool prepared = tp_prepare_spill( index_state, index_rel, spill, segment_capacity); tp_release_index_lock(index_state); if (prepared) { tp_build_prepared_spill(index_rel, spill); TP_INJECTION_POINT(TP_INJECTION_SPILL_BEFORE_FINALIZE); CHECK_FOR_INTERRUPTS(); if (tp_acquire_spill_index_lock(index_state, nowait)) { tp_publish_prepared_spill( index_state, index_rel, spill, out_segment_root, segment_capacity); tp_release_index_lock(index_state); success = true; } } } } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); tp_cleanup_prepared_spill(index_rel, spill); } PG_END_TRY(); pfree(spill); return success; } /* * Run one compaction pass if the maintenance lock is free, reporting * whether it ran. This runs inside an ordinary INSERT, so the lock is * taken conditionally: when another session already holds it, the * INSERT skips compaction instead of waiting. bm25_compact() and * bm25_compact_step() wait for the lock. */ static bool tp_compact_inline(TpLocalIndexState *index_state, Relation index_rel) { if (!tp_try_compaction_lock(index_rel)) return false; PG_TRY(); { (void)tp_compact_step(index_state, index_rel); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); tp_compaction_unlock(index_rel); } PG_END_TRY(); return true; } static void tp_compact_build_private(TpLocalIndexState *index_state, Relation index_rel) { Assert(index_state->lock_held); while (tp_compact_step(index_state, index_rel)) CHECK_FOR_INTERRUPTS(); } void tp_apply_compaction_policy( TpLocalIndexState *index_state, Relation index_rel, bool maintenance_opportunity) { if (!maintenance_opportunity) return; /* * Reached when a VACUUM (PARALLEL n) worker spills this index's * memtable during cleanup. Publication assigns an XID for its * reclaim stamp, which parallel mode forbids, so leave the new * segment in L0 for later serial maintenance to compact. */ if (IsInParallelMode() || IsParallelWorker()) return; pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_COMPACTING); switch (tp_index_compaction_mode(index_rel)) { case TP_COMPACTION_INLINE: tp_compact_inline(index_state, index_rel); break; case TP_COMPACTION_BACKGROUND: /* * Temporary indexes are unreachable from other sessions, and a * process that cannot dispatch would record a request that is * later discarded. Both compact inline. */ if (RelationUsesLocalBuffers(index_rel) || !tp_compaction_dispatch_possible()) tp_compact_inline(index_state, index_rel); else tp_compaction_request(RelationGetRelid(index_rel)); break; case TP_COMPACTION_MANUAL: break; } pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_LOADING); } /* Flags controlling tp_spill_memtable_if_needed_internal(). */ #define TP_SPILL_APPLY_POLICY 0x01 /* run the policy after spilling */ #define TP_SPILL_COMPACT_FOR_ROOM 0x02 /* compact a full L0 to make room */ #define TP_SPILL_REQUIRED 0x04 /* empty chain is the postcondition */ #define TP_SPILL_NOWAIT 0x08 /* skip rather than wait for the lock */ /* * Report whether level 0 already holds its maximum segment count, so * a spill would raise the capacity error instead of linking a segment. */ static bool tp_l0_at_capacity(Relation index_rel) { TpIndexMetaPage metap = tp_get_metapage(index_rel); bool at_capacity = metap->level_counts[0] >= tp_injected_segment_count_limit(); pfree(metap); return at_capacity; } /* * Make room at level 0 for a spill, and report whether the spill should be * attempted at all. * * A full level 0 is normally transient: compacting it merges the segments * into level 1 and the spill proceeds. Compaction runs here regardless of * the index's policy, because a background worker's pass comes too late for * a spill blocked now; manual indexes fall through to the capacity error. * * Only a denied maintenance admission leaves level 0 full without knowing * whether it is reducible; those records wait in the durable chain for a * later spill, unless the caller requires an empty chain. Every other * outcome falls through to the spill, so an irreducible level 0 fails closed * rather than growing the chain without bound. */ static bool tp_make_room_for_spill( Relation index_rel, TpLocalIndexState *index_state, int flags) { bool require_spill = (flags & TP_SPILL_REQUIRED) != 0; if (!tp_l0_at_capacity(index_rel)) return true; /* The shutdown spill never compacts, so it cannot make room. */ if (!(flags & TP_SPILL_COMPACT_FOR_ROOM)) return require_spill; if (tp_index_compaction_mode(index_rel) == TP_COMPACTION_MANUAL) return true; if (tp_compact_inline(index_state, index_rel)) return true; return require_spill || !tp_l0_at_capacity(index_rel); } /* * Spill memtable to an L0 segment. min_pages is a chain-page count below * which the spill is a no-op, used by VACUUM cleanup and the shutdown hook * to avoid runt L0 segments. The pre-lock read is a fast bailout; the * authoritative check runs after exclusive acquisition. * * TP_SPILL_REQUIRED makes an empty chain the postcondition: the spill runs * even at a full level 0, where it reports the capacity limit. VACUUM needs * this because it identifies dead documents from published segments alone. * * TP_SPILL_NOWAIT skips busy publication, writer-gate, and per-index locks. * Buffer, WAL, and I/O work may still wait after admission. */ static bool tp_spill_memtable_if_needed_internal( Relation index, TpLocalIndexState *index_state, uint32 min_pages, int flags) { bool spilled = false; volatile bool gate_acquired = false; /* Standby is read-only; spill is primary-only. */ if (RecoveryInProgress()) return false; if (!index_state || !index_state->shared) return false; if (!tp_reseed_chain_page_count_if_needed( index_state, index, (flags & TP_SPILL_NOWAIT) != 0)) return false; if (pg_atomic_read_u32(&index_state->shared->chain_page_count) < min_pages) return false; if (!tp_make_room_for_spill(index, index_state, flags)) return false; /* * Keep spill publication outside compaction's prepare/publish window. * A no-wait caller must not block here either, or the shutdown hook * would wait on the very index it promised to skip. */ if (flags & TP_SPILL_NOWAIT) { if (!tp_try_compaction_publication_lock(index, ShareLock)) return false; } else tp_compaction_publication_lock(index, ShareLock); PG_TRY(); { if (flags & TP_SPILL_NOWAIT) gate_acquired = tp_try_acquire_memtable_write_lock( index_state, LW_EXCLUSIVE); else { tp_acquire_memtable_write_lock(index_state, LW_EXCLUSIVE); gate_acquired = true; } if (gate_acquired && pg_atomic_read_u32(&index_state->shared->chain_page_count) >= min_pages) spilled = tp_do_spill( index_state, index, NULL, tp_injected_segment_count_limit(), (flags & TP_SPILL_NOWAIT) != 0); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); if (gate_acquired) tp_release_memtable_write_lock(index_state); tp_compaction_publication_unlock(index, ShareLock); } PG_END_TRY(); if (flags & TP_SPILL_APPLY_POLICY) tp_apply_compaction_policy(index_state, index, spilled); return spilled; } void tp_spill_memtable_if_needed( Relation index, TpLocalIndexState *index_state, uint32 min_pages) { (void)tp_spill_memtable_if_needed_internal( index, index_state, min_pages, TP_SPILL_APPLY_POLICY | TP_SPILL_COMPACT_FOR_ROOM); } /* * Spill for VACUUM: defer the compaction policy to the caller, and * make an empty chain the postcondition so Phase 2 can identify dead * documents from published segments alone. */ bool tp_spill_memtable_if_needed_deferred( Relation index, TpLocalIndexState *index_state, uint32 min_pages) { return tp_spill_memtable_if_needed_internal( index, index_state, min_pages, TP_SPILL_COMPACT_FOR_ROOM | TP_SPILL_REQUIRED); } void tp_spill_memtable_without_compaction_if_needed( Relation index, TpLocalIndexState *index_state, uint32 min_pages) { (void)tp_spill_memtable_if_needed_internal( index, index_state, min_pages, TP_SPILL_NOWAIT); } /* * Auto-spill the on-disk memtable when the chain grows past the * configured page threshold (issue #374). The shared spill helper owns * the threshold checks so every threshold-driven caller has identical * race behavior. */ static void tp_auto_spill_if_needed(TpLocalIndexState *index_state, Relation index_rel) { uint32 threshold = (uint32)tp_memtable_pages_threshold; if (threshold == 0) return; /* auto-spill disabled */ tp_spill_memtable_if_needed(index_rel, index_state, threshold); } /* * Flush build context to a segment and link as L0 chain head. * Used during serial CREATE INDEX with arena-based build. */ static void tp_build_flush_and_link(TpBuildContext *ctx, Relation index) { BlockNumber segment_root; segment_root = tp_write_segment_from_build_ctx(ctx, index); if (segment_root == InvalidBlockNumber) return; tp_link_l0_chain_head(index, segment_root); } /* * Link a newly-written segment as the L0 chain head. * * Reads the metapage, points the new segment's next_segment at the * current L0 head, then updates the metapage head and count. */ void tp_link_l0_chain_head(Relation index, BlockNumber segment_root) { Buffer metabuf; Buffer seg_buf = InvalidBuffer; GenericXLogState *state; Page metapage; TpIndexMetaPage metap; metabuf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); metapage = GenericXLogRegisterBuffer(state, metabuf, 0); metap = (TpIndexMetaPage)PageGetContents(metapage); tp_check_level_count_increment(metap, 0); if (metap->level_heads[0] != InvalidBlockNumber) { Page seg_page; TpSegmentHeader *seg_header; seg_buf = ReadBuffer(index, segment_root); LockBuffer(seg_buf, BUFFER_LOCK_EXCLUSIVE); seg_page = GenericXLogRegisterBuffer(state, seg_buf, 0); /* Ensure pd_lower covers content for GenericXLog */ ((PageHeader)seg_page)->pd_lower = BLCKSZ; seg_header = (TpSegmentHeader *)PageGetContents(seg_page); seg_header->next_segment = metap->level_heads[0]; } metap->level_heads[0] = segment_root; metap->level_counts[0]++; GenericXLogFinish(state); if (BufferIsValid(seg_buf)) UnlockReleaseBuffer(seg_buf); UnlockReleaseBuffer(metabuf); } /* * Truncate recyclable pages from the end of an index relation. * * Only a contiguous EOF suffix already carrying TP_FREE_PAGE_MAGIC is * safe to remove. That stamp proves the page was unlinked from every * owning structure and completed its reclaim protocol, including standby * conflict WAL where an old snapshot could still see it. * * Absence from the published graph is not proof of recyclability: DEAD * memtable pages remain protected by dead_fxid, and a crash or handled * error can leave unreachable structural pages. Stop at any DEAD, live, * unknown, or orphan page. * * Caller must hold maintenance admission, the exclusive memtable writer * gate, and the per-index LWLock EXCLUSIVE, in that order. Spill construction * allocates without the per-index lock, so the writer gate must also cover * suffix inspection through RelationTruncate(). */ void tp_truncate_dead_pages(Relation index) { BlockNumber nblocks = RelationGetNumberOfBlocks(index); BlockNumber truncate_to = nblocks; while (truncate_to > TP_METAPAGE_BLKNO + 1) { BlockNumber blk = truncate_to - 1; Buffer buf; Page page; bool recyclable; buf = ReadBuffer(index, blk); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); recyclable = tp_page_is_recyclable(page); UnlockReleaseBuffer(buf); if (!recyclable) break; truncate_to = blk; } if (truncate_to < nblocks) { TP_INJECTION_POINT(TP_INJECTION_TRUNCATE_AFTER_INSPECTION); RelationTruncate(index, truncate_to); } } /* * tp_spill_memtable - Force memtable flush to disk segment * * This function allows manual triggering of segment writes. * Returns the block number of the written segment, or NULL if memtable was * empty. */ PG_FUNCTION_INFO_V1(tp_spill_memtable); Datum tp_spill_memtable(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); Oid index_oid; Relation index_rel; TpLocalIndexState *index_state; BlockNumber segment_root; RangeVar *rv; bool spilled; /* Replica is read-only; spill is primary-only. Standby's * memtable changes only via WAL redo. */ if (RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("bm25_spill_index() cannot run during recovery"))); /* Parse index name (supports schema.index notation) */ rv = makeRangeVarFromNameList(stringToQualifiedNameList(index_name, NULL)); index_oid = RangeVarGetRelid(rv, AccessShareLock, false); if (!OidIsValid(index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" does not exist", index_name))); /* Check that caller owns the index */ if (!object_ownercheck(RelationRelationId, index_oid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, index_name); /* Open the index */ index_rel = index_open(index_oid, RowExclusiveLock); /* Get index state */ index_state = tp_get_local_index_state(RelationGetRelid(index_rel)); if (!index_state) { index_close(index_rel, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", index_name))); } /* Keep spill publication outside compaction's prepare/publish window. */ tp_compaction_publication_lock(index_rel, ShareLock); tp_acquire_memtable_write_lock(index_state, LW_EXCLUSIVE); PG_TRY(); { /* * Use the out-param so the legacy `bm25_spill_index()` SRF * returns the new L0 segment's block number even when the policy * immediately folds that segment into L1. */ segment_root = InvalidBlockNumber; spilled = tp_do_spill( index_state, index_rel, &segment_root, tp_injected_segment_count_limit(), false); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); tp_release_memtable_write_lock(index_state); tp_compaction_publication_unlock(index_rel, ShareLock); } PG_END_TRY(); tp_apply_compaction_policy(index_state, index_rel, spilled); index_close(index_rel, RowExclusiveLock); /* Return block number or NULL */ if (segment_root != InvalidBlockNumber) { PG_RETURN_INT32(segment_root); } else { PG_RETURN_NULL(); } } PG_FUNCTION_INFO_V1(tp_force_merge); /* * SQL-callable: bm25_force_merge(index_name text) → void * * Run one bounded best-effort compaction pass. Useful after bulk loads or * when benchmarking with a compact segment layout. */ Datum tp_force_merge(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); Oid index_oid; Relation index_rel; RangeVar *rv; /* Replica is read-only; compaction is primary-only. Standby's * segments change only via WAL redo. */ if (RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("bm25_force_merge() cannot run during recovery"))); rv = makeRangeVarFromNameList(stringToQualifiedNameList(index_name, NULL)); index_oid = RangeVarGetRelid(rv, AccessShareLock, false); if (!OidIsValid(index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" does not exist", index_name))); /* Check that caller owns the index */ if (!object_ownercheck(RelationRelationId, index_oid, GetUserId())) aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, index_name); index_rel = index_open(index_oid, RowExclusiveLock); /* * Serialize same-index maintenance before phase-specific LWLocks. * Waiting is safe: the maintenance lock lives in pg_am's object * namespace, outside the relation locks REINDEX INDEX CONCURRENTLY * takes. An explicit merge should queue behind concurrent * maintenance rather than fail. */ { TpLocalIndexState *index_state = tp_get_local_index_state(index_oid); volatile bool publication_locked = false; volatile bool gate_locked = false; if (index_state == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for " "\"%s\"", index_name))); tp_compaction_lock(index_rel); PG_TRY(); { tp_compaction_publication_lock(index_rel, ShareLock); publication_locked = true; tp_acquire_memtable_write_lock(index_state, LW_EXCLUSIVE); gate_locked = true; (void)tp_do_spill( index_state, index_rel, NULL, PG_UINT16_MAX, false); tp_release_memtable_write_lock(index_state); gate_locked = false; tp_compaction_publication_unlock(index_rel, ShareLock); publication_locked = false; tp_force_compact(index_state, index_rel); TP_INJECTION_POINT(TP_INJECTION_FORCE_MERGE_BEFORE_TRUNCATE); tp_acquire_memtable_write_lock(index_state, LW_EXCLUSIVE); gate_locked = true; tp_acquire_index_lock(index_state, LW_EXCLUSIVE); tp_truncate_dead_pages(index_rel); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); if (gate_locked) tp_release_memtable_write_lock(index_state); if (publication_locked) tp_compaction_publication_unlock(index_rel, ShareLock); tp_compaction_unlock(index_rel); } PG_END_TRY(); } index_close(index_rel, RowExclusiveLock); PG_RETURN_VOID(); } /* * Helper: Extract options from index relation */ static void tp_build_extract_options( Relation index, char **text_config_name, Oid *text_config_oid, double *k1, double *b) { TpOptions *options; *text_config_name = NULL; *text_config_oid = InvalidOid; /* Extract options from index */ options = (TpOptions *)index->rd_options; if (options) { if (options->text_config_offset > 0) { *text_config_name = pstrdup( (char *)options + options->text_config_offset); /* Convert text config name to OID */ { List *names = stringToQualifiedNameList(*text_config_name, NULL); *text_config_oid = get_ts_config_oid(names, false); list_free(names); } } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("text_config parameter is required for bm25 " "indexes"), errhint("Specify text_config when creating the index: " "CREATE INDEX ... USING " "bm25(column) WITH (text_config='english')"))); } *k1 = options->k1; *b = options->b; } else { /* No options provided - require text_config */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("text_config parameter is required for bm25 indexes"), errhint("Specify text_config when creating the index: " "CREATE INDEX ... USING " "bm25(column) WITH (text_config='english')"))); } } /* * Helper: Initialize metapage for new index */ static void tp_build_init_metapage( Relation index, Oid text_config_oid, double k1, double b) { Buffer metabuf; GenericXLogState *state; Page metapage; TpIndexMetaPage metap; /* Initialize metapage */ metabuf = ReadBuffer(index, P_NEW); Assert(BufferGetBlockNumber(metabuf) == TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); metapage = GenericXLogRegisterBuffer(state, metabuf, GENERIC_XLOG_FULL_IMAGE); tp_init_metapage(metapage, text_config_oid); metap = (TpIndexMetaPage)PageGetContents(metapage); metap->k1 = k1; metap->b = b; GenericXLogFinish(state); UnlockReleaseBuffer(metabuf); } /* * Extract terms and frequencies from a TSVector * Returns the document length (sum of all term frequencies) */ int tp_extract_terms_from_tsvector( TSVector tsvector, char ***terms_out, int32 **frequencies_out, int *term_count_out) { int term_count = tsvector->size; char **terms; int32 *frequencies; int doc_length = 0; int i; WordEntry *we; *term_count_out = term_count; if (term_count == 0) { *terms_out = NULL; *frequencies_out = NULL; return 0; } we = ARRPTR(tsvector); terms = palloc(term_count * sizeof(char *)); frequencies = palloc(term_count * sizeof(int32)); for (i = 0; i < term_count; i++) { char *lexeme_start = STRPTR(tsvector) + we[i].pos; int lexeme_len = we[i].len; char *lexeme; /* Always allocate on heap for terms array */ lexeme = palloc(lexeme_len + 1); memcpy(lexeme, lexeme_start, lexeme_len); lexeme[lexeme_len] = '\0'; terms[i] = lexeme; /* Get frequency from TSVector - count positions or default to 1 */ if (we[i].haspos) frequencies[i] = (int32)POSDATALEN(tsvector, &we[i]); else frequencies[i] = 1; doc_length += frequencies[i]; } *terms_out = terms; *frequencies_out = frequencies; return doc_length; } /* * Free memory allocated for terms array */ static void tp_free_terms_array(char **terms, int term_count) { int i; if (terms == NULL) return; for (i = 0; i < term_count; i++) pfree(terms[i]); pfree(terms); } /* * Maximum input bytes per to_tsvector call. Postgres's tsvector caps the * lexeme dictionary at 1 MB (MAXSTRPOS). 256 KB leaves comfortable * headroom for stemming/case-fold expansion. */ #define TP_TSVECTOR_CHUNK_BYTES (256 * 1024) /* * Tokenize a single chunk via to_tsvector_byid and extract terms. * Caller owns the returned arrays. doc_length is returned. * * Frees the per-chunk `text` wrapper and the tsvector backing before * returning, so callers running many chunks in a row (the chunked path * in tp_tokenize_text) do not retain ~256 KB + tsvector backing per * chunk in CurrentMemoryContext. The output term strings have already * been copied into freshly palloc'd buffers by * tp_extract_terms_from_tsvector, so they survive the frees. */ static int tp_tokenize_chunk( const char *chunk, int chunk_len, Oid text_config_oid, char ***terms_out, int32 **frequencies_out, int *term_count_out) { text *chunk_text; Datum tsvector_datum; TSVector tsvector; int doc_length; chunk_text = cstring_to_text_with_len(chunk, chunk_len); tsvector_datum = DirectFunctionCall2Coll( to_tsvector_byid, InvalidOid, ObjectIdGetDatum(text_config_oid), PointerGetDatum(chunk_text)); tsvector = DatumGetTSVector(tsvector_datum); doc_length = tp_extract_terms_from_tsvector( tsvector, terms_out, frequencies_out, term_count_out); pfree(chunk_text); pfree(tsvector); return doc_length; } /* * Find a chunk boundary inside the first `target` bytes of `data`. * * Prefers the byte index just past the last ASCII whitespace at or * before `target`. If no whitespace is found, returns the largest * UTF-8/multibyte codepoint boundary <= target (and never less than one * full character). * * `data` must be at least `target` bytes long. Returns 1..target. */ static int tp_find_chunk_boundary(const char *data, int target) { int i; int pos; for (i = target; i > 0; i--) { char c = data[i - 1]; if (c == ' ' || c == '\t' || c == '\n' || c == '\r' || c == '\f') return i; } pos = 0; while (pos < target) { int mblen = pg_mblen(data + pos); if (mblen <= 0) mblen = 1; if (pos + mblen > target) break; pos += mblen; } if (pos == 0) { /* Single character is larger than target; emit it anyway. */ pos = pg_mblen(data); if (pos <= 0) pos = 1; } return pos; } typedef struct TpTermEntry { char *term; int32 freq; } TpTermEntry; static int tp_term_entry_cmp(const void *a, const void *b) { const TpTermEntry *ea = (const TpTermEntry *)a; const TpTermEntry *eb = (const TpTermEntry *)b; return strcmp(ea->term, eb->term); } /* * Merge accumulated (term, freq) entries by sorting then collapsing * adjacent duplicates. Output arrays are palloc'd in the current * memory context. Frees `entries` itself; per-entry term strings are * either reassigned into the output or pfree'd as duplicates. */ static void tp_merge_term_entries( TpTermEntry *entries, int entry_count, char ***terms_out, int32 **frequencies_out, int *term_count_out) { int out; int i; char **terms; int32 *freqs; if (entry_count == 0) { *terms_out = NULL; *frequencies_out = NULL; *term_count_out = 0; if (entries != NULL) pfree(entries); return; } qsort(entries, entry_count, sizeof(TpTermEntry), tp_term_entry_cmp); terms = palloc(entry_count * sizeof(char *)); freqs = palloc(entry_count * sizeof(int32)); out = 0; terms[out] = entries[0].term; freqs[out] = entries[0].freq; for (i = 1; i < entry_count; i++) { if (strcmp(entries[i].term, terms[out]) == 0) { freqs[out] += entries[i].freq; pfree(entries[i].term); } else { out++; terms[out] = entries[i].term; freqs[out] = entries[i].freq; } } out++; pfree(entries); *terms_out = terms; *frequencies_out = freqs; *term_count_out = out; } int tp_tokenize_text( text *document_text, Oid text_config_oid, char ***terms_out, int32 **frequencies_out, int *term_count_out) { const char *data = VARDATA_ANY(document_text); int len = VARSIZE_ANY_EXHDR(document_text); int doc_length = 0; int offset; int cap; int used; TpTermEntry *acc; /* * Single-chunk fast path: pass document_text straight into * to_tsvector_byid. Avoids the cstring_to_text_with_len memcpy that * tp_tokenize_chunk would otherwise do on every small document. */ if (len <= TP_TSVECTOR_CHUNK_BYTES) { Datum tsvector_datum; TSVector tsvector; tsvector_datum = DirectFunctionCall2Coll( to_tsvector_byid, InvalidOid, ObjectIdGetDatum(text_config_oid), PointerGetDatum(document_text)); tsvector = DatumGetTSVector(tsvector_datum); return tp_extract_terms_from_tsvector( tsvector, terms_out, frequencies_out, term_count_out); } /* Chunked path: accumulate per-chunk (term, freq) into acc[]. */ cap = 1024; used = 0; acc = palloc(cap * sizeof(TpTermEntry)); offset = 0; while (offset < len) { int remaining = len - offset; int take = remaining <= TP_TSVECTOR_CHUNK_BYTES ? remaining : tp_find_chunk_boundary( data + offset, TP_TSVECTOR_CHUNK_BYTES); char **chunk_terms; int32 *chunk_freqs; int chunk_term_count; int i; doc_length += tp_tokenize_chunk( data + offset, take, text_config_oid, &chunk_terms, &chunk_freqs, &chunk_term_count); if (used + chunk_term_count > cap) { while (used + chunk_term_count > cap) cap *= 2; acc = repalloc(acc, cap * sizeof(TpTermEntry)); } for (i = 0; i < chunk_term_count; i++) { acc[used].term = chunk_terms[i]; /* takes ownership */ acc[used].freq = chunk_freqs[i]; used++; } if (chunk_terms != NULL) pfree(chunk_terms); if (chunk_freqs != NULL) pfree(chunk_freqs); offset += take; } tp_merge_term_entries( acc, used, terms_out, frequencies_out, term_count_out); return doc_length; } /* * Core document processing: convert text to terms and add to posting lists. * This is shared between CREATE INDEX build (heap scan) and DML inserts * (single-tuple aminsert). * * If index_rel is provided, auto-spill will occur when memory limit is * exceeded. If index_rel is NULL, no auto-spill occurs. */ bool tp_process_document_text( text *document_text, ItemPointer ctid, Oid text_config_oid, TpLocalIndexState *index_state, Relation index_rel, int32 *doc_length_out) { char *document_str; char **terms; int32 *frequencies; int term_count; int doc_length; if (!document_text || !index_state) return false; document_str = text_to_cstring(document_text); /* Validate the TID before processing */ if (!ItemPointerIsValid(ctid)) { elog(WARNING, "Invalid TID during document processing, skipping document"); pfree(document_str); return false; } /* Tokenize document (chunks oversized inputs to fit tsvector cap) */ doc_length = tp_tokenize_text( document_text, text_config_oid, &terms, &frequencies, &term_count); if (index_rel != NULL) { char *index_name = get_rel_name(RelationGetRelid(index_rel)); TpVector *tpvec; tpvec = create_tpvector_from_strings( index_name, term_count, (const char **)terms, frequencies); pfree(index_name); /* * Acquire exclusive lock for this transaction if not already held. * During index build, we acquire once and hold for the entire build. */ tp_acquire_index_lock(index_state, LW_EXCLUSIVE); tp_add_document_terms( index_state, index_rel, ctid, (const char *)tpvec, (uint32)VARSIZE(tpvec), term_count, doc_length); tp_auto_spill_if_needed(index_state, index_rel); pfree(tpvec); tp_free_terms_array(terms, term_count); pfree(frequencies); } else if (term_count > 0) { /* * Legacy recovery path with no Relation: under Phase 4 * the rebuild function is short-circuited and never * reaches here, but we keep the cleanup path correct. */ tp_free_terms_array(terms, term_count); pfree(frequencies); } if (doc_length_out) *doc_length_out = doc_length; pfree(document_str); return true; } /* * Callback state for table_index_build_scan during serial builds. */ typedef struct TpBuildCallbackState { TpBuildContext *build_ctx; TpLocalIndexState *index_state; Relation index; Oid text_config_oid; MemoryContext per_doc_ctx; bool is_text_array; uint64 total_docs; uint64 total_len; uint64 tuples_done; } TpBuildCallbackState; /* * Per-tuple callback for table_index_build_scan. * * Receives pre-evaluated index expression values from the scan, * tokenizes the document text, and adds it to the build context. */ static void tp_build_callback( Relation index, ItemPointer ctid, Datum *values, bool *isnull, bool tupleIsAlive, void *state) { TpBuildCallbackState *bs = (TpBuildCallbackState *)state; text *document_text; char **terms; int32 *frequencies; int term_count; int doc_length; MemoryContext oldctx; /* Suppress unused parameter warnings for callback signature */ (void)index; (void)tupleIsAlive; if (isnull[0]) return; if (!ItemPointerIsValid(ctid)) return; /* * Tokenize in temporary context to prevent * to_tsvector_byid memory from accumulating. * Detoasting also happens here so the copy is freed * by MemoryContextReset below. */ oldctx = MemoryContextSwitchTo(bs->per_doc_ctx); if (bs->is_text_array) document_text = tp_flatten_text_array(values[0]); else document_text = DatumGetTextPP(values[0]); doc_length = tp_tokenize_text( document_text, bs->text_config_oid, &terms, &frequencies, &term_count); MemoryContextSwitchTo(oldctx); tp_build_context_add_document( bs->build_ctx, terms, frequencies, term_count, doc_length, ctid); /* Reset per-doc context (frees tsvector, terms) */ MemoryContextReset(bs->per_doc_ctx); /* Budget-based flush */ if (tp_build_context_should_flush(bs->build_ctx)) { bs->total_docs += bs->build_ctx->num_docs; bs->total_len += bs->build_ctx->total_len; tp_build_flush_and_link(bs->build_ctx, bs->index); tp_build_context_reset(bs->build_ctx); pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_COMPACTING); if (tp_index_compaction_mode(bs->index) == TP_COMPACTION_INLINE) tp_compact_build_private(bs->index_state, bs->index); pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_LOADING); } bs->tuples_done++; if (bs->tuples_done % TP_PROGRESS_REPORT_INTERVAL == 0) { pgstat_progress_update_param( PROGRESS_CREATEIDX_TUPLES_DONE, bs->tuples_done); CHECK_FOR_INTERRUPTS(); } } /* * Build a new Tapir index */ IndexBuildResult * tp_build(Relation heap, Relation index, IndexInfo *indexInfo) { IndexBuildResult *result; char *text_config_name = NULL; Oid text_config_oid = InvalidOid; double k1, b; uint64 total_docs = 0; uint64 total_len = 0; TpLocalIndexState *index_state; TpBuildProgress *progress; TpLocalIndexState *build_state; SubTransactionId index_create_subid; bool is_text_array; tp_check_bm25_build_allowed(heap); tp_rls_note_bm25_build(); /* Show "started" for first partition only (suppresses duplicates) */ progress = current_build_progress(); if (progress == NULL || progress->partition_count == 0) elog(NOTICE, "BM25 index build started for relation %s", RelationGetRelationName(index)); /* * Determine if the index key yields a text array type. * If so, we flatten array elements into a single text value * before tokenization. */ is_text_array = tp_index_key_is_text_array(index); /* Report initialization phase */ pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, PROGRESS_CREATEIDX_SUBPHASE_INITIALIZE); /* Extract options from index */ tp_build_extract_options( index, &text_config_name, &text_config_oid, &k1, &b); /* Log configuration (only for first partition when active) */ if (progress == NULL || progress->partition_count == 0) { if (text_config_name) elog(NOTICE, "Using text search configuration: %s", text_config_name); elog(NOTICE, "Using index options: k1=%.2f, b=%.2f", k1, b); } index_create_subid = index->rd_createSubid; build_state = tp_create_build_index_state( index->rd_id, heap->rd_id, index_create_subid); /* Initialize metapage */ tp_build_init_metapage(index, text_config_oid, k1, b); /* * Check if parallel build is possible and beneficial. * * Postgres has already called plan_create_index_workers() and stored * the result in indexInfo->ii_ParallelWorkers. We use that value * directly to avoid redundant planning work and ensure consistency. * * We add our own minimum tuple threshold (100K) because for smaller * tables, the parallel coordination overhead exceeds the benefit. */ { int nworkers = indexInfo->ii_ParallelWorkers; double reltuples = heap->rd_rel->reltuples; /* * Only consider parallel build for tables with 100K+ estimated rows. * For smaller tables, the parallel coordination overhead exceeds * the benefit. * * If reltuples is -1 (table never analyzed), estimate from page count. * We use a conservative estimate of 50 tuples per 8KB page, which * assumes ~160 bytes per row (reasonable for text search workloads). */ #define TP_MIN_PARALLEL_TUPLES 100000 #define TP_TUPLES_PER_PAGE_ESTIMATE 50 if (reltuples < 0) { BlockNumber nblocks = RelationGetNumberOfBlocks(heap); reltuples = (double)nblocks * TP_TUPLES_PER_PAGE_ESTIMATE; } /* * Thresholds for warning about suboptimal parallelism. * These are conservative - we only warn when users could see * significant (>2x) speedup from more parallelism. */ #define TP_WARN_NO_PARALLEL_TUPLES 1000000 /* 1M tuples */ #define TP_WARN_FEW_WORKERS_TUPLES 5000000 /* 5M tuples */ #define TP_WARN_FEW_WORKERS_MIN 2 /* suggest more if <= this */ if (nworkers > 0 && reltuples >= TP_MIN_PARALLEL_TUPLES && !indexInfo->ii_Concurrent) { IndexBuildResult *par_result; /* * Warn if table is very large but parallelism is limited. * Suppress during partitioned builds to reduce noise. */ if (progress == NULL && reltuples >= TP_WARN_FEW_WORKERS_TUPLES && nworkers <= TP_WARN_FEW_WORKERS_MIN) { elog(NOTICE, "Large table (%.0f tuples) with only %d parallel " "workers. " "Consider increasing " "max_parallel_maintenance_workers " "and " "maintenance_work_mem (need 32MB per worker) " "for faster builds.", reltuples, nworkers); } par_result = tp_build_parallel( heap, index, indexInfo, text_config_oid, k1, b, is_text_array, nworkers); /* * Parallel workers kept all build data in DSM/BufFiles. * Invalidate the stable runtime cache only after the new * index relfilenode is complete. */ { Buffer metabuf; Page mpage; TpIndexMetaPage metap; tp_finalize_build_mode(build_state); metabuf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_SHARE); mpage = BufferGetPage(metabuf); metap = (TpIndexMetaPage)PageGetContents(mpage); record_or_report_build_completion( progress, (uint64)metap->total_docs, (uint64)metap->total_len); UnlockReleaseBuffer(metabuf); } return par_result; } if (progress == NULL && reltuples >= TP_WARN_NO_PARALLEL_TUPLES && nworkers == 0) { /* * Large table but no parallel workers available. * This is likely due to * max_parallel_maintenance_workers = 0. */ elog(NOTICE, "Large table (%.0f tuples) but parallel build " "disabled. " "Set max_parallel_maintenance_workers > 0 and " "increase maintenance_work_mem if PostgreSQL cannot " "fund a worker.", reltuples); } } /* * Serial build using arena-based build context. * * Uses table_index_build_scan() which automatically evaluates * index expressions and partial-index predicates, passing * pre-computed values to our callback. * maintenance_work_mem controls the per-batch memory budget. */ { TpBuildCallbackState bs; TpBuildContext *build_ctx; Size budget; double reltuples; index_state = build_state; tp_acquire_index_lock(index_state, LW_EXCLUSIVE); /* Budget: maintenance_work_mem (in KB) -> bytes */ budget = (Size)maintenance_work_mem * 1024L; /* * Clamp to the arena's addressable capacity so a large * maintenance_work_mem still spills instead of overrunning * the 4 GiB arena (issue #433). */ budget = tp_arena_clamp_budget(budget); build_ctx = tp_build_context_create(budget); /* Initialize callback state */ bs.build_ctx = build_ctx; bs.index_state = index_state; bs.index = index; bs.text_config_oid = text_config_oid; bs.is_text_array = is_text_array; bs.per_doc_ctx = AllocSetContextCreate( CurrentMemoryContext, "build per-doc temp", ALLOCSET_DEFAULT_SIZES); bs.total_docs = 0; bs.total_len = 0; bs.tuples_done = 0; /* Report loading phase */ pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_LOADING); { double rt = heap->rd_rel->reltuples; int64 tuples_est = (rt > 0) ? (int64)rt : 0; pgstat_progress_update_param( PROGRESS_CREATEIDX_TUPLES_TOTAL, tuples_est); } /* Scan table with expression evaluation */ reltuples = table_index_build_scan( heap, index, indexInfo, true, false, tp_build_callback, &bs, NULL); /* Accumulate final batch stats */ total_docs = bs.total_docs + build_ctx->num_docs; total_len = bs.total_len + build_ctx->total_len; pgstat_progress_update_param( PROGRESS_CREATEIDX_TUPLES_DONE, bs.tuples_done); /* Report writing phase */ pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_WRITING); /* Write final segment if data remains */ if (build_ctx->num_docs > 0) { tp_build_flush_and_link(build_ctx, index); if (tp_index_compaction_mode(index) == TP_COMPACTION_INLINE) { pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_COMPACTING); tp_compact_build_private(index_state, index); } } /* Update metapage with corpus statistics */ { Buffer metabuf; GenericXLogState *state; Page metapage; TpIndexMetaPage metap; metabuf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); metapage = GenericXLogRegisterBuffer(state, metabuf, 0); metap = (TpIndexMetaPage)PageGetContents(metapage); metap->total_docs = total_docs; metap->total_len = total_len; GenericXLogFinish(state); UnlockReleaseBuffer(metabuf); } /* Create index build result */ result = (IndexBuildResult *)palloc(sizeof(IndexBuildResult)); result->heap_tuples = reltuples; result->index_tuples = total_docs; record_or_report_build_completion(progress, total_docs, total_len); /* * Release the per-index lock before finalizing. * Critical for partitioned tables to avoid hitting * MAX_SIMUL_LWLOCKS limit. */ tp_release_index_lock(index_state); /* * The build output is complete; invalidate the derived runtime * cache in place without replacing its stable allocation. */ tp_finalize_build_mode(index_state); /* Cleanup */ tp_build_context_destroy(build_ctx); MemoryContextDelete(bs.per_doc_ctx); } return result; } /* * Build an empty Tapir index (for CREATE INDEX without data) */ void tp_buildempty(Relation index) { TpOptions *options; Buffer metabuf; Page metapage; TpIndexMetaPage metap; char *text_config_name = NULL; Oid text_config_oid = InvalidOid; /* Extract options from index */ options = (TpOptions *)index->rd_options; if (options) { if (options->text_config_offset > 0) { text_config_name = pstrdup( (char *)options + options->text_config_offset); { List *names = stringToQualifiedNameList(text_config_name, NULL); text_config_oid = get_ts_config_oid(names, false); list_free(names); } } else { ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("text_config parameter is required for bm25 " "indexes"), errhint("Specify text_config when creating the index: " "CREATE INDEX ... USING " "bm25(column) WITH (text_config='english')"))); } } else { /* No options provided - require text_config */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("text_config parameter is required for bm25 indexes"), errhint("Specify text_config when creating the index: " "CREATE INDEX ... USING " "bm25(column) WITH (text_config='english')"))); } /* Create and initialize the metapage */ metabuf = ReadBufferExtended(index, INIT_FORKNUM, P_NEW, RBM_NORMAL, NULL); Assert(BufferGetBlockNumber(metabuf) == TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); { GenericXLogState *state; state = GenericXLogStart(index); metapage = GenericXLogRegisterBuffer( state, metabuf, GENERIC_XLOG_FULL_IMAGE); tp_init_metapage(metapage, text_config_oid); /* Set additional parameters after init */ metap = (TpIndexMetaPage)PageGetContents(metapage); metap->k1 = TP_DEFAULT_K1; metap->b = TP_DEFAULT_B; GenericXLogFinish(state); } UnlockReleaseBuffer(metabuf); } /* * Insert a tuple into the Tapir index. * * Tokenization happens before any lock is acquired so that * CPU-intensive text processing does not serialize inserts. * The per-index lock is held as LW_SHARED for the memtable * chain append, then released before the auto-spill check * (which may need LW_EXCLUSIVE). */ bool tp_insert( Relation index, Datum *values, bool *isnull, ItemPointer ht_ctid, Relation heapRel, IndexUniqueCheck checkUnique, bool indexUnchanged, IndexInfo *indexInfo) { text *document_text; Datum vector_datum; TpVector *tpvec; TpVectorEntry *vector_entry; int32 *frequencies = NULL; int term_count; int doc_length = 0; int i; TpLocalIndexState *index_state; char **terms = NULL; (void)checkUnique; /* unused */ (void)indexUnchanged; /* unused */ (void)heapRel; /* unused */ (void)indexInfo; /* unused */ /* Skip NULL documents */ if (isnull[0]) return true; /* --- Phase 1: Tokenize (no lock held) --- */ if (tp_index_key_is_text_array(index)) document_text = tp_flatten_text_array(values[0]); else document_text = DatumGetTextPP(values[0]); { char *index_name; char *schema_name; Oid namespace_oid = RelationGetNamespace(index); schema_name = get_namespace_name(namespace_oid); index_name = quote_qualified_identifier( schema_name, RelationGetRelationName(index)); vector_datum = DirectFunctionCall2( to_tpvector, PointerGetDatum(document_text), CStringGetTextDatum(index_name)); pfree(index_name); pfree(schema_name); } tpvec = (TpVector *)DatumGetPointer(vector_datum); /* Extract terms and frequencies */ term_count = tpvec->entry_count; if (term_count > 0) { terms = palloc(term_count * sizeof(char *)); frequencies = palloc(term_count * sizeof(int32)); vector_entry = TPVECTOR_ENTRIES_PTR(tpvec); for (i = 0; i < term_count; i++) { TpVectorEntryView v; char *lexeme; vector_entry = tpvector_entry_decode_advance(vector_entry, &v); lexeme = palloc(v.lexeme_len + 1); memcpy(lexeme, v.lexeme, v.lexeme_len); lexeme[v.lexeme_len] = '\0'; terms[i] = lexeme; frequencies[i] = (int32)v.frequency; doc_length += v.frequency; } } /* --- Phase 2: Shared-memory + chain-page work (under lock) --- */ index_state = tp_get_local_index_state(RelationGetRelid(index)); if (index_state != NULL) { /* * Acquire per-index lock in SHARED mode. Phase 4 does * not require any hash-table initialization here — the * chain pages are extended on demand by * tp_memtable_append() while we hold per-buffer EXCL * on the tail. Spill holds LW_EXCLUSIVE to fence * inserts during the chain-reset step. */ tp_acquire_memtable_write_lock(index_state, LW_SHARED); tp_acquire_index_lock(index_state, LW_SHARED); /* Validate TID before appending to the chain */ if (!ItemPointerIsValid(ht_ctid)) elog(WARNING, "Invalid TID in tp_insert, skipping"); else { tp_add_document_terms( index_state, index, ht_ctid, (const char *)tpvec, (uint32)VARSIZE(tpvec), term_count, doc_length); } /* Release lock before spill check */ tp_release_index_lock(index_state); tp_release_memtable_write_lock(index_state); /* * Auto-spill check runs outside the lock — may * acquire LW_EXCLUSIVE if a spill is needed. */ tp_auto_spill_if_needed(index_state, index); } else if (ItemPointerIsValid(ht_ctid)) { /* * No shared state for this index -- nothing to do. * Memtable v2 (issue #374) writes go through * tp_memtable_append, which needs a registered * shared-state entry to take the per-index LWLock; if * the entry isn't there yet, the backend hasn't * touched the index successfully yet and isn't going * to insert anyway. */ } /* Free the terms array and individual lexemes */ if (terms) { for (i = 0; i < term_count; i++) pfree(terms[i]); pfree(terms); pfree(frequencies); } return true; } timescale-pg_textsearch-b63fe58/src/access/build_context.c000066400000000000000000001037661525762047700240350ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build_context.c - Arena-based build context for CREATE INDEX * * See build_context.h for design overview. */ #include #include #include #include #include #include #include #include #include "access/build_context.h" #include "constants.h" #include "memtable/arena.h" #include "memtable/expull.h" #include "segment/alive_bitset.h" #include "segment/compression.h" #include "segment/fieldnorm.h" #include "segment/io.h" #include "segment/pagemapper.h" #include "segment/segment.h" /* Forward declarations for hash table support */ static uint32 build_term_hash(const void *key, Size keysize); static int build_term_match(const void *key1, const void *key2, Size keysize); /* Comparison function for sorting terms */ static int build_term_info_cmp(const void *a, const void *b); /* GUC: compression for segments */ extern bool tp_compress_segments; typedef struct TpBuildTermBlockInfo { uint64 posting_offset; uint32 skip_entry_start; uint32 block_count; uint32 doc_freq; } TpBuildTermBlockInfo; static void build_context_initialize(TpBuildContext *ctx) { HASHCTL info; Assert(CurrentMemoryContext == ctx->memory_context); ctx->arena = tp_arena_create(); memset(&info, 0, sizeof(info)); info.keysize = sizeof(char *); info.entrysize = sizeof(TpBuildTermEntry); info.hash = build_term_hash; info.match = build_term_match; info.hcxt = ctx->memory_context; ctx->terms_ht = hash_create( "build_terms", 16384, &info, HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT); ctx->docs_capacity = TP_BUILD_INITIAL_DOCS; ctx->fieldnorms = palloc(ctx->docs_capacity * sizeof(uint8)); ctx->ctids = palloc(ctx->docs_capacity * sizeof(ItemPointerData)); ctx->num_docs = 0; ctx->num_terms = 0; ctx->posting_blocks = 0; ctx->total_len = 0; } /* * Create a new build context. */ TpBuildContext * tp_build_context_create(Size budget) { TpBuildContext *ctx; MemoryContext oldctx; ctx = palloc0(sizeof(TpBuildContext)); ctx->budget = budget; ctx->memory_context = AllocSetContextCreate( CurrentMemoryContext, "build batch", ALLOCSET_DEFAULT_SIZES); oldctx = MemoryContextSwitchTo(ctx->memory_context); PG_TRY(); { build_context_initialize(ctx); } PG_CATCH(); { MemoryContextSwitchTo(oldctx); MemoryContextDelete(ctx->memory_context); pfree(ctx); PG_RE_THROW(); } PG_END_TRY(); MemoryContextSwitchTo(oldctx); return ctx; } /* * Hash function: hash the string content pointed to by the key. */ static uint32 build_term_hash(const void *key, Size keysize) { const char *term = *(const char *const *)key; (void)keysize; return DatumGetUInt32(hash_any((const unsigned char *)term, strlen(term))); } /* * Match function: compare the string content pointed to by keys. */ static int build_term_match(const void *key1, const void *key2, Size keysize) { const char *t1 = *(const char *const *)key1; const char *t2 = *(const char *const *)key2; (void)keysize; return strcmp(t1, t2); } /* * Grow the flat document arrays when capacity is exceeded. */ static void build_context_grow_docs(TpBuildContext *ctx) { uint32 new_capacity; new_capacity = tp_grow_capacity( ctx->docs_capacity, TP_BUILD_INITIAL_DOCS, "documents"); ctx->fieldnorms = repalloc_huge( ctx->fieldnorms, mul_size((Size)new_capacity, sizeof(uint8))); ctx->ctids = repalloc_huge( ctx->ctids, mul_size((Size)new_capacity, sizeof(ItemPointerData))); ctx->docs_capacity = new_capacity; } /* * Add a single document's terms to the build context. */ uint32 tp_build_context_add_document( TpBuildContext *ctx, char **terms, int32 *frequencies, int term_count, int32 doc_length, ItemPointer ctid) { uint32 doc_id; uint8 norm; int i; MemoryContext oldctx; Assert(ctx != NULL); Assert(ctid != NULL); oldctx = MemoryContextSwitchTo(ctx->memory_context); PG_TRY(); { /* Assign sequential doc_id (UINT32_MAX reserved as sentinel) */ if (ctx->num_docs >= UINT32_MAX - 1) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("too many documents in segment (max %u)", UINT32_MAX - 1))); if (ctx->num_docs >= ctx->docs_capacity) build_context_grow_docs(ctx); doc_id = ctx->num_docs; norm = encode_fieldnorm(doc_length); /* Store fieldnorm and CTID */ ctx->fieldnorms[doc_id] = norm; ctx->ctids[doc_id] = *ctid; ctx->num_docs++; ctx->total_len += doc_length; /* Add each term to the hash table and EXPULL */ for (i = 0; i < term_count; i++) { TpBuildTermEntry *entry; bool found; char *term_key; /* * Look up or create the term entry. The hash table key * is a char* pointer. For new entries, we copy the term * string into the arena so it persists. */ term_key = terms[i]; entry = hash_search(ctx->terms_ht, &term_key, HASH_ENTER, &found); if (!found) { ArenaAddr str_addr; char *arena_str; uint32 len = strlen(terms[i]); /* Copy term string into arena */ str_addr = tp_arena_alloc(ctx->arena, len + 1); arena_str = tp_arena_get_ptr(ctx->arena, str_addr); memcpy(arena_str, terms[i], len + 1); /* Update entry to point to arena copy */ entry->term = arena_str; entry->term_len = len; tp_expull_init(&entry->expull); ctx->num_terms++; } if (entry->expull.num_entries % TP_BLOCK_SIZE == 0) ctx->posting_blocks++; /* Append posting to this term's EXPULL list */ tp_expull_append( ctx->arena, &entry->expull, doc_id, (uint16)frequencies[i], norm); } } PG_CATCH(); { MemoryContextSwitchTo(oldctx); PG_RE_THROW(); } PG_END_TRY(); MemoryContextSwitchTo(oldctx); return doc_id; } static Size build_scratch_allocation(Size bytes) { if (bytes == 0) return 0; return add_size(MAXALIGN(bytes), MAXALIGN(sizeof(void *))); } static Size build_context_serialization_scratch(TpBuildContext *ctx) { Size scratch; Size doc_capacity; uint32 skip_capacity; scratch = ALLOCSET_DEFAULT_INITSIZE; scratch = add_size( scratch, build_scratch_allocation( mul_size(ctx->num_terms, sizeof(TpBuildTermInfo)))); scratch = add_size( scratch, build_scratch_allocation( mul_size(ctx->num_terms, sizeof(uint32)))); scratch = add_size( scratch, build_scratch_allocation( mul_size(ctx->num_terms, sizeof(TpBuildTermBlockInfo)))); scratch = add_size( scratch, build_scratch_allocation(tp_dictionary_size(ctx->num_terms))); skip_capacity = 1024; while ((uint64)skip_capacity < ctx->posting_blocks) skip_capacity = tp_grow_capacity(skip_capacity, 1024, "posting blocks"); scratch = add_size( scratch, build_scratch_allocation( mul_size((Size)skip_capacity, sizeof(TpSkipEntry)))); doc_capacity = ctx->docs_capacity; scratch = add_size( scratch, build_scratch_allocation(mul_size(doc_capacity, sizeof(uint32)))); scratch = add_size( scratch, build_scratch_allocation(mul_size(doc_capacity, sizeof(uint16)))); scratch = add_size( scratch, build_scratch_allocation(tp_alive_bitset_size(doc_capacity))); return scratch; } bool tp_build_context_should_flush(TpBuildContext *ctx) { Size docs_usage; Size live_bytes; Size required_bytes; Size scratch_bytes; Assert(ctx != NULL); if (ctx->budget == 0) return false; if (!ctx->account_full_allocation) { docs_usage = mul_size( ctx->docs_capacity, add_size(sizeof(*ctx->fieldnorms), sizeof(*ctx->ctids))); required_bytes = add_size(tp_arena_mem_usage(ctx->arena), docs_usage); return required_bytes >= ctx->budget; } live_bytes = MemoryContextMemAllocated(ctx->memory_context, true); scratch_bytes = build_context_serialization_scratch(ctx); required_bytes = add_size(live_bytes, scratch_bytes); return required_bytes >= ctx->budget; } Size tp_build_context_minimum_budget(TpBuildContext *ctx) { Size live_bytes; Size required_bytes; Assert(ctx != NULL); live_bytes = MemoryContextMemAllocated(ctx->memory_context, true); required_bytes = add_size(live_bytes, build_context_serialization_scratch(ctx)); return required_bytes; } /* * Comparison function for sorting TpBuildTermInfo by term string. */ static int build_term_info_cmp(const void *a, const void *b) { const TpBuildTermInfo *ta = (const TpBuildTermInfo *)a; const TpBuildTermInfo *tb = (const TpBuildTermInfo *)b; return strcmp(ta->term, tb->term); } /* * Build a sorted term array from the build context. */ TpBuildTermInfo * tp_build_context_get_sorted_terms(TpBuildContext *ctx, uint32 *num_terms) { HASH_SEQ_STATUS status; TpBuildTermEntry *entry; TpBuildTermInfo *terms; uint32 count; uint32 i; count = hash_get_num_entries(ctx->terms_ht); if (count == 0) { *num_terms = 0; return NULL; } terms = palloc_extended(count * sizeof(TpBuildTermInfo), MCXT_ALLOC_HUGE); i = 0; hash_seq_init(&status, ctx->terms_ht); while ((entry = hash_seq_search(&status)) != NULL) { Assert(i < count); terms[i].term = entry->term; terms[i].term_len = entry->term_len; terms[i].expull = &entry->expull; terms[i].doc_freq = entry->expull.num_entries; i++; } Assert(i == count); /* Sort lexicographically */ qsort(terms, count, sizeof(TpBuildTermInfo), build_term_info_cmp); *num_terms = count; return terms; } static void validate_build_terms(TpBuildTermInfo *terms, uint32 num_terms) { uint64 string_pos = 0; uint64 skip_entries = 0; uint32 i; if (!tp_dictionary_offsets_fit(num_terms)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment dictionary exceeds %u terms", TP_MAX_DICTIONARY_TERMS))); for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); uint64 blocks = tp_posting_block_count(terms[i].doc_freq); if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment string pool exceeds the " "4 GiB format limit"))); string_pos += entry_size; if (blocks > TP_MAX_GROWABLE_CAPACITY - skip_entries) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u posting blocks", TP_MAX_GROWABLE_CAPACITY))); skip_entries += blocks; } } /* * Write a segment from the build context. * * This mirrors tp_write_segment() but reads from the arena/EXPULL * instead of DSA/dshash. Doc IDs are already sequential (assigned * during add_document), so no docmap hash lookup is needed. */ BlockNumber tp_write_segment_from_build_ctx(TpBuildContext *ctx, Relation index) { TpBuildTermInfo *terms; uint32 num_terms; BlockNumber header_block; BlockNumber page_index_root; TpSegmentWriter writer; TpSegmentHeader header; TpDictionary dict; uint32 *string_offsets; uint64 string_pos; uint32 i; Buffer header_buf; Page header_page; /* Per-term block tracking (same as tp_write_segment). */ TpBuildTermBlockInfo *term_blocks; /* Accumulated skip entries */ TpSkipEntry *all_skip_entries; uint32 skip_entries_count; uint32 skip_entries_capacity; /* Get sorted terms */ terms = tp_build_context_get_sorted_terms(ctx, &num_terms); validate_build_terms(terms, num_terms); /* Initialize writer */ tp_segment_writer_init(&writer, index); if (writer.pages_allocated == 0) elog(ERROR, "tp_write_segment_from_build_ctx: " "Failed to allocate first page"); header_block = writer.pages[0]; /* Initialize header */ memset(&header, 0, sizeof(TpSegmentHeader)); header.magic = TP_SEGMENT_MAGIC; header.version = TP_SEGMENT_FORMAT_VERSION; header.created_at = GetCurrentTimestamp(); header.num_pages = 0; header.num_terms = num_terms; header.level = 0; header.next_segment = InvalidBlockNumber; header.num_docs = ctx->num_docs; header.total_tokens = ctx->total_len; /* Dictionary immediately follows header */ header.dictionary_offset = sizeof(TpSegmentHeader); /* Write placeholder header */ tp_segment_writer_write(&writer, &header, sizeof(TpSegmentHeader)); /* Write dictionary section */ dict.num_terms = num_terms; tp_segment_writer_write( &writer, &dict, offsetof(TpDictionary, string_offsets)); /* Build string offsets */ string_offsets = palloc_extended( num_terms * sizeof(uint32), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); string_pos = 0; for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment string pool exceeds the " "4 GiB format limit"), errhint("The corpus vocabulary is too large for a " "single segment."))); string_offsets[i] = (uint32)string_pos; string_pos += entry_size; } /* Write string offsets array */ tp_segment_writer_write( &writer, string_offsets, num_terms * sizeof(uint32)); /* Write string pool */ header.strings_offset = writer.current_offset; for (i = 0; i < num_terms; i++) { uint32 length = terms[i].term_len; uint32 dict_offset = (uint32)((uint64)i * sizeof(TpDictEntry)); tp_segment_writer_write(&writer, &length, sizeof(uint32)); tp_segment_writer_write(&writer, terms[i].term, length); tp_segment_writer_write(&writer, &dict_offset, sizeof(uint32)); } /* Record entries offset */ header.entries_offset = writer.current_offset; /* Write placeholder dict entries */ { TpDictEntry placeholder; memset(&placeholder, 0, sizeof(TpDictEntry)); for (i = 0; i < num_terms; i++) tp_segment_writer_write( &writer, &placeholder, sizeof(TpDictEntry)); } /* Postings start here */ header.postings_offset = writer.current_offset; /* Initialize per-term tracking and skip entry accumulator */ term_blocks = palloc_extended( num_terms * sizeof(TpBuildTermBlockInfo), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); skip_entries_capacity = 1024; skip_entries_count = 0; all_skip_entries = palloc(skip_entries_capacity * sizeof(TpSkipEntry)); /* * Streaming pass: for each term, read postings from EXPULL * and write compressed blocks. */ for (i = 0; i < num_terms; i++) { TpExpullReader reader; uint32 doc_count; uint32 block_idx; uint32 num_blocks; /* Record where this term's postings start */ term_blocks[i].posting_offset = writer.current_offset; term_blocks[i].skip_entry_start = skip_entries_count; doc_count = terms[i].expull->num_entries; term_blocks[i].doc_freq = terms[i].doc_freq; if (doc_count == 0) { term_blocks[i].block_count = 0; continue; } num_blocks = (uint32)tp_posting_block_count(doc_count); term_blocks[i].block_count = num_blocks; /* Initialize EXPULL reader for this term */ tp_expull_reader_init(&reader, ctx->arena, terms[i].expull); /* Process posting blocks */ for (block_idx = 0; block_idx < num_blocks; block_idx++) { TpExpullEntry entries[TP_BLOCK_SIZE]; TpBlockPosting block_postings[TP_BLOCK_SIZE]; TpSkipEntry skip; uint32 nread; uint32 j; uint16 max_tf = 0; uint8 min_norm = 255; uint32 last_docid = 0; /* Read a block of entries from EXPULL */ nread = tp_expull_reader_read(&reader, entries, TP_BLOCK_SIZE); Assert(nread > 0); /* Convert to TpBlockPosting format */ for (j = 0; j < nread; j++) { block_postings[j].doc_id = entries[j].doc_id; block_postings[j].frequency = entries[j].frequency; block_postings[j].fieldnorm = entries[j].fieldnorm; block_postings[j].reserved = 0; /* Track block stats */ if (entries[j].doc_id > last_docid) last_docid = entries[j].doc_id; if (entries[j].frequency > max_tf) max_tf = entries[j].frequency; if (entries[j].fieldnorm < min_norm) min_norm = entries[j].fieldnorm; } /* Build skip entry */ skip.last_doc_id = last_docid; skip.doc_count = (uint8)nread; skip.block_max_tf = max_tf; skip.block_max_norm = min_norm; skip.posting_offset = writer.current_offset; memset(skip.reserved, 0, sizeof(skip.reserved)); /* Write posting block */ if (tp_compress_segments) { uint8 compressed[TP_MAX_COMPRESSED_BLOCK_SIZE]; uint32 compressed_size; compressed_size = tp_compress_block(block_postings, nread, compressed); skip.flags = TP_BLOCK_FLAG_DELTA; tp_segment_writer_write(&writer, compressed, compressed_size); } else { skip.flags = TP_BLOCK_FLAG_UNCOMPRESSED; tp_segment_writer_write( &writer, block_postings, nread * sizeof(TpBlockPosting)); } /* Accumulate skip entry */ if (skip_entries_count >= skip_entries_capacity) { skip_entries_capacity = tp_grow_capacity( skip_entries_capacity, 1024, "posting blocks"); all_skip_entries = repalloc_huge( all_skip_entries, mul_size( (Size)skip_entries_capacity, sizeof(TpSkipEntry))); } all_skip_entries[skip_entries_count++] = skip; } } /* Write skip index */ header.skip_index_offset = writer.current_offset; if (skip_entries_count > 0) tp_segment_writer_write( &writer, all_skip_entries, skip_entries_count * sizeof(TpSkipEntry)); /* Write fieldnorm table */ header.fieldnorm_offset = writer.current_offset; if (ctx->num_docs > 0) tp_segment_writer_write( &writer, ctx->fieldnorms, ctx->num_docs * sizeof(uint8)); /* Write CTID pages array (BlockNumber per doc) */ header.ctid_pages_offset = writer.current_offset; { uint32 *ctid_pages = palloc_extended( ctx->num_docs * sizeof(uint32), MCXT_ALLOC_HUGE); for (i = 0; i < ctx->num_docs; i++) ctid_pages[i] = ItemPointerGetBlockNumber(&ctx->ctids[i]); tp_segment_writer_write( &writer, ctid_pages, ctx->num_docs * sizeof(uint32)); pfree(ctid_pages); } /* Write CTID offsets array (OffsetNumber per doc) */ header.ctid_offsets_offset = writer.current_offset; { uint16 *ctid_offsets = palloc_extended( ctx->num_docs * sizeof(uint16), MCXT_ALLOC_HUGE); for (i = 0; i < ctx->num_docs; i++) ctid_offsets[i] = ItemPointerGetOffsetNumber(&ctx->ctids[i]); tp_segment_writer_write( &writer, ctid_offsets, ctx->num_docs * sizeof(uint16)); pfree(ctid_offsets); } /* Write alive bitset (all alive) */ header.alive_bitset_offset = writer.current_offset; header.alive_count = ctx->num_docs; if (ctx->num_docs > 0) { uint32 bitset_size = tp_alive_bitset_size(ctx->num_docs); uint8 *bitset_data = palloc(bitset_size); tp_alive_bitset_init_data(bitset_data, ctx->num_docs); tp_segment_writer_write(&writer, bitset_data, bitset_size); pfree(bitset_data); } /* Flush remaining buffered data */ tp_segment_writer_flush(&writer); /* * Prevent tp_segment_writer_finish from flushing again * (matches existing tp_write_segment pattern). */ writer.buffer_pos = SizeOfPageHeaderData; /* Write page index */ page_index_root = write_page_index(index, writer.pages, writer.pages_allocated); /* Finalize header */ header.page_index = page_index_root; header.data_size = writer.current_offset; header.num_pages = writer.pages_allocated; /* * Write dictionary entries with real skip_index_offset * values. This must happen BEFORE tp_segment_writer_finish * so writer.pages is still valid. * * Logged as GenericXLog deltas (not full-page images) since * each batch only mutates a handful of 16-byte dict entries * in pages that the writer flush already WAL-logged in full. */ { Buffer dict_buf = InvalidBuffer; Page dict_page = NULL; GenericXLogState *xlog_state = NULL; uint32 current_page = UINT32_MAX; for (i = 0; i < num_terms; i++) { TpDictEntry entry; uint64 entry_offset; uint32 logical_page; uint32 page_offset; BlockNumber physical_block; entry.skip_index_offset = header.skip_index_offset + ((uint64)term_blocks[i].skip_entry_start * sizeof(TpSkipEntry)); entry.block_count = term_blocks[i].block_count; entry.doc_freq = term_blocks[i].doc_freq; /* Locate this entry in the segment */ entry_offset = header.entries_offset + ((uint64)i * sizeof(TpDictEntry)); logical_page = (uint32)(entry_offset / SEGMENT_DATA_PER_PAGE); page_offset = (uint32)(entry_offset % SEGMENT_DATA_PER_PAGE); if (logical_page >= writer.pages_allocated) elog(ERROR, "dict entry %u: logical page %u >= " "pages %u", i, logical_page, writer.pages_allocated); /* Read page if different from current */ if (logical_page != current_page) { if (current_page != UINT32_MAX) { GenericXLogFinish(xlog_state); UnlockReleaseBuffer(dict_buf); } physical_block = writer.pages[logical_page]; dict_buf = ReadBuffer(index, physical_block); LockBuffer(dict_buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(index); dict_page = GenericXLogRegisterBuffer(xlog_state, dict_buf, 0); current_page = logical_page; } /* Write entry to working copy - handle page boundary spanning */ { uint32 bytes_remaining = SEGMENT_DATA_PER_PAGE - page_offset; if (bytes_remaining >= sizeof(TpDictEntry)) { char *dest = (char *)dict_page + SizeOfPageHeaderData + page_offset; memcpy(dest, &entry, sizeof(TpDictEntry)); } else { char *dest = (char *)dict_page + SizeOfPageHeaderData + page_offset; char *src = (char *)&entry; memcpy(dest, src, bytes_remaining); GenericXLogFinish(xlog_state); UnlockReleaseBuffer(dict_buf); logical_page++; if (logical_page >= writer.pages_allocated) elog(ERROR, "dict entry spans beyond " "allocated pages"); physical_block = writer.pages[logical_page]; dict_buf = ReadBuffer(index, physical_block); LockBuffer(dict_buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(index); dict_page = GenericXLogRegisterBuffer(xlog_state, dict_buf, 0); current_page = logical_page; dest = (char *)dict_page + SizeOfPageHeaderData; memcpy(dest, src + bytes_remaining, sizeof(TpDictEntry) - bytes_remaining); } } } if (current_page != UINT32_MAX) { GenericXLogFinish(xlog_state); UnlockReleaseBuffer(dict_buf); } } tp_segment_writer_finish(&writer); /* Flush all pages to disk */ FlushRelationBuffers(index); /* * Write final header. The header page already received a full-page * WAL image during the writer flush, so this final patch is logged * as a small GenericXLog delta. */ { GenericXLogState *xlog_state; header_buf = ReadBuffer(index, header_block); LockBuffer(header_buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(index); header_page = GenericXLogRegisterBuffer(xlog_state, header_buf, 0); { TpSegmentHeader *hdr = (TpSegmentHeader *)PageGetContents( header_page); hdr->strings_offset = header.strings_offset; hdr->entries_offset = header.entries_offset; hdr->postings_offset = header.postings_offset; hdr->skip_index_offset = header.skip_index_offset; hdr->fieldnorm_offset = header.fieldnorm_offset; hdr->ctid_pages_offset = header.ctid_pages_offset; hdr->ctid_offsets_offset = header.ctid_offsets_offset; hdr->alive_bitset_offset = header.alive_bitset_offset; hdr->alive_count = header.alive_count; hdr->num_docs = header.num_docs; hdr->data_size = header.data_size; hdr->num_pages = header.num_pages; hdr->page_index = header.page_index; } GenericXLogFinish(xlog_state); UnlockReleaseBuffer(header_buf); } FlushRelationBuffers(index); /* Cleanup */ if (term_blocks) pfree(term_blocks); pfree(all_skip_entries); if (string_offsets) pfree(string_offsets); if (terms) pfree(terms); if (writer.pages) pfree(writer.pages); return header_block; } /* * Write a segment from the build context to a BufFile as a flat * byte stream. No page boundaries, no page index. Used by parallel * build workers to write to SharedFileSet temp files. * * Returns the data_size (total bytes written for this segment). */ uint64 tp_write_segment_to_buffile(TpBuildContext *ctx, BufFile *file) { TpBuildTermInfo *terms; uint32 num_terms; TpSegmentHeader header; TpDictionary dict; /* * BufFile position tracking. BufFile uses (fileno, offset) * pairs where fileno is the 1GB file segment index. We save * the starting position and end position to handle seek-back * for header/dict entry updates. */ int base_fileno; off_t base_file_offset; uint32 *string_offsets; uint64 string_pos; uint32 i; /* Current write position in the flat stream */ uint64 current_offset; /* Per-term block tracking */ TpBuildTermBlockInfo *term_blocks; /* Accumulated skip entries */ TpSkipEntry *all_skip_entries; uint32 skip_entries_count; uint32 skip_entries_capacity; /* Get sorted terms */ terms = tp_build_context_get_sorted_terms(ctx, &num_terms); validate_build_terms(terms, num_terms); /* Record starting position for later seek-back */ BufFileTell(file, &base_fileno, &base_file_offset); current_offset = 0; /* Initialize header */ memset(&header, 0, sizeof(TpSegmentHeader)); header.magic = TP_SEGMENT_MAGIC; header.version = TP_SEGMENT_FORMAT_VERSION; header.created_at = GetCurrentTimestamp(); header.num_pages = 0; /* No pages in flat format */ header.num_terms = num_terms; header.level = 0; header.next_segment = InvalidBlockNumber; header.num_docs = ctx->num_docs; header.total_tokens = ctx->total_len; header.page_index = InvalidBlockNumber; /* Dictionary immediately follows header */ header.dictionary_offset = sizeof(TpSegmentHeader); /* Write placeholder header */ BufFileWrite(file, &header, sizeof(TpSegmentHeader)); current_offset += sizeof(TpSegmentHeader); /* Write dictionary section */ dict.num_terms = num_terms; BufFileWrite(file, &dict, offsetof(TpDictionary, string_offsets)); current_offset += offsetof(TpDictionary, string_offsets); /* Build string offsets */ string_offsets = palloc_extended( num_terms * sizeof(uint32), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); string_pos = 0; for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment string pool exceeds the " "4 GiB format limit"), errhint("The corpus vocabulary is too large for a " "single segment."))); string_offsets[i] = (uint32)string_pos; string_pos += entry_size; } /* Write string offsets array */ BufFileWrite(file, string_offsets, num_terms * sizeof(uint32)); current_offset += num_terms * sizeof(uint32); /* Write string pool */ header.strings_offset = current_offset; for (i = 0; i < num_terms; i++) { uint32 length = terms[i].term_len; uint32 dict_offset = (uint32)((uint64)i * sizeof(TpDictEntry)); BufFileWrite(file, &length, sizeof(uint32)); BufFileWrite(file, terms[i].term, length); BufFileWrite(file, &dict_offset, sizeof(uint32)); current_offset += sizeof(uint32) + length + sizeof(uint32); } /* Record entries offset */ header.entries_offset = current_offset; /* Write placeholder dict entries */ { TpDictEntry placeholder; memset(&placeholder, 0, sizeof(TpDictEntry)); for (i = 0; i < num_terms; i++) { BufFileWrite(file, &placeholder, sizeof(TpDictEntry)); current_offset += sizeof(TpDictEntry); } } /* Postings start here */ header.postings_offset = current_offset; /* Initialize per-term tracking and skip entry accumulator */ term_blocks = palloc_extended( num_terms * sizeof(TpBuildTermBlockInfo), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); skip_entries_capacity = 1024; skip_entries_count = 0; all_skip_entries = palloc(skip_entries_capacity * sizeof(TpSkipEntry)); /* Streaming pass: write posting blocks */ for (i = 0; i < num_terms; i++) { TpExpullReader reader; uint32 doc_count; uint32 block_idx; uint32 num_blocks; term_blocks[i].posting_offset = current_offset; term_blocks[i].skip_entry_start = skip_entries_count; doc_count = terms[i].expull->num_entries; term_blocks[i].doc_freq = terms[i].doc_freq; if (doc_count == 0) { term_blocks[i].block_count = 0; continue; } num_blocks = (uint32)tp_posting_block_count(doc_count); term_blocks[i].block_count = num_blocks; tp_expull_reader_init(&reader, ctx->arena, terms[i].expull); for (block_idx = 0; block_idx < num_blocks; block_idx++) { TpExpullEntry entries[TP_BLOCK_SIZE]; TpBlockPosting block_postings[TP_BLOCK_SIZE]; TpSkipEntry skip; uint32 nread; uint32 j; uint16 max_tf = 0; uint8 min_norm = 255; uint32 last_docid = 0; nread = tp_expull_reader_read(&reader, entries, TP_BLOCK_SIZE); Assert(nread > 0); for (j = 0; j < nread; j++) { block_postings[j].doc_id = entries[j].doc_id; block_postings[j].frequency = entries[j].frequency; block_postings[j].fieldnorm = entries[j].fieldnorm; block_postings[j].reserved = 0; if (entries[j].doc_id > last_docid) last_docid = entries[j].doc_id; if (entries[j].frequency > max_tf) max_tf = entries[j].frequency; if (entries[j].fieldnorm < min_norm) min_norm = entries[j].fieldnorm; } skip.last_doc_id = last_docid; skip.doc_count = (uint8)nread; skip.block_max_tf = max_tf; skip.block_max_norm = min_norm; skip.posting_offset = current_offset; memset(skip.reserved, 0, sizeof(skip.reserved)); if (tp_compress_segments) { uint8 compressed[TP_MAX_COMPRESSED_BLOCK_SIZE]; uint32 compressed_size; compressed_size = tp_compress_block(block_postings, nread, compressed); skip.flags = TP_BLOCK_FLAG_DELTA; BufFileWrite(file, compressed, compressed_size); current_offset += compressed_size; } else { skip.flags = TP_BLOCK_FLAG_UNCOMPRESSED; BufFileWrite( file, block_postings, nread * sizeof(TpBlockPosting)); current_offset += nread * sizeof(TpBlockPosting); } if (skip_entries_count >= skip_entries_capacity) { skip_entries_capacity = tp_grow_capacity( skip_entries_capacity, 1024, "posting blocks"); all_skip_entries = repalloc_huge( all_skip_entries, mul_size( (Size)skip_entries_capacity, sizeof(TpSkipEntry))); } all_skip_entries[skip_entries_count++] = skip; } } /* Write skip index */ header.skip_index_offset = current_offset; if (skip_entries_count > 0) { BufFileWrite( file, all_skip_entries, skip_entries_count * sizeof(TpSkipEntry)); current_offset += skip_entries_count * sizeof(TpSkipEntry); } /* Write fieldnorm table */ header.fieldnorm_offset = current_offset; if (ctx->num_docs > 0) { BufFileWrite(file, ctx->fieldnorms, ctx->num_docs * sizeof(uint8)); current_offset += ctx->num_docs * sizeof(uint8); } /* Write CTID pages array */ header.ctid_pages_offset = current_offset; { uint32 *ctid_pages = palloc_extended( ctx->num_docs * sizeof(uint32), MCXT_ALLOC_HUGE); for (i = 0; i < ctx->num_docs; i++) ctid_pages[i] = ItemPointerGetBlockNumber(&ctx->ctids[i]); BufFileWrite(file, ctid_pages, ctx->num_docs * sizeof(uint32)); current_offset += ctx->num_docs * sizeof(uint32); pfree(ctid_pages); } /* Write CTID offsets array */ header.ctid_offsets_offset = current_offset; { uint16 *ctid_offsets = palloc_extended( ctx->num_docs * sizeof(uint16), MCXT_ALLOC_HUGE); for (i = 0; i < ctx->num_docs; i++) ctid_offsets[i] = ItemPointerGetOffsetNumber(&ctx->ctids[i]); BufFileWrite(file, ctid_offsets, ctx->num_docs * sizeof(uint16)); current_offset += ctx->num_docs * sizeof(uint16); pfree(ctid_offsets); } /* Final data_size */ header.data_size = current_offset; /* * Seek back to write dict entries and header with real offsets. * * Save the end position first (BufFileWrite may have crossed a * 1GB file segment boundary), then seek back to early positions * within this segment (guaranteed to be in the same 1GB segment * since the header/dict entries are near the start), then * restore the end position. */ { TpDictEntry *dict_entries; int end_fileno; off_t end_file_offset; /* Save end position */ BufFileTell(file, &end_fileno, &end_file_offset); dict_entries = palloc_extended( tp_dictionary_size(num_terms), MCXT_ALLOC_HUGE); for (i = 0; i < num_terms; i++) { dict_entries[i].skip_index_offset = header.skip_index_offset + ((uint64)term_blocks[i].skip_entry_start * sizeof(TpSkipEntry)); dict_entries[i].block_count = term_blocks[i].block_count; dict_entries[i].doc_freq = term_blocks[i].doc_freq; } /* Seek back to dict entries position */ { uint64 base_abs = tp_buffile_composite_offset(base_fileno, base_file_offset); int dict_fileno; off_t dict_offset; tp_buffile_decompose_offset( base_abs + header.entries_offset, &dict_fileno, &dict_offset); BufFileSeek(file, dict_fileno, dict_offset, SEEK_SET); } BufFileWrite(file, dict_entries, tp_dictionary_size(num_terms)); if (dict_entries) pfree(dict_entries); /* Seek back to header position */ BufFileSeek(file, base_fileno, base_file_offset, SEEK_SET); BufFileWrite(file, &header, sizeof(TpSegmentHeader)); /* Restore end position for caller's next write */ BufFileSeek(file, end_fileno, end_file_offset, SEEK_SET); } /* Cleanup */ if (term_blocks) pfree(term_blocks); pfree(all_skip_entries); if (string_offsets) pfree(string_offsets); if (terms) pfree(terms); return current_offset; } /* * Reset the build context for reuse. */ void tp_build_context_reset(TpBuildContext *ctx) { MemoryContext oldctx; if (ctx == NULL) return; MemoryContextReset(ctx->memory_context); oldctx = MemoryContextSwitchTo(ctx->memory_context); PG_TRY(); { build_context_initialize(ctx); } PG_CATCH(); { MemoryContextSwitchTo(oldctx); PG_RE_THROW(); } PG_END_TRY(); MemoryContextSwitchTo(oldctx); } /* * Destroy the build context. */ void tp_build_context_destroy(TpBuildContext *ctx) { if (ctx == NULL) return; MemoryContextDelete(ctx->memory_context); pfree(ctx); } timescale-pg_textsearch-b63fe58/src/access/build_context.h000066400000000000000000000115521525762047700240310ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build_context.h - Arena-based build context for CREATE INDEX * * Replaces the DSA-based memtable during index builds with a * process-local arena + EXPULL structure. This eliminates DSA * fragmentation, dshash lock overhead, and posting list * reallocation copies. * * The build context is NOT shared between backends — it is used * only during serial CREATE INDEX (and per-worker in parallel * builds). Concurrent INSERT operations continue to use the * shared dshash/DSA memtable. */ #pragma once #include #include #include #include "memtable/arena.h" #include "memtable/expull.h" #include "segment/segment.h" /* * Hash table entry for one term during index build. * The term string is stored in the arena; the HTAB key * is a pointer to it. */ typedef struct TpBuildTermEntry { char *term; /* Key: pointer to arena-stored null-terminated string */ uint32 term_len; TpExpull expull; /* Inline EXPULL posting list header */ } TpBuildTermEntry; /* * Initial and growth sizes for flat arrays. */ #define TP_BUILD_INITIAL_DOCS 4096 /* * Build context for CREATE INDEX. * Holds all in-memory state for one spill batch. */ typedef struct TpBuildContext { /* * Stable wrapper ownership remains with the caller. Batch allocations * live in this resettable child context. */ MemoryContext memory_context; /* Arena for EXPULL blocks and term strings */ TpArena *arena; /* Local hash table: term string -> TpBuildTermEntry */ HTAB *terms_ht; /* Flat arrays indexed by doc_id (sequential assignment) */ uint8 *fieldnorms; /* Encoded fieldnorm per doc */ ItemPointerData *ctids; /* Heap CTID per doc */ uint32 num_docs; /* Documents in current batch */ uint32 docs_capacity; uint32 num_terms; /* Vocabulary in current batch */ uint64 posting_blocks; /* Serialized posting-block count */ /* Corpus statistics for this batch */ uint64 total_len; /* Sum of document lengths */ /* Budget for flush decisions */ Size budget; /* Max tracked bytes before flush */ /* * Parallel workers account actual child-context allocation plus * serialization scratch. Serial builds retain their existing estimator. */ bool account_full_allocation; } TpBuildContext; /* * Create a new build context. * budget: max tracked bytes before caller should flush (0 = no limit). */ extern TpBuildContext *tp_build_context_create(Size budget); /* * Add a single document's terms to the build context. * Assigns a sequential doc_id and stores fieldnorm + CTID. * * terms: array of null-terminated term strings * frequencies: parallel array of term frequencies * term_count: number of terms * doc_length: sum of all frequencies (for fieldnorm encoding) * ctid: heap tuple ID * * Returns the assigned doc_id. */ extern uint32 tp_build_context_add_document( TpBuildContext *ctx, char **terms, int32 *frequencies, int term_count, int32 doc_length, ItemPointer ctid); /* * Check if the build context should be flushed (budget exceeded). */ extern bool tp_build_context_should_flush(TpBuildContext *ctx); /* * Minimum useful worker budget: initial context allocations, one arena page, * and the serialization scratch needed for the initial document capacity. */ extern Size tp_build_context_minimum_budget(TpBuildContext *ctx); /* * Build a sorted term array from the build context. * Returns a palloc'd array of TermInfo-like structures. * Caller must pfree the returned array. * * The returned TpBuildTermInfo entries have: * - term, term_len: from the hash table entry * - expull: pointer to the TpExpull in the hash table entry * - doc_freq: number of distinct documents for the term */ typedef struct TpBuildTermInfo { char *term; uint32 term_len; TpExpull *expull; /* Points into HTAB entry */ uint32 doc_freq; /* Number of documents for this term */ } TpBuildTermInfo; extern TpBuildTermInfo * tp_build_context_get_sorted_terms(TpBuildContext *ctx, uint32 *num_terms); /* * Write a segment from the build context to an index relation. * Returns the header block number of the written segment. */ extern BlockNumber tp_write_segment_from_build_ctx(TpBuildContext *ctx, Relation index); /* * Write a segment from the build context to a BufFile as a flat * byte stream (no page boundaries). Used by parallel build workers. * Returns the data_size (total bytes written for this segment). */ extern uint64 tp_write_segment_to_buffile(TpBuildContext *ctx, BufFile *file); /* * Reset the build context for reuse (new spill batch). * Frees all arena memory and clears the hash table and arrays. */ extern void tp_build_context_reset(TpBuildContext *ctx); /* * Destroy the build context, freeing all memory. */ extern void tp_build_context_destroy(TpBuildContext *ctx); timescale-pg_textsearch-b63fe58/src/access/build_merge.c000066400000000000000000000352421525762047700234410ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build_merge.c - Streaming final merge for parallel index builds */ #include #include #include #include #include #include #include #include "access/build_merge.h" #include "constants.h" #include "segment/alive_bitset.h" #include "segment/io.h" #include "segment/segment.h" #define TP_BUILD_MERGE_COPY_BUFFER_SIZE (64 * 1024) #define TP_BUILD_MERGE_DICT_CHUNK 4096 static void build_merge_rewind(BufFile *file) { if (BufFileSeek(file, 0, 0, SEEK_SET) != 0) ereport(ERROR, (errcode_for_file_access(), errmsg("could not rewind parallel build merge stream"))); } static void build_merge_copy_file( BufFile *file, uint64 size, TpMergeSink *sink, char *buffer) { uint64 remaining = size; build_merge_rewind(file); while (remaining > 0) { Size chunk = (Size) Min(remaining, (uint64)TP_BUILD_MERGE_COPY_BUFFER_SIZE); BufFileReadExact(file, buffer, chunk); merge_sink_write(sink, buffer, chunk); remaining -= chunk; CHECK_FOR_INTERRUPTS(); } } static void build_merge_copy_source( TpSegmentReader *reader, uint64 source_offset, uint64 size, TpMergeSink *sink, char *buffer) { uint64 copied = 0; while (copied < size) { Size chunk = (Size) Min(size - copied, (uint64)TP_BUILD_MERGE_COPY_BUFFER_SIZE); tp_segment_read(reader, source_offset + copied, buffer, chunk); merge_sink_write(sink, buffer, chunk); copied += chunk; CHECK_FOR_INTERRUPTS(); } } static void build_merge_write_zeros(TpMergeSink *sink, uint64 size, char *buffer) { uint64 remaining = size; memset(buffer, 0, TP_BUILD_MERGE_COPY_BUFFER_SIZE); while (remaining > 0) { Size chunk = (Size) Min(remaining, (uint64)TP_BUILD_MERGE_COPY_BUFFER_SIZE); merge_sink_write(sink, buffer, chunk); remaining -= chunk; } } static void build_merge_write_alive(TpMergeSink *sink, uint32 num_docs, char *buffer) { uint32 remaining = tp_alive_bitset_size(num_docs); memset(buffer, 0xff, TP_BUILD_MERGE_COPY_BUFFER_SIZE); while (remaining > 0) { uint32 chunk = Min(remaining, TP_BUILD_MERGE_COPY_BUFFER_SIZE); if (remaining == chunk && num_docs % 8 != 0) buffer[chunk - 1] = (1 << (num_docs % 8)) - 1; merge_sink_write(sink, buffer, chunk); remaining -= chunk; } } static void build_merge_flush_block( TpMergeSink *sink, BufFile *skips, TpBlockPosting *block, uint32 count) { TpSkipEntry skip = merge_sink_write_posting_block(sink, block, count); BufFileWrite(skips, &skip, sizeof(skip)); } void tp_write_parallel_build_merge( TpMergeSink *sink, TpMergeSource *sources, uint32 num_sources, uint64 total_docs, uint64 total_tokens) { BufFile *string_offsets_file; BufFile *strings_file; BufFile *refs_file; BufFile *skips_file; BufFile *dict_entries_file; TpTermSegmentRef *term_refs; uint32 *source_doc_bases; char *copy_buffer; TpSegmentHeader header; TpDictionary dict; uint64 source_docs = 0; uint64 source_tokens = 0; uint64 string_pos = 0; uint64 skip_entries = 0; uint64 skip_bytes_written = 0; uint32 num_terms = 0; uint32 i; if (num_sources > INT_MAX) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: too many parallel build segments"))); if (!tp_document_count_fits(total_docs)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u documents", TP_MAX_GROWABLE_CAPACITY))); source_doc_bases = palloc_extended(num_sources * sizeof(uint32), MCXT_ALLOC_HUGE); for (i = 0; i < num_sources; i++) { TpSegmentHeader *source_header = sources[i].reader->header; if (sources[i].reader->buffile == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("parallel build merge received a non-BufFile " "source"))); if (source_header->alive_bitset_offset != 0 || source_header->alive_count != 0) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("parallel build worker segment has an unexpected " "alive bitset"))); source_doc_bases[i] = (uint32)source_docs; if (pg_add_u64_overflow( source_docs, source_header->num_docs, &source_docs) || !tp_document_count_fits(source_docs)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u documents", TP_MAX_GROWABLE_CAPACITY))); if (pg_add_u64_overflow( source_tokens, source_header->total_tokens, &source_tokens)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: token count overflow"))); } if (source_docs != total_docs || source_tokens != total_tokens) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("parallel build worker totals do not match leader " "totals"), errdetail( "Worker segments contain %" PRIu64 " documents and %" PRIu64 " tokens; leader collected %" PRIu64 " documents and %" PRIu64 " tokens.", source_docs, source_tokens, total_docs, total_tokens))); string_offsets_file = BufFileCreateTemp(false); strings_file = BufFileCreateTemp(false); refs_file = BufFileCreateTemp(false); skips_file = BufFileCreateTemp(false); dict_entries_file = BufFileCreateTemp(false); term_refs = palloc_extended( num_sources * sizeof(TpTermSegmentRef), MCXT_ALLOC_HUGE); copy_buffer = palloc(TP_BUILD_MERGE_COPY_BUFFER_SIZE); /* * Merge dictionaries once, retaining only one term's exact source * references while spooling compact metadata streams. */ while (true) { int min_idx; const char *term; uint32 term_len; uint32 num_refs = 0; uint64 term_postings = 0; uint64 term_blocks; uint32 dict_offset; uint32 string_offset; min_idx = merge_find_min_source(sources, num_sources); if (min_idx < 0) break; if (num_terms >= TP_MAX_DICTIONARY_TERMS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment dictionary exceeds %u " "terms", TP_MAX_DICTIONARY_TERMS))); term = sources[min_idx].current_term; term_len = strlen(term); if (tp_string_pool_offset_overflows( string_pos, tp_string_pool_entry_size(term_len))) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: merged string pool exceeds 4 " "GiB"))); string_offset = (uint32)string_pos; dict_offset = (uint32)((uint64)num_terms * sizeof(TpDictEntry)); BufFileWrite( string_offsets_file, &string_offset, sizeof(string_offset)); BufFileWrite(strings_file, &term_len, sizeof(term_len)); BufFileWrite(strings_file, term, term_len); BufFileWrite(strings_file, &dict_offset, sizeof(dict_offset)); string_pos += tp_string_pool_entry_size(term_len); for (i = 0; i < num_sources; i++) { if (sources[i].exhausted || strcmp(sources[i].current_term, term) != 0) continue; term_refs[num_refs].segment_idx = (int)i; term_refs[num_refs].entry = sources[i].current_entry; if (pg_add_u64_overflow( term_postings, sources[i].current_entry.doc_freq, &term_postings)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: posting count overflow"))); num_refs++; } if (!tp_document_count_fits(term_postings)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: term exceeds %u postings", TP_MAX_GROWABLE_CAPACITY))); term_blocks = tp_posting_block_count(term_postings); if (term_blocks > TP_MAX_GROWABLE_CAPACITY - skip_entries) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u posting blocks", TP_MAX_GROWABLE_CAPACITY))); skip_entries += term_blocks; BufFileWrite(refs_file, &num_refs, sizeof(num_refs)); BufFileWrite( refs_file, term_refs, num_refs * sizeof(TpTermSegmentRef)); for (i = 0; i < num_refs; i++) merge_source_advance(&sources[term_refs[i].segment_idx]); num_terms++; CHECK_FOR_INTERRUPTS(); } memset(&header, 0, sizeof(header)); header.magic = TP_SEGMENT_MAGIC; header.version = TP_SEGMENT_FORMAT_VERSION; header.created_at = GetCurrentTimestamp(); header.num_terms = num_terms; header.level = 0; header.next_segment = InvalidBlockNumber; header.num_docs = (uint32)total_docs; header.total_tokens = total_tokens; header.page_index = InvalidBlockNumber; merge_sink_write(sink, &header, sizeof(header)); header.dictionary_offset = sink->current_offset; memset(&dict, 0, sizeof(dict)); dict.num_terms = num_terms; merge_sink_write(sink, &dict, offsetof(TpDictionary, string_offsets)); build_merge_copy_file( string_offsets_file, (uint64)num_terms * sizeof(uint32), sink, copy_buffer); header.strings_offset = sink->current_offset; build_merge_copy_file(strings_file, string_pos, sink, copy_buffer); header.entries_offset = sink->current_offset; build_merge_write_zeros(sink, tp_dictionary_size(num_terms), copy_buffer); header.postings_offset = sink->current_offset; build_merge_rewind(refs_file); for (i = 0; i < num_terms; i++) { TpBlockPosting block[TP_BLOCK_SIZE]; TpDictEntry entry; uint32 num_refs; uint32 block_count = 0; uint32 doc_count = 0; uint32 in_block = 0; uint64 expected_docs = 0; uint32 ref_idx; int previous_source = -1; BufFileReadExact(refs_file, &num_refs, sizeof(num_refs)); if (num_refs == 0 || num_refs > num_sources) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid parallel build term reference count %u", num_refs))); BufFileReadExact( refs_file, term_refs, num_refs * sizeof(TpTermSegmentRef)); entry.skip_index_offset = skip_bytes_written; for (ref_idx = 0; ref_idx < num_refs; ref_idx++) { TpTermSegmentRef *ref = &term_refs[ref_idx]; TpMergeSource *source; TpPostingMergeSource posting_source; if (ref->segment_idx <= previous_source || ref->segment_idx < 0 || (uint32)ref->segment_idx >= num_sources) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid parallel build term source index"))); previous_source = ref->segment_idx; source = &sources[ref->segment_idx]; expected_docs += ref->entry.doc_freq; posting_source_init_fast( &posting_source, source->reader, &ref->entry); while (!posting_source.exhausted) { TpBlockPosting *input = &posting_source.block_postings [posting_source.current_in_block]; uint64 output_doc_id; if (input->doc_id >= source->reader->header->num_docs) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("parallel build posting document ID %u " "exceeds source document count %u", input->doc_id, source->reader->header->num_docs))); output_doc_id = (uint64)source_doc_bases[ref->segment_idx] + input->doc_id; if (output_doc_id >= total_docs) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("parallel build posting document ID " "exceeds merged document count"))); block[in_block].doc_id = (uint32)output_doc_id; block[in_block].frequency = input->frequency; block[in_block].fieldnorm = input->fieldnorm; block[in_block].reserved = 0; in_block++; doc_count++; posting_source_advance_fast(&posting_source); if (in_block == TP_BLOCK_SIZE) { build_merge_flush_block(sink, skips_file, block, in_block); in_block = 0; block_count++; } } posting_source_free(&posting_source); } if (in_block > 0) { build_merge_flush_block(sink, skips_file, block, in_block); block_count++; } if (doc_count != expected_docs) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("parallel build posting count mismatch"), errdetail( "Expected %" PRIu64 " postings but read %u.", expected_docs, doc_count))); entry.block_count = block_count; entry.doc_freq = doc_count; BufFileWrite(dict_entries_file, &entry, sizeof(entry)); skip_bytes_written += (uint64)block_count * sizeof(TpSkipEntry); if ((i % 1000) == 0) CHECK_FOR_INTERRUPTS(); } if (skip_bytes_written != skip_entries * sizeof(TpSkipEntry)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("parallel build skip count mismatch"))); header.skip_index_offset = sink->current_offset; build_merge_copy_file( skips_file, skip_entries * sizeof(TpSkipEntry), sink, copy_buffer); header.fieldnorm_offset = sink->current_offset; for (i = 0; i < num_sources; i++) build_merge_copy_source( sources[i].reader, sources[i].reader->header->fieldnorm_offset, (uint64)sources[i].reader->header->num_docs * sizeof(uint8), sink, copy_buffer); header.ctid_pages_offset = sink->current_offset; for (i = 0; i < num_sources; i++) build_merge_copy_source( sources[i].reader, sources[i].reader->header->ctid_pages_offset, (uint64)sources[i].reader->header->num_docs * sizeof(BlockNumber), sink, copy_buffer); header.ctid_offsets_offset = sink->current_offset; for (i = 0; i < num_sources; i++) build_merge_copy_source( sources[i].reader, sources[i].reader->header->ctid_offsets_offset, (uint64)sources[i].reader->header->num_docs * sizeof(OffsetNumber), sink, copy_buffer); header.alive_bitset_offset = sink->current_offset; header.alive_count = (uint32)total_docs; if (total_docs > 0) build_merge_write_alive(sink, (uint32)total_docs, copy_buffer); header.data_size = sink->current_offset; tp_segment_writer_flush(&sink->writer); sink->writer.buffer_pos = SizeOfPageHeaderData; build_merge_rewind(dict_entries_file); for (i = 0; i < num_terms;) { TpDictEntry entries[TP_BUILD_MERGE_DICT_CHUNK]; uint32 count = Min(TP_BUILD_MERGE_DICT_CHUNK, num_terms - i); uint32 j; BufFileReadExact( dict_entries_file, entries, count * sizeof(TpDictEntry)); for (j = 0; j < count; j++) { if (pg_add_u64_overflow( header.skip_index_offset, entries[j].skip_index_offset, &entries[j].skip_index_offset)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: skip index offset overflow"))); } merge_sink_write_at( sink, header.entries_offset + (uint64)i * sizeof(TpDictEntry), entries, count * sizeof(TpDictEntry)); i += count; } merge_sink_finish(sink, &header); BufFileClose(string_offsets_file); BufFileClose(strings_file); BufFileClose(refs_file); BufFileClose(skips_file); BufFileClose(dict_entries_file); pfree(copy_buffer); pfree(term_refs); pfree(source_doc_bases); } timescale-pg_textsearch-b63fe58/src/access/build_merge.h000066400000000000000000000007141525762047700234420ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build_merge.h - Streaming final merge for parallel index builds */ #pragma once #include #include "segment/merge.h" #include "segment/merge_internal.h" extern void tp_write_parallel_build_merge( TpMergeSink *sink, TpMergeSource *sources, uint32 num_sources, uint64 total_docs, uint64 total_tokens); timescale-pg_textsearch-b63fe58/src/access/build_parallel.c000066400000000000000000000604011525762047700241310ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build_parallel.c - Parallel index build with leader-only merge * * Architecture: * - Phase 1: Each worker scans a heap partition, builds a local * TpBuildContext, flushes L0 segments to BufFile. Workers * report segment info (offsets and sizes). * - Barrier: Leader opens all worker BufFiles, performs a single * N-way merge of ALL segments directly to paged storage. * - Leader links the single merged segment, updates metapage, * then wakes workers to exit. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "access/build_context.h" #include "access/build_merge.h" #include "access/build_parallel.h" #include "constants.h" #include "index/metapage.h" #include "segment/io.h" #include "segment/merge.h" #include "segment/merge_internal.h" #include "segment/pagemapper.h" #include "segment/segment.h" #include "types/array.h" /* ---------------------------------------------------------------- * Worker-local segment tracking * ---------------------------------------------------------------- */ typedef struct WorkerSegmentEntry { uint64 offset; /* BufFile offset */ uint64 data_size; /* Segment bytes */ } WorkerSegmentEntry; typedef struct WorkerSegmentTracker { WorkerSegmentEntry *entries; uint32 count; uint32 capacity; uint64 buffile_end; /* logical end of BufFile */ } WorkerSegmentTracker; static void tracker_init(WorkerSegmentTracker *tracker) { tracker->capacity = 32; tracker->count = 0; tracker->buffile_end = 0; tracker->entries = palloc(tracker->capacity * sizeof(WorkerSegmentEntry)); } static void tracker_add_segment( WorkerSegmentTracker *tracker, uint64 offset, uint64 data_size) { if (tracker->count >= tracker->capacity) { tracker->capacity = tp_grow_capacity(tracker->capacity, 32, "worker segments"); tracker->entries = repalloc( tracker->entries, mul_size(tracker->capacity, sizeof(WorkerSegmentEntry))); } tracker->entries[tracker->count].offset = offset; tracker->entries[tracker->count].data_size = data_size; tracker->count++; } static void tracker_destroy(WorkerSegmentTracker *tracker) { if (tracker->entries) pfree(tracker->entries); } /* ---------------------------------------------------------------- * Shared memory estimation and initialization * ---------------------------------------------------------------- */ Size tp_parallel_build_estimate_shmem( Relation heap, Snapshot snapshot, int nworkers) { Size size; (void)heap; (void)snapshot; /* Base shared structure (includes per-worker block ranges) */ size = MAXALIGN(sizeof(TpParallelBuildShared)); /* Per-worker result array */ size = add_size(size, MAXALIGN(sizeof(TpParallelWorkerResult) * nworkers)); return size; } static void tp_init_parallel_shared( TpParallelBuildShared *shared, Relation heap, Relation index, Oid text_config_oid, double k1, double b, bool is_text_array, int nworkers) { TpParallelWorkerResult *results; int i; memset(shared, 0, sizeof(TpParallelBuildShared)); /* Immutable configuration */ shared->heaprelid = RelationGetRelid(heap); shared->indexrelid = RelationGetRelid(index); shared->text_config_oid = text_config_oid; shared->k1 = k1; shared->b = b; shared->is_text_array = is_text_array; shared->nworkers = nworkers; /* Coordination */ ConditionVariableInit(&shared->all_done_cv); pg_atomic_init_u32(&shared->workers_done, 0); pg_atomic_init_u64(&shared->tuples_done, 0); /* TID range scan coordination */ shared->nworkers_launched = 0; pg_atomic_init_u32(&shared->scan_ready, 0); /* Phase coordination */ pg_atomic_init_u32(&shared->phase1_done, 0); ConditionVariableInit(&shared->phase2_cv); pg_atomic_init_u32(&shared->phase2_ready, 0); /* Initialize per-worker results */ results = TpParallelWorkerResults(shared); for (i = 0; i < nworkers; i++) memset(&results[i], 0, sizeof(TpParallelWorkerResult)); } /* ---------------------------------------------------------------- * Worker entry point * ---------------------------------------------------------------- */ PGDLLEXPORT void tp_parallel_build_worker_main(dsm_segment *seg, shm_toc *toc) { TpParallelBuildShared *shared; TpParallelWorkerResult *my_result; Relation heap; Relation index; IndexInfo *indexInfo; EState *estate; ExprContext *econtext; Datum idx_values[INDEX_MAX_KEYS]; bool idx_isnull[INDEX_MAX_KEYS]; TableScanDesc scan; Snapshot snap; TupleTableSlot *slot; TpBuildContext *build_ctx; MemoryContext build_tmpctx; MemoryContext oldctx; Size budget; int worker_id; BufFile *buffile; char file_name[64]; WorkerSegmentTracker tracker; /* Attach to shared memory */ shared = (TpParallelBuildShared *) shm_toc_lookup(toc, TP_PARALLEL_KEY_SHARED, false); worker_id = ParallelWorkerNumber; my_result = &TpParallelWorkerResults(shared)[worker_id]; /* Open heap and index relations */ heap = table_open(shared->heaprelid, AccessShareLock); index = index_open(shared->indexrelid, AccessExclusiveLock); /* Set up expression evaluation for index */ indexInfo = BuildIndexInfo(index); estate = CreateExecutorState(); econtext = GetPerTupleExprContext(estate); if (indexInfo->ii_Predicate != NIL) indexInfo->ii_PredicateState = ExecPrepareQual(indexInfo->ii_Predicate, estate); /* Attach to SharedFileSet and create worker's BufFile */ SharedFileSetAttach(&shared->fileset, seg); snprintf(file_name, sizeof(file_name), "tp_worker_%d", worker_id); buffile = BufFileCreateFileSet(&shared->fileset.fs, file_name); /* * Wait for leader to set up per-worker block ranges. * This is a brief spin (microseconds) after launch. */ while (pg_atomic_read_u32(&shared->scan_ready) == 0) pg_usleep(100); pg_read_barrier(); /* * Open a TID range scan limited to this worker's block range. * Each worker scans a contiguous, non-overlapping range of * heap blocks, ensuring disjoint CTIDs across workers. */ snap = GetTransactionSnapshot(); #if PG_VERSION_NUM >= 180000 snap = RegisterSnapshot(snap); #endif slot = table_slot_create(heap, NULL); { BlockNumber start_blk = shared->worker_start_block[worker_id]; BlockNumber end_blk = shared->worker_end_block[worker_id]; if (start_blk < end_blk) { ItemPointerData min_tid, max_tid; uint32 flags = SO_TYPE_TIDRANGESCAN | SO_ALLOW_PAGEMODE | SO_ALLOW_STRAT; ItemPointerSet(&min_tid, start_blk, FirstOffsetNumber); ItemPointerSet(&max_tid, end_blk - 1, MaxOffsetNumber); /* * table_beginscan_tidrange() omits SO_ALLOW_STRAT. These * one-pass build scans should reuse a bulk-read ring instead * of filling shared_buffers alongside the worker batches. * Keep synchronized scanning disabled for disjoint ranges. */ #if PG_VERSION_NUM >= 190000 scan = table_beginscan_common(heap, snap, 0, NULL, NULL, flags, 0); #else scan = heap->rd_tableam ->scan_begin(heap, snap, 0, NULL, NULL, flags); #endif heap->rd_tableam->scan_set_tidrange(scan, &min_tid, &max_tid); } else { /* Empty range: no blocks assigned to this worker */ scan = NULL; } } /* * The configured maintenance_work_mem is the total worker batch budget. * Divide it by the workers PostgreSQL actually launched; never multiply * a small setting through a per-worker floor. */ budget = (Size)maintenance_work_mem * 1024L / shared->nworkers_launched; /* * Clamp to the arena's addressable capacity: with few workers * launched, maintenance_work_mem / nworkers can still exceed the * 4 GiB arena, so the worker would overrun instead of spilling * (issue #433). */ budget = tp_arena_clamp_budget(budget); build_ctx = tp_build_context_create(budget); build_ctx->account_full_allocation = true; { Size minimum_budget = tp_build_context_minimum_budget(build_ctx); if (budget < minimum_budget) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("parallel index build memory budget is too small"), errdetail( "maintenance_work_mem provides %zu bytes per " "worker for %d workers, but at least %zu bytes " "are required for the initial batch.", budget, shared->nworkers_launched, minimum_budget), errhint("Increase maintenance_work_mem or reduce " "max_parallel_maintenance_workers."))); } tracker_init(&tracker); build_tmpctx = AllocSetContextCreate( CurrentMemoryContext, "parallel build per-doc temp", ALLOCSET_DEFAULT_SIZES); /* Process tuples from this worker's block range */ while (scan != NULL && table_scan_getnextslot_tidrange(scan, ForwardScanDirection, slot)) { text *document_text; ItemPointer ctid; char **terms; int32 *frequencies; int term_count; int doc_length; /* Evaluate index expression (or extract plain column) */ econtext->ecxt_scantuple = slot; FormIndexDatum(indexInfo, slot, estate, idx_values, idx_isnull); if (idx_isnull[0]) goto next_tuple; /* Check partial index predicate */ if (indexInfo->ii_Predicate != NIL && !ExecQual(indexInfo->ii_PredicateState, econtext)) goto next_tuple; ctid = &slot->tts_tid; if (!ItemPointerIsValid(ctid)) goto next_tuple; /* * Switch to temporary context before text extraction * so flatten allocations are freed per-document. */ oldctx = MemoryContextSwitchTo(build_tmpctx); if (shared->is_text_array) document_text = tp_flatten_text_array(idx_values[0]); else document_text = DatumGetTextPP(idx_values[0]); doc_length = tp_tokenize_text( document_text, shared->text_config_oid, &terms, &frequencies, &term_count); MemoryContextSwitchTo(oldctx); tp_build_context_add_document( build_ctx, terms, frequencies, term_count, doc_length, ctid); /* Reset per-doc context */ MemoryContextReset(build_tmpctx); /* Budget-based flush to BufFile */ if (tp_build_context_should_flush(build_ctx)) { uint64 data_size; uint64 seg_offset; my_result->total_docs += build_ctx->num_docs; my_result->total_len += build_ctx->total_len; /* L0 segment starts at current end of BufFile */ seg_offset = tracker.buffile_end; { int fileno; off_t file_offset; tp_buffile_decompose_offset(seg_offset, &fileno, &file_offset); BufFileSeek(buffile, fileno, file_offset, SEEK_SET); } data_size = tp_write_segment_to_buffile(build_ctx, buffile); /* Track this L0 segment */ tracker_add_segment(&tracker, seg_offset, data_size); tracker.buffile_end = seg_offset + data_size; tp_build_context_reset(build_ctx); } next_tuple: ResetExprContext(econtext); my_result->tuples_scanned++; if (my_result->tuples_scanned % TP_PROGRESS_REPORT_INTERVAL == 0) { pg_atomic_add_fetch_u64( &shared->tuples_done, TP_PROGRESS_REPORT_INTERVAL); CHECK_FOR_INTERRUPTS(); } } /* Flush remaining data */ if (build_ctx->num_docs > 0) { uint64 data_size; uint64 seg_offset; my_result->total_docs += build_ctx->num_docs; my_result->total_len += build_ctx->total_len; seg_offset = tracker.buffile_end; { int fileno; off_t file_offset; tp_buffile_decompose_offset(seg_offset, &fileno, &file_offset); BufFileSeek(buffile, fileno, file_offset, SEEK_SET); } data_size = tp_write_segment_to_buffile(build_ctx, buffile); tracker_add_segment(&tracker, seg_offset, data_size); tracker.buffile_end = seg_offset + data_size; } /* Append the segment directory instead of bounding it by the DSM size. */ { int fileno; off_t file_offset; tp_buffile_decompose_offset( tracker.buffile_end, &fileno, &file_offset); if (BufFileSeek(buffile, fileno, file_offset, SEEK_SET) != 0) elog(ERROR, "could not seek parallel worker segment directory"); BufFileWrite( buffile, tracker.entries, mul_size(tracker.count, sizeof(WorkerSegmentEntry))); my_result->segment_directory_offset = tracker.buffile_end; my_result->final_segment_count = tracker.count; } /* Export BufFile so leader can reopen */ BufFileExportFileSet(buffile); BufFileClose(buffile); /* Cleanup Phase 1 resources */ FreeExecutorState(estate); tp_build_context_destroy(build_ctx); tracker_destroy(&tracker); MemoryContextDelete(build_tmpctx); if (scan != NULL) table_endscan(scan); #if PG_VERSION_NUM >= 180000 UnregisterSnapshot(snap); #endif ExecDropSingleTupleTableSlot(slot); /* Signal Phase 1 done, wait for leader to finish merge */ pg_atomic_fetch_add_u32(&shared->phase1_done, 1); ConditionVariableBroadcast(&shared->all_done_cv); ConditionVariablePrepareToSleep(&shared->phase2_cv); while (pg_atomic_read_u32(&shared->phase2_ready) == 0) ConditionVariableSleep(&shared->phase2_cv, PG_WAIT_EXTENSION); ConditionVariableCancelSleep(); /* No Phase 2 work — leader did the merge. Just exit. */ index_close(index, AccessExclusiveLock); table_close(heap, AccessShareLock); pg_atomic_fetch_add_u32(&shared->workers_done, 1); ConditionVariableSignal(&shared->all_done_cv); } /* ---------------------------------------------------------------- * Leader: main parallel build entry point * ---------------------------------------------------------------- */ IndexBuildResult * tp_build_parallel( Relation heap, Relation index, IndexInfo *indexInfo, Oid text_config_oid, double k1, double b, bool is_text_array, int nworkers) { IndexBuildResult *result; ParallelContext *pcxt; TpParallelBuildShared *shared; Snapshot snapshot; Size shmem_size; int launched; uint64 total_docs = 0; uint64 total_len = 0; /* Workers reconstruct IndexInfo via BuildIndexInfo() */ (void)indexInfo; /* Ensure reasonable number of workers */ if (nworkers > TP_MAX_PARALLEL_WORKERS) nworkers = TP_MAX_PARALLEL_WORKERS; /* Report loading phase */ pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_LOADING); /* Report estimated tuple count for progress tracking */ { double reltuples = heap->rd_rel->reltuples; int64 tuples_est = (reltuples > 0) ? (int64)reltuples : 0; pgstat_progress_update_param( PROGRESS_CREATEIDX_TUPLES_TOTAL, tuples_est); } /* Get snapshot for parallel scan */ snapshot = GetTransactionSnapshot(); #if PG_VERSION_NUM >= 180000 /* PG18: Must register the snapshot for index builds */ snapshot = RegisterSnapshot(snapshot); #endif /* Calculate shared memory size */ shmem_size = tp_parallel_build_estimate_shmem(heap, snapshot, nworkers); /* Enter parallel mode and create context */ EnterParallelMode(); pcxt = CreateParallelContext( "pg_textsearch", "tp_parallel_build_worker_main", nworkers); /* Estimate and allocate shared memory */ shm_toc_estimate_chunk(&pcxt->estimator, shmem_size); shm_toc_estimate_keys(&pcxt->estimator, 1); InitializeParallelDSM(pcxt); /* Allocate and initialize shared state */ shared = (TpParallelBuildShared *)shm_toc_allocate(pcxt->toc, shmem_size); tp_init_parallel_shared( shared, heap, index, text_config_oid, k1, b, is_text_array, nworkers); /* Initialize SharedFileSet for worker temp files */ SharedFileSetInit(&shared->fileset, pcxt->seg); /* Insert shared state into TOC */ shm_toc_insert(pcxt->toc, TP_PARALLEL_KEY_SHARED, shared); /* Launch workers */ LaunchParallelWorkers(pcxt); launched = pcxt->nworkers_launched; if (launched == 0) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_RESOURCES), errmsg("parallel index build: could not launch " "any workers"), errhint("Increase max_worker_processes or " "reduce " "max_parallel_maintenance_workers."))); ereport(NOTICE, (errmsg("parallel index build: launched %d of %d " "requested workers", launched, nworkers))); /* * Compute per-worker block ranges for disjoint TID scans. * Must be done after launch so we know `launched`. Workers * spin on scan_ready until ranges are set. */ { BlockNumber nblocks = RelationGetNumberOfBlocks(heap); BlockNumber blocks_per_worker = nblocks / launched; BlockNumber remainder = nblocks % launched; BlockNumber cursor = 0; int i; for (i = 0; i < launched; i++) { BlockNumber count = blocks_per_worker + (i < (int)remainder ? 1 : 0); shared->worker_start_block[i] = cursor; shared->worker_end_block[i] = cursor + count; cursor += count; } shared->nworkers_launched = launched; pg_write_barrier(); pg_atomic_write_u32(&shared->scan_ready, 1); } /* * Phase 1 wait: wait for all workers to finish BufFile phase. * Workers signal phase1_done and then block on phase2_cv. */ ConditionVariablePrepareToSleep(&shared->all_done_cv); while (pg_atomic_read_u32(&shared->phase1_done) < (uint32)launched) { pgstat_progress_update_param( PROGRESS_CREATEIDX_TUPLES_DONE, (int64)pg_atomic_read_u64(&shared->tuples_done)); ConditionVariableTimedSleep( &shared->all_done_cv, 1000 /* ms */, PG_WAIT_EXTENSION); } ConditionVariableCancelSleep(); /* Collect corpus stats from all workers */ { TpParallelWorkerResult *results; int i; results = TpParallelWorkerResults(shared); for (i = 0; i < launched; i++) { if (pg_add_u64_overflow( total_docs, results[i].total_docs, &total_docs)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: document count overflow"))); total_len += results[i].total_len; } if (!tp_document_count_fits(total_docs)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u documents", TP_MAX_GROWABLE_CAPACITY))); } /* Report final tuple count */ pgstat_progress_update_param( PROGRESS_CREATEIDX_TUPLES_DONE, (int64)total_docs); /* Report writing phase */ pgstat_progress_update_param( PROGRESS_CREATEIDX_SUBPHASE, TP_PHASE_WRITING); /* * Leader-only merge: open all worker BufFile segments and * perform a single N-way merge directly to paged storage. */ { TpParallelWorkerResult *results = TpParallelWorkerResults(shared); BufFile **open_files; bool *file_opened; TpSegmentReader **readers; TpMergeSource *sources; uint32 total_segments = 0; uint32 num_sources = 0; int w; uint32 i; MemoryContext merge_ctx; MemoryContext old_ctx; /* Count total segments across all workers */ for (w = 0; w < launched; w++) { if (pg_add_u32_overflow( total_segments, results[w].final_segment_count, &total_segments)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("parallel index build: segment count " "overflow"))); } /* Open worker BufFiles and create segment readers */ open_files = palloc0(sizeof(BufFile *) * launched); file_opened = palloc0(sizeof(bool) * launched); readers = palloc0(mul_size(sizeof(TpSegmentReader *), total_segments)); sources = palloc0(mul_size(sizeof(TpMergeSource), total_segments)); { uint32 reader_idx = 0; for (w = 0; w < launched; w++) { uint32 s; char fname[64]; int fileno; off_t file_offset; Size directory_size; WorkerSegmentEntry *segments; if (results[w].final_segment_count == 0) continue; snprintf(fname, sizeof(fname), "tp_worker_%d", w); open_files[w] = BufFileOpenFileSet( &shared->fileset.fs, fname, O_RDONLY, false); file_opened[w] = true; directory_size = mul_size( results[w].final_segment_count, sizeof(WorkerSegmentEntry)); segments = palloc(directory_size); tp_buffile_decompose_offset( results[w].segment_directory_offset, &fileno, &file_offset); if (BufFileSeek( open_files[w], fileno, file_offset, SEEK_SET) != 0) elog(ERROR, "could not seek parallel worker segment directory"); BufFileReadExact(open_files[w], segments, directory_size); for (s = 0; s < results[w].final_segment_count; s++) { readers[reader_idx] = tp_segment_open_from_buffile( open_files[w], segments[s].offset); if (!readers[reader_idx]) { reader_idx++; continue; } if (merge_source_init_from_reader( &sources[num_sources], readers[reader_idx])) { /* * Source now owns this reader; clear * slot so cleanup won't double-close. */ readers[reader_idx] = NULL; num_sources++; } reader_idx++; } pfree(segments); } } if (num_sources > 0) { TpMergeSink sink; BlockNumber segment_root; merge_ctx = AllocSetContextCreate( CurrentMemoryContext, "Leader Merge", ALLOCSET_DEFAULT_SIZES); old_ctx = MemoryContextSwitchTo(merge_ctx); /* Write single merged segment to index pages */ merge_sink_init_pages(&sink, index); if (sink.writer.pages_allocated == 0) elog(ERROR, "merge: failed to allocate segment pages"); segment_root = sink.writer.pages[0]; tp_write_parallel_build_merge( &sink, sources, num_sources, total_docs, total_len); MemoryContextSwitchTo(old_ctx); /* * Flush dirty buffers before updating the metapage, * ensuring merged segment data is durable first. */ FlushRelationBuffers(index); /* Link as L0 head in metapage */ { Buffer metabuf; GenericXLogState *state; Page metapage; TpIndexMetaPage metap; metabuf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); metapage = GenericXLogRegisterBuffer(state, metabuf, 0); metap = (TpIndexMetaPage)PageGetContents(metapage); metap->level_heads[0] = segment_root; metap->level_counts[0] = 1; metap->total_docs = total_docs; metap->total_len = total_len; GenericXLogFinish(state); UnlockReleaseBuffer(metabuf); } if (sink.writer.pages) pfree(sink.writer.pages); if (sink.page_index_pages) pfree(sink.page_index_pages); MemoryContextDelete(merge_ctx); } else { /* No segments at all — just update metapage stats */ Buffer metabuf; GenericXLogState *state; Page metapage; TpIndexMetaPage metap; metabuf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); metapage = GenericXLogRegisterBuffer(state, metabuf, 0); metap = (TpIndexMetaPage)PageGetContents(metapage); metap->total_docs = total_docs; metap->total_len = total_len; GenericXLogFinish(state); UnlockReleaseBuffer(metabuf); } /* * Close merge sources. merge_source_close() frees term * data and closes the underlying reader, so we only need * to close readers that were not successfully wrapped in * a source (e.g. empty segments that were skipped). */ for (i = 0; i < num_sources; i++) merge_source_close(&sources[i]); for (i = 0; i < total_segments; i++) { if (readers[i]) tp_segment_close(readers[i]); } for (w = 0; w < launched; w++) { if (file_opened[w]) BufFileClose(open_files[w]); } pfree(readers); pfree(sources); pfree(open_files); pfree(file_opened); } /* Wake workers so they can exit */ pg_atomic_write_u32(&shared->phase2_ready, 1); ConditionVariableBroadcast(&shared->phase2_cv); /* Wait for all workers to complete */ ConditionVariablePrepareToSleep(&shared->all_done_cv); while (pg_atomic_read_u32(&shared->workers_done) < (uint32)launched) { ConditionVariableTimedSleep( &shared->all_done_cv, 1000 /* ms */, PG_WAIT_EXTENSION); } ConditionVariableCancelSleep(); WaitForParallelWorkersToFinish(pcxt); /* Build result */ result = palloc0(sizeof(IndexBuildResult)); result->heap_tuples = (double)total_docs; result->index_tuples = (double)total_docs; /* Cleanup */ DestroyParallelContext(pcxt); ExitParallelMode(); #if PG_VERSION_NUM >= 180000 UnregisterSnapshot(snapshot); #endif return result; } timescale-pg_textsearch-b63fe58/src/access/build_parallel.h000066400000000000000000000071561525762047700241460ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * build_parallel.h - Parallel index build structures * * Architecture (leader-only merge): * - Phase 1: Workers scan heap, flush L0 segments to BufFile. * Workers report segment offsets/sizes and signal phase1_done. * - Leader: Opens all worker BufFiles, performs single N-way merge * directly to paged storage. Updates metapage. * - Leader signals phase2_ready; workers wake and exit. */ #pragma once #include #include #include #include #include #include /* * Maximum parallel workers for index build. * Should not exceed max_parallel_maintenance_workers. */ #define TP_MAX_PARALLEL_WORKERS 32 /* * Shared memory keys for parallel build TOC */ #define TP_PARALLEL_KEY_SHARED UINT64CONST(0xB175DA7A00000001) /* * Per-worker result reported back to leader via shared memory. */ typedef struct TpParallelWorkerResult { /* Corpus statistics */ uint64 total_docs; /* Documents indexed */ uint64 total_len; /* Sum of document lengths */ uint64 tuples_scanned; /* Segment directory stored at the end of the worker's BufFile */ uint32 final_segment_count; uint64 segment_directory_offset; } TpParallelWorkerResult; /* * Shared state for parallel index build * * Stored in DSM segment, accessible to all workers and leader. */ typedef struct TpParallelBuildShared { /* Immutable configuration (set before workers launch) */ Oid heaprelid; /* Heap relation OID */ Oid indexrelid; /* Index relation OID */ Oid text_config_oid; /* Text search config OID */ double k1; /* BM25 k1 parameter */ double b; /* BM25 b parameter */ bool is_text_array; /* Indexed column is text[] */ int32 nworkers; /* Workers requested */ int32 nworkers_launched; /* Actual workers launched */ /* Per-worker heap block ranges for disjoint TID scan */ BlockNumber worker_start_block[TP_MAX_PARALLEL_WORKERS]; BlockNumber worker_end_block[TP_MAX_PARALLEL_WORKERS]; pg_atomic_uint32 scan_ready; /* 1 when block ranges set */ /* Temp files for worker segments */ SharedFileSet fileset; /* Coordination */ ConditionVariable all_done_cv; /* Workers signal when done */ pg_atomic_uint32 workers_done; /* Phase coordination */ pg_atomic_uint32 phase1_done; /* Workers done with BufFile */ ConditionVariable phase2_cv; /* Leader signals merge done */ pg_atomic_uint32 phase2_ready; /* 1 when leader merge done */ /* Progress reporting */ pg_atomic_uint64 tuples_done; /* * Per-worker results (variable-length array follows). * Workers write their own slot; leader reads after Phase 1. */ } TpParallelBuildShared; /* * Get pointer to worker results array */ static inline TpParallelWorkerResult * TpParallelWorkerResults(TpParallelBuildShared *shared) { return (TpParallelWorkerResult *)((char *)shared + MAXALIGN(sizeof(TpParallelBuildShared))); } /* * Function declarations */ /* Main parallel build entry point */ extern struct IndexBuildResult *tp_build_parallel( Relation heap, Relation index, struct IndexInfo *indexInfo, Oid text_config_oid, double k1, double b, bool is_text_array, int nworkers); /* Worker entry point (called by parallel infrastructure) */ extern PGDLLEXPORT void tp_parallel_build_worker_main(dsm_segment *seg, shm_toc *toc); /* Estimate shared memory size needed for parallel build */ extern Size tp_parallel_build_estimate_shmem( Relation heap, Snapshot snapshot, int nworkers); timescale-pg_textsearch-b63fe58/src/access/compaction_api.c000066400000000000000000000257351525762047700241560ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compaction_api.c - BM25 index compaction inspection and control */ #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "constants.h" #include "index/compaction_request.h" #include "index/metapage.h" #include "index/state.h" #include "segment/compaction.h" typedef struct TpCompactionTarget { Oid index_oid; Oid database_oid; Oid tablespace_oid; RelFileNumber relfilenumber; Oid owner_oid; } TpCompactionTarget; /* * Open a bm25 index by OID, validating that it is in fact a bm25 * index and (optionally) that the caller owns it. */ static Relation tp_open_bm25_index(Oid indexoid, LOCKMODE lockmode, bool need_owner) { Relation index_rel; /* * Reject nonowners before queuing for a heavyweight lock. Recheck after * opening because ALTER OWNER can complete while this caller waits. */ if (need_owner && !object_ownercheck(RelationRelationId, indexoid, GetUserId())) { char *relname = get_rel_name(indexoid); if (relname == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation with OID %u does not exist", indexoid))); aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, relname); } index_rel = relation_open(indexoid, lockmode); if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, lockmode); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a bm25 index", relname))); } if (index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, lockmode); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is a partitioned bm25 index", relname), errhint("Use a physical partition index instead."))); } if (need_owner && !object_ownercheck(RelationRelationId, indexoid, GetUserId())) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, lockmode); aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, relname); } return index_rel; } /* * Return NULL when a captured worker target no longer names the same * physical bm25 index. */ static Relation tp_open_current_bm25_target( const TpCompactionTarget *target, LOCKMODE lockmode, bool need_owner, bool need_background) { Relation index_rel; HeapTuple tuple; Form_pg_class classform; if (target->database_oid != MyDatabaseId) return NULL; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(target->index_oid)); if (!HeapTupleIsValid(tuple)) return NULL; classform = (Form_pg_class)GETSTRUCT(tuple); if (classform->relowner != target->owner_oid) { ReleaseSysCache(tuple); return NULL; } /* * Authorize against the captured owner so OID reuse cannot redirect the * lockless precheck to a replacement relation. */ if (need_owner && !has_privs_of_role(GetUserId(), target->owner_oid)) { char *relname = pstrdup(NameStr(classform->relname)); ReleaseSysCache(tuple); aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, relname); } ReleaseSysCache(tuple); index_rel = try_relation_open(target->index_oid, lockmode); if (index_rel == NULL) return NULL; if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || index_rel->rd_rel->relkind != RELKIND_INDEX || index_rel->rd_rel->relpersistence == RELPERSISTENCE_TEMP || index_rel->rd_locator.dbOid != target->database_oid || index_rel->rd_locator.spcOid != target->tablespace_oid || index_rel->rd_locator.relNumber != target->relfilenumber || index_rel->rd_rel->relowner != target->owner_oid || (need_background && tp_index_compaction_mode(index_rel) != TP_COMPACTION_BACKGROUND)) { relation_close(index_rel, lockmode); return NULL; } if (need_owner && !object_ownercheck(RelationRelationId, target->index_oid, GetUserId())) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, lockmode); aclcheck_error(ACLCHECK_NOT_OWNER, OBJECT_INDEX, relname); } return index_rel; } PG_FUNCTION_INFO_V1(tp_level_counts); Datum tp_level_counts(PG_FUNCTION_ARGS) { Oid indexoid = PG_GETARG_OID(0); Relation index_rel; TpIndexMetaPageData *metap; Datum elems[TP_MAX_LEVELS]; ArrayType *result; int i; index_rel = tp_open_bm25_index(indexoid, AccessShareLock, false); metap = tp_get_metapage(index_rel); for (i = 0; i < TP_MAX_LEVELS; i++) elems[i] = Int32GetDatum((int32)metap->level_counts[i]); pfree(metap); relation_close(index_rel, AccessShareLock); result = construct_array( elems, TP_MAX_LEVELS, INT4OID, sizeof(int32), true, TYPALIGN_INT); PG_RETURN_ARRAYTYPE_P(result); } PG_FUNCTION_INFO_V1(tp_needs_compaction); Datum tp_needs_compaction(PG_FUNCTION_ARGS) { Oid indexoid = PG_GETARG_OID(0); Relation index_rel; TpLocalIndexState *index_state; bool available; index_rel = tp_open_bm25_index(indexoid, AccessShareLock, false); index_state = tp_get_local_index_state(indexoid); if (index_state == NULL) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", relname))); } PG_TRY(); { available = tp_compaction_pass_available(index_state, index_rel); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); relation_close(index_rel, AccessShareLock); } PG_END_TRY(); PG_RETURN_BOOL(available); } PG_FUNCTION_INFO_V1(tp_compact_index); Datum tp_compact_index(PG_FUNCTION_ARGS) { Oid indexoid = PG_GETARG_OID(0); Relation index_rel; TpLocalIndexState *index_state; volatile bool pass_ran; if (RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("cannot compact a bm25 index during recovery"))); index_rel = tp_open_bm25_index(indexoid, RowExclusiveLock, true); if (!RelationUsesLocalBuffers(index_rel)) PreventCommandIfReadOnly("bm25 index compaction"); index_state = tp_get_local_index_state(indexoid); if (index_state == NULL) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", relname))); } PG_TRY(); { do { tp_compaction_lock(index_rel); PG_TRY(_pass); { pass_ran = tp_compact_step(index_state, index_rel); } PG_FINALLY(_pass); { if (index_state->lock_held) tp_release_index_lock(index_state); tp_compaction_unlock(index_rel); } PG_END_TRY(_pass); CHECK_FOR_INTERRUPTS(); } while (pass_ran); } PG_FINALLY(); { relation_close(index_rel, RowExclusiveLock); } PG_END_TRY(); PG_RETURN_VOID(); } PG_FUNCTION_INFO_V1(tp_compact_index_step); Datum tp_compact_index_step(PG_FUNCTION_ARGS) { Oid indexoid = PG_GETARG_OID(0); Relation index_rel; TpLocalIndexState *index_state; bool pass_ran; if (RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("cannot compact a bm25 index during recovery"))); index_rel = tp_open_bm25_index(indexoid, RowExclusiveLock, true); if (!RelationUsesLocalBuffers(index_rel)) PreventCommandIfReadOnly("bm25 index compaction"); index_state = tp_get_local_index_state(indexoid); if (index_state == NULL) { char *relname = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", relname))); } tp_compaction_lock(index_rel); PG_TRY(); { pass_ran = tp_compact_step(index_state, index_rel); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); tp_compaction_unlock(index_rel); relation_close(index_rel, RowExclusiveLock); } PG_END_TRY(); PG_RETURN_BOOL(pass_ran); } PG_FUNCTION_INFO_V1(tp_compact_index_step_if_current); Datum tp_compact_index_step_if_current(PG_FUNCTION_ARGS) { TpCompactionTarget target = { .index_oid = PG_GETARG_OID(0), .database_oid = PG_GETARG_OID(1), .tablespace_oid = PG_GETARG_OID(2), .relfilenumber = PG_GETARG_OID(3), .owner_oid = PG_GETARG_OID(4), }; Relation index_rel; TpLocalIndexState *volatile index_state = NULL; volatile bool pass_ran; if (RecoveryInProgress()) ereport(ERROR, (errcode(ERRCODE_READ_ONLY_SQL_TRANSACTION), errmsg("cannot compact a bm25 index during recovery"))); index_rel = tp_open_current_bm25_target(&target, RowExclusiveLock, true, true); if (index_rel == NULL) PG_RETURN_BOOL(false); PreventCommandIfReadOnly("bm25 index compaction"); /* * A dispatched worker defers instead of queueing: the schedule brings * it back, and a blocked worker would hold its slot and transaction * open for the length of an unrelated VACUUM or compaction. */ if (!tp_try_compaction_lock(index_rel)) { relation_close(index_rel, RowExclusiveLock); PG_RETURN_BOOL(false); } PG_TRY(); { Relation current_rel; /* * ALTER OWNER, ALTER INDEX, or REINDEX may complete between the * captured target and admission. Reopen without taking another * relation lock so relcache invalidations are processed, then * repeat the complete physical-target check. */ current_rel = tp_open_current_bm25_target(&target, NoLock, true, true); if (current_rel == NULL) pass_ran = false; else { relation_close(current_rel, NoLock); index_state = tp_get_local_index_state(target.index_oid); if (index_state == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", RelationGetRelationName(index_rel)))); pass_ran = tp_compact_step(index_state, index_rel); } } PG_FINALLY(); { if (index_state != NULL && index_state->lock_held) tp_release_index_lock(index_state); tp_compaction_unlock(index_rel); relation_close(index_rel, RowExclusiveLock); } PG_END_TRY(); PG_RETURN_BOOL(pass_ran); } PG_FUNCTION_INFO_V1(tp_background_target_is_current); Datum tp_background_target_is_current(PG_FUNCTION_ARGS) { TpCompactionTarget target = { .index_oid = PG_GETARG_OID(0), .database_oid = PG_GETARG_OID(1), .tablespace_oid = PG_GETARG_OID(2), .relfilenumber = PG_GETARG_OID(3), .owner_oid = PG_GETARG_OID(4), }; Relation index_rel; index_rel = tp_open_current_bm25_target(&target, AccessShareLock, true, true); if (index_rel == NULL) PG_RETURN_BOOL(false); relation_close(index_rel, AccessShareLock); PG_RETURN_BOOL(true); } timescale-pg_textsearch-b63fe58/src/access/handler.c000066400000000000000000000133261525762047700225770ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * handler.c - BM25 access method handler and options */ #include #include #include #include #include #include #include #include #include "access/am.h" #include "planner/cost.h" #include "tsearch/ts_utils.h" /* Relation options - initialized in mod.c */ extern relopt_kind tp_relopt_kind; /* * Report index properties for pg_index_column_has_property() etc. */ static bool tp_property( Oid index_oid, int attno, IndexAMProperty prop, const char *propname, bool *res, bool *isnull) { (void)index_oid; (void)attno; (void)propname; (void)isnull; switch (prop) { case AMPROP_DISTANCE_ORDERABLE: *res = true; return true; default: return false; /* Let core handle other properties */ } } /* * Access method handler - returns IndexAmRoutine with function pointers */ PG_FUNCTION_INFO_V1(tp_handler); Datum tp_handler(PG_FUNCTION_ARGS) { IndexAmRoutine *amroutine; (void)fcinfo; /* unused */ amroutine = makeNode(IndexAmRoutine); amroutine->amstrategies = TSearchStrategyNumber; amroutine->amsupport = 8; /* 8 for distance */ amroutine->amoptsprocnum = 0; amroutine->amcanorder = false; amroutine->amcanorderbyop = true; /* Supports ORDER BY operators */ #if PG_VERSION_NUM >= 180000 amroutine->amcanhash = false; amroutine->amconsistentequality = false; amroutine->amconsistentordering = true; /* Support consistent ordering for ORDER BY */ #endif amroutine->amcanbackward = false; /* Cannot scan backwards */ amroutine->amcanunique = false; /* Cannot enforce uniqueness */ amroutine->amcanmulticol = false; /* Single column only */ amroutine->amoptionalkey = true; /* Can scan without search key */ amroutine->amsearcharray = false; /* No array search support */ amroutine->amsearchnulls = false; /* Cannot search for NULLs */ amroutine->amstorage = false; /* No separate storage type */ amroutine->amclusterable = false; /* Cannot cluster on this index */ amroutine->ampredlocks = false; /* No predicate locking */ amroutine->amcanparallel = false; /* No parallel scan support yet */ amroutine->amcanbuildparallel = true; amroutine->amcaninclude = false; /* No INCLUDE columns */ amroutine->amusemaintenanceworkmem = false; /* Vacuum does not use maintenance work mem */ amroutine->amsummarizing = false; amroutine->amparallelvacuumoptions = VACUUM_OPTION_PARALLEL_BULKDEL; amroutine->amkeytype = InvalidOid; /* Interface functions */ amroutine->ambuild = tp_build; amroutine->ambuildempty = tp_buildempty; amroutine->aminsert = tp_insert; amroutine->aminsertcleanup = NULL; amroutine->ambulkdelete = tp_bulkdelete; amroutine->amvacuumcleanup = tp_vacuumcleanup; amroutine->amcanreturn = NULL; amroutine->amcostestimate = tp_costestimate; amroutine->amoptions = tp_options; amroutine->amproperty = tp_property; amroutine->ambuildphasename = tp_buildphasename; /* No build phase names */ amroutine->amvalidate = tp_validate; amroutine->amadjustmembers = NULL; /* No member adjustment */ amroutine->ambeginscan = tp_beginscan; amroutine->amrescan = tp_rescan; amroutine->amgettuple = tp_gettuple; amroutine->amgetbitmap = NULL; /* No bitmap scans - ORDER BY only */ amroutine->amendscan = tp_endscan; amroutine->ammarkpos = NULL; /* No mark/restore support */ amroutine->amrestrpos = NULL; amroutine->amestimateparallelscan = NULL; /* No parallel support yet */ amroutine->aminitparallelscan = NULL; amroutine->amparallelrescan = NULL; #if PG_VERSION_NUM >= 180000 amroutine->amtranslatestrategy = NULL; amroutine->amtranslatecmptype = NULL; #endif PG_RETURN_POINTER(amroutine); } /* * Parse and validate index options */ bytea * tp_options(Datum reloptions, bool validate) { static const relopt_parse_elt tab[] = {{.optname = "text_config", .opttype = RELOPT_TYPE_STRING, .offset = offsetof(TpOptions, text_config_offset)}, {.optname = "k1", .opttype = RELOPT_TYPE_REAL, .offset = offsetof(TpOptions, k1)}, {.optname = "b", .opttype = RELOPT_TYPE_REAL, .offset = offsetof(TpOptions, b)}, {.optname = "compaction_schedule", .opttype = RELOPT_TYPE_STRING, .offset = offsetof(TpOptions, compaction_schedule_offset)}, {.optname = "compaction_lineage", .opttype = RELOPT_TYPE_STRING, .offset = offsetof(TpOptions, compaction_lineage_offset)}, {.optname = "compaction", .opttype = RELOPT_TYPE_ENUM, .offset = offsetof(TpOptions, compaction)}}; return (bytea *)build_reloptions( reloptions, validate, tp_relopt_kind, sizeof(TpOptions), tab, lengthof(tab)); } /* * Validate BM25 index definition */ bool tp_validate(Oid opclassoid) { HeapTuple tup; Form_pg_opclass opclassform; Oid opcintype; bool result = true; /* Look up the opclass */ tup = SearchSysCache1(CLAOID, ObjectIdGetDatum(opclassoid)); if (!HeapTupleIsValid(tup)) { elog(WARNING, "cache lookup failed for operator class %u", opclassoid); return false; } opclassform = (Form_pg_opclass)GETSTRUCT(tup); opcintype = opclassform->opcintype; switch (opcintype) { case TEXTOID: case VARCHAROID: case BPCHAROID: /* char(n) */ case TEXTARRAYOID: /* text[] */ case 1015: /* varchar[] */ case 1014: /* bpchar[] */ result = true; break; default: elog(WARNING, "Tapir index can only be created on text, varchar, " "or char columns (or arrays thereof) " "(got type OID %u)", opcintype); result = false; break; } ReleaseSysCache(tup); return result; } timescale-pg_textsearch-b63fe58/src/access/rls.c000066400000000000000000000157751525762047700217740ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * rls.c - BM25 and row-level security policy checks */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/rls.h" #include "constants.h" static bool relation_has_rls(Oid relid) { HeapTuple tuple; bool has_rls; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relid)); if (!HeapTupleIsValid(tuple)) return false; has_rls = ((Form_pg_class)GETSTRUCT(tuple))->relrowsecurity; ReleaseSysCache(tuple); return has_rls; } static bool relation_has_bm25_index(Oid relid) { Relation indexrel; ScanKeyData key; SysScanDesc scan; HeapTuple tuple; Oid bm25_am_oid; bool found = false; bm25_am_oid = get_am_oid("bm25", true); if (!OidIsValid(bm25_am_oid)) return false; indexrel = table_open(IndexRelationId, AccessShareLock); ScanKeyInit( &key, Anum_pg_index_indrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan( indexrel, IndexIndrelidIndexId, true, SnapshotSelf, 1, &key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_index indexform = (Form_pg_index)GETSTRUCT(tuple); HeapTuple classtuple; if (!indexform->indislive) continue; classtuple = SearchSysCache1( RELOID, ObjectIdGetDatum(indexform->indexrelid)); if (HeapTupleIsValid(classtuple)) { found = ((Form_pg_class)GETSTRUCT(classtuple))->relam == bm25_am_oid; ReleaseSysCache(classtuple); } if (found) break; } systable_endscan(scan); table_close(indexrel, AccessShareLock); return found; } static Oid find_rls_ancestor(Oid relid) { Relation inhrel; List *pending = list_make1_oid(relid); List *visited = NIL; Oid result = InvalidOid; inhrel = table_open(InheritsRelationId, AccessShareLock); while (pending != NIL) { Oid current = linitial_oid(pending); ScanKeyData key; SysScanDesc scan; HeapTuple tuple; pending = list_delete_first(pending); if (list_member_oid(visited, current)) continue; visited = lappend_oid(visited, current); if (relation_has_rls(current)) { result = current; break; } ScanKeyInit( &key, Anum_pg_inherits_inhrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(current)); scan = systable_beginscan( inhrel, InheritsRelidSeqnoIndexId, true, NULL, 1, &key); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_inherits inhform = (Form_pg_inherits)GETSTRUCT(tuple); pending = lappend_oid(pending, inhform->inhparent); } systable_endscan(scan); } table_close(inhrel, AccessShareLock); list_free(pending); list_free(visited); return result; } static Oid find_bm25_indexed_relation(Oid relid) { List *relations; ListCell *lc; Oid result = InvalidOid; relations = find_all_inheritors(relid, NoLock, NULL); foreach (lc, relations) { Oid current = lfirst_oid(lc); if (relation_has_bm25_index(current)) result = current; if (OidIsValid(result)) break; } list_free(relations); return result; } static Oid find_index_heap_relation(Oid indexrelid) { Relation indexrel; ScanKeyData key; SysScanDesc scan; HeapTuple tuple; Oid heaprelid = InvalidOid; indexrel = table_open(IndexRelationId, AccessShareLock); ScanKeyInit( &key, Anum_pg_index_indexrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(indexrelid)); scan = systable_beginscan( indexrel, IndexRelidIndexId, true, SnapshotSelf, 1, &key); tuple = systable_getnext(scan); if (HeapTupleIsValid(tuple)) heaprelid = ((Form_pg_index)GETSTRUCT(tuple))->indrelid; systable_endscan(scan); table_close(indexrel, AccessShareLock); return heaprelid; } static void check_bm25_build_allowed(Oid heaprelid) { Oid rls_relid; if (tp_rls_allowed_for_current_utility()) return; rls_relid = find_rls_ancestor(heaprelid); if (!OidIsValid(rls_relid)) return; ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("BM25 indexes are not allowed on row-level security " "relation \"%s\" when " "\"pg_textsearch.allow_rls\" is off", get_rel_name(rls_relid)), errdetail( "Relation \"%s\" is in the protected relation's " "inheritance hierarchy.", get_rel_name(heaprelid)), errhint("Set pg_textsearch.allow_rls to on to accept the " "documented term-frequency leakage risk."))); } void tp_check_bm25_build_allowed(Relation heap) { check_bm25_build_allowed(RelationGetRelid(heap)); } bool tp_check_bm25_index_create_allowed(Oid indexrelid) { Oid bm25_am_oid; Oid heaprelid; Relation index; Relation heap; bool is_bm25; bm25_am_oid = get_am_oid("bm25", true); if (!OidIsValid(bm25_am_oid)) return false; index = relation_open(indexrelid, NoLock); is_bm25 = (index->rd_rel->relkind == RELKIND_INDEX || index->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) && index->rd_rel->relam == bm25_am_oid; relation_close(index, NoLock); if (!is_bm25) return false; heaprelid = find_index_heap_relation(indexrelid); if (!OidIsValid(heaprelid)) elog(ERROR, "could not find heap relation for newly created BM25 index %u", indexrelid); heap = table_open(heaprelid, NoLock); tp_check_bm25_build_allowed(heap); table_close(heap, NoLock); return true; } void tp_check_bm25_hierarchy_allowed(Oid relid) { List *relations; ListCell *lc; if (tp_rls_allowed_for_current_utility()) return; relations = find_all_inheritors(relid, NoLock, NULL); foreach (lc, relations) { Oid current = lfirst_oid(lc); if (relation_has_bm25_index(current)) check_bm25_build_allowed(current); } list_free(relations); } void tp_check_rls_enable_allowed(Oid relid) { Oid indexed_relid; if (tp_rls_allowed_for_current_utility()) return; if (!relation_has_rls(relid)) { return; } indexed_relid = find_bm25_indexed_relation(relid); if (!OidIsValid(indexed_relid)) return; if (indexed_relid == relid) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot enable row-level security on relation \"%s\" " "because it has a BM25 index while " "\"pg_textsearch.allow_rls\" is off", get_rel_name(relid)), errhint("Set pg_textsearch.allow_rls to on to accept the " "documented term-frequency leakage risk."))); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot enable row-level security on relation \"%s\" " "because descendant relation \"%s\" has a BM25 index " "while \"pg_textsearch.allow_rls\" is off", get_rel_name(relid), get_rel_name(indexed_relid)), errhint("Set pg_textsearch.allow_rls to on to accept the " "documented term-frequency leakage risk."))); } timescale-pg_textsearch-b63fe58/src/access/rls.h000066400000000000000000000010701525762047700217600ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #ifndef TP_ACCESS_RLS_H #define TP_ACCESS_RLS_H #include #include extern bool tp_rls_allowed_for_current_utility(void); extern void tp_rls_note_bm25_build(void); extern void tp_check_bm25_build_allowed(Relation heap); extern bool tp_check_bm25_index_create_allowed(Oid indexrelid); extern void tp_check_bm25_hierarchy_allowed(Oid relid); extern void tp_check_rls_enable_allowed(Oid relid); #endif /* TP_ACCESS_RLS_H */ timescale-pg_textsearch-b63fe58/src/access/scan.c000066400000000000000000000403061525762047700221040ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * scan.c - BM25 index scan operations */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "access/boolean.h" #include "constants.h" #include "index/limit.h" #include "index/metapage.h" #include "index/resolve.h" #include "index/state.h" #include "memtable/scan.h" #include "types/query.h" #include "types/vector.h" /* * Backend-local cached score for ORDER BY optimization. * * When tp_gettuple returns a row, the BM25 score is cached here. The * bm25_get_current_score() stub function returns this value, avoiding * re-computation of scores in resjunk ORDER BY expressions. */ static float8 tp_cached_score = 0.0; float8 tp_get_cached_score(void) { return tp_cached_score; } /* * Session-local count of BM25 scoring passes. * * Scan depth is invisible in query results -- the executor's Filter, * Limit and backoff re-drives produce the exact top-k regardless of how * deep the internal top-K was seeded -- so this counter is the only * regression-stable signal that a scan was seeded for its own filter * rather than someone else's (issue #435). A well-seeded scan costs * exactly one pass; each backoff re-drive adds another. */ static uint64 tp_scoring_passes = 0; PG_FUNCTION_INFO_V1(tp_debug_scoring_passes); Datum tp_debug_scoring_passes(PG_FUNCTION_ARGS) { bool reset = PG_GETARG_BOOL(0); uint64 result = tp_scoring_passes; if (reset) tp_scoring_passes = 0; PG_RETURN_INT64((int64)result); } /* Track CTIDs already emitted by this scan. */ static bool tp_ctid_seen_or_mark(TpScanOpaque so, ItemPointer tid) { bool found; if (so->returned_ctids == NULL) { HASHCTL ctl; long nelem; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(ItemPointerData); ctl.entrysize = sizeof(ItemPointerData); ctl.hcxt = so->scan_context; /* Use the current batch size as the initial dynahash hint. */ nelem = so->max_results_used > 0 ? so->max_results_used : 256; so->returned_ctids = hash_create( "Tapir scan returned ctids", nelem, &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); } (void)hash_search(so->returned_ctids, tid, HASH_ENTER, &found); return found; } /* Reset emitted-CTID tracking for a restarted scan. */ static void tp_returned_ctids_reset(TpScanOpaque so) { if (so && so->returned_ctids) { hash_destroy(so->returned_ctids); so->returned_ctids = NULL; } } /* * Clean up any previous scan results in the scan opaque structure */ static void tp_rescan_cleanup_results(TpScanOpaque so) { if (!so) return; Assert(so->scan_context != NULL); /* Clean up result CTIDs */ if (so->result_ctids) { MemoryContext oldcontext = MemoryContextSwitchTo(so->scan_context); pfree(so->result_ctids); so->result_ctids = NULL; MemoryContextSwitchTo(oldcontext); } /* Clean up result scores */ if (so->result_scores) { MemoryContext oldcontext = MemoryContextSwitchTo(so->scan_context); pfree(so->result_scores); so->result_scores = NULL; MemoryContextSwitchTo(oldcontext); } if (so->boolean_results) { BufFileClose(so->boolean_results); so->boolean_results = NULL; } } /* * Process ORDER BY scan keys for <@> operator * * Handles both bm25query and plain text arguments to support: * - ORDER BY content <@> 'query'::bm25query (explicit bm25query) * - ORDER BY content <@> 'query' (plain text, implicit index resolution) */ static void tp_rescan_process_orderby( IndexScanDesc scan, ScanKey orderbys, int norderbys, TpIndexMetaPage metap) { TpScanOpaque so = (TpScanOpaque)scan->opaque; for (int i = 0; i < norderbys; i++) { ScanKey orderby = &orderbys[i]; /* Check for <@> operator strategy */ if (orderby->sk_strategy == 1) /* Strategy 1: <@> operator */ { Datum query_datum = orderby->sk_argument; char *query_cstr; Oid query_index_oid = InvalidOid; int64 hint_k; double hint_selectivity; /* * Use sk_subtype to determine the argument type. * sk_subtype contains the right-hand operand's type OID. */ if (orderby->sk_subtype == TEXTOID) { /* Plain text - use text directly */ text *query_text = (text *)DatumGetPointer(query_datum); query_cstr = text_to_cstring(query_text); } else { /* bm25query - extract query text and index OID */ TpQuery *query = (TpQuery *)DatumGetPointer(query_datum); query_cstr = pstrdup(get_tpquery_text(query)); query_index_oid = get_tpquery_index_oid(query); if (tpquery_get_seed_hint(query, &hint_k, &hint_selectivity)) so->limit = tp_seed_limit_for_filter( (int)hint_k, hint_selectivity); /* Validate index OID if provided in query */ if (tpquery_has_index(query)) { tp_validate_query_index( query_index_oid, scan->indexRelation); } } /* Clear query vector since we're using text directly */ if (so->query_vector) { pfree(so->query_vector); so->query_vector = NULL; } /* Free old query text if it exists */ if (so->query_text) { MemoryContext oldcontext = MemoryContextSwitchTo( so->scan_context); pfree(so->query_text); MemoryContextSwitchTo(oldcontext); } /* Allocate new query text in scan context */ { MemoryContext oldcontext = MemoryContextSwitchTo( so->scan_context); so->query_text = pstrdup(query_cstr); MemoryContextSwitchTo(oldcontext); } /* Store index OID for this scan */ so->index_oid = RelationGetRelid(scan->indexRelation); /* Mark all docs as candidates for ORDER BY operation */ if (metap && metap->total_docs > 0) so->result_count = metap->total_docs; pfree(query_cstr); } } } /* * Begin a scan of the Tapir index */ IndexScanDesc tp_beginscan(Relation index, int nkeys, int norderbys) { IndexScanDesc scan; TpScanOpaque so; /* * Surface the durable "results may be incomplete" marker left by * an in-place pre-v1.3 upgrade so read-only workloads * see it; throttled to once per index per session. */ tp_warn_if_pending_docid(index); scan = RelationGetIndexScan(index, nkeys, norderbys); /* Allocate and initialize scan opaque data */ so = (TpScanOpaque)palloc0(sizeof(TpScanOpaqueData)); so->scan_context = AllocSetContextCreate( CurrentMemoryContext, "Tapir Scan Context", ALLOCSET_DEFAULT_SIZES); so->boolean_context = AllocSetContextCreate( so->scan_context, "Tapir Boolean Scan Context", ALLOCSET_DEFAULT_SIZES); so->limit = -1; /* Initialize limit to -1 (no limit) */ so->max_results_used = 0; scan->opaque = so; /* * Custom index AMs must allocate ORDER BY arrays themselves. */ if (norderbys > 0) { scan->xs_orderbyvals = (Datum *)palloc0(sizeof(Datum) * norderbys); scan->xs_orderbynulls = (bool *)palloc(sizeof(bool) * norderbys); /* Initialize all orderbynulls to true */ memset(scan->xs_orderbynulls, true, sizeof(bool) * norderbys); } return scan; } /* * Restart a scan with new keys */ void tp_rescan( IndexScanDesc scan, ScanKey keys, int nkeys, ScanKey orderbys, int norderbys) { TpScanOpaque so = (TpScanOpaque)scan->opaque; TpIndexMetaPage metap = NULL; Assert(scan != NULL); Assert(scan->opaque != NULL); if (nkeys > 0 && norderbys > 0) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("pg_textsearch does not support Boolean filtering " "and BM25 ordering in the same index scan"))); if (!so) return; so->limit = -1; /* Reset scan state */ if (so) { /* Clean up any previous results */ tp_rescan_cleanup_results(so); /* Release scratch allocations from the previous Boolean execution. */ MemoryContextReset(so->boolean_context); /* Drop the emitted-CTID dedup set from any prior scan */ tp_returned_ctids_reset(so); /* Reset scan position and state */ so->current_pos = 0; so->result_count = 0; so->eof_reached = false; so->query_vector = NULL; /* * NULL keys restart the scan with its previous keys. Only discard * the copied Boolean query when PostgreSQL supplies replacements. */ if (keys != NULL) { if (so->boolean_query != NULL) { pfree(so->boolean_query); so->boolean_query = NULL; } so->is_boolean_scan = false; so->boolean_recheck = false; } } if (nkeys > 0 && keys && so) { if (!metap) metap = tp_get_metapage(scan->indexRelation); tp_boolean_rescan(scan, keys, nkeys, metap); } /* Process ORDER BY scan keys for <@> operator */ if (norderbys > 0 && orderbys && so) { /* Get index metadata to check if we have documents */ if (!metap) metap = tp_get_metapage(scan->indexRelation); tp_rescan_process_orderby(scan, orderbys, norderbys, metap); } if (metap) pfree(metap); } /* * End a scan and cleanup resources */ void tp_endscan(IndexScanDesc scan) { TpScanOpaque so = (TpScanOpaque)scan->opaque; if (so) { tp_rescan_cleanup_results(so); if (so->scan_context) MemoryContextDelete(so->scan_context); /* Free query vector if it was allocated */ if (so->query_vector) pfree(so->query_vector); pfree(so); scan->opaque = NULL; } /* * Don't free ORDER BY arrays here - PostgreSQL's core code will free them. */ if (scan->numberOfOrderBys > 0) { scan->xs_orderbyvals = NULL; scan->xs_orderbynulls = NULL; } } /* * Execute BM25 scoring query to get ordered results */ static bool tp_execute_scoring_query(IndexScanDesc scan) { TpScanOpaque so = (TpScanOpaque)scan->opaque; bool success = false; TpLocalIndexState *index_state = NULL; TpVector *query_vector; if (!so || !so->query_text) return false; tp_scoring_passes++; Assert(so->scan_context != NULL); /* Clean up previous results */ if (so->result_ctids || so->result_scores) { MemoryContext oldcontext = MemoryContextSwitchTo(so->scan_context); if (so->result_ctids) { pfree(so->result_ctids); so->result_ctids = NULL; } if (so->result_scores) { pfree(so->result_scores); so->result_scores = NULL; } MemoryContextSwitchTo(oldcontext); } so->result_count = 0; so->current_pos = 0; /* Get the index state with posting lists */ index_state = tp_get_local_index_state( RelationGetRelid(scan->indexRelation)); if (!index_state) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for BM25 " "search"))); } /* Keep the memtable and segment-root snapshot in one index state. */ tp_acquire_index_lock(index_state, LW_SHARED); /* Use the original query vector or create one from text */ query_vector = so->query_vector; if (!query_vector && so->query_text) { /* * We have a text query - convert it to a vector using the index. */ char *index_name = tp_get_qualified_index_name(scan->indexRelation); text *index_name_text = cstring_to_text(index_name); text *query_text_datum = cstring_to_text(so->query_text); Datum query_vec_datum = DirectFunctionCall2( to_tpvector, PointerGetDatum(query_text_datum), PointerGetDatum(index_name_text)); query_vector = (TpVector *)DatumGetPointer(query_vec_datum); /* Free existing query vector if present */ if (so->query_vector) pfree(so->query_vector); /* Store the converted vector for this query execution */ so->query_vector = query_vector; } if (!query_vector) { ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("no query vector available in scan state"))); } /* Find documents matching the query using posting lists */ success = tp_memtable_search(scan, index_state, query_vector); /* Release the lock - we've extracted all CTIDs we need */ tp_release_index_lock(index_state); return success; } /* * Get next tuple from scan */ bool tp_gettuple(IndexScanDesc scan, ScanDirection dir) { TpScanOpaque so = (TpScanOpaque)scan->opaque; float4 bm25_score; BlockNumber blknum; (void)dir; /* BM25 index only supports forward scan */ Assert(scan != NULL); Assert(so != NULL); Assert(so->is_boolean_scan || so->query_text != NULL); /* Execute scoring query if we haven't done so yet */ if (so->result_ctids == NULL && so->boolean_results == NULL && !so->eof_reached) { /* Count index scan for pg_stat_user_indexes */ pgstat_count_index_scan(scan->indexRelation); #if PG_VERSION_NUM >= 180000 if (scan->instrument) scan->instrument->nsearches++; #endif if (so->is_boolean_scan) { TpLocalIndexState *index_state = tp_get_local_index_state( RelationGetRelid(scan->indexRelation)); if (!index_state) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for BM25 " "Boolean search"))); if (!tp_boolean_execute(scan, index_state)) { so->eof_reached = true; return false; } } else if (!tp_execute_scoring_query(scan)) { so->eof_reached = true; return false; } if (so->result_ctids == NULL && so->boolean_results == NULL) { so->eof_reached = true; return false; } } if (so->boolean_results != NULL) { if (!tp_boolean_next(scan)) { so->eof_reached = true; return false; } scan->xs_recheck = so->boolean_recheck; scan->xs_recheckorderby = false; return true; } /* Advance, growing the scoring batch if needed. */ for (;;) { if (so->current_pos >= so->result_count || so->eof_reached) { /* * If result_count hit the internal limit, there may be * more documents. Double the limit and re-execute the * scoring query. */ if (!so->is_boolean_scan && !so->eof_reached && so->result_count > 0 && so->result_count >= so->max_results_used) { int old_count = so->result_count; int new_limit; int max_limit = Min((Size)PG_INT32_MAX, MaxAllocSize / sizeof(ItemPointerData)); if (so->max_results_used >= max_limit) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("BM25 result batch is too large"))); new_limit = so->max_results_used > max_limit / 2 ? max_limit : so->max_results_used * 2; so->limit = new_limit; if (tp_execute_scoring_query(scan) && so->result_count > old_count) { /* * Re-scoring can reorder concurrent results, so * restart and filter by emitted CTID, not position. */ so->current_pos = 0; continue; } else { so->eof_reached = true; return false; } } else return false; } Assert(so->scan_context != NULL); Assert(so->result_ctids != NULL); Assert(so->current_pos < so->result_count); Assert(ItemPointerIsValid(&so->result_ctids[so->current_pos])); /* Skip results with invalid block numbers */ blknum = BlockIdGetBlockNumber( &(so->result_ctids[so->current_pos].ip_blkid)); if (blknum == InvalidBlockNumber) { so->current_pos++; continue; } /* Skip CTIDs already emitted by an earlier pass (dedup) */ if (tp_ctid_seen_or_mark(so, &so->result_ctids[so->current_pos])) { so->current_pos++; continue; } /* This position is emittable. */ break; } scan->xs_heaptid = so->result_ctids[so->current_pos]; scan->xs_recheck = so->boolean_recheck; scan->xs_recheckorderby = false; /* Set ORDER BY distance value */ if (scan->numberOfOrderBys > 0) { float4 raw_score; Assert(scan->numberOfOrderBys == 1); Assert(scan->xs_orderbyvals != NULL); Assert(scan->xs_orderbynulls != NULL); Assert(so->result_scores != NULL); /* Convert BM25 score to Datum (ensure negative for ASC sort) */ raw_score = so->result_scores[so->current_pos]; bm25_score = (raw_score > 0) ? -raw_score : raw_score; scan->xs_orderbyvals[0] = Float8GetDatum((float8)bm25_score); scan->xs_orderbynulls[0] = false; /* Log BM25 score if enabled */ elog(tp_log_scores ? NOTICE : DEBUG1, "BM25 index scan: tid=(%u,%u), BM25_score=%.4f", BlockIdGetBlockNumber(&scan->xs_heaptid.ip_blkid), scan->xs_heaptid.ip_posid, bm25_score); /* Cache score for stub function to retrieve */ tp_cached_score = (float8)bm25_score; } /* Move to next position */ so->current_pos++; return true; } timescale-pg_textsearch-b63fe58/src/access/vacuum.c000066400000000000000000001171441525762047700224650ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * vacuum.c - BM25 index vacuum and maintenance operations */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "access/build_context.h" #include "debug/injection.h" #include "index/freepage.h" #include "index/metapage.h" #include "index/state.h" #include "memtable/page.h" #include "segment/alive_bitset.h" #include "segment/compaction.h" #include "segment/dictionary.h" #include "segment/graph_snapshot.h" #include "segment/io.h" #include "segment/segment.h" #include "segment/tombstone.h" /* * Per-segment state for VACUUM dead tuple tracking. * Tracks which segments contain dead CTIDs without storing the * CTIDs themselves. */ typedef struct TpVacuumSegmentInfo { BlockNumber root_block; uint32 level; uint32 num_docs; /* segment header num_docs */ uint64 total_tokens; /* segment header total_tokens */ uint32 *dead_doc_ids; /* Array of dead doc_ids */ uint32 dead_count; bool affected; bool is_v5; /* true if segment has alive bitset */ bool already_empty; /* V5 header already has no live docs */ } TpVacuumSegmentInfo; /* * Build a quoted, schema-qualified index name for a REINDEX errhint, so the * hint stays pasteable for mixed-case names and independent of search_path. */ static char * tp_vacuum_reindex_hint_name(Relation index) { return quote_qualified_identifier( get_namespace_name(RelationGetNamespace(index)), RelationGetRelationName(index)); } /* * Convert a 32-bit xid (known to be in the allowable range when * nextFullXid was current) into a FullTransactionId. */ static inline FullTransactionId tp_full_xid_from_allowable_at(FullTransactionId nextFullXid, TransactionId xid) { #if PG_VERSION_NUM >= 170003 return FullTransactionIdFromAllowableAt(nextFullXid, xid); #else uint32 epoch; if (!TransactionIdIsNormal(xid)) return FullTransactionIdFromEpochAndXid(0, xid); Assert(TransactionIdPrecedesOrEquals( xid, XidFromFullTransactionId(nextFullXid))); epoch = EpochFromFullTransactionId(nextFullXid); if (xid > XidFromFullTransactionId(nextFullXid)) { Assert(epoch != 0); epoch--; } return FullTransactionIdFromEpochAndXid(epoch, xid); #endif } FullTransactionId tp_reclaim_horizon(Relation heaprel) { TransactionId oldest; /* A test may pin the horizon so no parked page is yet recyclable. */ if (tp_injected_reclaim_horizon_held()) return FirstNormalFullTransactionId; oldest = GetOldestNonRemovableTransactionId(heaprel); return tp_full_xid_from_allowable_at(ReadNextFullTransactionId(), oldest); } /* * Build a hash set of block numbers reachable from the current * memtable chain (metap->memtable_head_blkno). Used by * tp_reclaim_dead_memtable_pages to avoid freeing pages that are * still in the live chain — a crash-safety guard against the * scenario where a crash between tp_spill_finalize and * tp_memtable_mark_chain_dead leaves pages reachable but stamped * DEAD once global xmin advances. * * Walks both the main chain (via next_block) and any fragment * continuation sub-chains. Caller must hash_destroy() the result. */ static HTAB * tp_collect_reachable_chain_blocks(Relation indexrel) { HASHCTL info; HTAB *reachable; TpIndexMetaPage metap; BlockNumber cur; memset(&info, 0, sizeof(info)); info.keysize = sizeof(BlockNumber); info.entrysize = sizeof(BlockNumber); info.hcxt = CurrentMemoryContext; reachable = hash_create( "reachable chain blocks", 128, &info, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); metap = tp_get_metapage(indexrel); if (metap == NULL) return reachable; cur = metap->memtable_head_blkno; pfree(metap); while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpMemtablePageHeader *hdr; TpMemtableRecord *rec; BlockNumber next; bool found; CHECK_FOR_INTERRUPTS(); hash_search(reachable, &cur, HASH_ENTER, &found); buf = ReadBuffer(indexrel, cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); break; } hdr = tp_memtable_page_header(page); next = hdr->next_block; rec = tp_memtable_page_first(page); /* * If this page has a fragment record, its next_block points * to the first continuation page. Walk the continuation * chain and add those blocks too, then resume from the * first non-continuation block. */ if (rec != NULL && (rec->flags & TP_MEMTABLE_RECORD_FLAG_FRAGMENT) != 0) { BlockNumber cont = next; while (cont != InvalidBlockNumber) { Buffer cont_buf; Page cont_page; TpMemtablePageHeader *cont_hdr; hash_search(reachable, &cont, HASH_ENTER, &found); cont_buf = ReadBuffer(indexrel, cont); LockBuffer(cont_buf, BUFFER_LOCK_SHARE); cont_page = BufferGetPage(cont_buf); if (!tp_memtable_page_is_valid(cont_page) || !tp_memtable_page_is_continuation(cont_page)) { next = cont; UnlockReleaseBuffer(cont_buf); break; } cont_hdr = tp_memtable_page_header(cont_page); cont = cont_hdr->next_block; UnlockReleaseBuffer(cont_buf); } /* * If the inner loop exhausted (cont == Invalid), next * stays as the last continuation's next_block, which * is Invalid — outer loop terminates correctly. */ if (cont == InvalidBlockNumber) next = InvalidBlockNumber; } UnlockReleaseBuffer(buf); cur = next; } return reachable; } /* * Sum segment.alive_count across all on-disk segments. Used by * tp_vacuumcleanup to set stats->num_index_tuples: reltuples must * reflect the live-doc count, which may be strictly less than * metap->total_docs (that tracks Σ segment.num_docs, which for V5 * segments includes bitset-dead docs). * * For V5 segments the count is in the header; pre-V5 segments * have no alive-bitset so their alive count equals num_docs. */ static uint64 tp_count_live_docs( Relation index, const TpSegmentGraphSnapshot *segment_snapshot) { uint64 alive = 0; for (int level = 0; level < TP_MAX_LEVELS; level++) { const BlockNumber *roots; uint32 root_count; roots = tp_segment_graph_snapshot_level( segment_snapshot, level, &root_count); for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { BlockNumber seg = roots[root_idx]; TpSegmentReader *reader = tp_segment_open(index, seg); if (!reader || !reader->header) { if (reader) tp_segment_close(reader); break; } alive += (reader->header->alive_bitset_offset > 0) ? reader->header->alive_count : reader->header->num_docs; tp_segment_close(reader); } } return alive; } /* * Sum exact term frequencies from a current-format segment. * This is an exceptional consistency check, so it favors a complete * posting walk over trusting the header value under investigation. */ static uint64 tp_vacuum_measure_current_segment_tokens( Relation index, BlockNumber root_block) { TpSegmentReader *reader; TpDictionary dictionary; uint32 *string_offsets; uint64 total_tokens = 0; reader = tp_segment_open_ex(index, root_block, false); if (reader == NULL || reader->header == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open current segment at block %u", root_block))); if (reader->header->num_terms == 0) { tp_segment_close(reader); return 0; } tp_segment_read( reader, reader->header->dictionary_offset, &dictionary, sizeof(dictionary.num_terms)); if (dictionary.num_terms != reader->header->num_terms) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("current segment dictionary count is inconsistent"))); string_offsets = palloc(sizeof(uint32) * dictionary.num_terms); tp_segment_read( reader, reader->header->dictionary_offset + sizeof(dictionary.num_terms), string_offsets, sizeof(uint32) * dictionary.num_terms); for (uint32 i = 0; i < dictionary.num_terms; i++) { TpSegmentPostingIterator iterator; TpSegmentPosting *posting; char *term; term = tp_segment_read_term_at_index( reader, reader->header, string_offsets, i); if (!tp_segment_posting_iterator_init(&iterator, reader, term)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not read current segment term"))); while (tp_segment_posting_iterator_next(&iterator, &posting)) { if (pg_add_u64_overflow( total_tokens, (uint64)posting->frequency, &total_tokens)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("current segment token statistics overflow"))); } tp_segment_posting_iterator_free(&iterator); pfree(term); } pfree(string_offsets); tp_segment_close(reader); return total_tokens; } /* * Walk all segment docmaps and call the callback for each CTID. * Returns an array of TpVacuumSegmentInfo with affected flags set. * *num_segments_out receives the total segment count. */ static TpVacuumSegmentInfo * tp_vacuum_identify_affected( Relation index, const TpSegmentGraphSnapshot *segment_snapshot, IndexBulkDeleteCallback callback, void *callback_state, bool scan_ctids, int *num_segments_out, int64 *total_dead_out) { TpVacuumSegmentInfo *segments; /* Keep both CTID scratch arrays within roughly one PostgreSQL block. */ const uint32 ctid_batch_capacity = BLCKSZ / (sizeof(BlockNumber) + sizeof(OffsetNumber)); BlockNumber *ctid_pages; OffsetNumber *ctid_offsets; int capacity = 32; int count = 0; int64 total_dead = 0; bool identification_paused = false; segments = palloc(capacity * sizeof(TpVacuumSegmentInfo)); ctid_pages = palloc(ctid_batch_capacity * sizeof(BlockNumber)); ctid_offsets = palloc(ctid_batch_capacity * sizeof(OffsetNumber)); for (int level = 0; level < TP_MAX_LEVELS; level++) { const BlockNumber *roots; uint32 root_count; roots = tp_segment_graph_snapshot_level( segment_snapshot, level, &root_count); for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentReader *reader; uint32 seg_dead = 0; BlockNumber seg = roots[root_idx]; reader = tp_segment_open_ex(index, seg, false); if (!reader || !reader->header) { if (reader) tp_segment_close(reader); break; } /* * Check each CTID against the callback. A caller that cannot * report a dead CTID skips the walk: dead counts stay zero and * every other field recorded below comes from the segment * header, so scanning would cost O(num_docs) for nothing. */ { uint32 *dead_ids = NULL; uint32 dead_cap = 0; bool has_bitset = (reader->header->alive_bitset_offset > 0); for (uint32 batch_start = 0; scan_ctids && batch_start < reader->header->num_docs; batch_start += ctid_batch_capacity) { uint32 batch_count = Min(ctid_batch_capacity, reader->header->num_docs - batch_start); tp_segment_read( reader, reader->header->ctid_pages_offset + (uint64)batch_start * sizeof(BlockNumber), ctid_pages, batch_count * sizeof(BlockNumber)); tp_segment_read( reader, reader->header->ctid_offsets_offset + (uint64)batch_start * sizeof(OffsetNumber), ctid_offsets, batch_count * sizeof(OffsetNumber)); for (uint32 batch_pos = 0; batch_pos < batch_count; batch_pos++) { ItemPointerData ctid; uint32 i = batch_start + batch_pos; /* * Skip docs already marked dead in the alive * bitset. Without this, CTID reuse after a * previous VACUUM would double-count dead docs * and corrupt total_docs in the metapage. */ if (has_bitset && !tp_segment_is_alive(reader, i)) continue; ItemPointerSet( &ctid, ctid_pages[batch_pos], ctid_offsets[batch_pos]); if (ItemPointerIsValid(&ctid) && callback(&ctid, callback_state)) { if (seg_dead >= dead_cap) { dead_cap = (dead_cap == 0) ? 64 : dead_cap * 2; dead_ids = dead_ids ? repalloc( dead_ids, dead_cap * sizeof(uint32)) : palloc(dead_cap * sizeof(uint32)); } dead_ids[seg_dead] = i; seg_dead++; } } if (!identification_paused) { if (IsInParallelMode() && !IsParallelWorker()) TP_INJECTION_POINT( TP_INJECTION_PARALLEL_VACUUM_LEADER_ESTIMATE); TP_INJECTION_POINT( TP_INJECTION_COMPACTION_SOURCE_ESTIMATE); identification_paused = true; } } /* Record segment info */ if (count >= capacity) { capacity *= 2; segments = repalloc( segments, capacity * sizeof(TpVacuumSegmentInfo)); } segments[count].root_block = seg; segments[count].level = level; segments[count].num_docs = reader->header->num_docs; segments[count].total_tokens = reader->header->total_tokens; segments[count].dead_doc_ids = dead_ids; segments[count].dead_count = seg_dead; segments[count].affected = (seg_dead > 0); segments[count].is_v5 = (reader->header->alive_bitset_offset > 0); segments[count].already_empty = segments[count].is_v5 && reader->header->alive_count == 0; } total_dead += seg_dead; count++; tp_segment_close(reader); } } pfree(ctid_offsets); pfree(ctid_pages); *num_segments_out = count; *total_dead_out = total_dead; return segments; } /* * Rebuild a single segment, excluding dead CTIDs. * * Reads the segment's docmap, calls the callback for each CTID, * fetches live heap tuples, tokenizes them, and writes a new * segment via TpBuildContext. * * Returns the new segment's root block, or InvalidBlockNumber if * all docs were dead (segment should be removed from chain). * * *old_total_len only accounts for dead docs when * measure_dead_tokens is set; otherwise they are skipped without * being fetched. */ static BlockNumber tp_vacuum_rebuild_segment( Relation index, Relation heap, BlockNumber old_root, uint32 level pg_attribute_unused(), IndexBulkDeleteCallback callback, void *callback_state, bool measure_dead_tokens, uint64 *old_total_len, uint64 *new_total_docs, uint64 *new_total_len) { TpSegmentReader *reader; TpBuildContext *build_ctx; BlockNumber new_root; Oid text_config_oid; IndexInfo *indexInfo; EState *estate; ExprContext *econtext; TupleTableSlot *eval_slot; Datum idx_values[INDEX_MAX_KEYS]; bool idx_isnull[INDEX_MAX_KEYS]; MemoryContext per_doc_ctx; MemoryContext old_ctx; uint64 docs_added = 0; uint64 old_len = 0; uint64 len_added = 0; /* Get text config from metapage */ { TpIndexMetaPage mp = tp_get_metapage(index); text_config_oid = mp->text_config_oid; pfree(mp); } /* Open segment with CTID preloading */ reader = tp_segment_open_ex(index, old_root, true); if (!reader || !reader->header) { if (reader) tp_segment_close(reader); if (old_total_len) *old_total_len = 0; *new_total_docs = 0; if (new_total_len) *new_total_len = 0; return InvalidBlockNumber; } /* Set up expression evaluation for index */ indexInfo = BuildIndexInfo(index); estate = CreateExecutorState(); econtext = GetPerTupleExprContext(estate); eval_slot = MakeSingleTupleTableSlot( RelationGetDescr(heap), &TTSOpsBufferHeapTuple); if (indexInfo->ii_Predicate != NIL) indexInfo->ii_PredicateState = ExecPrepareQual(indexInfo->ii_Predicate, estate); /* Create build context (no budget limit for VACUUM rebuild) */ build_ctx = tp_build_context_create(0); per_doc_ctx = AllocSetContextCreate( CurrentMemoryContext, "VACUUM rebuild per-doc", ALLOCSET_DEFAULT_SIZES); /* Iterate docmap, skip dead, fetch+tokenize live */ for (uint32 i = 0; i < reader->header->num_docs; i++) { ItemPointerData ctid; HeapTupleData tuple_data; HeapTuple tuple = &tuple_data; Buffer heap_buf = InvalidBuffer; bool valid; text *document_text; char **terms; int32 *frequencies; int term_count; int doc_length; bool is_dead; tp_segment_lookup_ctid(reader, i, &ctid); if (!ItemPointerIsValid(&ctid)) continue; is_dead = callback(&ctid, callback_state); /* * A dead doc contributes nothing to the replacement segment, so * fetching and tokenizing it is only worthwhile when the caller * needs *old_total_len to split an unknowable legacy aggregate. */ if (is_dead && !measure_dead_tokens) continue; /* Fetch heap tuple */ tuple->t_self = ctid; valid = heap_fetch(heap, SnapshotAny, tuple, &heap_buf, true); if (!valid) { if (heap_buf != InvalidBuffer) ReleaseBuffer(heap_buf); continue; } /* Evaluate index expression */ ExecStoreBufferHeapTuple(tuple, eval_slot, heap_buf); econtext->ecxt_scantuple = eval_slot; FormIndexDatum(indexInfo, eval_slot, estate, idx_values, idx_isnull); if (idx_isnull[0]) { ExecClearTuple(eval_slot); ResetExprContext(econtext); ReleaseBuffer(heap_buf); continue; } /* Check partial index predicate */ if (indexInfo->ii_Predicate != NIL && !ExecQual(indexInfo->ii_PredicateState, econtext)) { ExecClearTuple(eval_slot); ResetExprContext(econtext); ReleaseBuffer(heap_buf); continue; } /* Tokenize in per-doc context (includes detoasting) */ old_ctx = MemoryContextSwitchTo(per_doc_ctx); document_text = DatumGetTextPP(idx_values[0]); doc_length = tp_tokenize_text( document_text, text_config_oid, &terms, &frequencies, &term_count); MemoryContextSwitchTo(old_ctx); old_len += doc_length; if (!is_dead) { tp_build_context_add_document( build_ctx, terms, frequencies, term_count, doc_length, &ctid); docs_added++; len_added += doc_length; } MemoryContextReset(per_doc_ctx); ExecClearTuple(eval_slot); ResetExprContext(econtext); ReleaseBuffer(heap_buf); } ExecDropSingleTupleTableSlot(eval_slot); FreeExecutorState(estate); tp_segment_close(reader); /* Write new segment if any docs survived */ if (build_ctx->num_docs > 0) new_root = tp_write_segment_from_build_ctx(build_ctx, index); else new_root = InvalidBlockNumber; tp_build_context_destroy(build_ctx); MemoryContextDelete(per_doc_ctx); if (old_total_len) *old_total_len = old_len; *new_total_docs = docs_added; if (new_total_len) *new_total_len = len_added; return new_root; } /* * Hand one rebuilt legacy source to the shared prepared-publication engine. * The compaction layer owns output linking, page collection, tombstones, * prefix validation, and publication. */ static void tp_vacuum_replace_segment( TpLocalIndexState *index_state, Relation index, uint32 level, BlockNumber old_root, BlockNumber new_root, uint64 docs_shrinkage, uint64 tokens_shrinkage, uint64 tokens_growth, bool parallel_context) { TpSegmentReplacementReclaimMode reclaim_mode = parallel_context ? TP_SEGMENT_REPLACEMENT_RECLAIM_NEXT_XID : TP_SEGMENT_REPLACEMENT_RECLAIM_CURRENT_XID; tp_publish_prepared_segment_replacement( index_state, index, level, old_root, new_root, docs_shrinkage, tokens_shrinkage, tokens_growth, reclaim_mode); } /* * Apply dead doc marks to a V5 segment's alive bitset. * * Loads the bitset, marks dead docs, and writes it back via GenericXLog. * Empty segments stay published until serial cleanup compacts their prefix. */ static void tp_vacuum_mark_dead( Relation index, BlockNumber root_block, uint32 *dead_doc_ids, uint32 dead_count) { TpSegmentReader *reader; TpAliveBitset *bitset; reader = tp_segment_open_ex(index, root_block, false); if (!reader || !reader->header) { if (reader) tp_segment_close(reader); return; } bitset = tp_alive_bitset_load(reader); if (!bitset) { tp_segment_close(reader); return; } for (uint32 i = 0; i < dead_count; i++) tp_alive_bitset_mark_dead(bitset, dead_doc_ids[i]); tp_alive_bitset_write(bitset, reader, index); tp_alive_bitset_free(bitset); tp_segment_close(reader); } /* * Bulk delete callback for vacuum and CREATE INDEX CONCURRENTLY * * Four-phase approach: * 1. Spill memtable to segments (all data in uniform format) * 2. Identify segments containing dead CTIDs (O(segments) memory) * 3. Mark dead docs or rebuild affected segments * 4. Update metapage statistics * * Also called during CREATE INDEX CONCURRENTLY validation with a * callback that returns false for all CTIDs (just collecting TIDs). * That path is normally a no-op; inconsistent legacy statistics still * trigger the same full legacy rebase used by VACUUM. */ static bool tp_vacuum_no_dead_callback(ItemPointer itemptr, void *state) { (void)itemptr; (void)state; return false; } static IndexBulkDeleteResult * tp_bulkdelete_internal( IndexVacuumInfo *info, IndexBulkDeleteResult *stats, IndexBulkDeleteCallback callback, void *callback_state, bool spill_memtable, bool scan_ctids) { TpIndexMetaPage metap; TpSegmentGraphSnapshot *segment_snapshot; TpLocalIndexState *index_state; TpVacuumSegmentInfo *segments; int num_segments; int64 total_dead; uint64 current_segment_tokens; uint64 header_total_tokens; uint64 legacy_source_tokens_remaining; uint64 snapshot_total_len; uint32 legacy_segment_count; uint32 legacy_segments_remaining; volatile bool index_lock_held = false; bool parallel_context = IsInParallelMode() || IsParallelWorker(); bool header_totals_match_metap; bool rebuild_all_legacy = false; bool current_tokens_overflow = false; bool needs_empty_cleanup = false; bool memtable_spilled = false; if (stats == NULL) stats = (IndexBulkDeleteResult *)palloc0( sizeof(IndexBulkDeleteResult)); metap = tp_get_metapage(info->index); if (!metap) { stats->num_pages = 1; stats->num_index_tuples = 0; stats->tuples_removed = 0; stats->pages_deleted = 0; elog(WARNING, "Tapir bulkdelete: couldn't read metapage for " "index %s", RelationGetRelationName(info->index)); return stats; } if (callback == NULL) { stats->num_pages = 1; stats->num_index_tuples = (double)metap->total_docs; stats->tuples_removed = 0; stats->pages_deleted = 0; pfree(metap); return stats; } /* * Phase 1: Normal bulk deletion spills the memtable so all data is in * segments. Phase 2 identifies dead documents from published segments * alone, so the chain must be empty on return -- a spill that cannot * run reports the level-0 capacity limit rather than stranding dead * records. Cleanup-only validation leaves a small memtable in place; * segment statistics exclude it and can be checked independently. */ index_state = tp_get_local_index_state(RelationGetRelid(info->index)); if (index_state == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", RelationGetRelationName(info->index)))); /* * Serialize source-derived maintenance before taking the per-index * lock. Compaction releases its LW_SHARED selection lock while it * derives replacement output, so this heavyweight lock is what keeps * VACUUM from changing selected alive bits or replacing a source until * compaction publishes. * * Acquire before Phase 1's required spill. That spill must succeed, * and at a full level 0 it can only do so by compacting first. The * make-room path uses conditional admission, so entering it without * maintenance would turn a busy maintenance lock into a spurious * level-0 capacity error instead of the wait VACUUM is entitled to. * Holding it here makes that conditional acquisition our own and * lets the compaction proceed. LW_SHARED is held only while copying * the published graph; the maintenance lock keeps those source * segments immutable during the callback walk, bitmap updates, * legacy rebuilds, and page collection, while concurrent spills may * safely prepend an L0 prefix. */ tp_compaction_lock(info->index); PG_TRY(); { if (spill_memtable) memtable_spilled = tp_spill_memtable_if_needed_deferred( info->index, index_state, 1); tp_acquire_index_lock(index_state, LW_SHARED); index_lock_held = true; /* Snapshot the published roots after spill and maintenance admission. */ pfree(metap); segment_snapshot = tp_segment_graph_snapshot_create(info->index); metap = &segment_snapshot->metapage; tp_release_index_lock(index_state); index_lock_held = false; /* Phase 2: Identify affected segments. */ segments = tp_vacuum_identify_affected( info->index, segment_snapshot, callback, callback_state, scan_ctids, &num_segments, &total_dead); header_total_tokens = 0; current_segment_tokens = 0; legacy_segment_count = 0; snapshot_total_len = tp_injected_vacuum_total_len(metap->total_len); header_totals_match_metap = true; for (int i = 0; i < num_segments; i++) { if (pg_add_u64_overflow( header_total_tokens, segments[i].total_tokens, &header_total_tokens)) header_totals_match_metap = false; if (segments[i].is_v5) { if (pg_add_u64_overflow( current_segment_tokens, segments[i].total_tokens, ¤t_segment_tokens)) { header_totals_match_metap = false; current_tokens_overflow = true; } } else legacy_segment_count++; } if (header_total_tokens != metap->total_len) header_totals_match_metap = false; if (current_tokens_overflow) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("current segment token statistics overflow"), errhint("Run REINDEX INDEX %s to rebuild the index.", tp_vacuum_reindex_hint_name(info->index)))); if (snapshot_total_len < current_segment_tokens || (legacy_segment_count == 0 && !header_totals_match_metap)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("index token statistics are inconsistent"), errhint("Run REINDEX INDEX %s to rebuild the index.", tp_vacuum_reindex_hint_name(info->index)))); legacy_source_tokens_remaining = 0; legacy_segments_remaining = 0; if (!header_totals_match_metap && legacy_segment_count > 0) { for (int i = 0; i < num_segments; i++) { uint64 measured_tokens; if (!segments[i].is_v5) continue; measured_tokens = tp_vacuum_measure_current_segment_tokens( info->index, segments[i].root_block); if (measured_tokens != segments[i].total_tokens) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("current segment token statistics are " "inconsistent"), errhint("Run REINDEX INDEX %s to rebuild the " "index.", tp_vacuum_reindex_hint_name( info->index)))); } /* * Individual legacy contributions are unknowable once their * aggregate disagrees with the metapage. Rebuild every legacy * source and distribute the known aggregate contribution across * their replacements so the final total is exact. */ rebuild_all_legacy = true; legacy_source_tokens_remaining = snapshot_total_len - current_segment_tokens; legacy_segments_remaining = legacy_segment_count; for (int i = 0; i < num_segments; i++) { if (!segments[i].is_v5) segments[i].affected = true; } } for (int i = 0; i < num_segments; i++) { if (segments[i].already_empty) { needs_empty_cleanup = true; break; } } if (total_dead == 0 && !needs_empty_cleanup && !rebuild_all_legacy) { stats->num_pages = 1; stats->num_index_tuples = (double)metap->total_docs; stats->tuples_removed = 0; stats->pages_deleted = 0; pfree(segments); tp_segment_graph_snapshot_free(segment_snapshot); goto bulkdelete_done; } tp_segment_graph_snapshot_free(segment_snapshot); /* * Phase 3: mark dead docs or rebuild selected legacy segments. * V5 mutations need only their segment-buffer locks. Legacy * replacements prepare their output and page list unlocked, then * validate and publish in a brief exclusive section. */ for (int level = 0; level < TP_MAX_LEVELS; level++) { for (int i = 0; i < num_segments; i++) { if ((int)segments[i].level != level || !segments[i].affected) continue; if (segments[i].is_v5) { /* * Persist an all-zero bitmap in both serial and * parallel VACUUM. Serial amvacuumcleanup removes * one empty-bearing prefix through the normal * prepared compaction publication path. */ (void)tp_vacuum_mark_dead( info->index, segments[i].root_block, segments[i].dead_doc_ids, segments[i].dead_count); } else { BlockNumber new_root; uint64 old_tokens = 0; uint64 source_tokens; uint64 new_docs = 0; uint64 new_tokens = 0; uint64 docs_shrinkage = 0; uint64 tokens_shrinkage = 0; uint64 tokens_growth = 0; new_root = tp_vacuum_rebuild_segment( info->index, info->heaprel, segments[i].root_block, level, callback, callback_state, rebuild_all_legacy, &old_tokens, &new_docs, &new_tokens); if (segments[i].num_docs > new_docs) docs_shrinkage = segments[i].num_docs - new_docs; if (rebuild_all_legacy) { Assert(legacy_segments_remaining > 0); if (legacy_segments_remaining == 1) source_tokens = legacy_source_tokens_remaining; else source_tokens = Min(old_tokens, legacy_source_tokens_remaining); legacy_source_tokens_remaining -= source_tokens; legacy_segments_remaining--; } else { /* * Inconsistent header totals either trigger the * legacy rebase above or raise before reaching * here, so the header total is authoritative. */ Assert(header_totals_match_metap); source_tokens = segments[i].total_tokens; } if (source_tokens > new_tokens) tokens_shrinkage = source_tokens - new_tokens; else tokens_growth = new_tokens - source_tokens; tp_vacuum_replace_segment( index_state, info->index, level, segments[i].root_block, new_root, docs_shrinkage, tokens_shrinkage, tokens_growth, parallel_context); } } } Assert(legacy_segments_remaining == 0); Assert(legacy_source_tokens_remaining == 0); /* * tp_vacuumcleanup will set num_index_tuples to the actual live * count; only tuples_removed needs to carry through from here. */ stats->num_pages = 1; stats->tuples_removed = (double)total_dead; stats->pages_deleted = 0; for (int i = 0; i < num_segments; i++) { if (segments[i].dead_doc_ids) pfree(segments[i].dead_doc_ids); } pfree(segments); bulkdelete_done:; } PG_FINALLY(); { if (index_lock_held) tp_release_index_lock(index_state); tp_compaction_unlock(info->index); } PG_END_TRY(); tp_apply_compaction_policy(index_state, info->index, memtable_spilled); return stats; } IndexBulkDeleteResult * tp_bulkdelete( IndexVacuumInfo *info, IndexBulkDeleteResult *stats, IndexBulkDeleteCallback callback, void *callback_state) { return tp_bulkdelete_internal( info, stats, callback, callback_state, true, true); } /* * Vacuum/cleanup the BM25 index */ IndexBulkDeleteResult * tp_vacuumcleanup(IndexVacuumInfo *info, IndexBulkDeleteResult *stats) { TpSegmentGraphSnapshot *segment_snapshot; TpLocalIndexState *index_state; int freed_pages; volatile bool maintenance_locked = false; volatile bool index_lock_held = false; /* Initialize stats if not provided */ if (stats == NULL) stats = tp_bulkdelete_internal( info, NULL, tp_vacuum_no_dead_callback, NULL, false, false); /* * Spill the memtable so the on-disk memtable chain doesn't * accumulate forever on insert-only tables. Insert-only * tables skip ambulkdelete (no dead tuples), so without this * call nothing would spill and the chain would keep growing. * Skip under TP_MIN_SPILL_PAGES — that few docs is cheaper * to read from the chain on every query than to compact a * runt L0 segment away. */ index_state = tp_get_local_index_state(RelationGetRelid(info->index)); if (index_state == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for \"%s\"", RelationGetRelationName(info->index)))); tp_spill_memtable_if_needed(info->index, index_state, TP_MIN_SPILL_PAGES); /* * Maintenance stabilizes segment payloads while cleanup removes at most * one empty-bearing prefix and counts live documents. Parallel VACUUM * cannot assign the reclaim XID needed by physical cleanup, so it leaves * zeroed segments for a later serial pass. */ tp_compaction_lock(info->index); maintenance_locked = true; PG_TRY(); { if (!IsInParallelMode() && !IsParallelWorker()) (void)tp_compact_empty_step(index_state, info->index); tp_acquire_index_lock(index_state, LW_SHARED); index_lock_held = true; segment_snapshot = tp_segment_graph_snapshot_create(info->index); tp_release_index_lock(index_state); index_lock_held = false; stats->num_pages = 1; /* * reltuples tracks live docs, which can be less than * metap->total_docs because V5 bitset flips with survivors * reduce alive_count without changing segment.num_docs. Sum * alive_count across segments for an accurate live count * regardless of whether tp_bulkdelete ran or this is a * no-deletes maintenance round. * * The maintenance lock keeps compaction and another VACUUM from * replacing snapshot sources. A concurrent spill may prepend L0 * but cannot invalidate the copied roots, so the per-index lock is * not held across this walk. */ stats->num_index_tuples = (double) tp_count_live_docs(info->index, segment_snapshot); tp_segment_graph_snapshot_free(segment_snapshot); /* * Return DEAD memtable orphan blocks to the index FSM once * their dead_fxid is older than the global visibility horizon. * Maintenance remains held so force-merge truncation cannot race * the captured fork size or page inspection. No per-index lock is * held: concurrent inserts and spill publication remain available. */ freed_pages = tp_reclaim_dead_memtable_pages(info->index, info->heaprel); tp_compaction_unlock(info->index); maintenance_locked = false; if (stats->pages_deleted == 0 && stats->tuples_removed == 0 && freed_pages == 0) stats->pages_free = 0; if (freed_pages != 0) { stats->pages_free += freed_pages; IndexFreeSpaceMapVacuum(info->index); } /* * Drain past-horizon displaced segment pages (issue #380). * Unlike the memtable reclaim above, this MUTATES the shared * tombstone chain (metap->pending_free_head and tombstone * next_page links), so it must run under LW_EXCLUSIVE, not * LW_SHARED. tp_tombstone_drain takes/releases the lock per * drained tombstone (own_lock=true) so concurrent reads never * wait only for the unlink, not its subsequent page frees. */ { uint32 drained = tp_tombstone_drain( info->index, index_state, tp_reclaim_horizon(info->heaprel), /* own_lock */ true); if (drained != 0) { stats->pages_free += drained; IndexFreeSpaceMapVacuum(info->index); } } } PG_FINALLY(); { if (index_lock_held) tp_release_index_lock(index_state); if (maintenance_locked) tp_compaction_unlock(info->index); } PG_END_TRY(); /* Deletion debt needs a maintenance opportunity even without a spill. */ tp_apply_compaction_policy(index_state, info->index, true); return stats; } /* * Scan the index main fork for memtable pages stamped DEAD at spill * and recycle blocks whose dead_fxid is older than the visibility * horizon for `heaprel` (FullTransactionId compare). Does not * WAL-log page bodies or clear DEAD flags; reuse overwrites via * tp_memtable_alloc_page. Caller holds the per-index maintenance * object lock to exclude compaction/force-merge truncation, but does * not hold the per-index LWLock across this O(index-pages) scan. * * Crash-safety guard: we build a set of blocks reachable from the * current memtable chain and skip freeing any page in that set. * This prevents corruption if a crash between tp_spill_finalize and * tp_memtable_mark_chain_dead left pages stamped DEAD but still * reachable via metap.head. See tp_collect_reachable_chain_blocks. * * A concurrent spill can make the reachable snapshot conservative: * pages reachable before publication remain in the set and are * retained for a later VACUUM. Live and newly allocated pages are * never DEAD, and each inspection is serialized by the page buffer * lock. If the snapshot observes the post-spill head, dead_fxid still * prevents reuse while an older primary or feedback-protected standby * snapshot can reference the retired chain. Before each page is free * stamped, stock conflict-only WAL cancels standby snapshots that outlived * feedback or a replication disconnect. */ int tp_reclaim_dead_memtable_pages(Relation indexrel, Relation heaprel) { int reclaimed_pages = 0; BlockNumber nblocks = RelationGetNumberOfBlocks(indexrel); BlockNumber blk; TransactionId oldest; FullTransactionId oldest_fxid; HTAB *reachable; bool reclaim_paused = false; oldest = GetOldestNonRemovableTransactionId(heaprel); oldest_fxid = tp_full_xid_from_allowable_at(ReadNextFullTransactionId(), oldest); /* * Build set of blocks still reachable from metap.head. * Pages in this set must not be freed even if stamped DEAD. */ reachable = tp_collect_reachable_chain_blocks(indexrel); for (blk = TP_METAPAGE_BLKNO + 1; blk < nblocks; blk++) { Buffer buf; Page page; TpMemtablePageHeader *hdr; CHECK_FOR_INTERRUPTS(); if (!reclaim_paused) { TP_INJECTION_POINT(TP_INJECTION_VACUUM_MEMTABLE_RECLAIM); reclaim_paused = true; } buf = ReadBuffer(indexrel, blk); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page) || !tp_memtable_page_is_dead(page)) { UnlockReleaseBuffer(buf); continue; } hdr = tp_memtable_page_header(page); if (FullTransactionIdPrecedes(hdr->dead_fxid, oldest_fxid)) { /* Skip pages still reachable from live chain */ bool found; hash_search(reachable, &blk, HASH_FIND, &found); if (!found) { FullTransactionId dead_fxid = hdr->dead_fxid; /* * Release the SHARE lock before returning the page to * the FSM: tp_record_free_index_page re-locks * EXCLUSIVE to write the recyclable free stamp so a * later allocator can tell this deliberately-freed * page from a live one. The page is DEAD (unlinked) * and unreachable, and is not yet in the FSM, so no * concurrent backend can allocate or resurrect it * between the release and the stamp. */ UnlockReleaseBuffer(buf); tp_log_page_reuse_conflict(indexrel, blk, dead_fxid); tp_record_free_index_page(indexrel, blk); reclaimed_pages++; continue; } } UnlockReleaseBuffer(buf); } hash_destroy(reachable); return reclaimed_pages; } timescale-pg_textsearch-b63fe58/src/compat.h000066400000000000000000000026241525762047700212100ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compat.h - Compiler and server-version portability helpers */ #pragma once #include /* * Packed struct marker. c.h defines pg_attribute_packed() only for * compilers that spell it as an attribute, and notes that MSVC can pack * a struct only by wrapping its whole definition. TP_PACKED therefore * expands to nothing on MSVC, and every struct that uses it wraps its * definition in #pragma pack(push, 1) / #pragma pack(pop). */ #ifdef _MSC_VER #define TP_PACKED #else #define TP_PACKED pg_attribute_packed() #endif /* * PG19 requires a hand-built TupleDesc to be finalized before it is * blessed or otherwise used; on a cassert build BlessTupleDesc() TRAPs * otherwise. Older servers have no such call, so define it away and * let call sites use the PG19 API unconditionally. */ #if PG_VERSION_NUM < 190000 #define TupleDescFinalize(tupdesc) ((void)(tupdesc)) #endif /* * post_parse_analyze_hook's JumbleState parameter became const in PG19. * A hook implementation has to match the typedef exactly — C does not * consider `const JumbleState *` and `JumbleState *` compatible * parameter types — so the qualifier must track the server version. */ #if PG_VERSION_NUM >= 190000 #define TP_JUMBLE_STATE const JumbleState #else #define TP_JUMBLE_STATE JumbleState #endif timescale-pg_textsearch-b63fe58/src/constants.h000066400000000000000000000234631525762047700217450ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * constants.h - Shared constants and configuration */ #pragma once /* * Page magic numbers - unique identifiers for each page type. * These are used to validate page contents and detect corruption. */ #define TP_METAPAGE_MAGIC 0x5450494D /* "TPIM" - Tapir Index Metapage */ #define TP_SEGMENT_MAGIC 0x54505347 /* "TPSG" - Tapir Segment */ #define TP_PAGE_INDEX_MAGIC 0x54505049 /* "TPPI" - Tapir Page Index */ #define TP_MEMTABLE_PAGE_MAGIC \ 0x5450544D /* "TPTM" - Tapir Memtable (replaces docid pages; introduced \ * with the on-disk memtable design, see issue #374) */ #define TP_TOMBSTONE_MAGIC \ 0x5450544F /* "TPTO" - Tapir Tombstone: parks displaced segment pages \ * for deferred, standby-safe FSM reclaim (issue #380) */ #define TP_TOMBSTONE_VERSION 1 #define TP_FREE_PAGE_MAGIC \ 0x54504650 /* "TPFP" - Tapir Free Page: a page deliberately returned \ * to the index FSM and safe to recycle (issues #426, #427) \ */ /* * Page format versions - bump when on-disk format changes. * Each page type has its own version for independent evolution. */ /* * v9 appends index capability flags. v8 adds the deferred-free * tombstone head. v7 is the on-disk memtable redesign (issue #374) * and appends * memtable_head_blkno and memtable_tail_blkno at the end of * TpIndexMetaPageData and retires the docid recovery pages. * * v6 is read-compatible (issue #383): tp_get_metapage accepts a * v6 layout in place and normalizes the missing chain head/tail * to InvalidBlockNumber. The first metapage mutation upgrades * the on-disk page to v7 atomically inside the surrounding * GenericXLog record (no extra WAL). * * A v6 index's only source of pending (unspilled) data is the * retired docid recovery chain anchored by * (now-renamed) _unused_docid_page: a v1.2.x clean shutdown * always spilled and cleared this pointer, but a SIGKILL'd * v1.2.x cluster could leave a non-Invalid pointer to ctids * that the v7 binary cannot re-tokenize. Such an index opens * fine; the first metapage mutation (via * tp_metapage_upgrade_to_current) emits a client-visible * WARNING recording the possibly-incomplete state and PRESERVES * the pointer as a durable marker, which * tp_warn_if_pending_docid() re-surfaces on the scan path, at most * once per session, until a REINDEX rebuilds the index from the heap * and clears it. Indexes from a clean v1.2.x shutdown have * _unused_docid_page == InvalidBlockNumber and upgrade silently * with no operator intervention. * * v5 and below are not read-compatible. v5 -> v6 changed BMW * scoring semantics with no on-disk-struct change, but * pre-v0.5.0 indexes carry an older segment format the current * binary cannot read; those continue to require REINDEX. */ #define TP_METAPAGE_VERSION 9 #define TP_METAPAGE_VERSION_V8 8 /* read-compatible: no capability flags */ #define TP_METAPAGE_VERSION_V7 \ 7 /* read-compatible: on-disk memtable, no \ * pending_free_head (issue #380) */ #define TP_METAPAGE_VERSION_V6 6 /* read-compatible (issue #383) */ #define TP_PAGE_INDEX_VERSION 1 /* Page index format version */ /* Initial version (introduced with the on-disk memtable design) */ #define TP_MEMTABLE_PAGE_VERSION 1 #define TP_METAPAGE_BLKNO 0 #define TP_METAPAGE_ALL_DOCUMENTS_INDEXED (1U << 0) /* Segment hierarchy configuration */ #define TP_MAX_LEVELS 8 /* Supports 8^8 = 16M segments */ #define TP_DEFAULT_SEGMENTS_PER_LEVEL 8 #define TP_MIN_SEGMENT_SIZE_MB 1 #define TP_DEFAULT_SEGMENT_SIZE_MB 4095 #define TP_MAX_SEGMENT_SIZE_MB 4095 #define TP_BASE_LEVEL_SIZE_BYTES ((uint64)8 * 1024 * 1024) /* BM25 scoring constants */ #define TP_DEFAULT_K1 1.2 #define TP_DEFAULT_B 0.75 /* Memory and capacity limits */ #define TP_QUERY_LIMITS_HASH_SIZE 128 #define TP_DEFAULT_QUERY_LIMIT 1000 #define TP_MAX_INITIAL_QUERY_LIMIT 100000 #define TP_DEFAULT_SEGMENT_THRESHOLD 10000 #define TP_DEFAULT_BULK_LOAD_THRESHOLD 100000 /* terms/xact trigger spill */ /* * Selectivity-seeded top-K for filtered BM25 search. When a filter * ("facet") sits above the BM25 index scan, seed the pushed-down * internal top-K to * ceil(margin * user_limit / filter_selectivity) so a single scoring * pass usually surfaces enough Filter-matching rows, avoiding the * executor's backoff re-drives. The margin absorbs variance around the * expected rank; the existing backoff remains the correctness net. */ #define TP_DEFAULT_FILTERED_SEED_MARGIN 3.0 #define TP_MIN_FILTERED_SEED_MARGIN 1.0 #define TP_MAX_FILTERED_SEED_MARGIN 1000.0 /* * Default for pg_textsearch.memtable_pages_threshold (on-disk * memtable auto-spill trigger). 64 chain pages * 8 KiB = ~512 * KiB of memtable per index before the next insert spills to an * L0 segment. Tuned for low latency without producing runt L0 * segments under typical insert traffic; 0 disables auto-spill. */ #define TP_DEFAULT_MEMTABLE_PAGES_THRESHOLD 64 /* * Lower bound on chain pages for VACUUM-cleanup and shutdown-hook * spills. Below this, leaving the documents on the in-relation * memtable chain is cheaper than writing a runt L0 segment — LSM * compaction would spend more work consolidating tiny segments * than the chain pages cost to scan during reads. */ #define TP_MIN_SPILL_PAGES 2 /* * Default for pg_textsearch.memory_limit (in kilobytes). Matches the * v1 in-memory memtable's 2 GiB ceiling. Used as both the GUC default * and the initial value of the backing variable so the two cannot * drift. A value of 0 means "no limit" (cache grows until backend * memory is exhausted); the cache implementation will treat any * non-zero value as a three-tier budget (per-index soft = limit/8, * global soft = limit/2, global hard = limit). */ #define TP_DEFAULT_MEMORY_LIMIT_KB (2 * 1024 * 1024) /* Hash table sizes */ #define TP_STRING_INTERNING_HASH_SIZE 1024 #define TP_POSTING_LIST_HASH_INITIAL_SIZE 32 #define TP_POSTING_LIST_HASH_MAX_SIZE 256 /* Posting list and array parameters */ #define TP_INITIAL_POSTING_LIST_CAPACITY 16 #define TP_POSTING_LIST_GROWTH_FACTOR 2 /* Query processing timeouts and limits */ #define TP_MAX_INDEX_NAME_LENGTH 1024 #define TP_MAX_TERM_LENGTH (1024 * 1024) /* 1MB sanity limit */ /* Cost estimation constants */ #define TP_DEFAULT_TUPLE_ESTIMATE 1000.0 #define TP_HIGH_STARTUP_COST 1000000.0 #define TP_INDEX_SCAN_COST_FACTOR 0.1 #define TP_DEFAULT_INDEX_SELECTIVITY 0.1 #define TP_DEFAULT_INDEX_PAGES 1000.0 /* * Fixed LWLock tranche IDs. * These must be consistent across all backends to allow DSA attachment. * We use high numbers (1001+) to avoid conflicts with core PostgreSQL * tranches. * * Using fixed tranche IDs is critical for supporting large numbers of indexes * (e.g., partitioned tables with 500+ partitions). If we called * LWLockNewTrancheId() for each index, we would quickly exhaust the available * tranche IDs and get "too many LWLocks taken" errors. */ #define TP_TRANCHE_STRING 1001 #define TP_TRANCHE_POSTING 1002 #define TP_TRANCHE_CORPUS 1003 #define TP_TRANCHE_DOC_LENGTHS 1004 #define TP_TRANCHE_INDEX_LOCK 1005 #define TP_TRANCHE_BUILD_DSA 1006 #define TP_TRANCHE_GLOBAL_DSA 1007 #define TP_TRANCHE_REGISTRY 1008 #define TP_TRANCHE_POSTING_LOCK 1009 /* * In-memory memtable cache LWLocks. * apply_lock serializes cache mutators (reader catchup, cold build, * spill catchup, tp_cache_clear). lock is the cache lifetime lock, * held SHARED for the lifetime of a served TpDataSource and EXCL only * for drop / evict. */ #define TP_TRANCHE_CACHE_APPLY_LOCK 1010 #define TP_TRANCHE_CACHE_LOCK 1011 /* * Global eviction mutex tranche. Serializes cache evictions * across backends. Acquired EXCL by evict_largest and by index * cleanup (DROP INDEX path) to prevent races between an in-flight * victim inspection and a concurrent dsa_free of the victim's * shared state. */ #define TP_TRANCHE_EVICTION_MUTEX 1012 #define TP_TRANCHE_MEMTABLE_WRITE_LOCK 1013 /* * Bounds of the contiguous fixed-tranche block above. TP_TRANCHE_COUNT * fixed IDs [TP_TRANCHE_FIRST, TP_TRANCHE_LAST] are defined; a handful * (POSTING, CORPUS) are currently reserved but still allocated so the * block stays contiguous. */ #define TP_TRANCHE_FIRST TP_TRANCHE_STRING #define TP_TRANCHE_LAST TP_TRANCHE_MEMTABLE_WRITE_LOCK #define TP_TRANCHE_COUNT ((TP_TRANCHE_LAST) - (TP_TRANCHE_FIRST) + 1) /* * Resolve a fixed logical tranche constant (TP_TRANCHE_*) to the actual * LWLock tranche ID to use at runtime. * * PG17/18: the fixed IDs are used directly (and named at startup via * LWLockRegisterTranche), so this is a compile-time identity. * * PG19+: LWLockRegisterTranche() was removed; a tranche name can only be * assigned when the ID is allocated via LWLockNewTrancheId(). We * allocate one contiguous, name-registered block of TP_TRANCHE_COUNT IDs * exactly once at shared-memory startup and map each fixed constant onto * it. See tp_registry_shmem_startup() and tp_tranche_id(). */ #if PG_VERSION_NUM >= 190000 extern int tp_tranche_id(int fixed_tranche_id); #else #define tp_tranche_id(fixed_tranche_id) (fixed_tranche_id) #endif /* * Global GUC variables declared in mod.c * Note: tp_relopt_kind is declared in index.c as it requires * access/reloptions.h */ extern bool tp_log_scores; extern bool tp_allow_rls; extern int tp_bulk_load_threshold; extern int tp_memtable_pages_threshold; extern int tp_segments_per_level; extern int tp_max_segment_size_mb; extern bool tp_filtered_seed; extern double tp_filtered_seed_margin; extern bool tp_compress_segments; extern bool tp_memtable_cache_enabled; extern bool tp_log_cache_state; extern int tp_memory_limit_kb; timescale-pg_textsearch-b63fe58/src/debug/000077500000000000000000000000001525762047700206365ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/debug/dump.c000066400000000000000000001150441525762047700217540ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * dump.c - Index dump and debugging utilities */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "debug/dump.h" #include "index/metapage.h" #include "index/resolve.h" #include "index/source.h" #include "index/state.h" #include "memtable/chain_source.h" #include "memtable/page.h" #include "segment/compaction.h" #include "segment/format.h" #include "segment/graph_snapshot.h" #include "segment/io.h" #include "segment/segment.h" #include "segment/tombstone.h" /* Output size limits for string mode */ #define MAX_OUTPUT_SIZE (256 * 1024) /* 256KB soft limit */ #define MAX_DOCS_TO_SHOW 20 /* Records to show in full */ /* Forward declaration - implemented in segment.c */ extern void tp_dump_segment_to_output( Relation index, BlockNumber segment_root, DumpOutput *out); /* * Dump memtable contents (v2 on-disk chain). * * Walks the on-disk page chain rooted at the metapage's * memtable_head_blkno field and prints per-page summaries plus * a per-document CTID + doc_length listing. In v2 there is no * in-memory term dictionary or posting list — those are * reconstructed by the chain source at query time. The TpVector * payload on each record contains the terms; we expose it via * vector_len here for sizing visibility but do not decode it * (see bm25_dump_index summarize section for term counts). */ static void dump_memtable(DumpOutput *out, TpLocalIndexState *index_state, Relation rel) { TpIndexMetaPage metap; BlockNumber blk; uint32 page_count = 0; uint32 rec_count = 0; uint32 docs_shown = 0; int max_docs; (void)index_state; if (rel == NULL) { dump_printf(out, "Memtable Pages:\n (relation not open)\n"); return; } metap = tp_get_metapage(rel); if (metap == NULL || !BlockNumberIsValid(metap->memtable_head_blkno)) { dump_printf(out, "Memtable Pages:\n (empty)\n"); if (metap) pfree(metap); return; } max_docs = out->full_dump ? INT_MAX : MAX_DOCS_TO_SHOW; dump_printf(out, "Memtable Pages:\n"); for (blk = metap->memtable_head_blkno; BlockNumberIsValid(blk);) { Buffer buf; Page page; BlockNumber next; int n; CHECK_FOR_INTERRUPTS(); buf = ReadBuffer(rel, blk); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { dump_printf(out, " block %u: INVALID (missing magic)\n", blk); UnlockReleaseBuffer(buf); break; } n = tp_memtable_page_n_records(page); next = tp_memtable_page_get_next(page); page_count++; dump_printf(out, " block %u: n_records=%d, next=%u\n", blk, n, next); if (!out->full_dump && out->str && out->str->len > (int)MAX_OUTPUT_SIZE) { rec_count += n; UnlockReleaseBuffer(buf); blk = next; continue; } { TpMemtableRecord *rec = tp_memtable_page_first(page); while (rec != NULL) { rec_count++; if (docs_shown < (uint32)max_docs) { dump_printf( out, " CTID (%u,%u): doc_length=%d, " "vector_len=%u\n", BlockIdGetBlockNumber(&rec->ctid.ip_blkid), rec->ctid.ip_posid, rec->doc_length, rec->vector_len); docs_shown++; } rec = tp_memtable_page_next(page, rec); } } UnlockReleaseBuffer(buf); blk = next; } if (docs_shown < rec_count) dump_printf( out, " ... (showing %u of %u records)\n", docs_shown, rec_count); dump_printf(out, "Total memtable pages: %u\n", page_count); dump_printf(out, "Total memtable records: %u\n", rec_count); pfree(metap); } /* * Summarize index statistics without dumping content */ void tp_summarize_index_to_output(const char *index_name, DumpOutput *out) { Oid index_oid; Relation index_rel = NULL; TpIndexMetaPage metap = NULL; TpSegmentGraphSnapshot *snapshot = NULL; TpLocalIndexState *index_state; uint32 memtable_terms = 0; uint32 memtable_docs = 0; int segment_count = 0; uint32 segment_terms = 0; uint32 segment_docs = 0; uint32 segment_alive = 0; bool acquired_lock = false; dump_printf(out, "Index: %s\n", index_name); index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) { dump_printf(out, "ERROR: Index '%s' not found\n", index_name); return; } /* Open the index */ index_rel = index_open(index_oid, AccessShareLock); /* Get index state */ index_state = tp_get_local_index_state(index_oid); if (index_state == NULL) { dump_printf( out, "ERROR: Could not get index state for '%s'\n", index_name); if (index_rel) index_close(index_rel, AccessShareLock); return; } /* * Issue #404: hold the per-index lock SHARED while consuming the * bounded segment-root snapshot so concurrent spill/merge cannot * recycle a root before it is opened. On error the end-of-xact * LWLockReleaseAll plus state cleanup reset the lock. */ if (index_state != NULL && !index_state->lock_held) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_lock = true; } snapshot = tp_segment_graph_snapshot_create(index_rel); metap = &snapshot->metapage; /* * Corpus statistics. * * docs_persisted / len_persisted come from the metapage and * are the durable source of truth: Σ segment.num_docs and * Σ segment.total_len respectively (see metapage.h). * * total_docs / total_len reported here are the live in-flight * totals (persisted + on-disk memtable chain), computed by * adding a fresh chain-source open onto the metapage values. * Tests that need a deterministic post-merge / post-vacuum * corpus count should grep for docs_persisted / len_persisted * instead. */ { TpDataSource *chain_src = tp_memtable_chain_source_create( index_state, index_rel, NULL, 0); uint64 metap_docs = metap ? metap->total_docs : 0; uint64 metap_len = metap ? metap->total_len : 0; uint64 chain_docs = chain_src ? (uint64)chain_src->total_docs : 0; uint64 chain_len = chain_src ? (uint64)chain_src->total_len : 0; uint64 total_docs = metap_docs + chain_docs; uint64 total_len = metap_len + chain_len; if (chain_src) tp_source_close(chain_src); dump_printf(out, "\nCorpus Statistics:\n"); dump_printf(out, " total_docs: " UINT64_FORMAT "\n", total_docs); dump_printf(out, " total_len: " UINT64_FORMAT "\n", total_len); if (metap) { dump_printf( out, " docs_persisted: " UINT64_FORMAT "\n", metap->total_docs); dump_printf( out, " len_persisted: " UINT64_FORMAT "\n", metap->total_len); } if (total_docs > 0) { float avg_doc_len = (float)total_len / (float)total_docs; dump_printf(out, " avg_doc_len: %.2f\n", avg_doc_len); } } /* BM25 parameters */ dump_printf(out, "\nBM25 Parameters:\n"); if (metap) { dump_printf(out, " k1: %.2f\n", metap->k1); dump_printf(out, " b: %.2f\n", metap->b); } /* Memory usage */ if (index_state->dsa) { size_t dsa_total_size = dsa_get_total_size(index_state->dsa); dump_printf(out, "\nMemory Usage:\n"); dump_printf( out, " DSA total size: %zu bytes (%.2f MB)\n", dsa_total_size, (double)dsa_total_size / (1024.0 * 1024.0)); } /* Count memtable documents from the on-disk page chain (v2). */ memtable_terms = 0; memtable_docs = 0; if (metap && BlockNumberIsValid(metap->memtable_head_blkno)) { BlockNumber blk = metap->memtable_head_blkno; while (BlockNumberIsValid(blk)) { Buffer buf; Page page; CHECK_FOR_INTERRUPTS(); buf = ReadBuffer(index_rel, blk); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (tp_memtable_page_is_valid(page)) { memtable_docs += tp_memtable_page_n_records(page); blk = tp_memtable_page_get_next(page); } else { /* Should not happen on a healthy chain. */ blk = InvalidBlockNumber; } UnlockReleaseBuffer(buf); } } dump_printf(out, "\nMemtable:\n"); dump_printf(out, " terms: %u\n", memtable_terms); dump_printf(out, " documents: %u\n", memtable_docs); /* Segment summary by level */ dump_printf(out, "\nSegments:\n"); { int level; uint64 segment_pages = 0; bool has_segments = false; for (level = 0; level < TP_MAX_LEVELS; level++) { const BlockNumber *roots; uint32 root_count; int level_segment_count = 0; roots = tp_segment_graph_snapshot_level( snapshot, level, &root_count); if (root_count == 0) continue; has_segments = true; for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentReader *reader; BlockNumber current_segment = roots[root_idx]; CHECK_FOR_INTERRUPTS(); reader = tp_segment_open(index_rel, current_segment); if (reader && reader->header) { TpSegmentHeader *header = reader->header; Size seg_size; segment_count++; level_segment_count++; segment_terms += header->num_terms; segment_docs += header->num_docs; segment_alive += header->alive_count; segment_pages += header->num_pages; seg_size = (Size)header->num_pages * BLCKSZ; if (header->alive_count < header->num_docs) dump_printf( out, " L%d Segment %d: block=%u, " "pages=%u, size=%.1fMB, " "terms=%u, docs=%u " "(alive=%u, dead=%u)\n", level, level_segment_count, current_segment, header->num_pages, (double)seg_size / (1024.0 * 1024.0), header->num_terms, header->num_docs, header->alive_count, header->num_docs - header->alive_count); else dump_printf( out, " L%d Segment %d: block=%u, " "pages=%u, size=%.1fMB, " "terms=%u, docs=%u\n", level, level_segment_count, current_segment, header->num_pages, (double)seg_size / (1024.0 * 1024.0), header->num_terms, header->num_docs); tp_segment_close(reader); } else { break; } } } if (has_segments) { if (segment_alive < segment_docs) dump_printf( out, " Total: %d segments, " UINT64_FORMAT " pages (%.1fMB), " "%u terms, %u docs " "(alive=%u, dead=%u)\n", segment_count, segment_pages, (double)(segment_pages * BLCKSZ) / (1024.0 * 1024.0), segment_terms, segment_docs, segment_alive, segment_docs - segment_alive); else dump_printf( out, " Total: %d segments, " UINT64_FORMAT " pages (%.1fMB), " "%u terms, %u docs\n", segment_count, segment_pages, (double)(segment_pages * BLCKSZ) / (1024.0 * 1024.0), segment_terms, segment_docs); } else { dump_printf(out, " (none)\n"); } } /* Index size */ dump_printf(out, "\nIndex Size:\n"); dump_printf( out, " on-disk: %lu bytes\n", (unsigned long)RelationGetNumberOfBlocks(index_rel) * BLCKSZ); /* Cleanup */ if (acquired_lock) tp_release_index_lock(index_state); tp_segment_graph_snapshot_free(snapshot); if (index_rel) index_close(index_rel, AccessShareLock); } /* * Main dump function - dumps entire index to output */ void tp_dump_index_to_output(const char *index_name, DumpOutput *out) { Oid index_oid; Relation index_rel = NULL; TpIndexMetaPage metap = NULL; TpSegmentGraphSnapshot *snapshot = NULL; TpLocalIndexState *index_state; bool acquired_lock = false; dump_printf(out, "Tapir Index Debug: %s\n", index_name); index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) { dump_printf(out, "ERROR: Index '%s' not found\n", index_name); return; } /* Open the index */ index_rel = index_open(index_oid, AccessShareLock); /* Get index state */ index_state = tp_get_local_index_state(index_oid); if (index_state == NULL) { dump_printf( out, "ERROR: Could not get index state for '%s'\n", index_name); if (index_rel) index_close(index_rel, AccessShareLock); return; } /* * Issue #404: hold the per-index lock (SHARED) while consuming the * bounded segment-root snapshot; see the matching comment in * tp_summarize_index_to_output. */ if (index_state != NULL && !index_state->lock_held) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_lock = true; } snapshot = tp_segment_graph_snapshot_create(index_rel); metap = &snapshot->metapage; /* * Corpus statistics. See the comment in * tp_summarize_index_to_output above for the docs_persisted / * len_persisted rationale (metapage = durable Σ). total_docs * / total_len here include any in-flight on-disk memtable * chain entries via a fresh chain-source open. */ { TpDataSource *chain_src = tp_memtable_chain_source_create( index_state, index_rel, NULL, 0); uint64 metap_docs = metap ? metap->total_docs : 0; uint64 metap_len = metap ? metap->total_len : 0; uint64 chain_docs = chain_src ? (uint64)chain_src->total_docs : 0; uint64 chain_len = chain_src ? (uint64)chain_src->total_len : 0; uint64 total_docs = metap_docs + chain_docs; uint64 total_len = metap_len + chain_len; if (chain_src) tp_source_close(chain_src); dump_printf(out, "Corpus Statistics:\n"); dump_printf(out, " total_docs: " UINT64_FORMAT "\n", total_docs); dump_printf(out, " total_len: " UINT64_FORMAT "\n", total_len); if (metap) { dump_printf( out, " docs_persisted: " UINT64_FORMAT "\n", metap->total_docs); dump_printf( out, " len_persisted: " UINT64_FORMAT "\n", metap->total_len); } if (total_docs > 0) { float avg_doc_len = (float)total_len / (float)total_docs; dump_printf(out, " avg_doc_len: %.4f\n", avg_doc_len); } else { dump_printf(out, " avg_doc_len: 0 (no documents)\n"); } } /* DSA memory */ if (index_state->dsa) { size_t dsa_total_size = dsa_get_total_size(index_state->dsa); dump_printf(out, "Memory Usage:\n"); dump_printf( out, " DSA total size: %zu bytes (%.2f MB)\n", dsa_total_size, (double)dsa_total_size / (1024.0 * 1024.0)); } /* BM25 parameters */ dump_printf(out, "BM25 Parameters:\n"); if (metap) { dump_printf(out, " k1: %.2f\n", metap->k1); dump_printf(out, " b: %.2f\n", metap->b); dump_printf(out, "Metapage Info:\n"); dump_printf(out, " magic: 0x%08X\n", metap->magic); dump_printf(out, " version: %u\n", metap->version); dump_printf( out, " memtable_head_blkno: %u\n", metap->memtable_head_blkno); dump_printf( out, " memtable_tail_blkno: %u\n", metap->memtable_tail_blkno); } /* Memtable contents */ dump_memtable(out, index_state, index_rel); /* Detailed segment dump (first 2 per level) */ { int level; int total_segments = snapshot->root_count; int dumped_count = 0; const int max_dump = 2; bool has_segments = snapshot->root_count > 0; /* Now dump first N segments from each level */ for (level = 0; level < TP_MAX_LEVELS; level++) { const BlockNumber *roots; uint32 root_count; int level_dumped = 0; roots = tp_segment_graph_snapshot_level( snapshot, level, &root_count); if (root_count == 0) continue; for (uint32 root_idx = 0; root_idx < root_count && level_dumped < max_dump; root_idx++) { BlockNumber current_segment = roots[root_idx]; CHECK_FOR_INTERRUPTS(); dump_printf(out, "\nL%d ", level); tp_dump_segment_to_output(index_rel, current_segment, out); dumped_count++; level_dumped++; } } if (total_segments > dumped_count) { dump_printf( out, "\n... %d more segments not shown\n", total_segments - dumped_count); } if (!has_segments) { dump_printf(out, "\nNo segments written yet\n"); } } /* Cleanup */ if (acquired_lock) tp_release_index_lock(index_state); tp_segment_graph_snapshot_free(snapshot); if (index_rel) index_close(index_rel, AccessShareLock); } /* * tp_dump_index - Debug function to show internal index structure * including both memtable and all segments * * Takes index name and optional filename. If filename is provided, * writes full dump to file (no truncation, includes hex dumps). * Otherwise returns truncated output as text. */ PG_FUNCTION_INFO_V1(tp_dump_index); Datum tp_dump_index(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); /* All dump functions require superuser */ if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to dump index"))); /* Check for optional filename parameter */ #ifdef DEBUG_DUMP_INDEX if (PG_NARGS() > 1 && !PG_ARGISNULL(1)) { /* File mode - full dump with hex */ text *filename_text = PG_GETARG_TEXT_PP(1); char *filename = text_to_cstring(filename_text); FILE *fp; fp = fopen(filename, "w"); if (!fp) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", filename))); } { DumpOutput out; dump_init_file(&out, fp); tp_dump_index_to_output(index_name, &out); } fclose(fp); elog(INFO, "Index dump written to %s", filename); PG_RETURN_TEXT_P(cstring_to_text_with_len(filename, strlen(filename))); } else #endif { /* String mode - truncated output for SQL return */ StringInfoData result; DumpOutput out; initStringInfo(&result); dump_init_string(&out, &result); tp_dump_index_to_output(index_name, &out); PG_RETURN_TEXT_P(cstring_to_text(result.data)); } } /* * tp_summarize_index - Show index statistics without dumping content * * A faster alternative to bm25_dump_index that shows only statistics: * corpus statistics, BM25 parameters, memory usage, memtable/segment counts. */ PG_FUNCTION_INFO_V1(tp_summarize_index); Datum tp_summarize_index(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); StringInfoData result; DumpOutput out; /* Superuser only */ if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to summarize index"))); initStringInfo(&result); dump_init_string(&out, &result); tp_summarize_index_to_output(index_name, &out); PG_RETURN_TEXT_P(cstring_to_text(result.data)); } PG_FUNCTION_INFO_V1(tp_pending_free_pages); Datum tp_pending_free_pages(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); Oid index_oid; Relation index_rel; TpLocalIndexState *index_state; uint64 count; if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to inspect index"))); index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" not found", index_name))); index_rel = index_open(index_oid, AccessShareLock); /* * Hold the per-index LWLock in shared mode across the tombstone * walk so a concurrent VACUUM drain / merge enqueue (which take * LW_EXCLUSIVE to mutate the chain and recycle tombstone pages via * the FSM) can't recycle a page we're about to read. Without it * the walk could follow a stale next_page link into a page that * has already been reinitialized as a different chain's tombstone * and return a silently wrong count. Mirrors tp_count_live_docs * in vacuum.c. */ index_state = tp_get_local_index_state(index_oid); if (index_state != NULL) tp_acquire_index_lock(index_state, LW_SHARED); count = tp_pending_free_block_count(index_rel); if (index_state != NULL) tp_release_index_lock(index_state); index_close(index_rel, AccessShareLock); PG_RETURN_INT64((int64)count); } /* * bm25_test_recycle_tombstone_head(index_name) -> block number * * INTERNAL-ONLY test scaffold (issues #426, #427). Returns the * still-live head tombstone page's block to the index FSM, then * rebuilds the FSM upper levels so GetFreeIndexPage() can hand the * block back out. This reproduces — without an actual crash — the * stale-FSM / non-atomic-claim hazard that lets an allocator pick up * a block that is still referenced by the deferred-free tombstone * chain (or, symmetrically, the on-disk memtable chain). * * Superuser-only; signature and existence are subject to change or * removal in ANY release without notice. Not part of the public API. */ PG_FUNCTION_INFO_V1(tp_test_recycle_tombstone_head); Datum tp_test_recycle_tombstone_head(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); Oid index_oid; Relation index_rel; BlockNumber head; if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to recycle tombstone head"))); index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" not found", index_name))); index_rel = index_open(index_oid, RowExclusiveLock); head = tp_tombstone_read_head(index_rel); if (head == InvalidBlockNumber) { index_close(index_rel, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index \"%s\" has no tombstone chain to recycle", index_name))); } RecordFreeIndexPage(index_rel, head); IndexFreeSpaceMapVacuum(index_rel); index_close(index_rel, RowExclusiveLock); PG_RETURN_INT64((int64)head); } /* * bm25_test_corrupt_tombstone_head(index_name) -> block number * * INTERNAL-ONLY test scaffold (issue #427). Overwrites the head * tombstone page's magic (WAL-logged) so it no longer validates as a * tombstone page, simulating a chain node that a page-reuse bug * already clobbered in production. Used to exercise the drain's * self-healing recovery path on an already-corrupt chain. * * Superuser-only; signature and existence are subject to change or * removal in ANY release without notice. Not part of the public API. */ PG_FUNCTION_INFO_V1(tp_test_corrupt_tombstone_head); Datum tp_test_corrupt_tombstone_head(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); char *index_name = text_to_cstring(index_name_text); Oid index_oid; Relation index_rel; BlockNumber head; Buffer buf; Page page; GenericXLogState *state; TpTombstonePage t; if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to corrupt tombstone head"))); index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" not found", index_name))); index_rel = index_open(index_oid, RowExclusiveLock); head = tp_tombstone_read_head(index_rel); if (head == InvalidBlockNumber) { index_close(index_rel, RowExclusiveLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index \"%s\" has no tombstone chain to corrupt", index_name))); } buf = ReadBuffer(index_rel, head); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index_rel); page = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE); t = tp_tombstone_page(page); t->magic = 0; /* no longer a valid tombstone page */ GenericXLogFinish(state); UnlockReleaseBuffer(buf); index_close(index_rel, RowExclusiveLock); PG_RETURN_INT64((int64)head); } /* * Page visualization support - only available in debug builds. * These functions write to arbitrary file paths, so they are gated * behind DEBUG_DUMP_INDEX to reduce attack surface in release builds. */ #ifdef DEBUG_DUMP_INDEX /* * Page map entry - tracks what each page is used for */ typedef struct PageMapEntry { uint8 level; /* Segment level (0-7), or 255 for special */ uint8 seg_in_lvl; /* Segment number within level */ uint8 page_type; /* See PAGE_* constants below */ uint8 data_region; /* See DATA_* constants below */ } PageMapEntry; /* Page types */ #define PAGE_UNUSED 0 /* Empty/free page */ #define PAGE_METAPAGE 1 /* Index metapage (block 0) */ #define PAGE_SEG_HDR 3 /* Segment header */ #define PAGE_SEG_INDEX 4 /* Segment page index */ #define PAGE_SEG_DATA 5 /* Segment data page */ /* Data regions within segment (for PAGE_SEG_DATA) */ #define DATA_DICTIONARY 0 /* Dictionary + strings + entries */ #define DATA_POSTING 1 /* Posting lists (compressed blocks) */ #define DATA_SKIP 2 /* Skip index entries */ #define DATA_DOCMAP 3 /* Fieldnorms + CTID mapping */ /* ANSI color codes - 256-color mode for segment backgrounds */ #define ANSI_RESET "\033[0m" #define ANSI_DIM "\033[2m" /* * Segment background colors (256-color palette) * Format: \033[48;5;Nm for background color N * Using distinct, visible colors that work with black text */ static const int segment_bg_colors[] = { 196, /* Red */ 46, /* Green */ 33, /* Blue */ 226, /* Yellow */ 201, /* Magenta */ 51, /* Cyan */ 208, /* Orange */ 141, /* Light purple */ 118, /* Lime */ 213, /* Pink */ 75, /* Sky blue */ 220, /* Gold */ 177, /* Violet */ 119, /* Light green */ 209, /* Salmon */ 147, /* Light blue */ }; #define NUM_SEGMENT_COLORS \ (sizeof(segment_bg_colors) / sizeof(segment_bg_colors[0])) /* Segment info for legend */ typedef struct SegmentInfo { int level; int seg_in_level; int global_idx; /* Index into segment_colors */ uint32 num_pages; BlockNumber root_block; } SegmentInfo; /* * Get character for page type/region * Returns single character mnemonic (space for postings to reduce clutter) */ static char get_page_char(PageMapEntry *e) { switch (e->page_type) { case PAGE_UNUSED: return '.'; case PAGE_METAPAGE: return 'M'; case PAGE_SEG_HDR: return 'H'; /* Header */ case PAGE_SEG_INDEX: return 'i'; /* Index */ case PAGE_SEG_DATA: switch (e->data_region) { case DATA_DICTIONARY: return 'd'; case DATA_POSTING: return ' '; /* Blank - most common, reduces clutter */ case DATA_SKIP: return 's'; case DATA_DOCMAP: return 'm'; default: return '?'; } default: return '?'; } } /* * Mark the metapage in the page map. */ static void mark_special_pages( Relation index_rel, TpIndexMetaPage metap, PageMapEntry *page_map, BlockNumber total_blocks) { (void)index_rel; (void)metap; (void)total_blocks; /* Mark metapage */ page_map[0].page_type = PAGE_METAPAGE; page_map[0].level = 255; } /* * Count segments in the index. */ static int count_segments(const TpSegmentGraphSnapshot *snapshot) { return snapshot ? (int)snapshot->root_count : 0; } /* * Collect segment info for the legend. * Caller must allocate segments array with enough space. */ static void collect_segment_info( Relation index_rel, const TpSegmentGraphSnapshot *snapshot, SegmentInfo *segments) { int num_segments = 0; int level; for (level = 0; level < TP_MAX_LEVELS; level++) { const BlockNumber *roots; uint32 root_count; roots = tp_segment_graph_snapshot_level(snapshot, level, &root_count); for (uint32 seg_in_level = 0; seg_in_level < root_count; seg_in_level++) { TpSegmentReader *reader; BlockNumber seg_root = roots[seg_in_level]; reader = tp_segment_open(index_rel, seg_root); if (!reader || !reader->header) { if (reader) tp_segment_close(reader); break; } segments[num_segments].level = level; segments[num_segments].seg_in_level = seg_in_level; segments[num_segments].global_idx = num_segments; segments[num_segments].num_pages = reader->num_pages; segments[num_segments].root_block = seg_root; num_segments++; tp_segment_close(reader); } } } /* * Classify a segment page by its data region based on byte offset. */ static void classify_segment_page( PageMapEntry *entry, TpSegmentReader *reader, uint32 page_idx, int global_idx, int seg_in_level) { uint64 page_start_offset; entry->level = global_idx; entry->seg_in_lvl = seg_in_level; /* First byte offset of this logical page */ page_start_offset = (uint64)page_idx * (BLCKSZ - SizeOfPageHeaderData); if (page_idx == 0) { entry->page_type = PAGE_SEG_HDR; } else if (page_start_offset < reader->header->postings_offset) { entry->page_type = PAGE_SEG_DATA; entry->data_region = DATA_DICTIONARY; } else if (page_start_offset < reader->header->skip_index_offset) { entry->page_type = PAGE_SEG_DATA; entry->data_region = DATA_POSTING; } else if (page_start_offset < reader->header->fieldnorm_offset) { entry->page_type = PAGE_SEG_DATA; entry->data_region = DATA_SKIP; } else { entry->page_type = PAGE_SEG_DATA; entry->data_region = DATA_DOCMAP; } } /* * Mark page index pages for a segment. */ static void mark_page_index_pages( Relation index_rel, TpSegmentReader *reader, PageMapEntry *page_map, BlockNumber total_blocks, int global_idx, int seg_in_level) { BlockNumber pi_blk; TpPageIndexSpecial pi_special; int pi_count = 0; if (reader->header->page_index == InvalidBlockNumber) return; pi_blk = reader->header->page_index; while (pi_blk != InvalidBlockNumber && pi_blk < total_blocks && pi_count < 1000) { Buffer buf; Page page; page_map[pi_blk].level = global_idx; page_map[pi_blk].seg_in_lvl = seg_in_level; page_map[pi_blk].page_type = PAGE_SEG_INDEX; buf = ReadBuffer(index_rel, pi_blk); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); memcpy(&pi_special, PageGetSpecialPointer(page), sizeof(TpPageIndexSpecial)); UnlockReleaseBuffer(buf); pi_blk = pi_special.next_page; pi_count++; } } /* * Mark all segment pages in the page map (second pass). */ static void mark_segment_pages( Relation index_rel, const TpSegmentGraphSnapshot *snapshot, PageMapEntry *page_map, BlockNumber total_blocks, SegmentInfo *segments, int num_segments) { int level; for (level = 0; level < TP_MAX_LEVELS; level++) { const BlockNumber *roots; uint32 root_count; roots = tp_segment_graph_snapshot_level(snapshot, level, &root_count); for (uint32 seg_in_level = 0; seg_in_level < root_count; seg_in_level++) { TpSegmentReader *reader; uint32 i; int global_idx = -1; BlockNumber seg_root = roots[seg_in_level]; reader = tp_segment_open(index_rel, seg_root); if (!reader || !reader->header) { if (reader) tp_segment_close(reader); break; } /* Find global index for this segment */ for (i = 0; i < (uint32)num_segments; i++) { if (segments[i].level == level && segments[i].seg_in_level == seg_in_level) { global_idx = i; break; } } /* Mark data pages */ for (i = 0; i < reader->num_pages; i++) { BlockNumber phys_page = reader->page_map[i]; if (phys_page < total_blocks) { classify_segment_page( &page_map[phys_page], reader, i, global_idx, seg_in_level); } } /* Mark page index pages */ mark_page_index_pages( index_rel, reader, page_map, total_blocks, global_idx, seg_in_level); tp_segment_close(reader); } } } /* Page counts structure for statistics */ typedef struct PageCounts { uint32 empty; uint32 header; uint32 dict; uint32 post; uint32 skip; uint32 docmap; uint32 idx; } PageCounts; /* * Count pages by type. */ static void count_page_types( PageMapEntry *page_map, BlockNumber total_blocks, PageCounts *counts) { BlockNumber blk; memset(counts, 0, sizeof(PageCounts)); for (blk = 0; blk < total_blocks; blk++) { PageMapEntry *e = &page_map[blk]; switch (e->page_type) { case PAGE_UNUSED: counts->empty++; break; case PAGE_METAPAGE: /* Metapage counted separately, not in output */ break; case PAGE_SEG_HDR: counts->header++; break; case PAGE_SEG_INDEX: counts->idx++; break; case PAGE_SEG_DATA: switch (e->data_region) { case DATA_DICTIONARY: counts->dict++; break; case DATA_POSTING: counts->post++; break; case DATA_SKIP: counts->skip++; break; case DATA_DOCMAP: counts->docmap++; break; } break; } } } /* * Write the header and legend to the output file. */ static void write_pageviz_header( FILE *fp, const char *index_name, BlockNumber total_blocks, SegmentInfo *segments, int num_segments, PageCounts *counts) { int current_level = -1; int i; fprintf(fp, "# Page Visualization: %s\n", index_name); fprintf(fp, "# Total: %u pages (%.1f MB), %d segments\n", total_blocks, (double)total_blocks * BLCKSZ / (1024.0 * 1024.0), num_segments); fprintf(fp, "#\n"); /* Segment legend organized by level */ fprintf(fp, "# Segments (background color indicates segment):\n"); for (i = 0; i < num_segments; i++) { SegmentInfo *seg = &segments[i]; int bg = segment_bg_colors[i % NUM_SEGMENT_COLORS]; double size_mb = (double)seg->num_pages * BLCKSZ / (1024.0 * 1024.0); if (seg->level != current_level) { if (current_level >= 0) fprintf(fp, "\n"); fprintf(fp, "# L%d: ", seg->level); current_level = seg->level; } else { fprintf(fp, " "); } fprintf(fp, "\033[48;5;%dm S%d " ANSI_RESET " (%u pg, %.1fMB)", bg, i, seg->num_pages, size_mb); } fprintf(fp, "\n"); fprintf(fp, "#\n"); fprintf(fp, "# Special: \033[48;5;15m\033[30mM" ANSI_RESET "=metapage\n"); fprintf(fp, "# Letters: H=header d=dictionary (blank)=postings " "s=skip m=docmap i=idx .=empty\n"); fprintf(fp, "#\n"); fprintf(fp, "# Page counts: empty=%u header=%u dict=%u post=%u " "skip=%u docmap=%u idx=%u\n", counts->empty, counts->header, counts->dict, counts->post, counts->skip, counts->docmap, counts->idx); fprintf(fp, "#\n"); } /* * Write the page map visualization. */ static void write_pageviz_map( FILE *fp, PageMapEntry *page_map, BlockNumber total_blocks, PageCounts *counts) { BlockNumber blk; int col = 0; for (blk = 0; blk < total_blocks; blk++) { PageMapEntry *e = &page_map[blk]; char c = get_page_char(e); switch (e->page_type) { case PAGE_UNUSED: fprintf(fp, ANSI_DIM "%c" ANSI_RESET, c); break; case PAGE_METAPAGE: fprintf(fp, "\033[48;5;15m\033[30m%c" ANSI_RESET, c); break; case PAGE_SEG_HDR: case PAGE_SEG_INDEX: case PAGE_SEG_DATA: { int bg = segment_bg_colors[e->level % NUM_SEGMENT_COLORS]; fprintf(fp, "\033[48;5;%dm%c" ANSI_RESET, bg, c); break; } default: fprintf(fp, "%c", c); break; } col++; if (col >= 128) { fprintf(fp, "\n"); col = 0; } } if (col > 0) fprintf(fp, "\n"); /* Summary line */ { uint32 used = total_blocks - counts->empty; fprintf(fp, "# Used: %u Empty: %u Utilization: %.1f%%\n", used, counts->empty, 100.0 * used / total_blocks); } } /* * Dump page visualization to file - main orchestrator. * * Output format: * - Legend at top showing segments organized by level with assigned colors * - Letter mnemonics indicate page type: * H=header d=dictionary (blank)=postings s=skip m=docmap i=idx .=empty * - Background colors distinguish segments (16-color palette, cycling) * - Line breaks every 128 characters */ static void tp_debug_pageviz_to_file(const char *index_name, const char *filename) { Oid index_oid; Relation index_rel = NULL; TpIndexMetaPage metap = NULL; TpSegmentGraphSnapshot *snapshot = NULL; TpLocalIndexState *index_state; bool acquired_lock = false; BlockNumber total_blocks = 0; PageMapEntry *page_map = NULL; SegmentInfo *segments = NULL; int num_segments = 0; PageCounts counts; FILE *fp; /* Open output file */ fp = fopen(filename, "w"); if (!fp) { ereport(ERROR, (errcode_for_file_access(), errmsg("could not open file \"%s\": %m", filename))); } /* Resolve index */ index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) { fprintf(fp, "ERROR: Index '%s' not found\n", index_name); fclose(fp); return; } /* Open index and get metapage */ index_rel = index_open(index_oid, AccessShareLock); total_blocks = RelationGetNumberOfBlocks(index_rel); index_state = tp_get_local_index_state(index_oid); if (index_state != NULL && !index_state->lock_held) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_lock = true; } snapshot = tp_segment_graph_snapshot_create(index_rel); metap = &snapshot->metapage; if (total_blocks == 0) { fprintf(fp, "ERROR: Index has no blocks\n"); goto cleanup; } /* Allocate and build page map */ page_map = palloc0(total_blocks * sizeof(PageMapEntry)); /* Count and allocate segments array */ num_segments = count_segments(snapshot); if (num_segments > 0) segments = palloc(num_segments * sizeof(SegmentInfo)); mark_special_pages(index_rel, metap, page_map, total_blocks); if (num_segments > 0) collect_segment_info(index_rel, snapshot, segments); mark_segment_pages( index_rel, snapshot, page_map, total_blocks, segments, num_segments); /* Count and write output */ count_page_types(page_map, total_blocks, &counts); write_pageviz_header( fp, index_name, total_blocks, segments, num_segments, &counts); write_pageviz_map(fp, page_map, total_blocks, &counts); cleanup: if (acquired_lock) tp_release_index_lock(index_state); if (segments) pfree(segments); if (page_map) pfree(page_map); tp_segment_graph_snapshot_free(snapshot); if (index_rel) index_close(index_rel, AccessShareLock); fclose(fp); } /* * tp_debug_pageviz - SQL function to dump page visualization * * Outputs ANSI-colored page layout (128 chars per line): * H=header d=dictionary (blank)=postings s=skip m=docmap i=idx .=empty * Background colors distinguish segments (16-color palette) * * Only available in debug builds (compile with -DDEBUG_DUMP_INDEX) * because it writes to arbitrary file paths. */ PG_FUNCTION_INFO_V1(tp_debug_pageviz); Datum tp_debug_pageviz(PG_FUNCTION_ARGS) { text *index_name_text = PG_GETARG_TEXT_PP(0); text *filename_text = PG_GETARG_TEXT_PP(1); char *index_name = text_to_cstring(index_name_text); char *filename = text_to_cstring(filename_text); /* Superuser only - writes to arbitrary file path */ if (!superuser()) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("must be superuser to write page visualization"))); tp_debug_pageviz_to_file(index_name, filename); elog(INFO, "Page visualization written to %s", filename); PG_RETURN_TEXT_P(cstring_to_text_with_len(filename, strlen(filename))); } #endif /* DEBUG_DUMP_INDEX */ timescale-pg_textsearch-b63fe58/src/debug/dump.h000066400000000000000000000042261525762047700217600ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * dump.h - Index dump and debugging utilities */ #pragma once #include #include /* * DumpOutput - abstraction for writing dump output to either * a StringInfo (for SQL return) or a FILE (for file output) */ typedef struct DumpOutput { StringInfo str; /* StringInfo for SQL return, NULL if file mode */ FILE *fp; /* FILE for file output, NULL if string mode */ bool full_dump; /* If true, no truncation (file mode) */ } DumpOutput; /* Initialize for string output (SQL return) */ static inline void pg_attribute_unused() dump_init_string(DumpOutput *out, StringInfo str) { out->str = str; out->fp = NULL; out->full_dump = false; } /* Initialize for file output */ static inline void pg_attribute_unused() dump_init_file(DumpOutput *out, FILE *fp) { out->str = NULL; out->fp = fp; out->full_dump = true; } /* Printf-style output */ static inline void pg_attribute_unused() dump_printf(DumpOutput *out, const char *fmt, ...) pg_attribute_printf(2, 3); static inline void pg_attribute_unused() dump_printf(DumpOutput *out, const char *fmt, ...) { va_list args; va_start(args, fmt); if (out->fp) { vfprintf(out->fp, fmt, args); } else if (out->str) { /* Use appendStringInfoVA for StringInfo */ for (;;) { va_list args_copy; int needed; va_copy(args_copy, args); needed = appendStringInfoVA(out->str, fmt, args_copy); va_end(args_copy); if (needed == 0) break; enlargeStringInfo(out->str, needed); } } va_end(args); } /* Check if we should truncate output (only in string mode) */ static inline bool pg_attribute_unused() dump_should_truncate(DumpOutput *out, size_t limit) { if (out->full_dump) return false; if (out->str && out->str->len > (int)limit) return true; return false; } /* Dump index function - declared in dump.c */ extern void tp_dump_index_to_output(const char *index_name, DumpOutput *out); /* Summarize index function - declared in dump.c */ extern void tp_summarize_index_to_output(const char *index_name, DumpOutput *out); timescale-pg_textsearch-b63fe58/src/debug/injection.c000066400000000000000000000124441525762047700227710ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #include #include "access/xlog.h" #include "debug/injection.h" #include "miscadmin.h" #include "storage/proc.h" static uint32 injected_segment_count_limit = PG_UINT16_MAX; static bool injected_reclaim_horizon_held = false; static bool injected_legacy_segment = false; static uint64 injected_legacy_total_tokens = 0; static bool injected_v5_segment_total_len = false; static uint64 injected_v5_segment_total_tokens = 0; static bool injected_vacuum_total_len = false; static uint64 injected_vacuum_total_tokens = 0; #if PG_VERSION_NUM >= 180000 #define TP_INJECTION_CALLBACK_ARGS \ const char *name, const void *private_data, void *arg #else #define TP_INJECTION_CALLBACK_ARGS const char *name, const void *private_data #endif #ifdef USE_INJECTION_POINTS extern PGDLLEXPORT void tp_injection_panic(TP_INJECTION_CALLBACK_ARGS); extern PGDLLEXPORT void tp_injection_segment_count_limit(TP_INJECTION_CALLBACK_ARGS); extern PGDLLEXPORT void tp_injection_reclaim_horizon_hold(TP_INJECTION_CALLBACK_ARGS); extern PGDLLEXPORT void tp_injection_legacy_segment(TP_INJECTION_CALLBACK_ARGS); extern PGDLLEXPORT void tp_injection_v5_segment_total_len(TP_INJECTION_CALLBACK_ARGS); extern PGDLLEXPORT void tp_injection_vacuum_total_len(TP_INJECTION_CALLBACK_ARGS); #endif uint32 tp_injected_segment_count_limit(void) { injected_segment_count_limit = PG_UINT16_MAX; TP_INJECTION_POINT(TP_INJECTION_SEGMENT_COUNT_LIMIT); return injected_segment_count_limit; } /* * Report whether a test is pinning the deferred-reclaim horizon so no * parked page is recyclable. */ bool tp_injected_reclaim_horizon_held(void) { injected_reclaim_horizon_held = false; TP_INJECTION_POINT(TP_INJECTION_RECLAIM_HORIZON); return injected_reclaim_horizon_held; } bool tp_injected_legacy_segment(uint64 *total_tokens) { injected_legacy_segment = false; injected_legacy_total_tokens = 0; TP_INJECTION_POINT(TP_INJECTION_LEGACY_SEGMENT); if (injected_legacy_segment) *total_tokens = injected_legacy_total_tokens; return injected_legacy_segment; } uint64 tp_injected_v5_segment_total_len(uint64 total_tokens) { injected_v5_segment_total_len = false; injected_v5_segment_total_tokens = 0; TP_INJECTION_POINT(TP_INJECTION_V5_SEGMENT_TOTAL_LEN); return injected_v5_segment_total_len ? injected_v5_segment_total_tokens : total_tokens; } uint64 tp_injected_vacuum_total_len(uint64 total_tokens) { injected_vacuum_total_len = false; injected_vacuum_total_tokens = 0; TP_INJECTION_POINT(TP_INJECTION_VACUUM_TOTAL_LEN); return injected_vacuum_total_len ? injected_vacuum_total_tokens : total_tokens; } #ifdef USE_INJECTION_POINTS /* * Report whether the attaching backend is this process or, when a * parallel worker fires the point, its lock group leader. */ static bool tp_injection_pid_matches(int pid) { if (pid == MyProcPid) return true; if (MyProc != NULL && MyProc->lockGroupLeader != NULL) return pid == MyProc->lockGroupLeader->pid; return false; } void tp_injection_panic(TP_INJECTION_CALLBACK_ARGS) { const TpInjectionCondition *condition = private_data; #if PG_VERSION_NUM >= 180000 (void)arg; #endif if ((condition->worker_only && condition->pid == MyProcPid) || !tp_injection_pid_matches(condition->pid)) return; XLogFlush(GetXLogInsertRecPtr()); if (condition->worker_only) elog(PANIC, "parallel worker panic triggered for injection point %s", name); else elog(PANIC, "panic triggered for injection point %s", name); } void tp_injection_segment_count_limit(TP_INJECTION_CALLBACK_ARGS) { const TpInjectionSegmentCountLimit *condition = private_data; #if PG_VERSION_NUM >= 180000 (void)arg; #endif if (!tp_injection_pid_matches(condition->pid)) return; if (condition->limit < 1 || condition->limit > PG_UINT16_MAX) elog(ERROR, "invalid segment count limit for injection point %s", name); injected_segment_count_limit = (uint32)condition->limit; } void tp_injection_reclaim_horizon_hold(TP_INJECTION_CALLBACK_ARGS) { const TpInjectionCondition *condition = private_data; #if PG_VERSION_NUM >= 180000 (void)arg; #endif (void)name; if (!tp_injection_pid_matches(condition->pid)) return; injected_reclaim_horizon_held = true; } void tp_injection_legacy_segment(TP_INJECTION_CALLBACK_ARGS) { const TpInjectionTokenTotal *condition = private_data; #if PG_VERSION_NUM >= 180000 (void)arg; #endif (void)name; if (!tp_injection_pid_matches(condition->pid)) return; injected_legacy_segment = true; injected_legacy_total_tokens = condition->total_tokens; } void tp_injection_v5_segment_total_len(TP_INJECTION_CALLBACK_ARGS) { const TpInjectionTokenTotal *condition = private_data; #if PG_VERSION_NUM >= 180000 (void)arg; #endif (void)name; if (!tp_injection_pid_matches(condition->pid)) return; injected_v5_segment_total_len = true; injected_v5_segment_total_tokens = condition->total_tokens; } void tp_injection_vacuum_total_len(TP_INJECTION_CALLBACK_ARGS) { const TpInjectionTokenTotal *condition = private_data; #if PG_VERSION_NUM >= 180000 (void)arg; #endif (void)name; if (!tp_injection_pid_matches(condition->pid)) return; injected_vacuum_total_len = true; injected_vacuum_total_tokens = condition->total_tokens; } #endif timescale-pg_textsearch-b63fe58/src/debug/injection.h000066400000000000000000000066421525762047700230010ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #pragma once #include #include #define TP_INJECTION_AFTER_SPILL_FINALIZE "pg-textsearch-after-spill-finalize" #define TP_INJECTION_SEGMENT_COUNT_LIMIT "pg-textsearch-segment-count-limit" #define TP_INJECTION_RECLAIM_HORIZON "pg-textsearch-reclaim-horizon" #define TP_INJECTION_LEGACY_SEGMENT "pg-textsearch-legacy-segment" #define TP_INJECTION_V5_SEGMENT_TOTAL_LEN "pg-textsearch-v5-segment-total-len" #define TP_INJECTION_VACUUM_TOTAL_LEN "pg-textsearch-vacuum-total-len" #define TP_INJECTION_VACUUM_MEMTABLE_RECLAIM \ "pg-textsearch-vacuum-memtable-reclaim" /* * Points that hold a backend at a chosen step of the spill and * compaction sequence so another session can race it. Tests attach * the "wait" action from the injection_points module and release the * backend with injection_points_wakeup(). */ #define TP_INJECTION_SPILL_BEFORE_FINALIZE \ "pg-textsearch-spill-before-finalize" #define TP_INJECTION_FORCE_MERGE_BEFORE_TRUNCATE \ "pg-textsearch-force-merge-before-truncate" #define TP_INJECTION_TRUNCATE_AFTER_INSPECTION \ "pg-textsearch-truncate-after-inspection" #define TP_INJECTION_BEFORE_COMPACTION_PUBLISH \ "pg-textsearch-before-compaction-publish" #define TP_INJECTION_AFTER_COMPACTION_PUBLISH \ "pg-textsearch-after-compaction-publish" #define TP_INJECTION_COMPACTION_AFTER_SELECT \ "pg-textsearch-compaction-after-select" #define TP_INJECTION_COMPACTION_SOURCE_ESTIMATE \ "pg-textsearch-compaction-source-estimate" #define TP_INJECTION_PARALLEL_VACUUM_LEADER_ESTIMATE \ "pg-textsearch-parallel-vacuum-leader-estimate" #define TP_INJECTION_COMPACTION_BEFORE_PUBLISH \ "pg-textsearch-compaction-before-publish" #define TP_INJECTION_COMPACTION_AFTER_RESTAMP \ "pg-textsearch-compaction-after-restamp" #define TP_INJECTION_COMPACTION_ALLOC_OUTPUT_DATA \ "pg-textsearch-compaction-alloc-output-data" #define TP_INJECTION_COMPACTION_ALLOC_PAGE_INDEX \ "pg-textsearch-compaction-alloc-page-index" #define TP_INJECTION_COMPACTION_ALLOC_TOMBSTONE \ "pg-textsearch-compaction-alloc-tombstone" #define TP_INJECTION_MEMTABLE_EXTEND "pg-textsearch-memtable-extend" #define TP_INJECTION_MEMTABLE_TAIL_READ "pg-textsearch-memtable-tail-read" #define TP_INJECTION_SCORING_SOURCE "pg-textsearch-scoring-source" #define TP_INJECTION_INDEX_LOCK_EXCLUSIVE_WAITER \ "pg-textsearch-index-lock-exclusive-waiter" #define TP_INJECTION_TOMBSTONE_AFTER_UNLINK \ "pg-textsearch-tombstone-after-unlink" #if PG_VERSION_NUM >= 180000 #define TP_INJECTION_POINT(name) INJECTION_POINT(name, NULL) #else #define TP_INJECTION_POINT(name) INJECTION_POINT(name) #endif typedef struct TpInjectionCondition { int pid; bool worker_only; } TpInjectionCondition; typedef struct TpInjectionSegmentCountLimit { int pid; int32 limit; } TpInjectionSegmentCountLimit; typedef struct TpInjectionTokenTotal { int pid; uint64 total_tokens; } TpInjectionTokenTotal; /* * tp_injected_* report a condition that only an attached injection * point can produce; without one they return the natural value. */ extern uint32 tp_injected_segment_count_limit(void); extern bool tp_injected_reclaim_horizon_held(void); extern bool tp_injected_legacy_segment(uint64 *total_tokens); extern uint64 tp_injected_v5_segment_total_len(uint64 total_tokens); extern uint64 tp_injected_vacuum_total_len(uint64 total_tokens); timescale-pg_textsearch-b63fe58/src/index/000077500000000000000000000000001525762047700206575ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/index/compaction_job.c000066400000000000000000002447131525762047700240240ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compaction_job.c - Managed pg_durable compaction jobs */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "index/compaction_job.h" #include "index/compaction_request.h" #define TP_JOB_LABEL_PREFIX "pg_textsearch:bg:v1:" /* * Required v0.2.8 entry points include df.wait_for_signal, * df.wait_for_schedule, and df.explain. Their catalog identities are resolved * below rather than trusting search_path. */ typedef struct TpCompactionJobTarget { Oid index_oid; Oid database_oid; Oid tablespace_oid; RelFileNumber relfilenumber; Oid owner_oid; Oid heap_oid; char *index_name; char *schedule; char *lineage; bool lineage_backfilled; char *history_prefix; char *family_prefix; } TpCompactionJobTarget; struct TpCompactionJobObjects { Oid durable_extension_oid; Oid durable_extension_owner; Oid durable_namespace_oid; Oid operator_namespace_oid; Oid textsearch_extension_oid; Oid textsearch_namespace_oid; Oid textsearch_extension_owner; Oid bm25_am_oid; Oid start_function_oid; Oid explain_function_oid; Oid signal_function_oid; Oid wait_signal_function_oid; Oid wait_schedule_function_oid; Oid loop_function_oid; Oid break_function_oid; Oid step_function_oid; Oid current_function_oid; Oid operator_oids[5]; Oid instances_relation_oid; Oid nodes_relation_oid; Oid vars_relation_oid; char *durable_schema; char *textsearch_schema; char *operator_schema; char *start_function; char *explain_function; char *signal_function; char *wait_signal_function; char *wait_schedule_function; char *loop_function; char *break_function; char *instances_relation; char *step_function; char *current_function; }; typedef struct TpHistoryHeapOwner { Oid heap_oid; Oid owner_oid; } TpHistoryHeapOwner; typedef enum TpJobObjectLookupMode { TP_JOB_OBJECTS_PREFLIGHT, TP_JOB_OBJECTS_LOCKED } TpJobObjectLookupMode; typedef struct TpJobObjectLock { Oid class_id; Oid object_id; LOCKMODE mode; } TpJobObjectLock; static char *tp_copy_spi_text( HeapTuple tuple, TupleDesc tuple_desc, int column, MemoryContext context); static int tp_set_safe_elevated_gucs(void) { int save_nestlevel = NewGUCNestLevel(); /* Never inherit caller-controlled name resolution across a user switch. */ (void)set_config_option( "search_path", "pg_catalog, pg_temp", PGC_USERSET, PGC_S_SESSION, GUC_ACTION_SAVE, true, 0, false); /* pg_dump emits row_security=off, which cannot be inherited here. */ (void)set_config_option( "row_security", "on", PGC_USERSET, PGC_S_SESSION, GUC_ACTION_SAVE, true, 0, false); return save_nestlevel; } static void tp_durable_required(void) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("background compaction requires pg_durable 0.2.8 or " "newer"))); } static void tp_durable_not_initialized(const char *detail) { if (detail != NULL) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_durable is not initialized for this database"), errdetail("%s", detail))); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_durable is not initialized for this database"))); } static bool tp_version_at_least_0_2_8(const char *version) { unsigned int major; unsigned int minor; unsigned int patch; char extra; if (sscanf(version, "%u.%u.%u%c", &major, &minor, &patch, &extra) != 3) return false; if (major != 0) return major > 0; if (minor != 2) return minor > 2; return patch >= 8; } static bool tp_library_is_preloaded(const char *library) { const char *preloads; char *raw; List *names = NIL; bool found = false; preloads = GetConfigOption("shared_preload_libraries", false, false); raw = pstrdup(preloads); if (!SplitIdentifierString(raw, ',', &names)) { pfree(raw); return false; } foreach_ptr(char, name, names) { if (strcmp(name, library) == 0) { found = true; break; } } list_free(names); pfree(raw); return found; } static void tp_lock_extension(Oid extension_oid) { LockDatabaseObject(ExtensionRelationId, extension_oid, 0, AccessShareLock); } static Oid tp_resolve_extension_function( Oid extension_oid, const char *schema_name, const char *function_name, int nargs, const Oid *argtypes, TpJobObjectLookupMode mode) { List *names; Oid function_oid; names = list_make2( makeString(pstrdup(schema_name)), makeString(pstrdup(function_name))); function_oid = LookupFuncName(names, nargs, argtypes, true); list_free_deep(names); if (!OidIsValid(function_oid) || getExtensionOfObject(ProcedureRelationId, function_oid) != extension_oid) tp_durable_not_initialized( "a required pg_durable function is missing"); if (mode == TP_JOB_OBJECTS_LOCKED) { LockDatabaseObject( ProcedureRelationId, function_oid, 0, AccessShareLock); if (getExtensionOfObject(ProcedureRelationId, function_oid) != extension_oid) tp_durable_not_initialized( "a required pg_durable function changed during admission"); } return function_oid; } static Oid tp_resolve_start_function(Oid extension_oid, TpJobObjectLookupMode mode) { Oid function_oid = InvalidOid; List *names; FuncCandidateList candidates; bool ambiguous = false; names = list_make2( makeString(pstrdup("df")), makeString(pstrdup("start"))); #if PG_VERSION_NUM >= 190000 candidates = FuncnameGetCandidates( names, 4, NIL, false, true, false, true, NULL); #else candidates = FuncnameGetCandidates(names, 4, NIL, false, true, false, true); #endif for (FuncCandidateList candidate = candidates; candidate != NULL; candidate = candidate->next) { bool matches = candidate->nominalnargs >= 4 && candidate->nargs >= 4 && candidate->nargs - candidate->ndargs == 4; for (int i = 0; matches && i < 4; i++) matches = candidate->args[i] == TEXTOID; if (!matches) continue; if (!OidIsValid(candidate->oid)) { ambiguous = true; continue; } if (get_func_rettype(candidate->oid) != TEXTOID || getExtensionOfObject(ProcedureRelationId, candidate->oid) != extension_oid) continue; if (OidIsValid(function_oid) && function_oid != candidate->oid) ambiguous = true; function_oid = candidate->oid; } list_free_deep(names); if (ambiguous) tp_durable_not_initialized( "df.start(text,text,text,text) is ambiguous"); if (!OidIsValid(function_oid)) tp_durable_not_initialized("df.start(text,text,text,text) is missing"); if (mode == TP_JOB_OBJECTS_LOCKED) { LockDatabaseObject( ProcedureRelationId, function_oid, 0, AccessShareLock); if (getExtensionOfObject(ProcedureRelationId, function_oid) != extension_oid) tp_durable_not_initialized( "df.start(text,text,text,text) changed during admission"); } return function_oid; } static Oid tp_resolve_extension_operator( Oid extension_oid, const char *schema_name, const char *operator_name, TpJobObjectLookupMode mode) { List *names; Oid operator_oid; names = list_make2( makeString(pstrdup(schema_name)), makeString(pstrdup(operator_name))); operator_oid = OpernameGetOprid(names, TEXTOID, TEXTOID); list_free_deep(names); if (!OidIsValid(operator_oid)) tp_durable_not_initialized(psprintf( "required text operator %s is missing", operator_name)); if (getExtensionOfObject(OperatorRelationId, operator_oid) != extension_oid) tp_durable_not_initialized(psprintf( "required text operator %s (OID %u) is not owned by " "pg_durable", operator_name, operator_oid)); if (mode == TP_JOB_OBJECTS_LOCKED) { LockDatabaseObject( OperatorRelationId, operator_oid, 0, AccessShareLock); if (getExtensionOfObject(OperatorRelationId, operator_oid) != extension_oid) tp_durable_not_initialized(psprintf( "required text operator %s changed during admission", operator_name)); } return operator_oid; } static Oid tp_resolve_extension_relation( Oid extension_oid, Oid namespace_oid, const char *relation_name, TpJobObjectLookupMode mode) { Oid relation_oid = get_relname_relid(relation_name, namespace_oid); if (!OidIsValid(relation_oid) || getExtensionOfObject(RelationRelationId, relation_oid) != extension_oid) tp_durable_not_initialized( psprintf("df.%s is missing", relation_name)); if (mode == TP_JOB_OBJECTS_LOCKED) { LockRelationOid(relation_oid, AccessShareLock); if (getExtensionOfObject(RelationRelationId, relation_oid) != extension_oid) tp_durable_not_initialized( psprintf("df.%s changed during admission", relation_name)); } return relation_oid; } static char * tp_qualified_function_name(Oid function_oid) { char *function_name = get_func_name(function_oid); char *schema_name = get_namespace_name(get_func_namespace(function_oid)); if (function_name == NULL || schema_name == NULL) tp_durable_not_initialized( "a required extension function has no catalog identity"); return quote_qualified_identifier(schema_name, function_name); } /* * Read the owner and version of a pg_extension row by OID. * * The EXTENSIONOID syscache was added in a later minor release, so scan the * catalog directly to stay portable across supported servers. Returns false * when no row exists. A requested version is palloc'd, or NULL when the * column is null. */ static bool tp_extension_lookup(Oid extension_oid, Oid *owner_out, char **version_out) { Relation rel; SysScanDesc scan; ScanKeyData entry[1]; HeapTuple tuple; bool found = false; rel = table_open(ExtensionRelationId, AccessShareLock); ScanKeyInit( &entry[0], Anum_pg_extension_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(extension_oid)); scan = systable_beginscan(rel, ExtensionOidIndexId, true, NULL, 1, entry); tuple = systable_getnext(scan); if (HeapTupleIsValid(tuple)) { found = true; if (owner_out != NULL) *owner_out = ((Form_pg_extension)GETSTRUCT(tuple))->extowner; if (version_out != NULL) { Datum datum; bool isnull; datum = heap_getattr( tuple, Anum_pg_extension_extversion, RelationGetDescr(rel), &isnull); *version_out = isnull ? NULL : TextDatumGetCString(datum); } } systable_endscan(scan); table_close(rel, AccessShareLock); return found; } static Oid tp_extension_owner(Oid extension_oid) { Oid owner_oid = InvalidOid; if (!tp_extension_lookup(extension_oid, &owner_oid, NULL)) tp_durable_not_initialized("an extension catalog row disappeared"); return owner_oid; } static List * tp_history_heap_owners(Oid heap_oid, Oid expected_owner_oid) { List *relation_oids; List *identities = NIL; ListCell *lc; bool partitioned; partitioned = get_rel_relkind(heap_oid) == RELKIND_PARTITIONED_TABLE; relation_oids = partitioned ? find_all_inheritors(heap_oid, NoLock, NULL) : list_make1_oid(heap_oid); foreach (lc, relation_oids) { Oid relation_oid = lfirst_oid(lc); HeapTuple tuple; Form_pg_class relation_form; TpHistoryHeapOwner *identity; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relation_oid)); if (!HeapTupleIsValid(tuple)) continue; relation_form = (Form_pg_class)GETSTRUCT(tuple); if ((partitioned && (!relation_form->relispartition || relation_form->relkind == RELKIND_PARTITIONED_TABLE)) || (!partitioned && relation_form->relowner != expected_owner_oid)) { ReleaseSysCache(tuple); continue; } identity = palloc(sizeof(*identity)); identity->heap_oid = relation_oid; identity->owner_oid = relation_form->relowner; identities = lappend(identities, identity); ReleaseSysCache(tuple); } list_free(relation_oids); return identities; } static bool tp_label_has_lineage( const char *label, const char *lineage, Oid submitted_by, List *identities) { unsigned int database_oid; unsigned int index_oid; unsigned int tablespace_oid; unsigned int relfilenumber; unsigned int label_owner_oid; unsigned int label_heap_oid; char parsed[TP_COMPACTION_LINEAGE_LENGTH + 1]; int consumed = 0; ListCell *lc; if (sscanf(label, TP_JOB_LABEL_PREFIX "%u:%u:%u:%u:%u:%u:%32[0-9a-f]:%n", &database_oid, &index_oid, &tablespace_oid, &relfilenumber, &label_owner_oid, &label_heap_oid, parsed, &consumed) != 7 || consumed <= 0 || database_oid != MyDatabaseId || label_owner_oid != submitted_by || strcmp(parsed, lineage) != 0) return false; foreach (lc, identities) { TpHistoryHeapOwner *identity = lfirst(lc); if (identity->heap_oid == label_heap_oid && identity->owner_oid == label_owner_oid) return true; } return false; } bool tp_compaction_job_lineage_exists( const TpCompactionJobObjects *objects, const char *lineage, Oid heap_oid, Oid owner_oid) { Oid save_userid; int save_sec_context; int save_nestlevel; bool spi_connected = false; bool found = false; StringInfoData sql; Oid argtypes[1] = {TEXTOID}; Datum values[1]; List *identities; char *schema; char *relation; char *prefix; tp_require_compaction_dependency_lock(); schema = get_namespace_name(objects->durable_namespace_oid); relation = get_rel_name(objects->instances_relation_oid); if (schema == NULL || relation == NULL) return false; identities = tp_history_heap_owners(heap_oid, owner_oid); prefix = psprintf(TP_JOB_LABEL_PREFIX "%u:", MyDatabaseId); values[0] = CStringGetTextDatum(prefix); initStringInfo(&sql); appendStringInfo( &sql, "SELECT instance.label, " "instance.submitted_by::pg_catalog.oid " "FROM %s AS instance " "WHERE instance.label OPERATOR(pg_catalog.~~) " "($1 OPERATOR(pg_catalog.||) '%%')", quote_qualified_identifier(schema, relation)); /* RLS must not hide a retained lineage from this internal collision check. */ save_nestlevel = tp_set_safe_elevated_gucs(); GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( objects->durable_extension_owner, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { int rc; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "SPI_connect failed"); spi_connected = true; rc = SPI_execute_with_args( sql.data, 1, argtypes, values, NULL, true, 0); if (rc != SPI_OK_SELECT) elog(ERROR, "could not inspect pg_durable lineage history"); for (uint64 i = 0; i < SPI_processed; i++) { bool isnull; Datum submitted_by; char *label = tp_copy_spi_text( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, CurrentMemoryContext); submitted_by = SPI_getbinval( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2, &isnull); found = !isnull && tp_label_has_lineage( label, lineage, DatumGetObjectId(submitted_by), identities); pfree(label); if (found) break; } SPI_finish(); spi_connected = false; } PG_FINALLY(); { if (spi_connected) SPI_finish(); AtEOXact_GUC(false, save_nestlevel); SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); pfree(prefix); pfree(sql.data); pfree(schema); pfree(relation); list_free_deep(identities); return found; } static void tp_populate_job_objects_as_owner( TpCompactionJobObjects *objects, TpJobObjectLookupMode mode) { char *version; Oid durable_oid; Oid durable_owner; Oid rechecked_oid; Oid start_oid; Oid durable_namespace_oid; char *durable_schema; const char *configured_database; char *database_name; Oid textsearch_oid; Oid textsearch_namespace_oid; char *textsearch_schema; Oid operator_schema_oid; char *operator_schema; AttrNumber submitted_by_attnum; Oid text_args[2] = {TEXTOID, TEXTOID}; Oid loop_args[3] = {TEXTOID, TEXTOID, BOOLOID}; Oid signal_args[3] = {TEXTOID, TEXTOID, TEXTOID}; Oid wait_signal_args[2] = {TEXTOID, INT4OID}; Oid oid_args[5] = {OIDOID, OIDOID, OIDOID, OIDOID, OIDOID}; memset(objects, 0, sizeof(*objects)); durable_oid = get_extension_oid("pg_durable", true); if (!OidIsValid(durable_oid)) tp_durable_required(); if (mode == TP_JOB_OBJECTS_LOCKED) { tp_lock_extension(durable_oid); rechecked_oid = get_extension_oid("pg_durable", true); if (rechecked_oid != durable_oid) tp_durable_required(); } if (!tp_extension_lookup(durable_oid, &durable_owner, &version)) tp_durable_required(); if (version == NULL) tp_durable_required(); if (!tp_version_at_least_0_2_8(version)) { pfree(version); tp_durable_required(); } pfree(version); if (!tp_library_is_preloaded("pg_durable")) tp_durable_not_initialized( "pg_durable is not present in shared_preload_libraries"); configured_database = GetConfigOption("pg_durable.database", true, false); database_name = get_database_name(MyDatabaseId); if (configured_database == NULL || database_name == NULL || strcmp(configured_database, database_name) != 0) tp_durable_not_initialized( "pg_durable.database does not name the current database"); start_oid = tp_resolve_start_function(durable_oid, mode); durable_namespace_oid = get_func_namespace(start_oid); durable_schema = get_namespace_name(durable_namespace_oid); if (durable_schema == NULL) tp_durable_not_initialized("the pg_durable schema is missing"); objects->durable_extension_oid = durable_oid; objects->durable_extension_owner = durable_owner; objects->durable_namespace_oid = durable_namespace_oid; objects->durable_schema = pstrdup(durable_schema); objects->start_function_oid = start_oid; objects->start_function = tp_qualified_function_name(start_oid); objects->explain_function_oid = tp_resolve_extension_function( durable_oid, durable_schema, "explain", 1, text_args, mode); objects->explain_function = tp_qualified_function_name( objects->explain_function_oid); objects->signal_function_oid = tp_resolve_extension_function( durable_oid, durable_schema, "signal", 3, signal_args, mode); objects->signal_function = tp_qualified_function_name( objects->signal_function_oid); objects->wait_signal_function_oid = tp_resolve_extension_function( durable_oid, durable_schema, "wait_for_signal", 2, wait_signal_args, mode); objects->wait_signal_function = tp_qualified_function_name( objects->wait_signal_function_oid); objects->wait_schedule_function_oid = tp_resolve_extension_function( durable_oid, durable_schema, "wait_for_schedule", 1, text_args, mode); objects->wait_schedule_function = tp_qualified_function_name( objects->wait_schedule_function_oid); objects->loop_function_oid = tp_resolve_extension_function( durable_oid, durable_schema, "loop", 3, loop_args, mode); objects->loop_function = tp_qualified_function_name( objects->loop_function_oid); objects->break_function_oid = tp_resolve_extension_function( durable_oid, durable_schema, "break", 1, text_args, mode); objects->break_function = tp_qualified_function_name( objects->break_function_oid); operator_schema_oid = get_extension_schema(durable_oid); operator_schema = get_namespace_name(operator_schema_oid); if (operator_schema == NULL) tp_durable_not_initialized( "the pg_durable operator schema is missing"); objects->operator_schema = pstrdup(operator_schema); objects->operator_namespace_oid = operator_schema_oid; objects->operator_oids[0] = tp_resolve_extension_operator( durable_oid, operator_schema, "|=>", mode); objects->operator_oids[1] = tp_resolve_extension_operator( durable_oid, operator_schema, "~>", mode); objects->operator_oids[2] = tp_resolve_extension_operator( durable_oid, operator_schema, "?>", mode); objects->operator_oids[3] = tp_resolve_extension_operator( durable_oid, operator_schema, "!>", mode); objects->operator_oids[4] = tp_resolve_extension_operator( durable_oid, operator_schema, "|", mode); objects->instances_relation_oid = tp_resolve_extension_relation( durable_oid, durable_namespace_oid, "instances", mode); objects->nodes_relation_oid = tp_resolve_extension_relation( durable_oid, durable_namespace_oid, "nodes", mode); objects->vars_relation_oid = tp_resolve_extension_relation( durable_oid, durable_namespace_oid, "vars", mode); objects->instances_relation = quote_qualified_identifier( durable_schema, get_rel_name(objects->instances_relation_oid)); submitted_by_attnum = get_attnum(objects->instances_relation_oid, "submitted_by"); if (submitted_by_attnum == InvalidAttrNumber || get_atttype(objects->instances_relation_oid, submitted_by_attnum) != REGROLEOID) tp_durable_not_initialized( "df.instances.submitted_by is not pg_catalog.regrole"); textsearch_oid = get_extension_oid("pg_textsearch", false); if (mode == TP_JOB_OBJECTS_LOCKED) tp_lock_extension(textsearch_oid); textsearch_namespace_oid = get_extension_schema(textsearch_oid); textsearch_schema = get_namespace_name(textsearch_namespace_oid); if (textsearch_schema == NULL) elog(ERROR, "pg_textsearch extension schema is missing"); objects->textsearch_extension_oid = textsearch_oid; objects->textsearch_extension_owner = tp_extension_owner(textsearch_oid); objects->textsearch_namespace_oid = textsearch_namespace_oid; objects->textsearch_schema = pstrdup(textsearch_schema); objects->bm25_am_oid = get_index_am_oid("bm25", false); objects->step_function_oid = tp_resolve_extension_function( textsearch_oid, objects->textsearch_schema, "bm25_compact_step_if_current", 5, oid_args, mode); objects->current_function_oid = tp_resolve_extension_function( textsearch_oid, objects->textsearch_schema, "bm25_background_target_is_current", 5, oid_args, mode); objects->step_function = tp_qualified_function_name( objects->step_function_oid); objects->current_function = tp_qualified_function_name( objects->current_function_oid); } static void tp_lookup_job_objects( TpCompactionJobObjects *objects, TpJobObjectLookupMode mode) { Oid durable_oid; Oid durable_owner = InvalidOid; Oid save_userid; int save_sec_context; if (tp_library_is_preloaded("pg_durable")) { const char *configured_database = GetConfigOption("pg_durable.database", true, false); char *database_name = get_database_name(MyDatabaseId); if (configured_database != NULL && database_name != NULL && strcmp(configured_database, database_name) != 0) tp_durable_not_initialized( "pg_durable.database does not name the current database"); if (database_name != NULL) pfree(database_name); } durable_oid = get_extension_oid("pg_durable", true); if (!OidIsValid(durable_oid) || !tp_extension_lookup(durable_oid, &durable_owner, NULL)) tp_durable_required(); GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( durable_owner, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { tp_populate_job_objects_as_owner(objects, mode); } PG_FINALLY(); { SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); } static void tp_preflight_job_objects(TpCompactionJobObjects *objects) { tp_lookup_job_objects(objects, TP_JOB_OBJECTS_PREFLIGHT); } static bool tp_job_object_is_member(Oid class_id, Oid object_id, Oid extension_oid) { return SearchSysCacheExists1( class_id == ProcedureRelationId ? PROCOID : class_id == OperatorRelationId ? OPEROID : RELOID, ObjectIdGetDatum(object_id)) && getExtensionOfObject(class_id, object_id) == extension_oid; } static bool tp_job_objects_still_match(const TpCompactionJobObjects *objects) { const char *function_names[] = { "start", "explain", "signal", "wait_for_signal", "wait_for_schedule", "loop", "break", "bm25_compact_step_if_current", "bm25_background_target_is_current", }; Oid function_oids[] = { objects->start_function_oid, objects->explain_function_oid, objects->signal_function_oid, objects->wait_signal_function_oid, objects->wait_schedule_function_oid, objects->loop_function_oid, objects->break_function_oid, objects->step_function_oid, objects->current_function_oid, }; const char *operator_names[] = {"|=>", "~>", "?>", "!>", "|"}; const char *relation_names[] = {"instances", "nodes", "vars"}; Oid relation_oids[] = { objects->instances_relation_oid, objects->nodes_relation_oid, objects->vars_relation_oid, }; Oid durable_oid; Oid durable_owner; Oid textsearch_oid; Oid textsearch_owner; char *durable_schema; char *operator_schema; char *textsearch_schema; durable_oid = get_extension_oid("pg_durable", true); textsearch_oid = get_extension_oid("pg_textsearch", true); if (durable_oid != objects->durable_extension_oid || textsearch_oid != objects->textsearch_extension_oid || get_index_am_oid("bm25", true) != objects->bm25_am_oid || !SearchSysCacheExists1( AMOID, ObjectIdGetDatum(objects->bm25_am_oid)) || getExtensionOfObject(AccessMethodRelationId, objects->bm25_am_oid) != textsearch_oid || !tp_extension_lookup(durable_oid, &durable_owner, NULL) || durable_owner != objects->durable_extension_owner || !tp_extension_lookup(textsearch_oid, &textsearch_owner, NULL) || textsearch_owner != objects->textsearch_extension_owner || get_extension_schema(textsearch_oid) != objects->textsearch_namespace_oid) return false; durable_schema = get_namespace_name(objects->durable_namespace_oid); operator_schema = get_namespace_name(objects->operator_namespace_oid); textsearch_schema = get_namespace_name(objects->textsearch_namespace_oid); if (durable_schema == NULL || operator_schema == NULL || textsearch_schema == NULL || strcmp(durable_schema, objects->durable_schema) != 0 || strcmp(operator_schema, objects->operator_schema) != 0 || strcmp(textsearch_schema, objects->textsearch_schema) != 0) return false; for (Size i = 0; i < lengthof(function_oids); i++) { Oid extension_oid = i < 7 ? durable_oid : textsearch_oid; Oid namespace_oid = i < 7 ? objects->durable_namespace_oid : objects->textsearch_namespace_oid; char *name; if (!tp_job_object_is_member( ProcedureRelationId, function_oids[i], extension_oid) || get_func_namespace(function_oids[i]) != namespace_oid) return false; name = get_func_name(function_oids[i]); if (name == NULL || strcmp(name, function_names[i]) != 0) return false; } for (Size i = 0; i < lengthof(objects->operator_oids); i++) { HeapTuple tuple; Form_pg_operator operator_form; char *name; if (!tp_job_object_is_member( OperatorRelationId, objects->operator_oids[i], durable_oid)) return false; tuple = SearchSysCache1( OPEROID, ObjectIdGetDatum(objects->operator_oids[i])); if (!HeapTupleIsValid(tuple)) return false; operator_form = (Form_pg_operator)GETSTRUCT(tuple); if (operator_form->oprnamespace != objects->operator_namespace_oid) { ReleaseSysCache(tuple); return false; } ReleaseSysCache(tuple); name = get_opname(objects->operator_oids[i]); if (name == NULL || strcmp(name, operator_names[i]) != 0) return false; } for (Size i = 0; i < lengthof(relation_oids); i++) { char *name; if (!tp_job_object_is_member( RelationRelationId, relation_oids[i], durable_oid) || get_rel_namespace(relation_oids[i]) != objects->durable_namespace_oid) return false; name = get_rel_name(relation_oids[i]); if (name == NULL || strcmp(name, relation_names[i]) != 0) return false; } return true; } static int tp_job_object_lock_cmp(const void *left, const void *right) { const TpJobObjectLock *a = left; const TpJobObjectLock *b = right; if (a->class_id < b->class_id) return -1; if (a->class_id > b->class_id) return 1; if (a->object_id < b->object_id) return -1; if (a->object_id > b->object_id) return 1; return 0; } TpCompactionJobObjects * tp_compaction_job_try_lock_objects(bool invalid_is_error) { TpCompactionJobObjects objects; TpCompactionJobObjects *result; TpJobObjectLock locks[20]; MemoryContext old_context = CurrentMemoryContext; ResourceOwner old_owner = CurrentResourceOwner; int acquired = 0; int count = 0; bool dependency_was_held; BeginInternalSubTransaction(NULL); PG_TRY(); { MemoryContextSwitchTo(old_context); tp_preflight_job_objects(&objects); ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(old_context); CurrentResourceOwner = old_owner; } PG_CATCH(); { ErrorData *edata; MemoryContextSwitchTo(old_context); edata = CopyErrorData(); FlushErrorState(); RollbackAndReleaseCurrentSubTransaction(); MemoryContextSwitchTo(old_context); CurrentResourceOwner = old_owner; if (invalid_is_error) ReThrowError(edata); FreeErrorData(edata); return NULL; } PG_END_TRY(); if (!ConditionalLockDatabaseObject( ExtensionRelationId, objects.durable_extension_oid, 0, AccessShareLock)) return NULL; if (!ConditionalLockDatabaseObject( ExtensionRelationId, objects.textsearch_extension_oid, 0, AccessShareLock)) { UnlockDatabaseObject( ExtensionRelationId, objects.durable_extension_oid, 0, AccessShareLock); return NULL; } dependency_was_held = tp_compaction_dependency_oid_lock_held( objects.bm25_am_oid); if (!tp_try_lock_compaction_dependency_oid(objects.bm25_am_oid)) goto unavailable; #define TP_ADD_JOB_OBJECT_LOCK(classid, objectid, lockmode) \ do \ { \ locks[count].class_id = (classid); \ locks[count].object_id = (objectid); \ locks[count].mode = (lockmode); \ count++; \ } while (0) TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.start_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.explain_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.signal_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.wait_signal_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.wait_schedule_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.loop_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.break_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.step_function_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( ProcedureRelationId, objects.current_function_oid, AccessShareLock); for (Size i = 0; i < lengthof(objects.operator_oids); i++) TP_ADD_JOB_OBJECT_LOCK( OperatorRelationId, objects.operator_oids[i], AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( RelationRelationId, objects.instances_relation_oid, RowExclusiveLock); TP_ADD_JOB_OBJECT_LOCK( RelationRelationId, objects.nodes_relation_oid, RowExclusiveLock); TP_ADD_JOB_OBJECT_LOCK( RelationRelationId, objects.vars_relation_oid, RowExclusiveLock); TP_ADD_JOB_OBJECT_LOCK( NamespaceRelationId, objects.durable_namespace_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( NamespaceRelationId, objects.textsearch_namespace_oid, AccessShareLock); TP_ADD_JOB_OBJECT_LOCK( NamespaceRelationId, objects.operator_namespace_oid, AccessShareLock); #undef TP_ADD_JOB_OBJECT_LOCK qsort(locks, count, sizeof(locks[0]), tp_job_object_lock_cmp); for (acquired = 0; acquired < count; acquired++) { if (locks[acquired].class_id == RelationRelationId) { if (!ConditionalLockRelationOid( locks[acquired].object_id, locks[acquired].mode)) goto unavailable; } else if (!ConditionalLockDatabaseObject( locks[acquired].class_id, locks[acquired].object_id, 0, locks[acquired].mode)) goto unavailable; } if (!tp_job_objects_still_match(&objects)) goto unavailable; result = palloc(sizeof(*result)); memcpy(result, &objects, sizeof(*result)); return result; unavailable: while (acquired > 0) { acquired--; if (locks[acquired].class_id == RelationRelationId) UnlockRelationOid(locks[acquired].object_id, locks[acquired].mode); else UnlockDatabaseObject( locks[acquired].class_id, locks[acquired].object_id, 0, locks[acquired].mode); } if (!dependency_was_held && tp_compaction_dependency_oid_lock_held(objects.bm25_am_oid)) tp_unlock_compaction_dependency_oid(objects.bm25_am_oid); UnlockDatabaseObject( ExtensionRelationId, objects.textsearch_extension_oid, 0, AccessShareLock); UnlockDatabaseObject( ExtensionRelationId, objects.durable_extension_oid, 0, AccessShareLock); return NULL; } static void tp_require_owner_login(Oid owner_oid) { HeapTuple tuple; Form_pg_authid role; bool can_login; tuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(owner_oid)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index owner role with OID %u does not exist", owner_oid))); role = (Form_pg_authid)GETSTRUCT(tuple); can_login = role->rolcanlogin; ReleaseSysCache(tuple); if (!can_login) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index owner must have LOGIN for background " "compaction"))); } static void tp_require_owner_database_connect(Oid owner_oid) { if (object_aclcheck( DatabaseRelationId, MyDatabaseId, owner_oid, ACL_CONNECT) != ACLCHECK_OK) ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("index owner cannot connect for background " "compaction"), errdetail( "Role \"%s\" lacks CONNECT privilege on database " "\"%s\".", GetUserNameFromId(owner_oid, false), get_database_name(MyDatabaseId)))); } static void tp_require_owner_superuser_policy(Oid owner_oid) { HeapTuple tuple; Form_pg_authid role; bool is_superuser; const char *superuser_setting; bool superuser_enabled; char *owner_name; tuple = SearchSysCache1(AUTHOID, ObjectIdGetDatum(owner_oid)); if (!HeapTupleIsValid(tuple)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index owner role with OID %u does not exist", owner_oid))); role = (Form_pg_authid)GETSTRUCT(tuple); is_superuser = role->rolsuper; ReleaseSysCache(tuple); superuser_setting = GetConfigOption( "pg_durable.enable_superuser_instances", true, false); if (is_superuser && superuser_setting != NULL && (!parse_bool(superuser_setting, &superuser_enabled) || !superuser_enabled)) { owner_name = GetUserNameFromId(owner_oid, false); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_durable superuser instances are disabled"), errdetail( "Index owner \"%s\" is a superuser, but " "pg_durable.enable_superuser_instances is off.", owner_name))); } } static void tp_owner_privilege_error( Oid owner_oid, const char *privilege, const char *object_name) { ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("index owner lacks required pg_durable privileges"), errdetail( "Role \"%s\" lacks %s privilege on %s.", GetUserNameFromId(owner_oid, false), privilege, object_name), errhint("Grant the role access with df.grant_usage()."))); } static void tp_owner_textsearch_schema_privilege_error( Oid owner_oid, const char *schema_name) { const char *owner_name = GetUserNameFromId(owner_oid, false); ereport(ERROR, (errcode(ERRCODE_INSUFFICIENT_PRIVILEGE), errmsg("index owner lacks required pg_textsearch schema " "privilege"), errdetail( "Role \"%s\" lacks USAGE privilege on schema %s.", owner_name, quote_identifier(schema_name)), errhint("Grant access with GRANT USAGE ON SCHEMA %s TO %s.", quote_identifier(schema_name), quote_identifier(owner_name)))); } static bool tp_owner_has_column_privilege( Oid relation_oid, Oid owner_oid, AclMode privilege, const char *column_name) { AttrNumber attnum; if (pg_class_aclcheck(relation_oid, owner_oid, privilege) == ACLCHECK_OK) return true; attnum = get_attnum(relation_oid, column_name); if (attnum == InvalidAttrNumber) tp_durable_not_initialized(psprintf( "required column %s.%s is missing", get_rel_name(relation_oid), column_name)); return pg_attribute_aclcheck(relation_oid, attnum, owner_oid, privilege) == ACLCHECK_OK; } static void tp_require_owner_column_privileges( Oid relation_oid, Oid owner_oid, AclMode privilege, const char *privilege_name, const char *const *columns, Size ncolumns) { for (Size i = 0; i < ncolumns; i++) { if (!tp_owner_has_column_privilege( relation_oid, owner_oid, privilege, columns[i])) tp_owner_privilege_error( owner_oid, privilege_name, quote_qualified_identifier( "df", get_rel_name(relation_oid))); } } static void tp_require_owner_function_privilege( Oid owner_oid, Oid function_oid, const char *function_name) { if (object_aclcheck( ProcedureRelationId, function_oid, owner_oid, ACL_EXECUTE) != ACLCHECK_OK) tp_owner_privilege_error(owner_oid, "EXECUTE", function_name); } static void tp_require_owner_durable_privileges( const TpCompactionJobObjects *objects, Oid owner_oid) { static const char *const instance_select_columns[] = {"id", "label", "status", "submitted_by", "created_at"}; static const char *const instance_insert_columns[] = {"id", "label", "root_node", "submitted_by", "database"}; static const char *const node_insert_columns[] = {"id", "instance_id", "node_type", "query", "result_name", "left_node", "right_node", "submitted_by", "database"}; static const char *const vars_select_columns[] = {"name", "value", "owner"}; if (object_aclcheck( NamespaceRelationId, objects->textsearch_namespace_oid, owner_oid, ACL_USAGE) != ACLCHECK_OK) tp_owner_textsearch_schema_privilege_error( owner_oid, objects->textsearch_schema); if (object_aclcheck( NamespaceRelationId, objects->durable_namespace_oid, owner_oid, ACL_USAGE) != ACLCHECK_OK) tp_owner_privilege_error(owner_oid, "USAGE", "schema df"); tp_require_owner_function_privilege( owner_oid, objects->start_function_oid, objects->start_function); tp_require_owner_function_privilege( owner_oid, objects->explain_function_oid, objects->explain_function); tp_require_owner_function_privilege( owner_oid, objects->signal_function_oid, objects->signal_function); tp_require_owner_column_privileges( objects->instances_relation_oid, owner_oid, ACL_SELECT, "SELECT", instance_select_columns, lengthof(instance_select_columns)); tp_require_owner_column_privileges( objects->instances_relation_oid, owner_oid, ACL_INSERT, "INSERT", instance_insert_columns, lengthof(instance_insert_columns)); tp_require_owner_column_privileges( objects->nodes_relation_oid, owner_oid, ACL_INSERT, "INSERT", node_insert_columns, lengthof(node_insert_columns)); tp_require_owner_column_privileges( objects->vars_relation_oid, owner_oid, ACL_SELECT, "SELECT", vars_select_columns, lengthof(vars_select_columns)); } static char * tp_build_history_prefix(const TpCompactionJobTarget *target) { return psprintf(TP_JOB_LABEL_PREFIX "%u:", target->database_oid); } static char * tp_build_family_prefix(const TpCompactionJobTarget *target) { return psprintf( TP_JOB_LABEL_PREFIX "%u:%u:%u:%u:", target->database_oid, target->index_oid, target->tablespace_oid, (Oid)target->relfilenumber); } static char * tp_hex_encode(const char *value) { static const char digits[] = "0123456789abcdef"; Size length = strlen(value); char *encoded; if (length > (MaxAllocSize - 1) / 2) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("background compaction schedule is too long"))); encoded = palloc(length * 2 + 1); for (Size i = 0; i < length; i++) { unsigned char byte = (unsigned char)value[i]; encoded[i * 2] = digits[byte >> 4]; encoded[i * 2 + 1] = digits[byte & 0x0f]; } encoded[length * 2] = '\0'; return encoded; } static int tp_hex_value(char value) { if (value >= '0' && value <= '9') return value - '0'; if (value >= 'a' && value <= 'f') return value - 'a' + 10; return -1; } static char * tp_build_label(const TpCompactionJobTarget *target, const char *schedule) { char *encoded = tp_hex_encode(schedule); char *label = psprintf( "%s%u:%u:%s:%s", target->family_prefix, target->owner_oid, target->heap_oid, target->lineage, encoded); pfree(encoded); return label; } static char * tp_decode_schedule(const char *encoded, MemoryContext result_context) { Size encoded_length = strlen(encoded); char *schedule; MemoryContext old_context; if ((encoded_length & 1) != 0) return NULL; old_context = MemoryContextSwitchTo(result_context); schedule = palloc(encoded_length / 2 + 1); MemoryContextSwitchTo(old_context); for (Size i = 0; i < encoded_length; i += 2) { int high = tp_hex_value(encoded[i]); int low = tp_hex_value(encoded[i + 1]); if (high < 0 || low < 0 || (high == 0 && low == 0)) { pfree(schedule); return NULL; } schedule[i / 2] = (char)((high << 4) | low); } schedule[encoded_length / 2] = '\0'; return schedule; } static char * tp_schedule_from_label( const TpCompactionJobTarget *target, const char *label, MemoryContext result_context) { char *owner_prefix = psprintf("%s%u:", target->family_prefix, target->owner_oid); Size prefix_length = strlen(owner_prefix); unsigned int heap_oid; char lineage[TP_COMPACTION_LINEAGE_LENGTH + 1]; int consumed = 0; if (strncmp(label, owner_prefix, prefix_length) != 0) { pfree(owner_prefix); return NULL; } pfree(owner_prefix); if (sscanf(label + prefix_length, "%u:%32[0-9a-f]:%n", &heap_oid, lineage, &consumed) != 2 || consumed <= 0 || heap_oid != target->heap_oid || strcmp(lineage, target->lineage) != 0) return NULL; return tp_decode_schedule( label + prefix_length + consumed, result_context); } static char * tp_schedule_from_legacy_label( const TpCompactionJobTarget *target, const char *label, MemoryContext result_context) { char *owner_prefix = psprintf("%s%u:", target->family_prefix, target->owner_oid); Size prefix_length = strlen(owner_prefix); const char *encoded; if (strncmp(label, owner_prefix, prefix_length) != 0) { pfree(owner_prefix); return NULL; } pfree(owner_prefix); encoded = label + prefix_length; if (strchr(encoded, ':') != NULL) return NULL; return tp_decode_schedule(encoded, result_context); } static void tp_capture_target( Oid indexoid, bool refresh_default, TpCompactionJobTarget *target) { Relation index_rel; const char *schedule; char *lineage; bool lineage_backfilled; memset(target, 0, sizeof(*target)); lineage = tp_ensure_index_compaction_lineage(indexoid, &lineage_backfilled); index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation with OID %u does not exist", indexoid))); if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || index_rel->rd_rel->relkind != RELKIND_INDEX) { char *index_name = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a physical bm25 index", index_name))); } if (RelationUsesLocalBuffers(index_rel)) { relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("background compaction is not supported for " "temporary indexes"))); } if (index_rel->rd_index == NULL || !index_rel->rd_index->indisvalid || !index_rel->rd_index->indisready || !index_rel->rd_index->indislive) { char *index_name = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index \"%s\" is not ready for background " "compaction", index_name))); } if (tp_index_compaction_mode(index_rel) != TP_COMPACTION_BACKGROUND) { char *index_name = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index \"%s\" is not configured for background " "compaction", index_name))); } target->index_oid = indexoid; target->database_oid = MyDatabaseId; target->tablespace_oid = index_rel->rd_locator.spcOid; target->relfilenumber = index_rel->rd_locator.relNumber; target->owner_oid = index_rel->rd_rel->relowner; target->heap_oid = index_rel->rd_index->indrelid; target->index_name = pstrdup(RelationGetRelationName(index_rel)); if (lineage == NULL) { relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index \"%s\" has no background compaction lineage", target->index_name), errhint("Recreate the index before using managed background " "compaction."))); } target->lineage = lineage; target->lineage_backfilled = lineage_backfilled; if (refresh_default || lineage_backfilled) { schedule = tp_index_compaction_schedule(index_rel); if (schedule == NULL) schedule = tp_background_compaction_schedule; if (schedule == NULL) elog(ERROR, "background compaction schedule is not initialized"); target->schedule = pstrdup(schedule); } /* * Keep the relation lock until transaction end. The workflow identity * and dependency are admitted against this physical generation. */ relation_close(index_rel, NoLock); target->history_prefix = tp_build_history_prefix(target); target->family_prefix = tp_build_family_prefix(target); } static bool tp_dependency_exists( const ObjectAddress *dependent, const ObjectAddress *referenced) { Relation depend_rel; SysScanDesc scan; ScanKeyData keys[3]; HeapTuple tuple; bool found = false; depend_rel = table_open(DependRelationId, AccessShareLock); ScanKeyInit( &keys[0], Anum_pg_depend_classid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(dependent->classId)); ScanKeyInit( &keys[1], Anum_pg_depend_objid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(dependent->objectId)); ScanKeyInit( &keys[2], Anum_pg_depend_objsubid, BTEqualStrategyNumber, F_INT4EQ, Int32GetDatum(dependent->objectSubId)); scan = systable_beginscan( depend_rel, DependDependerIndexId, true, NULL, 3, keys); while (HeapTupleIsValid(tuple = systable_getnext(scan))) { Form_pg_depend dependency = (Form_pg_depend)GETSTRUCT(tuple); if (dependency->refclassid == referenced->classId && dependency->refobjid == referenced->objectId && dependency->refobjsubid == referenced->objectSubId && dependency->deptype == DEPENDENCY_NORMAL) { found = true; break; } } systable_endscan(scan); table_close(depend_rel, AccessShareLock); return found; } static void tp_pin_durable_dependency(const TpCompactionJobObjects *objects) { ObjectAddress bm25_am = { .classId = AccessMethodRelationId, .objectId = objects->bm25_am_oid, .objectSubId = 0, }; ObjectAddress durable_ext = { .classId = ExtensionRelationId, .objectId = objects->durable_extension_oid, .objectSubId = 0, }; if (!tp_dependency_exists(&bm25_am, &durable_ext)) { recordDependencyOn(&bm25_am, &durable_ext, DEPENDENCY_NORMAL); CommandCounterIncrement(); } } static void tp_grant_helper_access(const TpCompactionJobObjects *objects, Oid owner_oid) { AclResult step_acl; AclResult current_acl; Oid save_userid; int save_sec_context; int save_nestlevel; bool spi_connected = false; StringInfoData sql; const char *owner_name; const char *quoted_owner; step_acl = object_aclcheck( ProcedureRelationId, objects->step_function_oid, owner_oid, ACL_EXECUTE); current_acl = object_aclcheck( ProcedureRelationId, objects->current_function_oid, owner_oid, ACL_EXECUTE); if (step_acl == ACLCHECK_OK && current_acl == ACLCHECK_OK) return; /* * pg_durable executes SQL nodes by logging in as submitted_by. PUBLIC * remains revoked from the physical-target helpers, so grant only this * index owner. The helpers still enforce ownership and the complete * captured physical identity on every call. */ owner_name = GetUserNameFromId(owner_oid, false); quoted_owner = quote_identifier(owner_name); initStringInfo(&sql); appendStringInfo( &sql, "GRANT EXECUTE ON FUNCTION " "%s(pg_catalog.oid, pg_catalog.oid, pg_catalog.oid, " "pg_catalog.oid, pg_catalog.oid), " "%s(pg_catalog.oid, pg_catalog.oid, pg_catalog.oid, " "pg_catalog.oid, pg_catalog.oid) TO %s", objects->step_function, objects->current_function, quoted_owner); GetUserIdAndSecContext(&save_userid, &save_sec_context); save_nestlevel = tp_set_safe_elevated_gucs(); SetUserIdAndSecContext( objects->textsearch_extension_owner, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "SPI_connect failed"); spi_connected = true; if (SPI_execute(sql.data, false, 0) != SPI_OK_UTILITY) elog(ERROR, "could not grant managed compaction helper access"); SPI_finish(); spi_connected = false; } PG_FINALLY(); { if (spi_connected) SPI_finish(); AtEOXact_GUC(false, save_nestlevel); SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); pfree(sql.data); } static char * tp_copy_spi_text( HeapTuple tuple, TupleDesc tuple_desc, int column, MemoryContext context) { bool isnull; Datum value; char *temporary; char *copy; MemoryContext old_context; value = SPI_getbinval(tuple, tuple_desc, column, &isnull); if (isnull) return NULL; temporary = TextDatumGetCString(value); old_context = MemoryContextSwitchTo(context); copy = pstrdup(temporary); MemoryContextSwitchTo(old_context); pfree(temporary); return copy; } static char * tp_find_exact_instance( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, const char *label, MemoryContext result_context) { StringInfoData sql; Oid argtypes[2] = {TEXTOID, OIDOID}; Datum values[2] = {CStringGetTextDatum(label), ObjectIdGetDatum(target->owner_oid)}; char *instance_id = NULL; int rc; initStringInfo(&sql); appendStringInfo( &sql, "SELECT instance.id::pg_catalog.text " "FROM %s AS instance " "WHERE instance.label OPERATOR(pg_catalog.=) $1 " "AND instance.submitted_by::pg_catalog.oid " "OPERATOR(pg_catalog.=) $2 " "AND instance.status OPERATOR(pg_catalog.=) " "ANY (ARRAY['pending', 'running']::pg_catalog.text[]) " "ORDER BY instance.created_at DESC, instance.id DESC", objects->instances_relation); rc = SPI_execute_with_args(sql.data, 2, argtypes, values, NULL, true, 0); pfree(sql.data); if (rc != SPI_OK_SELECT) elog(ERROR, "could not search pg_durable instances"); if (SPI_processed > 0) instance_id = tp_copy_spi_text( SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, result_context); if (SPI_processed > 1) ereport(WARNING, (errmsg("multiple active pg_textsearch background " "compaction jobs share one canonical label"), errdetail("The newest job was selected deterministically."))); return instance_id; } static char * tp_find_family_instance( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, bool terminal, char **schedule, MemoryContext result_context) { StringInfoData sql; Oid argtypes[2] = {TEXTOID, OIDOID}; Datum values[2] = {CStringGetTextDatum(target->family_prefix), ObjectIdGetDatum(target->owner_oid)}; char *instance_id = NULL; uint64 managed_count = 0; int rc; *schedule = NULL; initStringInfo(&sql); appendStringInfo( &sql, "SELECT instance.id::pg_catalog.text, instance.label " "FROM %s AS instance " "WHERE instance.label OPERATOR(pg_catalog.~~) " "($1 OPERATOR(pg_catalog.||) '%%') " "AND instance.submitted_by::pg_catalog.oid " "OPERATOR(pg_catalog.=) $2 " "AND instance.status OPERATOR(pg_catalog.=) " "ANY (ARRAY[%s]::pg_catalog.text[]) " "ORDER BY instance.created_at DESC, instance.id DESC", objects->instances_relation, terminal ? "'completed', 'failed', 'cancelled'" : "'pending', 'running'"); rc = SPI_execute_with_args(sql.data, 2, argtypes, values, NULL, true, 0); pfree(sql.data); if (rc != SPI_OK_SELECT) elog(ERROR, "could not search pg_durable instance history"); for (uint64 i = 0; i < SPI_processed; i++) { char *label; char *decoded_schedule; label = tp_copy_spi_text( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2, CurrentMemoryContext); decoded_schedule = tp_schedule_from_label(target, label, result_context); pfree(label); if (decoded_schedule == NULL) continue; managed_count++; if (instance_id == NULL) { instance_id = tp_copy_spi_text( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, result_context); *schedule = decoded_schedule; } else pfree(decoded_schedule); } if (!terminal && managed_count > 1) ereport(WARNING, (errmsg("multiple active pg_textsearch background " "compaction jobs exist for one physical index"), errdetail("The newest job was selected deterministically."))); return instance_id; } static char * tp_find_current_family_instance( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, MemoryContext result_context) { StringInfoData sql; Oid argtypes[2] = {TEXTOID, OIDOID}; Datum values[2] = {CStringGetTextDatum(target->family_prefix), ObjectIdGetDatum(target->owner_oid)}; char *instance_id = NULL; int rc; initStringInfo(&sql); appendStringInfo( &sql, "SELECT instance.id::pg_catalog.text " "FROM %s AS instance " "WHERE instance.label OPERATOR(pg_catalog.~~) " "($1 OPERATOR(pg_catalog.||) '%%') " "AND instance.submitted_by::pg_catalog.oid " "OPERATOR(pg_catalog.=) $2 " "AND instance.status OPERATOR(pg_catalog.=) " "ANY (ARRAY['pending', 'running']::pg_catalog.text[]) " "ORDER BY instance.created_at DESC, instance.id DESC " "LIMIT 1", objects->instances_relation); rc = SPI_execute_with_args(sql.data, 2, argtypes, values, NULL, true, 1); pfree(sql.data); if (rc != SPI_OK_SELECT) elog(ERROR, "could not select current pg_durable instance"); if (SPI_processed > 0) instance_id = tp_copy_spi_text( SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, result_context); return instance_id; } static char * tp_find_legacy_family_instance( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, bool terminal, char **schedule, MemoryContext result_context) { StringInfoData sql; Oid argtypes[2] = {TEXTOID, OIDOID}; Datum values[2] = {CStringGetTextDatum(target->family_prefix), ObjectIdGetDatum(target->owner_oid)}; char *instance_id = NULL; int rc; *schedule = NULL; initStringInfo(&sql); appendStringInfo( &sql, "SELECT instance.id::pg_catalog.text, instance.label " "FROM %s AS instance " "WHERE instance.label OPERATOR(pg_catalog.~~) " "($1 OPERATOR(pg_catalog.||) '%%') " "AND instance.submitted_by::pg_catalog.oid " "OPERATOR(pg_catalog.=) $2 " "AND instance.status OPERATOR(pg_catalog.=) " "ANY (ARRAY[%s]::pg_catalog.text[]) " "ORDER BY instance.created_at DESC, instance.id DESC", objects->instances_relation, terminal ? "'completed', 'failed', 'cancelled'" : "'pending', 'running'"); rc = SPI_execute_with_args(sql.data, 2, argtypes, values, NULL, true, 0); pfree(sql.data); if (rc != SPI_OK_SELECT) elog(ERROR, "could not search legacy pg_durable instance history"); for (uint64 i = 0; i < SPI_processed; i++) { char *label; char *decoded_schedule; label = tp_copy_spi_text( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 2, CurrentMemoryContext); decoded_schedule = tp_schedule_from_legacy_label(target, label, result_context); pfree(label); if (decoded_schedule == NULL) continue; instance_id = tp_copy_spi_text( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, result_context); *schedule = decoded_schedule; break; } return instance_id; } static char * tp_schedule_from_prior_label( const TpCompactionJobTarget *target, const char *label, MemoryContext result_context) { const char *suffix; const char *encoded; char lineage[TP_COMPACTION_LINEAGE_LENGTH + 1]; unsigned int index_oid; unsigned int tablespace_oid; unsigned int relfilenumber; unsigned int owner_oid; unsigned int heap_oid; int consumed = 0; Size encoded_length; char *schedule; MemoryContext old_context; if (strncmp(label, target->history_prefix, strlen(target->history_prefix)) != 0) return NULL; suffix = label + strlen(target->history_prefix); if (sscanf(suffix, "%u:%u:%u:%u:%u:%32[0-9a-f]:%n", &index_oid, &tablespace_oid, &relfilenumber, &owner_oid, &heap_oid, lineage, &consumed) != 6 || consumed <= 0 || owner_oid != target->owner_oid || heap_oid != target->heap_oid || strcmp(lineage, target->lineage) != 0) return NULL; if (index_oid == target->index_oid && tablespace_oid == target->tablespace_oid && relfilenumber == (unsigned int)target->relfilenumber) return NULL; encoded = suffix + consumed; encoded_length = strlen(encoded); if ((encoded_length & 1) != 0) return NULL; old_context = MemoryContextSwitchTo(result_context); schedule = palloc(encoded_length / 2 + 1); MemoryContextSwitchTo(old_context); for (Size i = 0; i < encoded_length; i += 2) { int high = tp_hex_value(encoded[i]); int low = tp_hex_value(encoded[i + 1]); if (high < 0 || low < 0 || (high == 0 && low == 0)) { pfree(schedule); return NULL; } schedule[i / 2] = (char)((high << 4) | low); } schedule[encoded_length / 2] = '\0'; return schedule; } static char * tp_find_prior_generation_schedule( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, MemoryContext result_context) { StringInfoData sql; Oid argtypes[2] = {TEXTOID, OIDOID}; Datum values[2] = {CStringGetTextDatum(target->history_prefix), ObjectIdGetDatum(target->owner_oid)}; char *schedule = NULL; int rc; initStringInfo(&sql); appendStringInfo( &sql, "SELECT instance.label " "FROM %s AS instance " "WHERE instance.label OPERATOR(pg_catalog.~~) " "($1 OPERATOR(pg_catalog.||) '%%') " "AND instance.submitted_by::pg_catalog.oid " "OPERATOR(pg_catalog.=) $2 " "AND instance.status OPERATOR(pg_catalog.=) " "ANY (ARRAY['pending', 'running', 'completed', 'failed', " "'cancelled']::pg_catalog.text[]) " "ORDER BY instance.created_at DESC, instance.id DESC", objects->instances_relation); rc = SPI_execute_with_args(sql.data, 2, argtypes, values, NULL, true, 0); pfree(sql.data); if (rc != SPI_OK_SELECT) elog(ERROR, "could not search prior pg_durable instance history"); for (uint64 i = 0; i < SPI_processed; i++) { char *label = tp_copy_spi_text( SPI_tuptable->vals[i], SPI_tuptable->tupdesc, 1, CurrentMemoryContext); schedule = tp_schedule_from_prior_label(target, label, result_context); pfree(label); if (schedule != NULL) break; } return schedule; } static void tp_build_worker_queries( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, char **step_sql, char **current_sql) { char *family_literal = quote_literal_cstr(target->family_prefix); char *step_signature; char *step_signature_literal; char *current_signature; char *current_signature_literal; step_signature = psprintf( "%s(pg_catalog.oid,pg_catalog.oid,pg_catalog.oid," "pg_catalog.oid,pg_catalog.oid)", objects->step_function); step_signature_literal = quote_literal_cstr(step_signature); current_signature = psprintf( "%s(pg_catalog.oid,pg_catalog.oid,pg_catalog.oid," "pg_catalog.oid,pg_catalog.oid)", objects->current_function); current_signature_literal = quote_literal_cstr(current_signature); *step_sql = psprintf( "SELECT coalesce((WITH helper_args(" "index_oid, database_oid, tablespace_oid, relfilenumber, " "owner_oid) AS MATERIALIZED (SELECT %u::pg_catalog.oid, " "%u::pg_catalog.oid, %u::pg_catalog.oid, %u::pg_catalog.oid, " "%u::pg_catalog.oid WHERE " "pg_catalog.to_regprocedure(%s)::pg_catalog.oid " "OPERATOR(pg_catalog.=) %u::pg_catalog.oid) SELECT " "%s(helper_args.index_oid, helper_args.database_oid, " "helper_args.tablespace_oid, helper_args.relfilenumber, " "helper_args.owner_oid) FROM helper_args), false) AS ran", target->index_oid, target->database_oid, target->tablespace_oid, (Oid)target->relfilenumber, target->owner_oid, step_signature_literal, objects->step_function_oid, objects->step_function); *current_sql = psprintf( "SELECT (coalesce((WITH helper_args(" "index_oid, database_oid, tablespace_oid, relfilenumber, " "owner_oid) AS MATERIALIZED (SELECT %u::pg_catalog.oid, " "%u::pg_catalog.oid, %u::pg_catalog.oid, %u::pg_catalog.oid, " "%u::pg_catalog.oid WHERE " "pg_catalog.to_regprocedure(%s)::pg_catalog.oid " "OPERATOR(pg_catalog.=) %u::pg_catalog.oid) SELECT " "%s(helper_args.index_oid, helper_args.database_oid, " "helper_args.tablespace_oid, helper_args.relfilenumber, " "helper_args.owner_oid) FROM helper_args), false) " "AND coalesce((" "SELECT instance.id OPERATOR(pg_catalog.=) " "'{sys_instance_id}' " "FROM %s AS instance " "WHERE pg_catalog.left(instance.label, %zu) " "OPERATOR(pg_catalog.=) %s " "AND instance.submitted_by::pg_catalog.oid " "OPERATOR(pg_catalog.=) %u::pg_catalog.oid " "AND instance.status OPERATOR(pg_catalog.=) " "ANY (ARRAY['pending', 'running']::pg_catalog.text[]) " "ORDER BY instance.created_at DESC, instance.id DESC " "LIMIT 1), false)) AS current", target->index_oid, target->database_oid, target->tablespace_oid, (Oid)target->relfilenumber, target->owner_oid, current_signature_literal, objects->current_function_oid, objects->current_function, objects->instances_relation, strlen(target->family_prefix), family_literal, target->owner_oid); pfree(family_literal); pfree(step_signature); pfree(step_signature_literal); pfree(current_signature); pfree(current_signature_literal); } static void tp_append_guard(StringInfo sql, const TpCompactionJobObjects *objects) { appendStringInfo( sql, "(($2::pg_catalog.text OPERATOR(%s.|=>) 'current') " "OPERATOR(%s.~>) " "('SELECT $current.current' OPERATOR(%s.?>) 'SELECT true' " "OPERATOR(%s.!>) %s('stale'::pg_catalog.text)))", quote_identifier(objects->operator_schema), quote_identifier(objects->operator_schema), quote_identifier(objects->operator_schema), quote_identifier(objects->operator_schema), objects->break_function); } static void tp_append_cascade(StringInfo sql, const TpCompactionJobObjects *objects) { appendStringInfo( sql, "%s((($1::pg_catalog.text OPERATOR(%s.|=>) 'step') " "OPERATOR(%s.~>) " "('SELECT $step.ran' OPERATOR(%s.?>) 'SELECT true' " "OPERATOR(%s.!>) %s('false'::pg_catalog.text))), " "NULL::pg_catalog.text, false)", objects->loop_function, quote_identifier(objects->operator_schema), quote_identifier(objects->operator_schema), quote_identifier(objects->operator_schema), quote_identifier(objects->operator_schema), objects->break_function); } static void tp_append_job_graph(StringInfo sql, const TpCompactionJobObjects *objects) { const char *operator_schema = quote_identifier(objects->operator_schema); appendStringInfo(sql, "%s((", objects->loop_function); tp_append_guard(sql, objects); appendStringInfo(sql, " OPERATOR(%s.~>) ", operator_schema); tp_append_cascade(sql, objects); appendStringInfo( sql, " OPERATOR(%s.~>) %s((" "%s('compact'::pg_catalog.text, NULL::pg_catalog.int4) " "OPERATOR(%s.|) " "%s($3::pg_catalog.text) " "OPERATOR(%s.~>) ", operator_schema, objects->loop_function, objects->wait_signal_function, operator_schema, objects->wait_schedule_function, operator_schema); tp_append_guard(sql, objects); appendStringInfo(sql, " OPERATOR(%s.~>) ", operator_schema); tp_append_cascade(sql, objects); appendStringInfo( sql, "), NULL::pg_catalog.text, true) " "OPERATOR(%s.~>) %s('stale'::pg_catalog.text)), " "NULL::pg_catalog.text, true)", operator_schema, objects->break_function); } static char * tp_start_job( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, const char *schedule, const char *label, MemoryContext result_context) { StringInfoData sql; char *step_sql; char *current_sql; Oid argtypes[4] = {TEXTOID, TEXTOID, TEXTOID, TEXTOID}; Datum values[4]; char *instance_id; int rc; tp_build_worker_queries(objects, target, &step_sql, ¤t_sql); initStringInfo(&sql); appendStringInfo(&sql, "SELECT %s(", objects->start_function); tp_append_job_graph(&sql, objects); appendStringInfo( &sql, ", $4::pg_catalog.text, pg_catalog.current_database(), " "'caller'::pg_catalog.text)"); values[0] = CStringGetTextDatum(step_sql); values[1] = CStringGetTextDatum(current_sql); values[2] = CStringGetTextDatum(schedule); values[3] = CStringGetTextDatum(label); rc = SPI_execute_with_args(sql.data, 4, argtypes, values, NULL, false, 1); if (rc != SPI_OK_SELECT || SPI_processed != 1) elog(ERROR, "could not start pg_durable compaction workflow"); instance_id = tp_copy_spi_text( SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, result_context); if (instance_id == NULL) elog(ERROR, "pg_durable returned no compaction workflow identifier"); { Oid save_userid; int save_sec_context; Oid stamp_argtypes[1] = {TEXTOID}; Datum stamp_values[1] = {CStringGetTextDatum(instance_id)}; GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( objects->durable_extension_owner, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { resetStringInfo(&sql); appendStringInfo( &sql, "UPDATE %s SET created_at = " "pg_catalog.statement_timestamp(), updated_at = " "pg_catalog.statement_timestamp() " "WHERE id OPERATOR(pg_catalog.=) $1", objects->instances_relation); rc = SPI_execute_with_args( sql.data, 1, stamp_argtypes, stamp_values, NULL, false, 0); if (rc != SPI_OK_UPDATE || SPI_processed != 1) elog(ERROR, "could not timestamp pg_durable compaction workflow"); } PG_FINALLY(); { SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); } pfree(sql.data); pfree(step_sql); pfree(current_sql); return instance_id; } static void tp_validate_graph_as_owner( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target) { static const char *failure_prefixes[] = { "Cannot explain input", "Expression returned NULL", "Failed to evaluate expression", "Failed to parse Durofut JSON", "Invalid durable function graph", }; MemoryContext result_context = CurrentMemoryContext; Oid save_userid; int save_sec_context; int save_nestlevel; bool spi_connected = false; StringInfoData sql; char *step_sql; char *current_sql; char *explanation = NULL; Oid argtypes[3] = {TEXTOID, TEXTOID, TEXTOID}; Datum values[3]; int rc; GetUserIdAndSecContext(&save_userid, &save_sec_context); save_nestlevel = tp_set_safe_elevated_gucs(); SetUserIdAndSecContext( target->owner_oid, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "SPI_connect failed"); spi_connected = true; tp_build_worker_queries(objects, target, &step_sql, ¤t_sql); initStringInfo(&sql); appendStringInfo(&sql, "SELECT %s(", objects->explain_function); tp_append_job_graph(&sql, objects); appendStringInfoChar(&sql, ')'); values[0] = CStringGetTextDatum(step_sql); values[1] = CStringGetTextDatum(current_sql); values[2] = CStringGetTextDatum(target->schedule); rc = SPI_execute_with_args( sql.data, 3, argtypes, values, NULL, true, 1); if (rc != SPI_OK_SELECT || SPI_processed != 1) elog(ERROR, "could not validate pg_durable compaction workflow"); explanation = tp_copy_spi_text( SPI_tuptable->vals[0], SPI_tuptable->tupdesc, 1, result_context); pfree(sql.data); pfree(step_sql); pfree(current_sql); SPI_finish(); spi_connected = false; } PG_FINALLY(); { if (spi_connected) SPI_finish(); AtEOXact_GUC(false, save_nestlevel); SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); if (explanation == NULL) elog(ERROR, "pg_durable returned no workflow validation result"); for (Size i = 0; i < lengthof(failure_prefixes); i++) { if (strncmp(explanation, failure_prefixes[i], strlen(failure_prefixes[i])) == 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("pg_durable rejected the background compaction " "workflow"), errdetail("%s", explanation))); } pfree(explanation); } static char * tp_reconcile_job( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, bool refresh_default, MemoryContext result_context) { char *instance_id; char *current_instance_id = NULL; char *schedule = NULL; char *label = NULL; bool publish_new = false; if (refresh_default) { label = tp_build_label(target, target->schedule); instance_id = tp_find_exact_instance(objects, target, label, result_context); current_instance_id = tp_find_current_family_instance( objects, target, result_context); if (instance_id == NULL || current_instance_id == NULL || strcmp(instance_id, current_instance_id) != 0) { if (instance_id != NULL) pfree(instance_id); instance_id = tp_start_job( objects, target, target->schedule, label, result_context); } if (current_instance_id != NULL) pfree(current_instance_id); pfree(label); return instance_id; } instance_id = tp_find_family_instance( objects, target, false, &schedule, result_context); if (instance_id != NULL) { current_instance_id = tp_find_current_family_instance( objects, target, result_context); if (current_instance_id != NULL && strcmp(instance_id, current_instance_id) == 0) { pfree(current_instance_id); pfree(schedule); return instance_id; } if (current_instance_id != NULL) pfree(current_instance_id); pfree(instance_id); instance_id = NULL; publish_new = true; } else { instance_id = tp_find_family_instance( objects, target, true, &schedule, result_context); if (instance_id == NULL) { if (target->lineage_backfilled) { instance_id = tp_find_legacy_family_instance( objects, target, false, &schedule, result_context); if (instance_id == NULL) instance_id = tp_find_legacy_family_instance( objects, target, true, &schedule, result_context); } if (instance_id != NULL) pfree(instance_id); if (schedule == NULL) schedule = tp_find_prior_generation_schedule( objects, target, result_context); if (schedule == NULL && target->lineage_backfilled) schedule = MemoryContextStrdup(result_context, target->schedule); if (schedule == NULL) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("background compaction for index \"%s\" " "requires explicit adoption", target->index_name))); } else pfree(instance_id); } label = tp_build_label(target, schedule); if (publish_new) instance_id = tp_start_job(objects, target, schedule, label, result_context); else { instance_id = tp_find_exact_instance(objects, target, label, result_context); if (instance_id == NULL) instance_id = tp_start_job( objects, target, schedule, label, result_context); } pfree(label); pfree(schedule); return instance_id; } static void tp_signal_instance( const TpCompactionJobObjects *objects, const char *instance_id) { StringInfoData sql; Oid argtypes[3] = {TEXTOID, TEXTOID, TEXTOID}; Datum values[3] = {CStringGetTextDatum(instance_id), CStringGetTextDatum("compact"), CStringGetTextDatum("{}")}; int rc; initStringInfo(&sql); appendStringInfo( &sql, "SELECT %s($1::pg_catalog.text, $2::pg_catalog.text, " "$3::pg_catalog.text)", objects->signal_function); rc = SPI_execute_with_args(sql.data, 3, argtypes, values, NULL, false, 1); pfree(sql.data); if (rc != SPI_OK_SELECT || SPI_processed != 1) elog(ERROR, "could not signal pg_durable compaction workflow"); } static char * tp_reconcile_as_owner( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, bool refresh_default, bool signal, MemoryContext result_context) { Oid save_userid; int save_sec_context; int save_nestlevel; bool spi_connected = false; bool snapshot_pushed = false; char *instance_id = NULL; save_nestlevel = tp_set_safe_elevated_gucs(); GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( target->owner_oid, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { PushActiveSnapshot(GetLatestSnapshot()); snapshot_pushed = true; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "SPI_connect failed"); spi_connected = true; instance_id = tp_reconcile_job( objects, target, refresh_default, result_context); if (signal) tp_signal_instance(objects, instance_id); SPI_finish(); spi_connected = false; PopActiveSnapshot(); snapshot_pushed = false; } PG_FINALLY(); { if (spi_connected) SPI_finish(); if (snapshot_pushed) PopActiveSnapshot(); AtEOXact_GUC(false, save_nestlevel); SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); return instance_id; } static char * tp_schedule_as_trusted( const TpCompactionJobObjects *objects, const TpCompactionJobTarget *target, MemoryContext result_context) { Oid save_userid; int save_sec_context; int save_nestlevel; bool spi_connected = false; bool snapshot_pushed = false; char *instance_id = NULL; char *schedule = NULL; save_nestlevel = tp_set_safe_elevated_gucs(); GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( objects->durable_extension_owner, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { PushActiveSnapshot(GetLatestSnapshot()); snapshot_pushed = true; if (SPI_connect() != SPI_OK_CONNECT) elog(ERROR, "SPI_connect failed"); spi_connected = true; instance_id = tp_find_family_instance( objects, target, false, &schedule, result_context); if (instance_id == NULL) instance_id = tp_find_family_instance( objects, target, true, &schedule, result_context); if (instance_id == NULL && target->lineage_backfilled) { instance_id = tp_find_legacy_family_instance( objects, target, false, &schedule, result_context); if (instance_id == NULL) instance_id = tp_find_legacy_family_instance( objects, target, true, &schedule, result_context); } if (instance_id != NULL) pfree(instance_id); SPI_finish(); spi_connected = false; PopActiveSnapshot(); snapshot_pushed = false; } PG_FINALLY(); { if (spi_connected) SPI_finish(); if (snapshot_pushed) PopActiveSnapshot(); AtEOXact_GUC(false, save_nestlevel); SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); return schedule; } void tp_compaction_job_preflight(Oid owner_oid, const char *schedule) { TpCompactionJobTarget target; TpCompactionJobObjects objects; if (schedule == NULL) elog(ERROR, "background compaction schedule is not initialized"); tp_require_owner_login(owner_oid); tp_require_owner_database_connect(owner_oid); tp_preflight_job_objects(&objects); tp_require_owner_superuser_policy(owner_oid); tp_require_owner_durable_privileges(&objects, owner_oid); memset(&target, 0, sizeof(target)); target.database_oid = MyDatabaseId; target.owner_oid = owner_oid; target.schedule = pstrdup(schedule); target.family_prefix = tp_build_family_prefix(&target); tp_validate_graph_as_owner(&objects, &target); } static void tp_activate_captured_target( const TpCompactionJobObjects *objects, TpCompactionJobTarget *target, bool refresh_default) { char *instance_id PG_USED_FOR_ASSERTS_ONLY; tp_require_owner_login(target->owner_oid); tp_require_owner_database_connect(target->owner_oid); tp_require_owner_superuser_policy(target->owner_oid); tp_require_owner_durable_privileges(objects, target->owner_oid); tp_pin_durable_dependency(objects); tp_grant_helper_access(objects, target->owner_oid); instance_id = tp_reconcile_as_owner( objects, target, refresh_default, false, CurrentMemoryContext); Assert(instance_id != NULL); ereport(WARNING, (errmsg("pg_textsearch background compaction is a preview " "feature"))); } void tp_compaction_job_activate( const TpCompactionJobObjects *objects, Oid indexoid, bool refresh_default) { TpCompactionJobTarget target; tp_require_compaction_index_lock(indexoid); tp_require_compaction_dependency_lock(); tp_capture_target(indexoid, refresh_default, &target); tp_activate_captured_target(objects, &target, refresh_default); } void tp_compaction_job_activate_with_schedule( const TpCompactionJobObjects *objects, Oid indexoid, const char *schedule) { TpCompactionJobTarget target; if (schedule == NULL) elog(ERROR, "background compaction schedule is not initialized"); tp_require_compaction_index_lock(indexoid); tp_require_compaction_dependency_lock(); tp_capture_target(indexoid, true, &target); pfree(target.schedule); target.schedule = pstrdup(schedule); tp_activate_captured_target(objects, &target, true); } void tp_compaction_job_capture(Oid indexoid, TpCompactionJobIdentity *identity) { Relation index_rel; const char *lineage; const char *schedule; index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("relation with OID %u does not exist", indexoid))); if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || index_rel->rd_rel->relkind != RELKIND_INDEX || index_rel->rd_index == NULL || !index_rel->rd_index->indisvalid || !index_rel->rd_index->indisready || !index_rel->rd_index->indislive || tp_index_compaction_mode(index_rel) != TP_COMPACTION_BACKGROUND) { char *index_name = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("index \"%s\" is not ready for background " "compaction", index_name))); } memset(identity, 0, sizeof(*identity)); identity->heap_oid = index_rel->rd_index->indrelid; identity->namespace_oid = RelationGetNamespace(index_rel); identity->index_name = pstrdup(RelationGetRelationName(index_rel)); identity->index_oid = indexoid; identity->tablespace_oid = index_rel->rd_locator.spcOid; identity->relfilenumber = index_rel->rd_locator.relNumber; identity->owner_oid = index_rel->rd_rel->relowner; lineage = tp_index_compaction_lineage(index_rel); if (lineage == NULL) { identity->lineage = tp_new_compaction_lineage(); identity->lineage_backfilled = true; } else identity->lineage = pstrdup(lineage); schedule = tp_index_compaction_schedule(index_rel); if (schedule == NULL) schedule = tp_background_compaction_schedule; identity->schedule = pstrdup(schedule); identity->schedule_resolved = false; relation_close(index_rel, AccessShareLock); } void tp_compaction_job_resolve_schedule( const TpCompactionJobObjects *objects, Oid indexoid, TpCompactionJobIdentity *identity, MemoryContext result_context) { TpCompactionJobTarget target; char *schedule; if (identity->schedule_resolved) return; tp_require_compaction_index_lock(indexoid); memset(&target, 0, sizeof(target)); target.database_oid = MyDatabaseId; target.index_oid = identity->index_oid; target.tablespace_oid = identity->tablespace_oid; target.relfilenumber = identity->relfilenumber; target.owner_oid = identity->owner_oid; target.heap_oid = identity->heap_oid; target.lineage = identity->lineage; target.schedule = identity->schedule; target.lineage_backfilled = identity->lineage_backfilled; target.history_prefix = tp_build_history_prefix(&target); target.family_prefix = tp_build_family_prefix(&target); tp_require_compaction_dependency_lock(); schedule = tp_schedule_as_trusted(objects, &target, result_context); if (schedule != NULL) { pfree(identity->schedule); identity->schedule = schedule; } identity->schedule_resolved = true; pfree(target.history_prefix); pfree(target.family_prefix); } void tp_compaction_job_signal(const TpCompactionJobObjects *objects, Oid indexoid) { TpCompactionJobTarget target; char *instance_id PG_USED_FOR_ASSERTS_ONLY; tp_require_compaction_index_lock(indexoid); tp_require_compaction_dependency_lock(); tp_capture_target(indexoid, false, &target); tp_require_owner_login(target.owner_oid); tp_require_owner_database_connect(target.owner_oid); tp_require_owner_superuser_policy(target.owner_oid); tp_require_owner_durable_privileges(objects, target.owner_oid); tp_pin_durable_dependency(objects); tp_grant_helper_access(objects, target.owner_oid); instance_id = tp_reconcile_as_owner( objects, &target, false, true, CurrentMemoryContext); Assert(instance_id != NULL); } timescale-pg_textsearch-b63fe58/src/index/compaction_job.h000066400000000000000000000041431525762047700240200ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compaction_job.h - Managed pg_durable compaction jobs */ #pragma once #include typedef struct TpCompactionJobObjects TpCompactionJobObjects; typedef struct TpCompactionJobIdentity { Oid heap_oid; Oid namespace_oid; char *index_name; Oid index_oid; Oid tablespace_oid; Oid relfilenumber; Oid owner_oid; char *lineage; char *schedule; bool lineage_backfilled; bool schedule_resolved; } TpCompactionJobIdentity; typedef enum TpManagedIntentFlags { TP_MANAGED_INTENT_REFRESH_DEFAULT = 1 << 0, TP_MANAGED_INTENT_RECONCILE_OPTIONS = 1 << 1, TP_MANAGED_INTENT_PRESERVE_SCHEDULE = 1 << 2, TP_MANAGED_INTENT_LINEAGE_SUPPLIED = 1 << 3, TP_MANAGED_INTENT_DISABLE = 1 << 4, TP_MANAGED_INTENT_POST_PUBLICATION = 1 << 5, TP_MANAGED_INTENT_RECONCILE_LINEAGE = 1 << 6 } TpManagedIntentFlags; typedef struct TpManagedIndexIntent { Oid index_oid; TpCompactionJobIdentity source; char *schedule; char *lineage; int flags; SubTransactionId subid; } TpManagedIndexIntent; extern void tp_compaction_job_preflight(Oid owner_oid, const char *schedule); extern TpCompactionJobObjects * tp_compaction_job_try_lock_objects(bool invalid_is_error); extern void tp_compaction_job_activate( const TpCompactionJobObjects *objects, Oid indexoid, bool refresh_default); extern void tp_compaction_job_activate_with_schedule( const TpCompactionJobObjects *objects, Oid indexoid, const char *schedule); extern void tp_compaction_job_capture(Oid indexoid, TpCompactionJobIdentity *identity); extern void tp_compaction_job_resolve_schedule( const TpCompactionJobObjects *objects, Oid indexoid, TpCompactionJobIdentity *identity, MemoryContext result_context); extern void tp_compaction_job_signal(const TpCompactionJobObjects *objects, Oid indexoid); extern bool tp_compaction_job_lineage_exists( const TpCompactionJobObjects *objects, const char *lineage, Oid heap_oid, Oid owner_oid); timescale-pg_textsearch-b63fe58/src/index/compaction_request.c000066400000000000000000000656711525762047700247460ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compaction_request.c - Background compaction request tracking */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "index/compaction_job.h" #include "index/compaction_request.h" char *tp_background_compaction_schedule = NULL; /* * Pending requests live in TopTransactionContext, which PostgreSQL frees * at commit, prepare, and abort alike, so no transaction callback has to * clear them. A reset callback nulls this pointer when that happens. * The context outlives subtransactions, so a request from a rolled-back * savepoint survives, matching the spill it recorded. */ static List *tp_pending_compactions = NIL; static bool tp_pending_registered = false; /* True while tp_compaction_flush_requests() is signaling managed jobs. */ static bool tp_dispatch_active = false; #define TP_COMPACTION_INDEX_LOCK_SUBID 1 #define TP_COMPACTION_LINEAGE_LOCK_SUBID 2 static void tp_set_compaction_locktag(LOCKTAG *tag, Oid object_id, uint16 discriminator) { /* * pg_am has no subobjects, so its object-subid space is private to the * access method. LOCKTAG_OBJECT also keeps these locks disjoint from * SQL-visible advisory locks. */ SET_LOCKTAG_OBJECT( *tag, MyDatabaseId, AccessMethodRelationId, object_id, discriminator); } static bool tp_compaction_lock_held(Oid object_id, uint16 discriminator, LOCKMODE lockmode) { LOCKTAG tag; tp_set_compaction_locktag(&tag, object_id, discriminator); return LockHeldByMe(&tag, lockmode, true); } bool tp_compaction_dependency_lock_held(void) { Oid bm25_am_oid = get_index_am_oid("bm25", true); if (!OidIsValid(bm25_am_oid)) return false; return tp_compaction_dependency_oid_lock_held(bm25_am_oid); } bool tp_compaction_dependency_oid_lock_held(Oid bm25_am_oid) { return tp_compaction_lock_held(bm25_am_oid, 0, ShareRowExclusiveLock); } static bool tp_take_compaction_lock( Oid object_id, uint16 discriminator, LOCKMODE lockmode, bool nowait) { LOCKTAG tag; tp_set_compaction_locktag(&tag, object_id, discriminator); if (LockHeldByMe(&tag, lockmode, true)) return true; if (nowait) return ConditionalLockDatabaseObject( AccessMethodRelationId, object_id, discriminator, lockmode); LockDatabaseObject( AccessMethodRelationId, object_id, discriminator, lockmode); return true; } void tp_lock_compaction_index(Oid indexoid) { (void)tp_take_compaction_lock( indexoid, TP_COMPACTION_INDEX_LOCK_SUBID, ExclusiveLock, false); } void tp_lock_compaction_dependency(void) { Oid bm25_am_oid = get_index_am_oid("bm25", false); if (tp_compaction_dependency_lock_held()) return; LockDatabaseObject( AccessMethodRelationId, bm25_am_oid, 0, ShareRowExclusiveLock); } bool tp_try_lock_compaction_dependency(void) { Oid bm25_am_oid = get_index_am_oid("bm25", false); return tp_try_lock_compaction_dependency_oid(bm25_am_oid); } bool tp_try_lock_compaction_dependency_oid(Oid bm25_am_oid) { return tp_take_compaction_lock( bm25_am_oid, 0, ShareRowExclusiveLock, true); } void tp_unlock_compaction_dependency(void) { Oid bm25_am_oid = get_index_am_oid("bm25", false); tp_unlock_compaction_dependency_oid(bm25_am_oid); } void tp_unlock_compaction_dependency_oid(Oid bm25_am_oid) { UnlockDatabaseObject( AccessMethodRelationId, bm25_am_oid, 0, ShareRowExclusiveLock); } void tp_require_compaction_dependency_lock(void) { if (!tp_compaction_dependency_lock_held()) elog(ERROR, "managed compaction dependency lock is not held"); } void tp_require_compaction_index_lock(Oid indexoid) { if (!tp_compaction_lock_held( indexoid, TP_COMPACTION_INDEX_LOCK_SUBID, ExclusiveLock)) elog(ERROR, "managed compaction admission lock is not held"); } void tp_lock_compaction_lineage(const char *lineage) { uint32 hash; hash = hash_bytes( (const unsigned char *)lineage, TP_COMPACTION_LINEAGE_LENGTH); (void)tp_take_compaction_lock( hash, TP_COMPACTION_LINEAGE_LOCK_SUBID, ExclusiveLock, false); } bool tp_try_lock_compaction_lineage(const char *lineage) { uint32 hash; hash = hash_bytes( (const unsigned char *)lineage, TP_COMPACTION_LINEAGE_LENGTH); return tp_take_compaction_lock( hash, TP_COMPACTION_LINEAGE_LOCK_SUBID, ExclusiveLock, true); } List * tp_try_prelock_compaction_indexes(List *indexoids) { List *sorted = list_copy(indexoids); List *locked = NIL; ListCell *lc; Oid previous = InvalidOid; list_sort(sorted, list_oid_cmp); foreach (lc, sorted) { Oid indexoid = lfirst_oid(lc); if (!OidIsValid(indexoid) || indexoid == previous) continue; previous = indexoid; if ((tp_compaction_dependency_lock_held() && !tp_compaction_lock_held( indexoid, TP_COMPACTION_INDEX_LOCK_SUBID, ExclusiveLock)) || !ConditionalLockRelationOid(indexoid, ShareUpdateExclusiveLock)) continue; if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(indexoid))) { UnlockRelationOid(indexoid, ShareUpdateExclusiveLock); continue; } locked = lappend_oid(locked, indexoid); } list_free(sorted); foreach (lc, locked) { Oid indexoid = lfirst_oid(lc); if (!tp_take_compaction_lock( indexoid, TP_COMPACTION_INDEX_LOCK_SUBID, ExclusiveLock, true)) { UnlockRelationOid(indexoid, ShareUpdateExclusiveLock); locked = foreach_delete_current(locked, lc); } } return locked; } /* * Can this process hand a compaction request off at commit? Callers * that cannot compact inline instead, so the debt is never dropped. */ bool tp_compaction_dispatch_possible(void) { return !( IsParallelWorker() || IsInParallelMode() || RecoveryInProgress() || AmAutoVacuumWorkerProcess() || tp_dispatch_active); } /* Spill-time compaction policy. No reloption means inline. */ int tp_index_compaction_mode(Relation index_rel) { TpOptions *options = (TpOptions *)index_rel->rd_options; if (options == NULL) return TP_COMPACTION_INLINE; return options->compaction; } /* Per-index schedule override. No reloption means use the global schedule. */ const char * tp_index_compaction_schedule(Relation index_rel) { TpOptions *options = (TpOptions *)index_rel->rd_options; if (options == NULL || options->compaction_schedule_offset == 0) return NULL; return (const char *)options + options->compaction_schedule_offset; } const char * tp_index_compaction_lineage(Relation index_rel) { TpOptions *options = (TpOptions *)index_rel->rd_options; if (options == NULL || options->compaction_lineage_offset == 0) return NULL; return (const char *)options + options->compaction_lineage_offset; } char * tp_new_compaction_lineage(void) { static const char digits[] = "0123456789abcdef"; unsigned char bytes[TP_COMPACTION_LINEAGE_BYTES]; char *lineage; if (!pg_strong_random(bytes, sizeof(bytes))) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not generate background compaction lineage"))); lineage = palloc(sizeof(bytes) * 2 + 1); for (Size i = 0; i < sizeof(bytes); i++) { lineage[i * 2] = digits[bytes[i] >> 4]; lineage[i * 2 + 1] = digits[bytes[i] & 0x0f]; } lineage[sizeof(bytes) * 2] = '\0'; return lineage; } static bool tp_heaps_share_partition_family(Oid first_heap_oid, Oid second_heap_oid) { List *ancestors; Oid first_root = first_heap_oid; Oid second_root = second_heap_oid; if (get_rel_relispartition(first_heap_oid)) { ancestors = get_partition_ancestors(first_heap_oid); if (ancestors != NIL) first_root = llast_oid(ancestors); list_free(ancestors); } if (get_rel_relispartition(second_heap_oid)) { ancestors = get_partition_ancestors(second_heap_oid); if (ancestors != NIL) second_root = llast_oid(ancestors); list_free(ancestors); } return first_root == second_root && get_rel_relkind(first_root) == RELKIND_PARTITIONED_TABLE; } static bool tp_live_compaction_lineage_conflicts( const char *lineage, Oid excluded_index_oid, Oid heap_oid) { Relation class_rel; SysScanDesc scan; HeapTuple tuple; bool found = false; class_rel = table_open(RelationRelationId, AccessShareLock); scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 0, NULL); while ((tuple = systable_getnext(scan)) != NULL) { Form_pg_class class_form = (Form_pg_class)GETSTRUCT(tuple); Datum reloptions; bool isnull; List *options; ListCell *lc; if (class_form->relkind != RELKIND_INDEX && class_form->relkind != RELKIND_PARTITIONED_INDEX) continue; if (class_form->oid == excluded_index_oid) continue; reloptions = heap_getattr( tuple, Anum_pg_class_reloptions, RelationGetDescr(class_rel), &isnull); if (isnull) continue; options = untransformRelOptions(reloptions); foreach (lc, options) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, "compaction_lineage") == 0 && strcmp(defGetString(option), lineage) == 0) { Oid existing_heap_oid = IndexGetRelation(class_form->oid, true); if (!OidIsValid(existing_heap_oid) || existing_heap_oid == heap_oid || !tp_heaps_share_partition_family( existing_heap_oid, heap_oid)) { found = true; break; } } } list_free_deep(options); if (found) break; } systable_endscan(scan); table_close(class_rel, AccessShareLock); return found; } bool tp_compaction_lineage_in_use(const char *lineage, Oid heap_oid, Oid owner_oid) { (void)owner_oid; return tp_live_compaction_lineage_conflicts(lineage, InvalidOid, heap_oid); } bool tp_compaction_lineage_in_use_by_other( const char *lineage, Oid indexoid, Oid heap_oid) { return tp_live_compaction_lineage_conflicts(lineage, indexoid, heap_oid); } char * tp_new_available_compaction_lineage(Oid heap_oid, Oid owner_oid) { char *lineage; do { lineage = tp_new_compaction_lineage(); if (tp_compaction_lineage_in_use(lineage, heap_oid, owner_oid)) { pfree(lineage); continue; } tp_lock_compaction_lineage(lineage); if (!tp_live_compaction_lineage_conflicts( lineage, InvalidOid, heap_oid)) return lineage; pfree(lineage); } while (true); } char * tp_ensure_index_compaction_lineage(Oid indexoid, bool *created) { AlterTableCmd *cmd; Relation index_rel; const char *existing; char *lineage; Oid heap_oid; Oid owner_oid; Oid save_userid; int save_sec_context; if (created != NULL) *created = false; index_rel = try_index_open(indexoid, AccessShareLock); if (index_rel == NULL) return NULL; if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || index_rel->rd_rel->relkind != RELKIND_INDEX || tp_index_compaction_mode(index_rel) != TP_COMPACTION_BACKGROUND) { index_close(index_rel, AccessShareLock); return NULL; } existing = tp_index_compaction_lineage(index_rel); if (existing != NULL) { lineage = pstrdup(existing); index_close(index_rel, NoLock); return lineage; } index_close(index_rel, AccessShareLock); index_rel = try_index_open(indexoid, ShareUpdateExclusiveLock); if (index_rel == NULL) return NULL; tp_lock_compaction_index(indexoid); if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || index_rel->rd_rel->relkind != RELKIND_INDEX || tp_index_compaction_mode(index_rel) != TP_COMPACTION_BACKGROUND) { index_close(index_rel, NoLock); return NULL; } existing = tp_index_compaction_lineage(index_rel); if (existing != NULL) { lineage = pstrdup(existing); index_close(index_rel, NoLock); return lineage; } /* Upgrade a pre-lineage background index under its physical owner. */ heap_oid = index_rel->rd_index->indrelid; owner_oid = index_rel->rd_rel->relowner; index_close(index_rel, NoLock); lineage = tp_new_available_compaction_lineage(heap_oid, owner_oid); cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetRelOptions; cmd->def = (Node *)list_make1(makeDefElem( "compaction_lineage", (Node *)makeString(lineage), -1)); cmd->behavior = DROP_RESTRICT; GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( owner_oid, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { AlterTableInternal(indexoid, list_make1(cmd), false); CommandCounterIncrement(); } PG_FINALLY(); { SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); if (created != NULL) *created = true; return lineage; } static void tp_set_partitioned_index_compaction_options( Relation index_rel, List *set_options, bool reset_schedule) { Relation class_rel; HeapTuple tuple; HeapTuple new_tuple; Datum old_options; Datum new_options; Datum values[Natts_pg_class]; bool isnull; bool nulls[Natts_pg_class]; bool replaces[Natts_pg_class]; class_rel = table_open(RelationRelationId, RowExclusiveLock); tuple = SearchSysCacheLocked1( RELOID, ObjectIdGetDatum(RelationGetRelid(index_rel))); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %u", RelationGetRelid(index_rel)); old_options = SysCacheGetAttr(RELOID, tuple, Anum_pg_class_reloptions, &isnull); new_options = transformRelOptions( isnull ? (Datum)0 : old_options, set_options, NULL, NULL, false, false); if (reset_schedule) new_options = transformRelOptions( new_options, list_make1(makeDefElem("compaction_schedule", NULL, -1)), NULL, NULL, false, true); (void)index_reloptions(index_rel->rd_indam->amoptions, new_options, true); memset(values, 0, sizeof(values)); memset(nulls, false, sizeof(nulls)); memset(replaces, false, sizeof(replaces)); if (new_options != (Datum)0) values[Anum_pg_class_reloptions - 1] = new_options; else nulls[Anum_pg_class_reloptions - 1] = true; replaces[Anum_pg_class_reloptions - 1] = true; new_tuple = heap_modify_tuple( tuple, RelationGetDescr(class_rel), values, nulls, replaces); CatalogTupleUpdate(class_rel, &new_tuple->t_self, new_tuple); UnlockTuple(class_rel, &tuple->t_self, InplaceUpdateTupleLock); InvokeObjectPostAlterHook( RelationRelationId, RelationGetRelid(index_rel), 0); heap_freetuple(new_tuple); ReleaseSysCache(tuple); table_close(class_rel, RowExclusiveLock); } void tp_reconcile_index_compaction_lineage(Oid indexoid, const char *lineage) { AlterTableCmd *cmd; Relation index_rel; const char *existing_lineage; List *set_options; Oid owner_oid; Oid save_userid; int save_sec_context; index_rel = try_index_open(indexoid, ShareUpdateExclusiveLock); if (index_rel == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("partition index with OID %u disappeared", indexoid))); if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || (index_rel->rd_rel->relkind != RELKIND_INDEX && index_rel->rd_rel->relkind != RELKIND_PARTITIONED_INDEX) || index_rel->rd_index == NULL) { char *index_name = pstrdup(RelationGetRelationName(index_rel)); index_close(index_rel, ShareUpdateExclusiveLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot reconcile partition index \"%s\" for " "background compaction", index_name), errdetail("The detached index is not a BM25 index."))); } existing_lineage = tp_index_compaction_lineage(index_rel); if (existing_lineage != NULL && strcmp(existing_lineage, lineage) == 0) { index_close(index_rel, NoLock); return; } set_options = list_make1(makeDefElem( "compaction_lineage", (Node *)makeString(pstrdup(lineage)), -1)); if (index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) { tp_set_partitioned_index_compaction_options( index_rel, set_options, false); index_close(index_rel, NoLock); CommandCounterIncrement(); return; } owner_oid = index_rel->rd_rel->relowner; index_close(index_rel, NoLock); cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetRelOptions; cmd->def = (Node *)set_options; cmd->behavior = DROP_RESTRICT; GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( owner_oid, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { AlterTableInternal(indexoid, list_make1(cmd), false); CommandCounterIncrement(); } PG_FINALLY(); { SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); } void tp_reconcile_index_compaction_options( Oid indexoid, const char *schedule, const char *lineage) { AlterTableCmd *cmd; Relation index_rel; const char *existing_lineage; const char *existing_schedule; List *commands = NIL; List *set_options = NIL; Oid owner_oid; Oid save_userid; int save_sec_context; bool schedule_matches; bool reset_schedule; index_rel = try_index_open(indexoid, ShareUpdateExclusiveLock); if (index_rel == NULL) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("partition index with OID %u disappeared", indexoid))); if (index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || (index_rel->rd_rel->relkind != RELKIND_INDEX && index_rel->rd_rel->relkind != RELKIND_PARTITIONED_INDEX) || index_rel->rd_index == NULL || (index_rel->rd_rel->relkind == RELKIND_INDEX && (!index_rel->rd_index->indisvalid || !index_rel->rd_index->indisready || !index_rel->rd_index->indislive))) { char *index_name = pstrdup(RelationGetRelationName(index_rel)); index_close(index_rel, ShareUpdateExclusiveLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("cannot reconcile partition index \"%s\" for " "background compaction", index_name), errdetail("The attached index is not a valid BM25 index."), errhint("Drop or rebuild the incompatible child index before " "creating the partitioned background index."))); } existing_lineage = tp_index_compaction_lineage(index_rel); existing_schedule = tp_index_compaction_schedule(index_rel); schedule_matches = schedule == NULL ? existing_schedule == NULL : existing_schedule != NULL && strcmp(existing_schedule, schedule) == 0; reset_schedule = schedule == NULL && existing_schedule != NULL; if (tp_index_compaction_mode(index_rel) == TP_COMPACTION_BACKGROUND && existing_lineage != NULL && strcmp(existing_lineage, lineage) == 0 && schedule_matches) { index_close(index_rel, NoLock); return; } set_options = lappend( set_options, makeDefElem("compaction", (Node *)makeString("background"), -1)); set_options = lappend(set_options, makeDefElem( "compaction_lineage", (Node *)makeString(pstrdup(lineage)), -1)); if (schedule != NULL) set_options = lappend(set_options, makeDefElem( "compaction_schedule", (Node *)makeString(pstrdup(schedule)), -1)); cmd = makeNode(AlterTableCmd); cmd->subtype = AT_SetRelOptions; cmd->def = (Node *)set_options; cmd->behavior = DROP_RESTRICT; commands = lappend(commands, cmd); if (reset_schedule) { cmd = makeNode(AlterTableCmd); cmd->subtype = AT_ResetRelOptions; cmd->def = (Node *)list_make1( makeDefElem("compaction_schedule", NULL, -1)); cmd->behavior = DROP_RESTRICT; commands = lappend(commands, cmd); } if (index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) { tp_set_partitioned_index_compaction_options( index_rel, set_options, reset_schedule); index_close(index_rel, NoLock); CommandCounterIncrement(); return; } owner_oid = index_rel->rd_rel->relowner; index_close(index_rel, NoLock); GetUserIdAndSecContext(&save_userid, &save_sec_context); SetUserIdAndSecContext( owner_oid, save_sec_context | SECURITY_LOCAL_USERID_CHANGE); PG_TRY(); { AlterTableInternal(indexoid, commands, false); CommandCounterIncrement(); } PG_FINALLY(); { SetUserIdAndSecContext(save_userid, save_sec_context); } PG_END_TRY(); } static void tp_pending_compactions_reset(void *arg pg_attribute_unused()) { tp_pending_compactions = NIL; tp_pending_registered = false; } void tp_compaction_request(Oid indexoid) { MemoryContext oldcxt; /* * Called after spill durability completes and the per-index LWLock is * released. Still defer SPI and callback execution until pre-commit; * only append to this list here. */ if (list_member_oid(tp_pending_compactions, indexoid)) return; oldcxt = MemoryContextSwitchTo(TopTransactionContext); if (!tp_pending_registered) { MemoryContextCallback *cb = palloc0(sizeof(*cb)); cb->func = tp_pending_compactions_reset; MemoryContextRegisterResetCallback(TopTransactionContext, cb); tp_pending_registered = true; } tp_pending_compactions = lappend_oid(tp_pending_compactions, indexoid); MemoryContextSwitchTo(oldcxt); } static void tp_run_request(Oid indexoid) { MemoryContext oldcxt = CurrentMemoryContext; ResourceOwner oldowner = CurrentResourceOwner; const char *idxname; /* * Resolve the index name before entering the error shield so a * dropped request can still produce a stable diagnostic. */ idxname = get_rel_name(indexoid); if (idxname == NULL) idxname = "?"; /* * PRE_COMMIT leaves the top-level block in TBLOCK_END. An outer * subtransaction keeps the recoverable inner transaction on a normal * TBLOCK_SUBINPROGRESS parent. A successful call releases the inner * subtransaction into the writer transaction. pg_durable delivers the * event through an independent connection, so its worker may observe the * already-published index state before the writer commits. An ordinary * failure rolls back only the inner transaction and becomes a warning. */ BeginInternalSubTransaction(NULL); BeginInternalSubTransaction(NULL); PG_TRY(); { TpCompactionJobObjects *objects; PushActiveSnapshot(GetLatestSnapshot()); objects = tp_compaction_job_try_lock_objects(false); if (objects != NULL) tp_compaction_job_signal(objects, indexoid); PopActiveSnapshot(); ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcxt); } PG_CATCH(); { ErrorData *edata; MemoryContextSwitchTo(oldcxt); edata = CopyErrorData(); FlushErrorState(); RollbackAndReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcxt); if (edata->sqlerrcode == ERRCODE_QUERY_CANCELED || edata->sqlerrcode == ERRCODE_ADMIN_SHUTDOWN || edata->sqlerrcode == ERRCODE_CRASH_SHUTDOWN) { ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcxt); CurrentResourceOwner = oldowner; ReThrowError(edata); } ereport(WARNING, (errmsg("bm25: managed background compaction request for " "index \"%s\" failed: %s", idxname, edata->message))); FreeErrorData(edata); } PG_END_TRY(); ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(oldcxt); CurrentResourceOwner = oldowner; } static bool tp_request_target_is_background(Oid indexoid) { Relation index_rel; bool background; index_rel = try_relation_open(indexoid, NoLock); if (index_rel == NULL) return false; background = index_rel->rd_rel->relkind == RELKIND_INDEX && index_rel->rd_indam != NULL && index_rel->rd_indam->ambuild == tp_build && index_rel->rd_index != NULL && index_rel->rd_index->indisvalid && index_rel->rd_index->indisready && index_rel->rd_index->indislive && tp_index_compaction_mode(index_rel) == TP_COMPACTION_BACKGROUND; relation_close(index_rel, NoLock); return background; } static List * tp_prelock_requests(List *pending) { List *sorted = list_copy(pending); List *targets = NIL; ListCell *lc; Oid previous = InvalidOid; list_sort(sorted, list_oid_cmp); foreach (lc, sorted) { Oid indexoid = lfirst_oid(lc); Relation index_rel; if (!OidIsValid(indexoid) || indexoid == previous) continue; previous = indexoid; /* * Dispatch is best effort and the on-disk debt is durable. A * terminal reconciliation batch may already hold the dependency; * never add a new admission after that point. */ if (tp_compaction_dependency_lock_held() && !tp_compaction_lock_held( indexoid, TP_COMPACTION_INDEX_LOCK_SUBID, ExclusiveLock)) continue; if (!ConditionalLockRelationOid(indexoid, ShareUpdateExclusiveLock)) continue; index_rel = try_relation_open(indexoid, NoLock); if (index_rel == NULL || index_rel->rd_rel->relkind != RELKIND_INDEX || index_rel->rd_indam == NULL || index_rel->rd_indam->ambuild != tp_build || index_rel->rd_index == NULL || !index_rel->rd_index->indisvalid || !index_rel->rd_index->indisready || !index_rel->rd_index->indislive || tp_index_compaction_mode(index_rel) != TP_COMPACTION_BACKGROUND) { if (index_rel != NULL) relation_close(index_rel, NoLock); UnlockRelationOid(indexoid, ShareUpdateExclusiveLock); continue; } relation_close(index_rel, NoLock); targets = lappend_oid(targets, indexoid); } list_free(sorted); foreach (lc, targets) { Oid indexoid = lfirst_oid(lc); if (!tp_take_compaction_lock( indexoid, TP_COMPACTION_INDEX_LOCK_SUBID, ExclusiveLock, true)) { UnlockRelationOid(indexoid, ShareUpdateExclusiveLock); targets = foreach_delete_current(targets, lc); } } return targets; } void tp_compaction_flush_requests(void) { ListCell *lc; List *pending; List *targets = NIL; if (tp_pending_compactions == NIL) return; /* * Defensive: such a process never records a request, because the * spill path compacts inline. Discard rather than run callback SQL * where it is unsafe; the debt is on disk and re-detected later. */ if (!tp_compaction_dispatch_possible()) { list_free(tp_pending_compactions); tp_pending_compactions = NIL; return; } /* Take ownership so callback re-entry cannot loop. */ pending = tp_pending_compactions; tp_pending_compactions = NIL; /* * A spill caused during dispatch compacts inline: its request would * land in a list this flush has stopped reading and be freed at * commit, while the original spill debt remains durable. */ tp_dispatch_active = true; PG_TRY(); { targets = tp_prelock_requests(pending); foreach (lc, targets) { Oid indexoid = lfirst_oid(lc); bool dispatch; dispatch = SearchSysCacheExists1( RELOID, ObjectIdGetDatum(indexoid)) && tp_request_target_is_background(indexoid); /* Let the signaled worker compact while private admission remains. */ UnlockRelationOid(indexoid, ShareUpdateExclusiveLock); if (!dispatch) continue; tp_run_request(indexoid); } } PG_FINALLY(); { tp_dispatch_active = false; list_free(targets); list_free(pending); } PG_END_TRY(); } timescale-pg_textsearch-b63fe58/src/index/compaction_request.h000066400000000000000000000043321525762047700247360ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compaction_request.h - Background compaction request tracking */ #pragma once #include #include typedef enum TpCompactionMode { TP_COMPACTION_INLINE = 0, TP_COMPACTION_BACKGROUND, TP_COMPACTION_MANUAL } TpCompactionMode; #define TP_COMPACTION_LINEAGE_BYTES 16 #define TP_COMPACTION_LINEAGE_LENGTH (TP_COMPACTION_LINEAGE_BYTES * 2) extern char *tp_background_compaction_schedule; extern int tp_index_compaction_mode(Relation index_rel); extern const char *tp_index_compaction_schedule(Relation index_rel); extern const char *tp_index_compaction_lineage(Relation index_rel); extern char *tp_new_compaction_lineage(void); extern char *tp_ensure_index_compaction_lineage(Oid indexoid, bool *created); extern void tp_reconcile_index_compaction_lineage(Oid indexoid, const char *lineage); extern void tp_reconcile_index_compaction_options( Oid indexoid, const char *schedule, const char *lineage); extern char *tp_new_available_compaction_lineage(Oid heap_oid, Oid owner_oid); extern bool tp_compaction_lineage_in_use(const char *lineage, Oid heap_oid, Oid owner_oid); extern bool tp_compaction_lineage_in_use_by_other( const char *lineage, Oid indexoid, Oid heap_oid); extern void tp_lock_compaction_index(Oid indexoid); extern void tp_lock_compaction_dependency(void); extern bool tp_try_lock_compaction_dependency(void); extern bool tp_try_lock_compaction_dependency_oid(Oid bm25_am_oid); extern bool tp_compaction_dependency_lock_held(void); extern bool tp_compaction_dependency_oid_lock_held(Oid bm25_am_oid); extern void tp_unlock_compaction_dependency(void); extern void tp_unlock_compaction_dependency_oid(Oid bm25_am_oid); extern void tp_lock_compaction_lineage(const char *lineage); extern void tp_require_compaction_index_lock(Oid indexoid); extern void tp_require_compaction_dependency_lock(void); extern bool tp_try_lock_compaction_lineage(const char *lineage); extern List *tp_try_prelock_compaction_indexes(List *indexoids); extern bool tp_compaction_dispatch_possible(void); extern void tp_compaction_request(Oid indexoid); extern void tp_compaction_flush_requests(void); timescale-pg_textsearch-b63fe58/src/index/freepage.c000066400000000000000000000135151525762047700226060ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * freepage.c - Recyclable free-page stamping for safe FSM reuse. * * See freepage.h for the rationale (issues #380, #426, #427). */ #include #include #include #include #include #include #include #include #include "constants.h" #include "index/freepage.h" bool tp_page_is_recyclable(Page page) { TpFreePageData *f = (TpFreePageData *)PageGetContents(page); return f->magic == TP_FREE_PAGE_MAGIC; } void tp_log_page_reuse_conflict( Relation index, BlockNumber block, FullTransactionId horizon) { xl_btree_reuse_page xlrec; if (!RelationNeedsWAL(index) || !XLogStandbyInfoActive()) return; xlrec.locator = index->rd_locator; xlrec.block = block; xlrec.snapshotConflictHorizon = horizon; xlrec.isCatalogRel = false; XLogBeginInsert(); XLogRegisterData((char *)&xlrec, SizeOfBtreeReusePage); XLogInsert(RM_BTREE_ID, XLOG_BTREE_REUSE_PAGE); } void tp_record_free_index_page(Relation index, BlockNumber blk) { Buffer buf; Page page; PageHeader ph; GenericXLogState *state; TpFreePageData *f; if (blk == TP_METAPAGE_BLKNO) elog(ERROR, "pg_textsearch: refusing to free metapage (block 0)"); buf = ReadBuffer(index, blk); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); page = GenericXLogRegisterBuffer(state, buf, 0); f = (TpFreePageData *)PageGetContents(page); f->magic = TP_FREE_PAGE_MAGIC; f->flags = 0; f->freed_fxid = ReadNextFullTransactionId(); /* * Collapse the GenericXLog page hole so the stamp above lands in * the WAL-logged lower region: computeDelta() diffs only * [0, pd_lower) and [pd_upper, BLCKSZ), ignoring the hole in * between. Setting pd_lower = pd_upper = pd_special = BLCKSZ * records the whole page while leaving the body bytes untouched, * so the delta is just the header + stamp (a few dozen bytes) even * though a merge/vacuum may free many pages. Same page-hole * convention as tp_tombstone_page_init. */ ph = (PageHeader)page; ph->pd_lower = BLCKSZ; ph->pd_upper = BLCKSZ; ph->pd_special = BLCKSZ; GenericXLogFinish(state); UnlockReleaseBuffer(buf); RecordFreeIndexPage(index, blk); } Buffer tp_fsm_claim_free_buffer(Relation index) { for (;;) { BlockNumber blk; Buffer buf; Page page; CHECK_FOR_INTERRUPTS(); blk = GetFreeIndexPage(index); if (blk == InvalidBlockNumber) return InvalidBuffer; /* FSM empty: caller extends */ /* * Drop obviously bogus FSM entries. GetFreeIndexPage() has * already marked `blk` used, so simply skipping it removes the * entry from circulation (no infinite loop, no re-offer). */ if (blk == TP_METAPAGE_BLKNO || blk >= RelationGetNumberOfBlocks(index)) continue; buf = ReadBuffer(index, blk); if (!ConditionalLockBuffer(buf)) { /* * Someone already holds this buffer's content lock — a * concurrent claimer mid-flight, or (under a stale FSM * offer) this very backend still holding the block as its * live memtable tail. Blocking here would risk a * self-deadlock in the latter case, so treat it like a * non-recyclable page and try the next candidate. */ ReleaseBuffer(buf); continue; } page = BufferGetPage(buf); if (tp_page_is_recyclable(page)) return buf; /* caller reinitializes + WAL-logs under the lock */ /* * The FSM offered a block that is NOT a deliberately-freed * page — a stale (crash) or double-allocated (non-atomic * GetFreeIndexPage) entry that still points at a live * structure page. Reusing it would corrupt that structure * (issues #426, #427). GetFreeIndexPage already cleared the * FSM slot, so release the page and try the next candidate. */ UnlockReleaseBuffer(buf); } } BlockNumber tp_fsm_claim_free_block(Relation index) { Buffer buf = tp_fsm_claim_free_buffer(index); BlockNumber blk; GenericXLogState *state; Page page; TpFreePageData *f; if (!BufferIsValid(buf)) return InvalidBlockNumber; blk = BufferGetBlockNumber(buf); /* * This variant releases the buffer lock before returning the block — * the caller reinitializes the page later, under a fresh lock. To * keep the claim atomic against a concurrent allocator that the * non-atomic GetFreeIndexPage() handed the same block, clear the free * stamp under the lock now (WAL-logged). That concurrent allocator * fails the conditional lock, or observes a page that is no longer * recyclable, and skips it, so it cannot double-allocate the block. * A crash between here and the caller's reinitialization only leaks * the block (reclaimed by REINDEX); it is out of the FSM and linked * into no structure. pd_lower is already BLCKSZ from the stamp, so * the cleared magic lands in GenericXLog's logged region. */ state = GenericXLogStart(index); page = GenericXLogRegisterBuffer(state, buf, 0); f = (TpFreePageData *)PageGetContents(page); f->magic = 0; GenericXLogFinish(state); UnlockReleaseBuffer(buf); return blk; } BlockNumber tp_fsm_claim_or_extend_block(Relation index) { Buffer buffer; BlockNumber block = tp_fsm_claim_free_block(index); if (block != InvalidBlockNumber) return block; /* * FSM has no reusable page: extend through the same bulk-extension API * as the memtable allocator. Mixing ReadBufferExtended(P_NEW) with * ExtendBufferedRel lets concurrent compaction and memtable growth * reserve the same block on PostgreSQL 17. EB_LOCK_FIRST returns the * zero-filled page pinned and exclusively locked. */ buffer = ExtendBufferedRel( BMR_REL(index), MAIN_FORKNUM, NULL, EB_LOCK_FIRST); block = BufferGetBlockNumber(buffer); UnlockReleaseBuffer(buffer); return block; } timescale-pg_textsearch-b63fe58/src/index/freepage.h000066400000000000000000000117421525762047700226130ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * freepage.h - Recyclable free-page stamping for FSM reuse. * * The index FSM (free space map) is a non-crash-safe hint whose * GetFreeIndexPage() claim is not atomic across backends. Either * property can offer a block that is still referenced by a live * pg_textsearch structure — the on-disk memtable chain (issue #426), * the deferred-free tombstone chain (issue #380 / #427), a segment, * or a page index. Blindly reinitializing such a block corrupts the * structure that still owns it. * * To make FSM reuse safe, every page returned to the FSM is first * WAL-stamped with TP_FREE_PAGE_MAGIC (tp_record_free_index_page). * Allocators then reuse a block ONLY when it still carries that stamp * (tp_fsm_claim_free_buffer / tp_fsm_claim_free_block). A block the * FSM offers that is NOT stamped (a stale or double-allocated entry * pointing at a live page) is skipped rather than overwritten — the * same recyclability discipline PostgreSQL's B-tree uses via * _bt_page_recyclable. * * Pages that were visible to standby readers also require * tp_log_page_reuse_conflict() before the free stamp. Its stock btree WAL * record cancels snapshots that outlived feedback or a replication * disconnect before later WAL can overwrite the page. */ #pragma once #include #include #include #include #include #include /* * Free-page stamp, written at PageGetContents() of a page that has * been deliberately returned to the index FSM. The rest of the page * body is left intact (a reuse fully reinitializes it), keeping the * WAL delta tiny. */ typedef struct TpFreePageData { uint32 magic; /* TP_FREE_PAGE_MAGIC */ uint32 flags; /* reserved, 0 */ FullTransactionId freed_fxid; /* xid at free time (diagnostic) */ } TpFreePageData; /* True iff `page` carries the recyclable free-page stamp. */ extern bool tp_page_is_recyclable(Page page); /* * Emit PostgreSQL's stock btree conflict-only WAL record before a page whose * contents may still be read by an old standby snapshot is reused. Redo does * not inspect the page; it only cancels snapshots at or before `horizon`. * Generic WAL carries no conflict horizon, and a custom rmgr would make replay * depend on loading this library. pg_waldump reports these as * Btree/REUSE_PAGE. */ extern void tp_log_page_reuse_conflict( Relation index, BlockNumber block, FullTransactionId horizon); /* * WAL-stamp `blk` as a recyclable free page, then return it to the * index FSM. Use in place of a bare RecordFreeIndexPage() so a later * allocator can distinguish a deliberately-freed page from a live one * whose block the FSM offered by mistake. Takes the buffer's * EXCLUSIVE lock; the caller must already have removed `blk` from * every live structure (it is unreferenced once freed). */ extern void tp_record_free_index_page(Relation index, BlockNumber blk); /* * Claim a recyclable free page from the FSM. Returns a pinned, * EXCLUSIVE-locked buffer whose page the caller must reinitialize and * WAL-log before releasing the lock, or InvalidBuffer when the FSM * offers no reusable page (the caller then extends the relation). * * Holding the buffer lock from the recyclability check through the * caller's reinitialization makes the claim atomic: a second backend * handed the same block by a non-atomic GetFreeIndexPage() fails the * conditional lock (or observes the now-live page) and skips it. The * conditional lock also avoids a self-deadlock when a stale FSM offer * hands back a block the caller itself already holds locked. This is * the path memtable inserts use (they run under the per-index lock * only in SHARED mode, so they race each other — issue #426). */ extern Buffer tp_fsm_claim_free_buffer(Relation index); /* * Block-returning wrapper over tp_fsm_claim_free_buffer for callers * that reopen the block later under their own lock (the segment * writer buffers pages and flushes them afterward). Because the lock * is dropped before the block is returned, this variant clears the * free stamp under the claim lock so the claim stays atomic even when * the caller holds the per-index lock only in SHARED mode and races a * concurrent allocator (e.g. VACUUM's pre-V5 segment rebuild vs. a * memtable insert). Returns InvalidBlockNumber when the FSM offers no * reusable page. */ extern BlockNumber tp_fsm_claim_free_block(Relation index); /* * Claim a recyclable free page via tp_fsm_claim_free_block, falling back * to ExtendBufferedRel when the FSM offers none -- the same extension API * as the memtable allocator, so the two cannot reserve the same block. * Always returns a valid block number. Shared by the block-oriented * allocators (segment and tombstone pages) that reopen the block later * under their own lock. */ extern BlockNumber tp_fsm_claim_or_extend_block(Relation index); timescale-pg_textsearch-b63fe58/src/index/limit.c000066400000000000000000000011341525762047700221400ustar00rootroot00000000000000/* Query LIMIT optimization shared by the planner and index scan. */ #include #include #include "constants.h" #include "index/limit.h" int tp_default_limit = TP_DEFAULT_QUERY_LIMIT; int tp_seed_limit_for_filter(int user_limit, double selectivity) { double seeded; if (!tp_filtered_seed || selectivity <= 0.0 || selectivity >= 1.0) return user_limit; seeded = ceil(tp_filtered_seed_margin * (double)user_limit / selectivity); if (seeded > (double)TP_MAX_INITIAL_QUERY_LIMIT) seeded = TP_MAX_INITIAL_QUERY_LIMIT; return seeded > user_limit ? (int)seeded : user_limit; } timescale-pg_textsearch-b63fe58/src/index/limit.h000066400000000000000000000013671525762047700221550ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * limit.h - Query LIMIT optimization interface */ #pragma once #include #include "constants.h" /* * Query LIMIT Optimization * * When a query has a LIMIT clause and uses ORDER BY with a BM25 score, * only the top N results need to be computed. This module holds the * formula that turns a user LIMIT plus a filter selectivity into the * scan's internal top-K, and the default used when no LIMIT is known. * * The planner carries scan-local hints in private bm25query values. */ /* Default limit when none detected */ extern int tp_default_limit; int tp_seed_limit_for_filter(int user_limit, double selectivity); timescale-pg_textsearch-b63fe58/src/index/metapage.c000066400000000000000000000320171525762047700226110ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * metapage.c - Index metapage operations * * Handles metapage initialization, reading, and management. The metapage * stores index configuration, statistics, and segment chain heads. */ #include #include #include #include #include #include #include #include "constants.h" #include "debug/injection.h" #include "index/metapage.h" /* * Initialize Tapir index metapage */ void tp_init_metapage(Page page, Oid text_config_oid) { TpIndexMetaPage metap; PageHeader phdr; int i; /* * Initialize page with no special space - metapage uses page content area */ PageInit(page, BLCKSZ, 0); metap = (TpIndexMetaPage)PageGetContents(page); metap->magic = TP_METAPAGE_MAGIC; metap->version = TP_METAPAGE_VERSION; metap->text_config_oid = text_config_oid; metap->total_docs = 0; metap->_unused_total_terms = 0; metap->total_len = 0; metap->root_blkno = InvalidBlockNumber; metap->_unused_docid_page = InvalidBlockNumber; /* Initialize hierarchical segment levels */ for (i = 0; i < TP_MAX_LEVELS; i++) { metap->level_heads[i] = InvalidBlockNumber; metap->level_counts[i] = 0; } /* * Memtable v2 chain: explicitly InvalidBlockNumber (= * 0xFFFFFFFF) — NOT zero, since block 0 is the metapage * itself. See TP_METAPAGE_VERSION in constants.h. */ metap->memtable_head_blkno = InvalidBlockNumber; metap->memtable_tail_blkno = InvalidBlockNumber; metap->pending_free_head = InvalidBlockNumber; metap->capabilities = TP_METAPAGE_ALL_DOCUMENTS_INDEXED; /* Update page header to reflect that we've used space for metapage */ phdr = (PageHeader)page; phdr->pd_lower = SizeOfPageHeaderData + sizeof(TpIndexMetaPageData); } void tp_check_level_count_increment(TpIndexMetaPage metap, uint32 level) { Assert(metap != NULL); if (level >= TP_MAX_LEVELS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid bm25 segment level %u", level))); if (metap->level_counts[level] >= tp_injected_segment_count_limit()) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("bm25 segment count limit reached at level %u", level))); } static void tp_validate_metapage_version(Relation index, TpIndexMetaPage metap) { /* Validate magic number */ if (metap->magic != TP_METAPAGE_MAGIC) elog(ERROR, "Tapir index metapage is corrupted for index \"%s\": expected " "magic " "0x%08X, found 0x%08X", RelationGetRelationName(index), TP_METAPAGE_MAGIC, metap->magic); /* * Check version compatibility. * * v9 appends capability flags. v8 and v7 are read-compatible; * v7 is the on-disk memtable redesign (issue #374), while v8 * adds pending_free_head. v6 is read-compatible (issue #383): * the layout is * byte-identical up through level_counts[]; v7 only appends * memtable_head_blkno and memtable_tail_blkno. We accept * v6 here and normalize the missing fields to * InvalidBlockNumber in the returned copy. The first * metapage mutation upgrades the on-disk page to v7 * atomically via tp_metapage_upgrade_to_current() inside * the surrounding GenericXLog record. * * v6 with a non-Invalid _unused_docid_page (formerly * first_docid_page) is also accepted here. It indicates * either (a) the operator called bm25_spill_index() before * the binary swap and the pointer is stale bookkeeping, or * (b) v1.2.x was SIGKILL'd with in-flight documents in the * chain and those documents are now lost. We cannot * distinguish those cases from the on-disk bytes alone, so * the upgrade helper emits a client-visible WARNING (once, * on first write) and PRESERVES the pointer as a durable * marker; the scan path re-surfaces it via * tp_warn_if_pending_docid() until a REINDEX clears it. * Operators who suspect lost documents should REINDEX. * * v5 and earlier are not read-compatible and require * REINDEX as before. */ if (metap->version != TP_METAPAGE_VERSION && metap->version != TP_METAPAGE_VERSION_V8 && metap->version != TP_METAPAGE_VERSION_V7 && metap->version != TP_METAPAGE_VERSION_V6) { ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("incompatible pg_textsearch index version for " "\"%s\": found %u, expected %u", RelationGetRelationName(index), metap->version, TP_METAPAGE_VERSION), errhint("This index was created by a previous release of " "pg_textsearch and uses an incompatible on-disk " "format. Run REINDEX INDEX %s to rebuild it.", RelationGetRelationName(index)))); } } TpIndexMetaPage tp_metapage_copy_from_page(Relation index, Page page) { TpIndexMetaPage metap = (TpIndexMetaPage)PageGetContents(page); TpIndexMetaPage result; Assert(RelationIsValid(index)); Assert(page != NULL); tp_validate_metapage_version(index, metap); /* * Note: a v6 metapage with a non-Invalid _unused_docid_page * is accepted here as well — the operator-visible WARNING * fires once, from tp_metapage_upgrade_to_current() on the * first write, at the point we upgrade the page and preserve * the pointer as a durable marker. See the rationale block * in that function. The scan path re-surfaces the marker via * tp_warn_if_pending_docid(); this function stays silent * because it is on the hot read path. */ /* * Copy metapage data to avoid buffer issues. On v6 we only * copy the v6-sized prefix and then explicitly initialize * the v7-new fields, since the on-disk bytes at those * offsets are unrelated (PageInit zero-fill, NOT * InvalidBlockNumber = 0xFFFFFFFF). */ result = palloc0(sizeof(TpIndexMetaPageData)); if (metap->version == TP_METAPAGE_VERSION) { memcpy(result, metap, sizeof(TpIndexMetaPageData)); } else if (metap->version == TP_METAPAGE_VERSION_V8) { memcpy(result, metap, TP_INDEX_METAPAGE_DATA_SIZE_V8); result->capabilities = 0; } else if (metap->version == TP_METAPAGE_VERSION_V7) { /* v7 has memtable head/tail but no pending_free_head. */ memcpy(result, metap, TP_INDEX_METAPAGE_DATA_SIZE_V7); result->pending_free_head = InvalidBlockNumber; result->capabilities = 0; } else { Assert(metap->version == TP_METAPAGE_VERSION_V6); memcpy(result, metap, TP_INDEX_METAPAGE_DATA_SIZE_V6); result->memtable_head_blkno = InvalidBlockNumber; result->memtable_tail_blkno = InvalidBlockNumber; result->pending_free_head = InvalidBlockNumber; result->capabilities = 0; } return result; } /* * Get Tapir index metapage */ TpIndexMetaPage tp_get_metapage(Relation index) { Buffer buf; Page page; TpIndexMetaPage result; if (!RelationIsValid(index)) elog(ERROR, "invalid relation passed to tp_get_metapage"); buf = ReadBuffer(index, TP_METAPAGE_BLKNO); if (!BufferIsValid(buf)) elog(ERROR, "failed to read metapage buffer for BM25 index \"%s\"", RelationGetRelationName(index)); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); result = tp_metapage_copy_from_page(index, page); UnlockReleaseBuffer(buf); return result; } /* * Read memtable_head_blkno from a metapage buffer page in a * version-tolerant way (issue #383). See header for contract. */ BlockNumber tp_metapage_read_memtable_head(Page page) { TpIndexMetaPage metap = (TpIndexMetaPage)PageGetContents(page); if (metap->version == TP_METAPAGE_VERSION_V6) return InvalidBlockNumber; return metap->memtable_head_blkno; } /* * Read memtable_tail_blkno from a metapage buffer page in a * version-tolerant way (issue #383). See header for contract. */ BlockNumber tp_metapage_read_memtable_tail(Page page) { TpIndexMetaPage metap = (TpIndexMetaPage)PageGetContents(page); if (metap->version == TP_METAPAGE_VERSION_V6) return InvalidBlockNumber; return metap->memtable_tail_blkno; } /* * In-place upgrade of read-compatible metapages. See header for * caller contract. No-op when the page is already current. */ void tp_metapage_upgrade_to_current(Relation index, Page page) { TpIndexMetaPage metap; PageHeader phdr; metap = (TpIndexMetaPage)PageGetContents(page); if (metap->version == TP_METAPAGE_VERSION) return; if (metap->version != TP_METAPAGE_VERSION_V6 && metap->version != TP_METAPAGE_VERSION_V7 && metap->version != TP_METAPAGE_VERSION_V8) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: cannot upgrade metapage " "with unknown version %u", metap->version), errhint("Caller must validate the metapage version " "via tp_get_metapage() before calling " "tp_metapage_upgrade_to_current()."))); /* * v6 lacks the memtable chain fields and may carry a stale * docid pointer; v7 already has the chain fields. Only the * v6 path initializes memtable head/tail and inspects the * legacy docid pointer. */ if (metap->version == TP_METAPAGE_VERSION_V6) { BlockNumber orphan_docid_page = metap->_unused_docid_page; metap->memtable_head_blkno = InvalidBlockNumber; metap->memtable_tail_blkno = InvalidBlockNumber; if (BlockNumberIsValid(orphan_docid_page) && !RecoveryInProgress()) ereport(WARNING, (errmsg("pg_textsearch: upgraded index \"%s\" " "from a pre-v1.3 on-disk format that may " "have held unspilled documents", RelationGetRelationName(index)), errdetail( "The pre-v1.3 docid chain (first page " "%u) replayed in-flight documents from " "the heap on restart; v1.3+ has no " "equivalent, so any documents not " "spilled before the binary swap are " "absent from the index.", orphan_docid_page), errhint("Run \"REINDEX INDEX %s;\" to rebuild " "from the table and clear this warning.", RelationGetRelationName(index)))); /* * Preserve orphan_docid_page as a durable "results may be * incomplete" marker. It rides through v8 in the * _unused_docid_page slot and is re-surfaced to clients on * the scan path (at most once per session) by * tp_warn_if_pending_docid() until a REINDEX rebuilds the * index from the heap and clears it. Do NOT reset it to * InvalidBlockNumber here. */ } if (metap->version < TP_METAPAGE_VERSION_V8) metap->pending_free_head = InvalidBlockNumber; metap->capabilities = 0; metap->version = TP_METAPAGE_VERSION; /* * Bump pd_lower so every newly-included field is inside the * "used" area; GenericXLog skips the [pd_lower, pd_upper) hole * and would otherwise zero the new fields on replay. */ { Size v8_pd_lower = SizeOfPageHeaderData + sizeof(TpIndexMetaPageData); phdr = (PageHeader)page; if (phdr->pd_lower < v8_pd_lower) phdr->pd_lower = v8_pd_lower; } } /* * Per-backend set of index OIDs already inspected for the * "results may be incomplete" marker this session. Lives in * TopMemoryContext so it survives the transient scan contexts. An OID * is recorded on the FIRST inspection regardless of outcome, so a * clean index costs at most one metapage read per session and a marked * index warns at most once -- neither floods a plan that reopens the * scan many times (e.g. the inner side of a nested loop). */ static List *tp_pending_docid_seen = NIL; /* * tp_warn_if_pending_docid * * Read-path guard for the pre-v1.3 upgrade marker. When a pre-v1.3 * index was upgraded while it still carried unspilled documents, * tp_metapage_upgrade_to_current() preserves the legacy docid pointer * in _unused_docid_page as a durable "results may be incomplete" * marker. This helper surfaces that marker to the client as a WARNING * on the scan path, so read-only workloads -- which never trigger the * write-path upgrade WARNING -- still learn that a REINDEX is needed. * * Inspects the metapage at most once per index per backend session * (see tp_pending_docid_seen). Runs during recovery too: a hot * standby serving reads against an upgraded primary must still see the * advisory, even though the REINDEX that clears it has to run on the * primary. */ void tp_warn_if_pending_docid(Relation index) { Oid index_oid = RelationGetRelid(index); TpIndexMetaPage metap; BlockNumber marker; MemoryContext old; if (list_member_oid(tp_pending_docid_seen, index_oid)) return; metap = tp_get_metapage(index); marker = metap->_unused_docid_page; pfree(metap); /* * Record the index before (possibly) warning so a clean index is * never re-read and a marked index warns only once this session. */ old = MemoryContextSwitchTo(TopMemoryContext); tp_pending_docid_seen = lappend_oid(tp_pending_docid_seen, index_oid); MemoryContextSwitchTo(old); if (!BlockNumberIsValid(marker)) return; ereport(WARNING, (errmsg("pg_textsearch: index \"%s\" may return " "incomplete results after upgrade from a " "pre-v1.3 on-disk format", RelationGetRelationName(index)), errhint("Some documents inserted before the " "upgrade may be missing. Run \"REINDEX " "INDEX %s;\" to rebuild from the table and " "clear this warning.", RelationGetRelationName(index)))); } timescale-pg_textsearch-b63fe58/src/index/metapage.h000066400000000000000000000214271525762047700226210ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * metapage.h - Index metapage structures and operations */ #pragma once #include #include #include #include #include #include "constants.h" /* * Tapir Index Metapage Structure * * The metapage is stored on block 0 of every Tapir index and contains * configuration parameters and global statistics needed for BM25 scoring. * * Segment hierarchy: LSM-style tiered compaction with TP_MAX_LEVELS levels. * Level 0 receives segments from memtable spills (~8MB each). * When a level reaches segments_per_level segments, they are merged into * a single segment at the next level. This provides exponentially larger * segments at higher levels while bounding write amplification. */ typedef struct TpIndexMetaPageData { uint32 magic; /* Magic number for validation */ uint32 version; /* Index format version */ Oid text_config_oid; /* Text search configuration OID */ /* * Corpus size for BM25 scoring. Satisfies the exact invariant * * total_docs = Σ segment.num_docs * * where the sum is over all on-disk segments reachable from * level_heads[] and num_docs is the segment header field (fixed * at segment creation). VACUUM keeps the invariant by * decrementing total_docs whenever a segment's num_docs changes * or the segment leaves the chain: pre-V5 rebuild shrinks * num_docs from old to docs_added; a V5 segment dropped because * all docs are dead contributes zero. V5 bitset flips that * leave survivors are invisible here — they shrink alive_count, * not num_docs. * * Per-segment dictionary doc_freq(t) ≤ segment.num_docs by * construction, so * * total_docs = Σ segment.num_docs * ≥ Σ segment.doc_freq(t) * = doc_freq(t) globally * * and BM25's N ≥ df(t) precondition for tp_calculate_idf holds. * * Unflushed memtable docs are NOT counted here; the on-disk * memtable chain (issue #374) is the in-flight buffer and * scoring composes its corpus totals from this metapage value * plus a chain-source open at read time. */ uint64 total_docs; uint64 _unused_total_terms; /* Unused, retained for on-disk compat */ uint64 total_len; /* Σ segment.total_len, same frame as total_docs */ float4 k1; /* BM25 k1 parameter */ float4 b; /* BM25 b parameter */ BlockNumber root_blkno; /* Root page of the index tree */ BlockNumber term_stats_root; /* Root page of term statistics B-tree */ BlockNumber _unused_docid_page; /* Reserved: was first_docid_page, * retired by issue #374 (the * on-disk memtable obsoletes * docid pages). Normally * InvalidBlockNumber, but on an * index upgraded in place from a * pre-v1.3 (v6) format that still * held unspilled docs it retains * the legacy pointer as a durable * "results may be incomplete" * marker until REINDEX. */ /* Hierarchical segment storage (LSM-style) */ BlockNumber level_heads[TP_MAX_LEVELS]; /* Head of segment chain per level */ uint16 level_counts[TP_MAX_LEVELS]; /* Segment count per level */ /* * On-disk memtable (issue #374): head/tail of the page * chain. Both fields hold InvalidBlockNumber when the * chain is empty; tail_blkno may differ from head_blkno * once the chain has more than one page. All mutations * to these fields go through GenericXLog atomic with the * corresponding chain-page mutation. Introduced in * TP_METAPAGE_VERSION 7. */ BlockNumber memtable_head_blkno; BlockNumber memtable_tail_blkno; /* * Head of the deferred-free tombstone chain (issue #380). * InvalidBlockNumber when empty. Each tombstone page lists * displaced segment blocks plus a merge horizon; pages are * returned to the FSM only once that horizon is past * GetOldestNonRemovableTransactionId. Introduced in * TP_METAPAGE_VERSION 8. All mutations go through * GenericXLog atomic with the corresponding chain mutation. */ BlockNumber pending_free_head; /* * Index capabilities. Introduced in version 9. Older indexes did * not retain documents whose configured tsvector was empty. */ uint32 capabilities; } TpIndexMetaPageData; typedef TpIndexMetaPageData *TpIndexMetaPage; /* * V6 metapage size (bytes), used by: * - tp_metapage_upgrade_to_current() when bumping pd_lower on * in-place upgrade. * - Tests / debug code that downgrades a v7 page to v6 for * compat-path exercise. * * v6 had no memtable_head_blkno / memtable_tail_blkno fields; * everything else is byte-identical. Use offsetof() so we never * drift from the real prefix layout of the live struct. */ #define TP_INDEX_METAPAGE_DATA_SIZE_V6 \ offsetof(TpIndexMetaPageData, memtable_head_blkno) /* * Pin the historical v6 prefix size at 108 bytes. v6 was frozen * by the v1.2.x releases; any future struct edit that changes the * layout of bytes [0, 108) would silently break v6 read compat * (issue #383). v7 only appends new fields after this prefix. */ StaticAssertDecl( TP_INDEX_METAPAGE_DATA_SIZE_V6 == 108, "v6 metapage prefix layout changed - " "backward-compat is broken"); /* * v7 prefix size: everything up to (but not including) * pending_free_head. v7 was frozen by v1.3.x; v8 only appends * pending_free_head after this prefix. See issue #380. */ #define TP_INDEX_METAPAGE_DATA_SIZE_V7 \ offsetof(TpIndexMetaPageData, pending_free_head) StaticAssertDecl( TP_INDEX_METAPAGE_DATA_SIZE_V7 == 116, "v7 metapage prefix layout changed - " "backward-compat is broken"); #define TP_INDEX_METAPAGE_DATA_SIZE_V8 \ offsetof(TpIndexMetaPageData, capabilities) StaticAssertDecl( TP_INDEX_METAPAGE_DATA_SIZE_V8 == 120, "v8 metapage prefix layout changed - " "backward-compat is broken"); /* * Metapage operations */ extern void tp_init_metapage(Page page, Oid text_config_oid); /* Caller must hold at least BUFFER_LOCK_SHARE on the metapage buffer. */ extern TpIndexMetaPage tp_metapage_copy_from_page(Relation index, Page page); extern TpIndexMetaPage tp_get_metapage(Relation index); extern void tp_check_level_count_increment(TpIndexMetaPage metap, uint32 level); /* * Read the on-disk memtable chain head/tail from a buffer page, * tolerating v6 metapages (where the fields do not exist on disk * and the underlying bytes are zero — which would otherwise be * misinterpreted as block 0, i.e. the metapage itself, rather * than InvalidBlockNumber). * * Caller must hold at least BUFFER_LOCK_SHARE on the metapage * buffer. These helpers are for direct-buffer reader sites that * bypass tp_get_metapage() (which already normalizes its palloc'd * copy). See src/memtable/log.c and src/memtable/chain_source.c. */ extern BlockNumber tp_metapage_read_memtable_head(Page page); extern BlockNumber tp_metapage_read_memtable_tail(Page page); /* * In-place upgrade of read-compatible metapages. Must be * called on a GenericXLogRegisterBuffer-returned page copy under * BUFFER_LOCK_EXCLUSIVE, BEFORE setting any newer field on the * page. On the current version this is a no-op. On older pages the * helper: * - initializes memtable_head_blkno = memtable_tail_blkno = * InvalidBlockNumber, * - PRESERVES any non-Invalid _unused_docid_page pointer as a * durable "results may be incomplete" marker and emits a * client-visible WARNING (see the rationale block in * tp_metapage_upgrade_to_current()); the marker persists in * v8 and is re-surfaced on the scan path (at most once per * session) by tp_warn_if_pending_docid() until a REINDEX * clears it, * - preserves fields already present in that version, * - initializes missing fields and clears capability flags, * - bumps the version field to TP_METAPAGE_VERSION, * - bumps pd_lower to cover the new fields so GenericXLog * records them in the page image. * * The upgrade piggybacks on the surrounding GenericXLog record; * there is no extra WAL traffic. Concurrent writers cannot race * the upgrade — the EXCLUSIVE buffer lock serializes them, and a * later writer simply observes v7 and the helper is a no-op. * * The Relation is only used to format the marker WARNING with the * index name; the function does not otherwise touch it. */ extern void tp_metapage_upgrade_to_current(Relation index, Page page); /* * Emit a client-visible WARNING (at most once per index per backend * session) when the index carries the durable "results may be * incomplete" marker left by an in-place pre-v1.3 (v6) upgrade that * still held unspilled documents. Called on the scan path * so read-only workloads also learn a REINDEX is required. No-op on * a clean index. */ extern void tp_warn_if_pending_docid(Relation index); timescale-pg_textsearch-b63fe58/src/index/registry.c000066400000000000000000000437651525762047700227120ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * registry.c - Global registry mapping database-qualified index OIDs * to shared state * * Uses a dshash (dynamic shared hash table) for O(1) lookups and no * limit on the number of indexes (beyond available memory). */ #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "index/metapage.h" #include "index/registry.h" #include "segment/io.h" #include "segment/merge.h" #include "segment/segment.h" /* Backend-local pointer to the registry in shared memory */ static TpGlobalRegistry *tapir_registry = NULL; /* Backend-local DSA area pointer */ static dsa_area *tapir_dsa = NULL; /* * Wait-event name for each fixed tranche constant, keyed by the * constant itself so the table cannot silently drift if a TP_TRANCHE_* * value is added, removed or renumbered in constants.h. Each name * mirrors its constant so the two read the same way in * pg_stat_activity.wait_event. */ #define TP_TRANCHE_SLOT(name) [TP_TRANCHE_##name - TP_TRANCHE_FIRST] static const char *const tp_tranche_names[TP_TRANCHE_COUNT] = { TP_TRANCHE_SLOT(STRING) = "tapir_string", TP_TRANCHE_SLOT(POSTING) = "tapir_posting", TP_TRANCHE_SLOT(CORPUS) = "tapir_corpus", TP_TRANCHE_SLOT(DOC_LENGTHS) = "tapir_doc_lengths", TP_TRANCHE_SLOT(INDEX_LOCK) = "tapir_index_lock", TP_TRANCHE_SLOT(BUILD_DSA) = "tapir_build_dsa", TP_TRANCHE_SLOT(GLOBAL_DSA) = "tapir_global_dsa", TP_TRANCHE_SLOT(REGISTRY) = "tapir_registry", TP_TRANCHE_SLOT(POSTING_LOCK) = "tapir_posting_lock", TP_TRANCHE_SLOT(CACHE_APPLY_LOCK) = "tapir_cache_apply_lock", TP_TRANCHE_SLOT(CACHE_LOCK) = "tapir_cache_lock", TP_TRANCHE_SLOT(EVICTION_MUTEX) = "tapir_eviction_mutex", TP_TRANCHE_SLOT(MEMTABLE_WRITE_LOCK) = "tapir_memtable_write_lock", }; #undef TP_TRANCHE_SLOT /* * Name of a fixed tranche constant, or a placeholder if constants.h * grew an entry that was never named above (which would otherwise leave * a NULL in the table). */ static const char * tp_tranche_name(int fixed_tranche_id) { const char *name = tp_tranche_names[fixed_tranche_id - TP_TRANCHE_FIRST]; Assert(name != NULL); return name != NULL ? name : "tapir_unnamed"; } #if PG_VERSION_NUM >= 190000 /* * PG19+ removed LWLockRegisterTranche(); tranche names can only be * assigned when the ID is allocated via LWLockNewTrancheId(). Allocate * one contiguous, name-registered block of TP_TRANCHE_COUNT IDs exactly * once (first backend, under AddinShmemInitLock) and record them in * shared memory so every backend resolves the same IDs. */ static void tp_init_tranche_ids(void) { for (int i = 0; i < TP_TRANCHE_COUNT; i++) tapir_registry->tranche_ids[i] = LWLockNewTrancheId( tp_tranche_name(TP_TRANCHE_FIRST + i)); } /* * Resolve a fixed logical tranche constant to its runtime tranche ID. * See constants.h for the PG17/18 (compile-time identity) counterpart. */ int tp_tranche_id(int fixed_tranche_id) { int idx = fixed_tranche_id - TP_TRANCHE_FIRST; Assert(tapir_registry != NULL); Assert(idx >= 0 && idx < TP_TRANCHE_COUNT); return tapir_registry->tranche_ids[idx]; } #else /* * PG17/18: the fixed IDs are used as-is, so the names just have to be * registered in every backend that may report a wait event on them. */ static void tp_register_tranche_names(void) { for (int i = 0; i < TP_TRANCHE_COUNT; i++) LWLockRegisterTranche( TP_TRANCHE_FIRST + i, tp_tranche_name(TP_TRANCHE_FIRST + i)); } #endif /* * Hash function for database-qualified index keys */ static uint32 registry_hash_fn(const void *key, size_t keysize, void *arg) { const TpRegistryKey *registry_key = (const TpRegistryKey *)key; Oid key_parts[2] = { registry_key->database_oid, registry_key->index_oid, }; (void)keysize; (void)arg; return hash_bytes((const unsigned char *)key_parts, sizeof(key_parts)); } /* * Compare function for database-qualified index keys */ static int registry_compare_fn(const void *a, const void *b, size_t keysize, void *arg) { const TpRegistryKey *key_a = (const TpRegistryKey *)a; const TpRegistryKey *key_b = (const TpRegistryKey *)b; (void)keysize; (void)arg; if (key_a->database_oid < key_b->database_oid) return -1; if (key_a->database_oid > key_b->database_oid) return 1; if (key_a->index_oid < key_b->index_oid) return -1; if (key_a->index_oid > key_b->index_oid) return 1; return 0; } /* * Copy function for database-qualified index keys */ static void registry_copy_fn(void *dest, const void *src, size_t keysize, void *arg) { (void)keysize; (void)arg; *(TpRegistryKey *)dest = *(const TpRegistryKey *)src; } /* * Fill in dshash parameters for the registry */ static void get_registry_params(dshash_parameters *params) { params->key_size = sizeof(TpRegistryKey); params->entry_size = sizeof(TpRegistryEntry); params->compare_function = registry_compare_fn; params->hash_function = registry_hash_fn; params->copy_function = registry_copy_fn; params->tranche_id = tp_tranche_id(TP_REGISTRY_HASH_TRANCHE_ID); } /* * Create the registry dshash table */ static dshash_table * registry_create(dsa_area *area) { dshash_parameters params; get_registry_params(¶ms); return dshash_create(area, ¶ms, NULL); } /* * Attach to the registry dshash table */ static dshash_table * registry_attach(dsa_area *area, dshash_table_handle handle) { dshash_parameters params; get_registry_params(¶ms); return dshash_attach(area, ¶ms, handle, NULL); } /* * Request shared memory for the registry * Called during shared_preload_libraries processing */ void tp_registry_init(void) { /* Request shared memory for the registry control structure */ RequestAddinShmemSpace(sizeof(TpGlobalRegistry)); } /* * Create or attach to the registry in shared memory * This is called during backend startup */ void tp_registry_shmem_startup(void) { bool found; LWLockAcquire(AddinShmemInitLock, LW_EXCLUSIVE); tapir_registry = ShmemInitStruct( "Tapir Index Registry", sizeof(TpGlobalRegistry), &found); if (!found) { /* First time initialization */ memset(tapir_registry, 0, sizeof(TpGlobalRegistry)); #if PG_VERSION_NUM >= 190000 /* * Allocate the runtime LWLock tranche-ID block before any lock * or DSA is initialized with a tranche ID (PG19 validates the * tranche ID in LWLockInitialize). */ tp_init_tranche_ids(); #endif /* * Initialize the registry lock using a fixed tranche ID. * Using a fixed ID avoids exhausting tranche IDs when creating many * indexes (e.g., partitioned tables with 500+ partitions). */ LWLockInitialize( &tapir_registry->lock, tp_tranche_id(TP_TRANCHE_REGISTRY)); /* * In-memory memtable cache fields. eviction_mutex * serializes evict_largest invocations across backends; * estimated_total_bytes is the live sum of per-index * cache estimated_bytes. */ LWLockInitialize( &tapir_registry->eviction_mutex, tp_tranche_id(TP_TRANCHE_EVICTION_MUTEX)); pg_atomic_init_u64(&tapir_registry->estimated_total_bytes, 0); /* Initialize handles as invalid - DSA/dshash created on first use */ tapir_registry->dsa_handle = DSA_HANDLE_INVALID; tapir_registry->registry_handle = DSHASH_HANDLE_INVALID; } LWLockRelease(AddinShmemInitLock); #if PG_VERSION_NUM < 190000 /* * Register the lock tranche names in this backend. On PG19+ the * names are supplied to LWLockNewTrancheId() at allocation time * (see tp_init_tranche_ids()) and LWLockRegisterTranche() no longer * exists. */ tp_register_tranche_names(); #endif } /* * Get or create the shared DSA area * * This function is called by any backend that needs access to the DSA. * The first backend creates it, others attach to it. */ dsa_area * tp_registry_get_dsa(void) { /* Quick check if already attached in this backend */ if (tapir_dsa != NULL) return tapir_dsa; /* Registry must be initialized via shared_preload_libraries */ if (!tapir_registry) elog(ERROR, "Tapir registry not initialized. " "Is pg_textsearch in shared_preload_libraries?"); /* Check if DSA exists, create if needed */ LWLockAcquire(&tapir_registry->lock, LW_EXCLUSIVE); if (tapir_registry->dsa_handle == DSA_HANDLE_INVALID) { /* First backend - create the DSA */ MemoryContext oldcontext; dshash_table *registry_hash; oldcontext = MemoryContextSwitchTo(TopMemoryContext); /* * Create DSA using fixed tranche ID. Using a fixed ID avoids * exhausting tranche IDs when creating many indexes (e.g., * partitioned tables with 500+ partitions). */ tapir_dsa = dsa_create(tp_tranche_id(TP_TRANCHE_GLOBAL_DSA)); MemoryContextSwitchTo(oldcontext); if (tapir_dsa == NULL) { LWLockRelease(&tapir_registry->lock); elog(ERROR, "Failed to create DSA area"); } /* Pin the DSA to keep it alive across backends */ dsa_pin(tapir_dsa); /* Pin the mapping for this backend */ dsa_pin_mapping(tapir_dsa); /* Store handle for other backends */ tapir_registry->dsa_handle = dsa_get_handle(tapir_dsa); /* Create the registry dshash */ registry_hash = registry_create(tapir_dsa); tapir_registry->registry_handle = dshash_get_hash_table_handle( registry_hash); dshash_detach(registry_hash); } else { /* DSA exists - attach to it */ MemoryContext oldcontext; oldcontext = MemoryContextSwitchTo(TopMemoryContext); tapir_dsa = dsa_attach(tapir_registry->dsa_handle); MemoryContextSwitchTo(oldcontext); if (tapir_dsa == NULL) { LWLockRelease(&tapir_registry->lock); elog(ERROR, "Failed to attach to pg_textsearch " "shared DSA"); } /* Pin the mapping for this backend */ dsa_pin_mapping(tapir_dsa); } LWLockRelease(&tapir_registry->lock); return tapir_dsa; } /* * Register an index in the global registry * Returns true on success (always succeeds with dshash - no limit) */ bool tp_registry_register( TpRegistryKey key, TpSharedIndexState *shared_state, dsa_pointer shared_dp) { dshash_table *registry_hash; TpRegistryEntry *entry; bool found; (void)shared_state; /* Not stored - we use DSA pointer instead */ /* Ensure DSA and registry are initialized */ tp_registry_get_dsa(); if (!tapir_registry || tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) { elog(ERROR, "Failed to initialize Tapir registry for database %u index %u", key.database_oid, key.index_oid); } registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) elog(ERROR, "Failed to attach to registry hash table"); /* Insert or update the entry */ entry = (TpRegistryEntry *) dshash_find_or_insert(registry_hash, &key, &found); entry->key = key; entry->shared_state_dp = shared_dp; dshash_release_lock(registry_hash, entry); dshash_detach(registry_hash); return true; } /* * Atomic register-if-absent. Returns true if we created a fresh * entry (caller's shared_dp is now in the registry); false if an * entry already existed (in which case *existing_dp is set to its * shared_state_dp and the caller should free its own allocation). */ bool tp_registry_register_if_absent( TpRegistryKey key, dsa_pointer shared_dp, dsa_pointer *existing_dp) { dshash_table *registry_hash; TpRegistryEntry *entry; bool found; tp_registry_get_dsa(); if (!tapir_registry || tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) { elog(ERROR, "Failed to initialize Tapir registry for database %u index %u", key.database_oid, key.index_oid); } registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) elog(ERROR, "Failed to attach to registry hash table"); entry = (TpRegistryEntry *) dshash_find_or_insert(registry_hash, &key, &found); if (found) { if (existing_dp) *existing_dp = entry->shared_state_dp; } else { entry->key = key; entry->shared_state_dp = shared_dp; } dshash_release_lock(registry_hash, entry); dshash_detach(registry_hash); return !found; } /* * Look up an index in the registry * Returns the shared state pointer (as DSA pointer cast) or NULL if not found */ TpSharedIndexState * tp_registry_lookup(TpRegistryKey key) { dshash_table *registry_hash; TpRegistryEntry *entry; dsa_pointer result = InvalidDsaPointer; /* Ensure DSA is initialized */ tp_registry_get_dsa(); if (!tapir_registry || tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) { return NULL; } registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) return NULL; entry = (TpRegistryEntry *)dshash_find(registry_hash, &key, false); if (entry) { result = entry->shared_state_dp; dshash_release_lock(registry_hash, entry); } dshash_detach(registry_hash); /* Return DSA pointer cast as TpSharedIndexState pointer * The caller will convert it back */ return (TpSharedIndexState *)(uintptr_t)result; } /* * Look up an index's DSA pointer in the registry * Returns the DSA pointer if found, InvalidDsaPointer otherwise */ dsa_pointer tp_registry_lookup_dsa(TpRegistryKey key) { dshash_table *registry_hash; TpRegistryEntry *entry; dsa_pointer result = InvalidDsaPointer; /* Ensure DSA is initialized */ tp_registry_get_dsa(); if (!tapir_registry || tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) { return InvalidDsaPointer; } registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) return InvalidDsaPointer; entry = (TpRegistryEntry *)dshash_find(registry_hash, &key, false); if (entry) { result = entry->shared_state_dp; dshash_release_lock(registry_hash, entry); } dshash_detach(registry_hash); return result; } /* * Check if an index is registered * Returns true if the index is in the registry, false otherwise */ bool tp_registry_is_registered(TpRegistryKey key) { dshash_table *registry_hash; TpRegistryEntry *entry; bool result = false; /* * If registry is not initialized, no indexes can be registered. * This function is called from object access hook which may fire * before any index has been created. */ if (!tapir_registry) return false; if (tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) return false; /* Ensure DSA is attached */ tp_registry_get_dsa(); if (!tapir_dsa) return false; registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) return false; entry = (TpRegistryEntry *)dshash_find(registry_hash, &key, false); if (entry) { result = true; dshash_release_lock(registry_hash, entry); } dshash_detach(registry_hash); return result; } /* * Unregister an index from the registry * Called when an index is dropped */ void tp_registry_unregister(TpRegistryKey key) { dshash_table *registry_hash; bool deleted; if (!tapir_registry || tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) { return; } /* Ensure DSA is attached */ tp_registry_get_dsa(); if (!tapir_dsa) return; registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) return; deleted = dshash_delete_key(registry_hash, &key); (void)deleted; /* Ignore if not found */ dshash_detach(registry_hash); } typedef struct DatabaseKeyCollector { Oid database_oid; TpRegistryKey *keys; Size count; Size capacity; } DatabaseKeyCollector; static bool collect_database_key_cb(TpRegistryKey key, dsa_pointer shared_dp, void *ctx) { DatabaseKeyCollector *collector = (DatabaseKeyCollector *)ctx; (void)shared_dp; if (key.database_oid != collector->database_oid) return false; if (collector->count == collector->capacity) { Size new_capacity; if (collector->capacity == 0) new_capacity = 16; else { if (collector->capacity > MaxAllocSize / (2 * sizeof(TpRegistryKey))) elog(ERROR, "too many pg_textsearch registry entries"); new_capacity = collector->capacity * 2; } if (new_capacity > MaxAllocSize / sizeof(TpRegistryKey)) elog(ERROR, "too many pg_textsearch registry entries"); if (collector->keys == NULL) collector->keys = palloc(new_capacity * sizeof(TpRegistryKey)); else collector->keys = repalloc( collector->keys, new_capacity * sizeof(TpRegistryKey)); collector->capacity = new_capacity; } collector->keys[collector->count++] = key; return false; } Size tp_registry_collect_database_keys(Oid database_oid, TpRegistryKey **keys) { DatabaseKeyCollector collector = { .database_oid = database_oid, .keys = NULL, .count = 0, .capacity = 0, }; Assert(keys != NULL); *keys = NULL; if (!OidIsValid(database_oid)) return 0; tp_registry_walk(collect_database_key_cb, &collector); *keys = collector.keys; return collector.count; } /* * Cache memory accounting accessors. Both return pointers into * the shmem registry control struct, valid for the lifetime of * the postmaster. */ pg_atomic_uint64 * tp_registry_estimated_total_bytes(void) { Assert(tapir_registry != NULL); return &tapir_registry->estimated_total_bytes; } LWLock * tp_registry_eviction_mutex(void) { Assert(tapir_registry != NULL); return &tapir_registry->eviction_mutex; } void tp_registry_walk(TpRegistryWalkCb cb, void *ctx) { dshash_table *registry_hash; dshash_seq_status *status; TpRegistryEntry *entry; Assert(cb != NULL); tp_registry_get_dsa(); if (!tapir_registry || tapir_registry->registry_handle == DSHASH_HANDLE_INVALID) return; status = palloc(sizeof(*status)); registry_hash = registry_attach(tapir_dsa, tapir_registry->registry_handle); if (!registry_hash) { pfree(status); return; } dshash_seq_init(status, registry_hash, false); PG_TRY(); { while ((entry = (TpRegistryEntry *)dshash_seq_next(status)) != NULL) { if (cb(entry->key, entry->shared_state_dp, ctx)) break; } } PG_FINALLY(); { dshash_seq_term(status); dshash_detach(registry_hash); pfree(status); } PG_END_TRY(); } timescale-pg_textsearch-b63fe58/src/index/registry.h000066400000000000000000000127521525762047700227070ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * registry.h - Global registry for shared index states * * Uses a dshash (dynamic shared hash table) to map database-qualified * index OIDs to their shared state DSA pointers. This allows unlimited * indexes (bounded only by available memory) with O(1) lookup performance. */ #pragma once #include #include #include #include #include #include "index/state.h" /* * LWLock tranche ID for the registry dshash is defined in constants.h * as TP_TRANCHE_REGISTRY. We use fixed tranche IDs to avoid exhausting * tranche IDs when creating many indexes (e.g., partitioned tables with * 500+ partitions). */ #include "constants.h" #define TP_REGISTRY_HASH_TRANCHE_ID TP_TRANCHE_REGISTRY /* * Cluster-wide registry key. Relation OIDs are unique only within a * database, so every registry operation must carry both identifiers. */ typedef struct TpRegistryKey { Oid database_oid; Oid index_oid; } TpRegistryKey; static inline TpRegistryKey tp_registry_key(Oid database_oid, Oid index_oid) { TpRegistryKey key = { .database_oid = database_oid, .index_oid = index_oid, }; return key; } /* * Registry entry stored in dshash * The key is the first field, value is shared_state_dp. */ typedef struct TpRegistryEntry { TpRegistryKey key; /* Hash key - must be first */ dsa_pointer shared_state_dp; /* DSA pointer to TpSharedIndexState */ } TpRegistryEntry; /* * Global registry control structure stored in shared memory. * The actual entries are in a dshash stored in DSA. * * The eviction_mutex and estimated_total_bytes fields are part * of the in-memory memtable cache memory-cap protocol. * estimated_total_bytes tracks the sum of TpMemtable.estimated_bytes * across all registered caches in this shmem segment; * eviction_mutex serializes tp_cache_evict_largest invocations * (and DROP-time shared-state teardown) so a victim's * TpSharedIndexState cannot be dsa_freed while another backend * is inspecting it. */ typedef struct TpGlobalRegistry { LWLock lock; /* Protects initialization */ dsa_handle dsa_handle; /* Handle for shared DSA area */ dshash_table_handle registry_handle; /* Handle for the registry dshash */ LWLock eviction_mutex; /* Serializes cache eviction */ pg_atomic_uint64 estimated_total_bytes; /* Σ per-index est bytes */ /* * PG19+: runtime LWLock tranche IDs, one per fixed TP_TRANCHE_* * constant, allocated once via LWLockNewTrancheId() at shmem startup * (LWLockRegisterTranche() no longer exists). Indexed by * (fixed_id - TP_TRANCHE_FIRST); see tp_tranche_id(). Unused on * PG17/18, where fixed IDs are used directly. */ int tranche_ids[TP_TRANCHE_COUNT]; } TpGlobalRegistry; /* Registry management functions */ extern void tp_registry_init(void); extern void tp_registry_shmem_startup(void); /* DSA management */ extern dsa_area *tp_registry_get_dsa(void); /* Registry operations */ extern bool tp_registry_register( TpRegistryKey key, TpSharedIndexState *shared_state, dsa_pointer shared_dp); /* * Register `shared_dp` for `key` only if no entry exists. * Returns true if a fresh entry was created, false if an existing * entry was found (in which case `*existing_dp` is set to the * existing shared_state_dp and the caller is expected to free its * own allocation and use the existing entry). */ extern bool tp_registry_register_if_absent( TpRegistryKey key, dsa_pointer shared_dp, dsa_pointer *existing_dp); extern TpSharedIndexState *tp_registry_lookup(TpRegistryKey key); extern dsa_pointer tp_registry_lookup_dsa(TpRegistryKey key); extern bool tp_registry_is_registered(TpRegistryKey key); extern void tp_registry_unregister(TpRegistryKey key); /* * Collect every key qualified by `database_oid` in one registry walk. * The returned array is allocated in CurrentMemoryContext and must be * pfree'd by the caller. No entries are removed while the iterator holds * a dshash partition lock. */ extern Size tp_registry_collect_database_keys(Oid database_oid, TpRegistryKey **keys); /* * Cache memory accounting helpers. Increment/decrement the * registry's estimated_total_bytes alongside per-index updates. * Internally these route through TpGlobalRegistry; cache code that * mutates TpMemtable.estimated_bytes MUST go through these so the * global counter stays in lockstep with the per-index counter. * * tp_registry_eviction_mutex returns the address of the global * eviction LWLock; tp_cache_evict_largest is the only legitimate * caller, and tp_cleanup_index_shared_memory takes it EXCL before * dsa_freeing a victim shared state. */ extern pg_atomic_uint64 *tp_registry_estimated_total_bytes(void); extern LWLock *tp_registry_eviction_mutex(void); /* * Callback-based registry iterator. For each registered index, * invokes `cb(key, shared_state_dp, ctx)`. A true callback return * stops the void iterator early. Bucket locks are held while the * callback runs; the callback MUST NOT acquire arbitrary LWLocks * (the dshash partition lock is held). Reading fields via * dsa_get_address from the supplied DSA pointer is safe. * * Used by tp_cache_evict_largest to scan for the largest-bytes * cache. Reads only — no entry mutation from within the * callback. */ typedef bool (*TpRegistryWalkCb)( TpRegistryKey key, dsa_pointer shared_dp, void *ctx); extern void tp_registry_walk(TpRegistryWalkCb cb, void *ctx); timescale-pg_textsearch-b63fe58/src/index/resolve.c000066400000000000000000000326641525762047700225150ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * resolve.c - Index name resolution and query validation */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "index/resolve.h" /* * Maximum depth for walking inheritance hierarchies. * Prevents infinite loops in case of catalog corruption. */ #define MAX_INHERITANCE_DEPTH 32 /* * Get the appropriate index name for the given index relation. * Returns a qualified name (schema.index) if the index is not visible * in the search path, otherwise returns just the index name. */ char * tp_get_qualified_index_name(Relation indexRelation) { Oid index_namespace = RelationGetNamespace(indexRelation); /* * If the index is not visible in the search path, use a qualified name */ if (!RelationIsVisible(RelationGetRelid(indexRelation))) { char *namespace_name = get_namespace_name(index_namespace); char *relation_name = RelationGetRelationName(indexRelation); return quote_qualified_identifier(namespace_name, relation_name); } else { return RelationGetRelationName(indexRelation); } } /* * Resolve index name to OID with schema support. * Returns the OID of the index, or InvalidOid if not found. * Handles both schema-qualified names (schema.index) and unqualified names. */ Oid tp_resolve_index_name_shared(const char *index_name) { Oid index_oid; if (strchr(index_name, '.') != NULL) { /* Contains a dot - try to parse as schema.relation */ List *namelist = stringToQualifiedNameList(index_name, NULL); if (list_length(namelist) == 2) { char *schemaname = strVal(linitial(namelist)); char *relname = strVal(lsecond(namelist)); /* Validate that schema name is not empty */ if (schemaname == NULL || strlen(schemaname) == 0) { index_oid = InvalidOid; } else { Oid namespace_oid = get_namespace_oid(schemaname, true); if (OidIsValid(namespace_oid)) index_oid = get_relname_relid(relname, namespace_oid); else index_oid = InvalidOid; } } else { index_oid = InvalidOid; } list_free_deep(namelist); } else { /* No schema specified - use search path */ index_oid = RelnameGetRelid(index_name); } return index_oid; } /* * Enforce that the current user may read the contents of the given BM25 * index before any index-derived data is returned. * * Standalone scoring (text <@> bm25query) and related helpers open a * caller-named index relation directly via index_open(). That path * bypasses the executor's range-table permission checks, so without this * guard a role could obtain BM25 scores - and therefore infer term * membership and corpus statistics - for an index whose underlying table * or column it is not permitted to read. * * The check mirrors PostgreSQL's index-scan authorization: the invoking * role must hold USAGE on the schema containing the indexed table, and * either SELECT on the table itself or SELECT on every heap column the * index reads (plain key columns plus any columns referenced by an * expression index or a partial-index predicate). */ void tp_check_index_read_privilege(Oid index_oid) { HeapTuple indtup; Form_pg_index indform; Oid heap_oid; Oid heap_nsp; Oid userid = GetUserId(); AclResult aclresult; Bitmapset *attnos = NULL; bool wholerow = false; int i; int nkeycols; Datum datum; bool isnull; indtup = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (!HeapTupleIsValid(indtup)) return; /* not an index; nothing to check */ indform = (Form_pg_index)GETSTRUCT(indtup); heap_oid = indform->indrelid; /* * Reading through the index requires USAGE on the schema containing the * indexed table, exactly as a direct index scan would. Enforce it before * any table- or column-level decision so a withheld schema cannot be * probed via scoring. */ heap_nsp = get_rel_namespace(heap_oid); aclresult = object_aclcheck(NamespaceRelationId, heap_nsp, userid, ACL_USAGE); if (aclresult != ACLCHECK_OK) { ReleaseSysCache(indtup); aclcheck_error(aclresult, OBJECT_SCHEMA, get_namespace_name(heap_nsp)); } /* Table-level SELECT is sufficient. */ aclresult = pg_class_aclcheck(heap_oid, userid, ACL_SELECT); if (aclresult == ACLCHECK_OK) { ReleaseSysCache(indtup); return; } /* * No table-level SELECT: fall back to per-column checks. Collect every * heap attribute the index reads - plain key columns, plus any columns * referenced by an expression index or a partial-index predicate - * offsetting by FirstLowInvalidHeapAttributeNumber to match * pull_varattnos(). */ nkeycols = indform->indnkeyatts; for (i = 0; i < nkeycols; i++) { AttrNumber attno = indform->indkey.values[i]; if (attno != 0) attnos = bms_add_member( attnos, attno - FirstLowInvalidHeapAttributeNumber); } /* Expression-index columns, if any. */ datum = SysCacheGetAttr( INDEXRELID, indtup, Anum_pg_index_indexprs, &isnull); if (!isnull) { char *str = TextDatumGetCString(datum); Node *node = stringToNode(str); pull_varattnos(node, 1, &attnos); pfree(str); } /* Partial-index predicate columns, if any. */ datum = SysCacheGetAttr( INDEXRELID, indtup, Anum_pg_index_indpred, &isnull); if (!isnull) { char *str = TextDatumGetCString(datum); Node *node = stringToNode(str); pull_varattnos(node, 1, &attnos); pfree(str); } ReleaseSysCache(indtup); /* * If we could not pin down any concrete column, fail closed using the * table-level result. */ if (bms_is_empty(attnos)) aclcheck_error( aclresult, get_relkind_objtype(get_rel_relkind(heap_oid)), get_rel_name(heap_oid)); i = -1; while ((i = bms_next_member(attnos, i)) >= 0) { AttrNumber attno = i + FirstLowInvalidHeapAttributeNumber; /* * A whole-row (attno == 0) or system-column (attno < 0) reference * cannot be resolved to a single named column; defer it and require * SELECT on every column below, matching core's whole-row handling * in ExecCheckOneRelPerms(). */ if (attno <= InvalidAttrNumber) { wholerow = true; continue; } aclresult = pg_attribute_aclcheck(heap_oid, attno, userid, ACL_SELECT); if (aclresult != ACLCHECK_OK) aclcheck_error_col( aclresult, get_relkind_objtype(get_rel_relkind(heap_oid)), get_rel_name(heap_oid), get_attname(heap_oid, attno, false)); } if (wholerow) { aclresult = pg_attribute_aclcheck_all( heap_oid, userid, ACL_SELECT, ACLMASK_ALL); if (aclresult != ACLCHECK_OK) aclcheck_error( aclresult, get_relkind_objtype(get_rel_relkind(heap_oid)), get_rel_name(heap_oid)); } bms_free(attnos); } /* * Check if child_oid inherits from ancestor_oid via pg_inherits. * Walks up the inheritance chain to handle multi-level partitions. */ static bool oid_inherits_from(Oid child_oid, Oid ancestor_oid) { Relation inhrel; Oid current_oid = child_oid; bool found = false; int depth = MAX_INHERITANCE_DEPTH; if (child_oid == ancestor_oid) return true; inhrel = table_open(InheritsRelationId, AccessShareLock); while (depth-- > 0) { SysScanDesc scan; ScanKeyData key; HeapTuple tuple; Oid parent_oid = InvalidOid; ScanKeyInit( &key, Anum_pg_inherits_inhrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(current_oid)); scan = systable_beginscan( inhrel, InheritsRelidSeqnoIndexId, true, NULL, 1, &key); tuple = systable_getnext(scan); if (HeapTupleIsValid(tuple)) { Form_pg_inherits inhform = (Form_pg_inherits)GETSTRUCT(tuple); parent_oid = inhform->inhparent; } systable_endscan(scan); if (!OidIsValid(parent_oid)) break; /* Reached top of hierarchy */ if (parent_oid == ancestor_oid) { found = true; break; } current_oid = parent_oid; } table_close(inhrel, AccessShareLock); return found; } /* * Check if two BM25 indexes match by attribute (for hypertables). * * This handles cases where chunk indexes don't have pg_inherits relationships * to the parent index (e.g., TimescaleDB hypertables). Instead we check: * 1. Both indexes use the BM25 access method * 2. The scan index's table inherits from the query index's table * 3. Both indexes are on the same column attribute number */ static bool indexes_match_by_attribute(Oid scan_index_oid, Oid query_index_oid) { HeapTuple scan_idx_tuple; HeapTuple query_idx_tuple; HeapTuple scan_class_tuple; HeapTuple query_class_tuple; Oid scan_heap_oid; Oid query_heap_oid; Oid bm25_am_oid; bool result = false; AttrNumber scan_attnum; AttrNumber query_attnum; HeapTuple am_tuple; /* Look up bm25 access method OID */ am_tuple = SearchSysCache1(AMNAME, CStringGetDatum("bm25")); if (!HeapTupleIsValid(am_tuple)) return false; bm25_am_oid = ((Form_pg_am)GETSTRUCT(am_tuple))->oid; ReleaseSysCache(am_tuple); /* Get pg_index entries for both indexes */ scan_idx_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(scan_index_oid)); if (!HeapTupleIsValid(scan_idx_tuple)) return false; query_idx_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(query_index_oid)); if (!HeapTupleIsValid(query_idx_tuple)) { ReleaseSysCache(scan_idx_tuple); return false; } /* Get heap OIDs from pg_index */ scan_heap_oid = ((Form_pg_index)GETSTRUCT(scan_idx_tuple))->indrelid; query_heap_oid = ((Form_pg_index)GETSTRUCT(query_idx_tuple))->indrelid; /* Get attribute numbers (assume single-column BM25 indexes) */ scan_attnum = ((Form_pg_index)GETSTRUCT(scan_idx_tuple))->indkey.values[0]; query_attnum = ((Form_pg_index)GETSTRUCT(query_idx_tuple))->indkey.values[0]; /* Check if both indexes use BM25 access method */ scan_class_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(scan_index_oid)); query_class_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(query_index_oid)); if (HeapTupleIsValid(scan_class_tuple) && HeapTupleIsValid(query_class_tuple)) { Oid scan_am = ((Form_pg_class)GETSTRUCT(scan_class_tuple))->relam; Oid query_am = ((Form_pg_class)GETSTRUCT(query_class_tuple))->relam; if (scan_am == bm25_am_oid && query_am == bm25_am_oid && oid_inherits_from(scan_heap_oid, query_heap_oid)) { if (scan_attnum != 0 && query_attnum != 0) { /* * Plain column: compare by column name rather * than raw attnum. Dropped columns can cause * parent and child tables to have different * physical attnums for the same logical column * (e.g., TimescaleDB hypertables or inheritance * after ALTER TABLE DROP COLUMN). */ char *scan_colname = get_attname(scan_heap_oid, scan_attnum, true); char *query_colname = get_attname(query_heap_oid, query_attnum, true); if (scan_colname && query_colname && strcmp(scan_colname, query_colname) == 0) { result = true; } } else if (scan_attnum == 0 && query_attnum == 0) { /* * Expression indexes: compare stored * expression trees from pg_index. */ Datum scan_expr_d, query_expr_d; bool scan_null, query_null; scan_expr_d = SysCacheGetAttr( INDEXRELID, scan_idx_tuple, Anum_pg_index_indexprs, &scan_null); query_expr_d = SysCacheGetAttr( INDEXRELID, query_idx_tuple, Anum_pg_index_indexprs, &query_null); if (!scan_null && !query_null) { char *scan_str = TextDatumGetCString(scan_expr_d); char *query_str = TextDatumGetCString(query_expr_d); List *scan_exprs = (List *)stringToNode(scan_str); List *query_exprs = (List *)stringToNode(query_str); pfree(scan_str); pfree(query_str); if (equal(scan_exprs, query_exprs)) result = true; } } } } /* Cleanup */ if (HeapTupleIsValid(scan_class_tuple)) ReleaseSysCache(scan_class_tuple); if (HeapTupleIsValid(query_class_tuple)) ReleaseSysCache(query_class_tuple); ReleaseSysCache(scan_idx_tuple); ReleaseSysCache(query_idx_tuple); return result; } /* * Validate that the query index OID matches the scan index. * Allows partitioned index queries to run on partition indexes. */ void tp_validate_query_index(Oid query_index_oid, Relation indexRelation) { Oid scan_index_oid = RelationGetRelid(indexRelation); /* Direct match - OK */ if (query_index_oid == scan_index_oid) return; /* * Check if query references a partitioned index and scan is on a * partition index (child of the partitioned index). */ if (get_rel_relkind(query_index_oid) == RELKIND_PARTITIONED_INDEX && oid_inherits_from(scan_index_oid, query_index_oid)) return; /* * Attribute-based matching for TimescaleDB hypertables and other cases * where chunk indexes don't have pg_inherits relationships to the parent. */ if (indexes_match_by_attribute(scan_index_oid, query_index_oid)) return; ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("tpquery index mismatch"), errhint("Query specifies index OID %u but scan is on " "index \"%s\" (OID %u)", query_index_oid, RelationGetRelationName(indexRelation), scan_index_oid))); } timescale-pg_textsearch-b63fe58/src/index/resolve.h000066400000000000000000000010051525762047700225030ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * resolve.h - Index name resolution and query validation */ #pragma once #include #include extern char *tp_get_qualified_index_name(Relation indexRelation); extern Oid tp_resolve_index_name_shared(const char *index_name); extern void tp_validate_query_index(Oid query_index_oid, Relation indexRelation); extern void tp_check_index_read_privilege(Oid index_oid); timescale-pg_textsearch-b63fe58/src/index/source.c000066400000000000000000000016561525762047700223330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * source.c - Common helpers for data source interface */ #include #include #include "index/source.h" /* * Allocate posting data with given capacity. * Arrays are allocated but not initialized. */ TpPostingData * tp_alloc_posting_data(int32 capacity) { TpPostingData *data; data = (TpPostingData *)palloc(sizeof(TpPostingData)); data->ctids = (ItemPointerData *)palloc( capacity * sizeof(ItemPointerData)); data->frequencies = (int32 *)palloc(capacity * sizeof(int32)); data->count = 0; data->doc_freq = 0; return data; } /* * Free posting data allocated by tp_alloc_posting_data(). */ void tp_free_posting_data(TpPostingData *data) { if (data) { if (data->ctids) pfree(data->ctids); if (data->frequencies) pfree(data->frequencies); pfree(data); } } timescale-pg_textsearch-b63fe58/src/index/source.h000066400000000000000000000060741525762047700223370ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * source.h - Abstract data source interface for posting lists * * Defines a columnar interface that both memtable and segment implement. * This allows scoring code to be agnostic to the underlying storage. */ #pragma once #include #include /* * Columnar posting data for a term. * Arrays are parallel - ctids[i] corresponds to frequencies[i]. */ typedef struct TpPostingData { ItemPointerData *ctids; /* Array of document CTIDs */ int32 *frequencies; /* Array of term frequencies */ int32 count; /* Number of entries */ int32 doc_freq; /* Document frequency (for IDF) */ } TpPostingData; /* * Abstract data source interface. * Both memtable and segment implement this interface. */ typedef struct TpDataSource TpDataSource; typedef void (*TpDocumentCallback)(ItemPointer ctid, void *arg); typedef struct TpDataSourceOps { /* * Get posting data for a term. * Returns NULL if term not found. * Caller must free with free_postings(). */ TpPostingData *(*get_postings)(TpDataSource *source, const char *term); /* * Free posting data returned by get_postings(). */ void (*free_postings)(TpDataSource *source, TpPostingData *data); /* * Get document length for a CTID. * Returns -1 if not found. */ int32 (*get_doc_length)(TpDataSource *source, ItemPointer ctid); /* * Get the per-term document frequency without materializing * the posting list. Returns 0 if the term is unknown. * Implementations should be O(1) lookup on an already-built * index/htab — this is on the BM25 IDF hot path and must not * allocate or copy postings. */ uint32 (*get_doc_freq)(TpDataSource *source, const char *term); /* * Invoke callback once for every document known to the source. */ void (*foreach_document)( TpDataSource *source, TpDocumentCallback callback, void *arg); /* * Close and free the data source. */ void (*close)(TpDataSource *source); } TpDataSourceOps; struct TpDataSource { const TpDataSourceOps *ops; int32 total_docs; /* Corpus document count */ int64 total_len; /* Corpus total length */ }; /* * Convenience macros for calling interface methods */ #define tp_source_get_postings(src, term) \ ((src)->ops->get_postings((src), (term))) #define tp_source_free_postings(src, data) \ ((src)->ops->free_postings((src), (data))) #define tp_source_get_doc_length(src, ctid) \ ((src)->ops->get_doc_length((src), (ctid))) #define tp_source_get_doc_freq(src, term) \ ((src)->ops->get_doc_freq((src), (term))) #define tp_source_foreach_document(src, callback, arg) \ ((src)->ops->foreach_document((src), (callback), (arg))) #define tp_source_close(src) ((src)->ops->close((src))) /* * Helper to allocate posting data with given capacity. * Allocates in CurrentMemoryContext. */ extern TpPostingData *tp_alloc_posting_data(int32 capacity); /* * Helper to free posting data allocated by tp_alloc_posting_data(). */ extern void tp_free_posting_data(TpPostingData *data); timescale-pg_textsearch-b63fe58/src/index/state.c000066400000000000000000001204041525762047700221440ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * state.c - Index state management * * Manages TpLocalIndexState and TpSharedIndexState structures for * coordinating index state across backends. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "constants.h" #include "debug/injection.h" #include "index/metapage.h" #include "index/registry.h" #include "index/source.h" #include "index/state.h" #include "memtable/cache.h" #include "memtable/chain_source.h" #include "memtable/log.h" #include "segment/io.h" #include "segment/merge.h" #include "segment/segment.h" /* Cache of local index states */ static HTAB *local_state_cache = NULL; typedef struct LocalStateCacheEntry { Oid index_oid; /* Hash key */ TpLocalIndexState *local_state; /* Cached local state */ } LocalStateCacheEntry; /* Registered with before_shmem_exit at most once per backend */ static bool shutdown_hook_registered = false; /* * Spill one cache entry, catching and swallowing any error so the * outer loop can continue with the remaining entries. Split out of * tp_shutdown_spill_callback so its PG_TRY doesn't nest in the same * lexical scope (which would shadow the outer PG_TRY's locals and * trip -Werror=shadow=compatible-local). */ static void tp_shutdown_spill_one(LocalStateCacheEntry *entry) { Relation index_rel = NULL; if (entry->local_state == NULL) return; if (!ConditionalLockRelationOid(entry->index_oid, RowExclusiveLock)) return; PG_TRY(); { index_rel = try_index_open(entry->index_oid, NoLock); if (index_rel != NULL) { tp_spill_memtable_without_compaction_if_needed( index_rel, entry->local_state, TP_MIN_SPILL_PAGES); index_close(index_rel, NoLock); index_rel = NULL; } } PG_CATCH(); { /* Don't leak the per-index LWLock to racing shutdown hooks */ if (entry->local_state->lock_held) tp_release_index_lock(entry->local_state); FlushErrorState(); if (index_rel != NULL) index_close(index_rel, NoLock); } PG_END_TRY(); UnlockRelationOid(entry->index_oid, RowExclusiveLock); } /* * before_shmem_exit hook: spill the memtable of every index this * backend has touched, so the on-disk memtable chain doesn't * accumulate runt pages that would have to be merge-compacted * later. Skipped on clean client exit (code == 0); fires on any * FATAL (cluster shutdown, pg_terminate_backend, etc.). */ static void tp_shutdown_spill_callback(int code, Datum arg pg_attribute_unused()) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; bool started_txn = false; if (local_state_cache == NULL || code == 0) return; /* catalog access needs a non-aborted transaction */ if (IsAbortedTransactionBlockState()) return; PG_TRY(); { if (!IsTransactionState()) { StartTransactionCommand(); started_txn = true; } hash_seq_init(&status, local_state_cache); while ((entry = (LocalStateCacheEntry *)hash_seq_search(&status)) != NULL) tp_shutdown_spill_one(entry); if (started_txn) CommitTransactionCommand(); } PG_CATCH(); { /* proc_exit still has to complete; swallow and move on */ if (IsTransactionState()) AbortCurrentTransaction(); FlushErrorState(); } PG_END_TRY(); } /* * Initialize the local state cache */ static void init_local_state_cache(void) { HASHCTL ctl; if (local_state_cache != NULL) return; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(Oid); ctl.entrysize = sizeof(LocalStateCacheEntry); ctl.hcxt = TopMemoryContext; local_state_cache = hash_create( "Tapir Local State Cache", 8, /* initial size */ &ctl, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); if (!shutdown_hook_registered) { before_shmem_exit(tp_shutdown_spill_callback, (Datum)0); shutdown_hook_registered = true; } } static void init_memtable(TpMemtable *memtable) { memtable->string_hash_handle = DSHASH_HANDLE_INVALID; memtable->doc_lengths_handle = DSHASH_HANDLE_INVALID; LWLockInitialize( &memtable->apply_lock, tp_tranche_id(TP_TRANCHE_CACHE_APPLY_LOCK)); LWLockInitialize(&memtable->lock, tp_tranche_id(TP_TRANCHE_CACHE_LOCK)); memtable->cursor_gen_spill_count = 0; memtable->cursor_next_blkno = InvalidBlockNumber; memtable->cursor_next_off = 0; memset(&memtable->cursor_locator, 0, sizeof(memtable->cursor_locator)); memtable->cursor_locator_valid = false; pg_atomic_init_u64(&memtable->cursor_seq, 0); pg_atomic_init_u64(&memtable->estimated_bytes, 0); } static dsa_pointer allocate_memtable(dsa_area *dsa) { dsa_pointer memtable_dp = dsa_allocate(dsa, sizeof(TpMemtable)); TpMemtable *memtable; if (!DsaPointerIsValid(memtable_dp)) elog(ERROR, "Failed to allocate memtable in DSA"); memtable = (TpMemtable *)dsa_get_address(dsa, memtable_dp); init_memtable(memtable); return memtable_dp; } static void init_local_state( TpLocalIndexState *local_state, TpSharedIndexState *shared_state, dsa_pointer shared_dp, dsa_area *dsa) { local_state->shared = shared_state; local_state->shared_dp = shared_dp; local_state->dsa = dsa; local_state->build_created_shared_state = false; local_state->lock_held = false; local_state->lock_mode = 0; local_state->terms_added_this_xact = 0; local_state->docs_since_global_check = 0; local_state->created_in_subxact = InvalidSubTransactionId; } /* * Get or create a local index state for the given index OID * * This function: * 1. Checks if we already have a cached local state * 2. If not, looks up the shared state in the registry * 3. Attaches to the shared DSA if needed * 4. Creates and caches the local state */ TpLocalIndexState * tp_get_local_index_state(Oid index_oid) { LocalStateCacheEntry *entry; TpLocalIndexState *local_state; TpSharedIndexState *shared_state; bool found; dsa_area *dsa; /* Initialize cache if needed */ init_local_state_cache(); /* Check cache first */ entry = (LocalStateCacheEntry *) hash_search(local_state_cache, &index_oid, HASH_FIND, NULL); if (entry != NULL) return entry->local_state; /* Look up shared state in registry */ shared_state = tp_registry_lookup( tp_registry_key(MyDatabaseId, index_oid)); if (shared_state == NULL) { /* * No registry entry found. This could mean: * 1. The index was just dropped * 2. We're in crash recovery after a restart * 3. The index doesn't exist * 4. The index is being built right now * 5. Backend startup when other backends created the index * * Check if the index exists and needs to be rebuilt. */ Relation index_rel; bool index_exists = false; index_rel = try_index_open(index_oid, AccessShareLock); if (index_rel != NULL) { index_exists = true; index_close(index_rel, AccessShareLock); } if (index_exists) { /* * Index exists on disk but not in the registry. This can * occur after: * 1. PostgreSQL crash/restart (shared memory was cleared) * 2. Extension reload after DROP/CREATE EXTENSION * 3. Backend startup when other backends created the index * * We rebuild the index state from the on-disk metapage to * recover the memtable and posting lists. */ local_state = tp_rebuild_index_from_disk(index_oid); if (local_state != NULL) return local_state; /* Recovery failed - index might be corrupted or stale */ } /* Index not found in registry and doesn't exist on disk */ return NULL; } /* * If we reach here, shared_state is set (non-NULL) and is actually a DSA * pointer that needs to be converted to an address. */ { /* shared_state is actually a DSA pointer - need to attach to DSA and * convert to address */ dsa_pointer shared_dp = (dsa_pointer)(uintptr_t)shared_state; /* Get the shared DSA area */ dsa = tp_registry_get_dsa(); /* Convert DSA pointer to memory address in this backend */ shared_state = (TpSharedIndexState *)dsa_get_address(dsa, shared_dp); /* Allocate local state */ local_state = (TpLocalIndexState *)MemoryContextAlloc( TopMemoryContext, sizeof(TpLocalIndexState)); init_local_state(local_state, shared_state, shared_dp, dsa); /* Cache the local state */ entry = (LocalStateCacheEntry *) hash_search(local_state_cache, &index_oid, HASH_ENTER, &found); entry->local_state = local_state; return local_state; } } /* * Register a fresh runtime state or attach to the existing state for the * same database-qualified index identity. REINDEX must never replace the * shared allocation because wrappers retained by other backends keep its * address. The calling backend's wrapper is replaced below. */ static TpLocalIndexState * create_or_attach_index_state( Oid index_oid, Oid heap_oid, SubTransactionId index_create_subid) { TpRegistryKey key = tp_registry_key(MyDatabaseId, index_oid); TpSharedIndexState *shared_state; TpLocalIndexState *local_state; TpLocalIndexState *old_local_state = NULL; dsa_area *dsa; dsa_pointer shared_dp; dsa_pointer memtable_dp; dsa_pointer existing_dp = InvalidDsaPointer; LocalStateCacheEntry *entry; bool found; dsa = tp_registry_get_dsa(); shared_dp = dsa_allocate(dsa, sizeof(TpSharedIndexState)); if (!DsaPointerIsValid(shared_dp)) elog(ERROR, "Failed to allocate DSA memory for shared state (index OID: %u, " "size: %zu)", index_oid, sizeof(TpSharedIndexState)); shared_state = (TpSharedIndexState *)dsa_get_address(dsa, shared_dp); shared_state->index_oid = index_oid; shared_state->heap_oid = heap_oid; pg_atomic_init_u64(&shared_state->estimated_bytes, 0); pg_atomic_init_u32(&shared_state->chain_page_count, 0); pg_atomic_init_u32(&shared_state->chain_page_count_needs_reseed, 0); pg_atomic_init_u32(&shared_state->chain_page_count_spc_oid, InvalidOid); pg_atomic_init_u32(&shared_state->chain_page_count_db_oid, InvalidOid); pg_atomic_init_u32( &shared_state->chain_page_count_rel_number, InvalidRelFileNumber); LWLockInitialize( &shared_state->lock, tp_tranche_id(TP_TRANCHE_INDEX_LOCK)); pg_atomic_init_u32(&shared_state->exclusive_waiters, 0); ConditionVariableInit(&shared_state->exclusive_waiters_cv); LWLockInitialize( &shared_state->memtable_write_lock, tp_tranche_id(TP_TRANCHE_MEMTABLE_WRITE_LOCK)); pg_atomic_init_u32(&shared_state->memtable_write_exclusive_waiters, 0); ConditionVariableInit(&shared_state->memtable_write_exclusive_waiters_cv); pg_atomic_init_u64(&shared_state->spill_generation, 0); memtable_dp = allocate_memtable(dsa); shared_state->memtable_dp = memtable_dp; if (!tp_registry_register_if_absent(key, shared_dp, &existing_dp)) { dsa_free(dsa, memtable_dp); dsa_free(dsa, shared_dp); shared_dp = existing_dp; shared_state = (TpSharedIndexState *)dsa_get_address(dsa, existing_dp); } /* * The wrapper cache is backend-local and can outlive a committed DROP * performed by another backend. The registry state resolved above is * authoritative; never inspect the old wrapper's potentially stale * shared pointer. Release only the known-live shared lock, if this * backend already acquired it earlier in the transaction. */ if (LWLockHeldByMe(&shared_state->lock)) LWLockRelease(&shared_state->lock); local_state = (TpLocalIndexState *) MemoryContextAlloc(TopMemoryContext, sizeof(TpLocalIndexState)); init_local_state(local_state, shared_state, shared_dp, dsa); if (index_create_subid != InvalidSubTransactionId) { local_state->build_created_shared_state = true; local_state->created_in_subxact = index_create_subid; } init_local_state_cache(); entry = (LocalStateCacheEntry *) hash_search(local_state_cache, &index_oid, HASH_ENTER, &found); if (found) old_local_state = entry->local_state; /* * REINDEX replaces this backend's wrapper while retaining the same * registry allocation. Carry transaction-local scalar accounting * forward only when the saved DSA identity matches the authoritative * state resolved above. Never dereference old_local_state->shared. */ if (old_local_state != NULL && old_local_state->shared_dp == shared_dp) { local_state->terms_added_this_xact = old_local_state->terms_added_this_xact; local_state->docs_since_global_check = old_local_state->docs_since_global_check; } entry->local_state = local_state; if (old_local_state != NULL) pfree(old_local_state); return local_state; } TpLocalIndexState * tp_create_shared_index_state( Oid index_oid, Oid heap_oid, SubTransactionId index_create_subid) { return create_or_attach_index_state( index_oid, heap_oid, index_create_subid); } TpLocalIndexState * tp_create_build_index_state( Oid index_oid, Oid heap_oid, SubTransactionId index_create_subid) { return create_or_attach_index_state( index_oid, heap_oid, index_create_subid); } /* * A successful serial or parallel build replaces the on-disk relfilenode. * Discard the derived cache in place while preserving the registry entry, * per-index LWLock, TpMemtable allocation, and memtable_dp. */ void tp_finalize_build_mode(TpLocalIndexState *local_state) { TpMemtable *memtable; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(local_state->dsa != NULL); Assert(DsaPointerIsValid(local_state->shared->memtable_dp)); tp_acquire_index_lock(local_state, LW_EXCLUSIVE); memtable = (TpMemtable *)dsa_get_address( local_state->dsa, local_state->shared->memtable_dp); LWLockAcquire(&memtable->apply_lock, LW_EXCLUSIVE); LWLockAcquire(&memtable->lock, LW_EXCLUSIVE); tp_cache_clear(local_state->dsa, memtable); pg_atomic_add_fetch_u64(&local_state->shared->spill_generation, 1); pg_atomic_write_u32( &local_state->shared->chain_page_count_needs_reseed, 1); pg_atomic_write_u32(&local_state->shared->chain_page_count, 0); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); tp_release_index_lock(local_state); } static void cleanup_owned_build_state(LocalStateCacheEntry *entry, dsa_area *global_dsa) { TpLocalIndexState *local_state = entry->local_state; TpRegistryKey key; dsa_pointer shared_dp; LWLock *eviction_mutex; Assert(local_state != NULL); Assert(local_state->build_created_shared_state); Assert(local_state->shared != NULL); key = tp_registry_key(MyDatabaseId, local_state->shared->index_oid); shared_dp = tp_registry_lookup_dsa(key); eviction_mutex = tp_registry_eviction_mutex(); if (eviction_mutex != NULL) LWLockAcquire(eviction_mutex, LW_EXCLUSIVE); tp_registry_unregister(key); if (DsaPointerIsValid(shared_dp) && global_dsa != NULL) { TpMemtable *memtable = (TpMemtable *) dsa_get_address(global_dsa, local_state->shared->memtable_dp); tp_cache_clear(global_dsa, memtable); dsa_free(global_dsa, local_state->shared->memtable_dp); dsa_free(global_dsa, shared_dp); } if (eviction_mutex != NULL) LWLockRelease(eviction_mutex); pfree(local_state); entry->local_state = NULL; } /* * Abort removes only a shared state registered by this transaction's * initial CREATE INDEX. REINDEX reuses pre-existing state; failure before * finalization leaves its cache untouched, while abort after finalization * may leave the stable cache empty. */ void tp_cleanup_build_mode_on_abort(void) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; dsa_area *global_dsa; if (local_state_cache == NULL) return; global_dsa = tp_registry_get_dsa(); hash_seq_init(&status, local_state_cache); while ((entry = hash_seq_search(&status)) != NULL) { TpLocalIndexState *local_state = entry->local_state; if (local_state == NULL) continue; if (!local_state->build_created_shared_state) continue; cleanup_owned_build_state(entry, global_dsa); } } void tp_commit_build_states(void) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; if (local_state_cache == NULL) return; hash_seq_init(&status, local_state_cache); while ((entry = hash_seq_search(&status)) != NULL) { if (entry->local_state != NULL) { entry->local_state->build_created_shared_state = false; entry->local_state->created_in_subxact = InvalidSubTransactionId; } } } bool tp_has_initial_create_ownership(void) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; if (local_state_cache == NULL) return false; hash_seq_init(&status, local_state_cache); while ((entry = hash_seq_search(&status)) != NULL) { if (entry->local_state != NULL && entry->local_state->build_created_shared_state) return true; } return false; } /* * Clean up index state on subtransaction abort * * Called from the SubXactCallback when a subtransaction aborts * (e.g., ROLLBACK TO SAVEPOINT). This handles two issues: * * 1. Registry/shared memory leak: If CREATE INDEX completed within * the subtransaction, the OAT_DROP hook won't fire during * subtransaction abort, so we must clean up manually. * * 2. LWLock tracking desync: AbortSubTransaction calls * LWLockReleaseAll(), releasing all locks including those * acquired before the savepoint. We must reset our lock_held * tracking for ALL entries to match. */ void tp_cleanup_subxact_abort(SubTransactionId mySubid) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; dsa_area *global_dsa; if (local_state_cache == NULL) return; global_dsa = tp_registry_get_dsa(); hash_seq_init(&status, local_state_cache); while ((entry = hash_seq_search(&status)) != NULL) { TpLocalIndexState *ls = entry->local_state; if (ls == NULL) continue; /* * Reset lock tracking for ALL entries, not just ones * from the aborting subtransaction. LWLockReleaseAll() * releases every lock held by this backend. */ ls->lock_held = false; ls->lock_mode = 0; /* Only process build state created in the aborting subxact. */ if (ls->created_in_subxact != mySubid) continue; if (ls->build_created_shared_state) { ls->terms_added_this_xact = 0; cleanup_owned_build_state(entry, global_dsa); } else { ls->created_in_subxact = InvalidSubTransactionId; } } } /* * Promote subtransaction state on subtransaction commit * * When a subtransaction commits (RELEASE SAVEPOINT), states created * within it are "promoted" to the parent subtransaction. This ensures * they get cleaned up if the parent later aborts. */ void tp_promote_subxact_states( SubTransactionId mySubid, SubTransactionId parentSubid) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; if (local_state_cache == NULL) return; hash_seq_init(&status, local_state_cache); while ((entry = hash_seq_search(&status)) != NULL) { TpLocalIndexState *ls = entry->local_state; if (ls == NULL) continue; if (ls->created_in_subxact == mySubid) ls->created_in_subxact = parentSubid; } } /* * Clean up shared memory allocations for an index * * Called only after the drop commits. We free the DSA allocations but keep * the DSA area itself since it's shared by all indices. */ static void cleanup_index_shared_memory(TpRegistryKey key, bool cleanup_local_state) { dsa_area *dsa; dsa_pointer shared_dp; TpSharedIndexState *shared_state; LocalStateCacheEntry *entry = NULL; bool found; /* * Release any per-index LWLock this backend still holds for * this index BEFORE we free the DSA memory that backs the * lock. Without this step, a chain_source or build that * leaks the lock (e.g., via an error path before its close() * runs) would leave a dangling held_lwlocks[] entry pointing * into freed shared memory; the end-of-xact LWLockReleaseAll() * would then trip an assertion in debug/sanitizer builds and * potentially corrupt LWLock accounting in production. * * Look up the local_state up front for this lookup; do NOT * remove it from the cache yet — the later block below will * dispose of it after the DSA frees. */ Assert(!cleanup_local_state || key.database_oid == MyDatabaseId); if (cleanup_local_state && local_state_cache != NULL) { entry = hash_search( local_state_cache, &key.index_oid, HASH_FIND, &found); if (found && entry != NULL && entry->local_state != NULL && entry->local_state->lock_held) tp_release_index_lock(entry->local_state); } entry = NULL; /* Look up the DSA pointer in registry */ LWLockAcquire(tp_registry_eviction_mutex(), LW_EXCLUSIVE); shared_dp = tp_registry_lookup_dsa(key); if (!DsaPointerIsValid(shared_dp)) { /* Still unregister even if no shared state found */ tp_registry_unregister(key); LWLockRelease(tp_registry_eviction_mutex()); return; /* Nothing to clean up */ } /* Get the shared DSA area */ dsa = tp_registry_get_dsa(); /* Get shared state */ shared_state = (TpSharedIndexState *)dsa_get_address(dsa, shared_dp); /* * The in-memory cache may have populated the dshash tables * hanging off the TpMemtable; drop them first so dsa_free on * the TpMemtable allocation does not leak the dshash internals. * Safe with an empty cache: * tp_cache_clear is a no-op when both handles are INVALID. * * The drop has committed and still holds its relation locks, so no * concurrent backend can be reading the cache here. A rolled-back * drop must never free shared state retained by other backends. * DROP DATABASE instead excludes all connections to its database. * * Memtable-cache eviction accesses victim shared states by DSA * pointer without holding the index relation lock. Take the * global eviction mutex EXCL across the unregister + dsa_free * so a concurrent evict_largest cannot deref a victim->lock * that we are about to free. Unregister FIRST so no new walker * can find the entry, then free under the same mutex so any * walker currently iterating completes before we recycle the * memory. The mutex order is global before per-index, matching * evict_largest's acquire sequence. */ /* Unregister first so no new walker can find us */ tp_registry_unregister(key); if (DsaPointerIsValid(shared_state->memtable_dp)) { TpMemtable *mt = (TpMemtable *) dsa_get_address(dsa, shared_state->memtable_dp); tp_cache_clear(dsa, mt); dsa_free(dsa, shared_state->memtable_dp); } /* Free shared_state */ dsa_free(dsa, shared_dp); LWLockRelease(tp_registry_eviction_mutex()); /* Clean up local state if we have it cached */ if (cleanup_local_state && local_state_cache != NULL) { entry = hash_search( local_state_cache, &key.index_oid, HASH_FIND, &found); if (found && entry != NULL && entry->local_state != NULL) { TpLocalIndexState *ls = entry->local_state; /* Remove from cache first, before detaching DSA */ hash_search( local_state_cache, &key.index_oid, HASH_REMOVE, &found); /* Don't detach DSA - it's shared and still in use by registry */ /* Just nullify the reference */ ls->dsa = NULL; ls->shared = NULL; /* Free the local state */ pfree(ls); } } } void tp_cleanup_index_shared_memory(Oid index_oid) { cleanup_index_shared_memory( tp_registry_key(MyDatabaseId, index_oid), true); } /* * Clean up every shared allocation qualified by a successfully dropped * database. DROP DATABASE cannot target the current connection, so this * path deliberately leaves the backend-local wrapper cache untouched. */ void tp_cleanup_database_shared_memory(Oid database_oid) { TpRegistryKey *keys; Size key_count; Size i; if (!OidIsValid(database_oid)) return; Assert(database_oid != MyDatabaseId); key_count = tp_registry_collect_database_keys(database_oid, &keys); for (i = 0; i < key_count; i++) cleanup_index_shared_memory(keys[i], false); if (keys != NULL) pfree(keys); } void tp_set_chain_page_count_for_relation( TpLocalIndexState *local_state, Relation index_rel, uint32 chain_pages) { RelFileLocator locator; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(local_state->lock_held); Assert(local_state->lock_mode == LW_EXCLUSIVE); Assert(index_rel != NULL); locator = index_rel->rd_locator; pg_atomic_write_u32(&local_state->shared->chain_page_count, chain_pages); pg_atomic_write_u32( &local_state->shared->chain_page_count_spc_oid, locator.spcOid); pg_atomic_write_u32( &local_state->shared->chain_page_count_db_oid, locator.dbOid); pg_atomic_write_u32( &local_state->shared->chain_page_count_rel_number, locator.relNumber); pg_atomic_write_u32( &local_state->shared->chain_page_count_needs_reseed, 0); } static bool chain_page_count_matches_relation( TpLocalIndexState *local_state, Relation index_rel) { RelFileLocator locator = index_rel->rd_locator; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(index_rel != NULL); if (pg_atomic_read_u32( &local_state->shared->chain_page_count_needs_reseed) != 0) return false; return pg_atomic_read_u32( &local_state->shared->chain_page_count_spc_oid) == locator.spcOid && pg_atomic_read_u32(&local_state->shared->chain_page_count_db_oid) == locator.dbOid && pg_atomic_read_u32( &local_state->shared->chain_page_count_rel_number) == locator.relNumber; } static void reseed_chain_page_count_locked( TpLocalIndexState *local_state, Relation index_rel) { TpDataSource *chain_src; uint32 chain_pages = 0; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(local_state->lock_held); Assert(local_state->lock_mode == LW_EXCLUSIVE); Assert(index_rel != NULL); chain_src = tp_memtable_chain_source_create(local_state, index_rel, NULL, 0); if (chain_src != NULL) { chain_pages = tp_memtable_chain_source_page_count(chain_src); tp_source_close(chain_src); } tp_set_chain_page_count_for_relation(local_state, index_rel, chain_pages); } /* * Rebuild finalization cannot know whether commit or rollback will select the * new or old relfilenode. Defer page-count reconstruction until a threshold * consumer has the current Relation, then recount under the per-index lock. */ bool tp_reseed_chain_page_count_if_needed( TpLocalIndexState *local_state, Relation index_rel, bool nowait) { bool acquired_lock = false; if (local_state == NULL || local_state->shared == NULL || index_rel == NULL) return true; if (chain_page_count_matches_relation(local_state, index_rel)) return true; if (!local_state->lock_held) { if (nowait) { if (!tp_try_acquire_index_lock(local_state, LW_EXCLUSIVE)) return false; } else tp_acquire_index_lock(local_state, LW_EXCLUSIVE); acquired_lock = true; } else Assert(local_state->lock_mode == LW_EXCLUSIVE); if (!chain_page_count_matches_relation(local_state, index_rel)) reseed_chain_page_count_locked(local_state, index_rel); if (acquired_lock) tp_release_index_lock(local_state); return true; } /* * Rebuild index state from disk on the first backend access of an * index whose registry entry is empty (server restart, hot * standby cold start, PITR-recovered cluster). * * Post-#374: the durable record of the in-flight memtable * lives entirely on the index relation's chain pages. * Standard GenericXLog replay restores both the chain pages and * the metapage's `memtable_head_blkno`/`memtable_tail_blkno` * pointers before any backend gets here, so this function only * has to (re)register shared state for lock coordination and * seed the chain-page counter used by the auto-spill heuristic. * * The flow at a high level: * * 1. validate metapage magic * 2. tp_create_shared_index_state with reuse_if_exists=true * (atomic register-or-attach; concurrent rebuilds resolve * to the same shared state) * 3. walk the on-disk chain to seed `chain_page_count` * * No WAL drain and no recovery-time corpus rebuild are needed: * the on-disk metapage + chain pages + segments already encode * every committed insert. The in-memory memtable cache is * derived state and lazily built on the first query after rebuild; * readers consult the chain source for in-flight statistics in * the meantime. */ TpLocalIndexState * tp_rebuild_index_from_disk(Oid index_oid) { Relation index_rel; TpIndexMetaPage early_metap; TpLocalIndexState *local_state; Oid heap_oid; SubTransactionId index_create_subid; /* Open the index relation */ index_rel = index_open(index_oid, AccessShareLock); if (index_rel == NULL) { elog(WARNING, "Could not open index %u for recovery", index_oid); return NULL; } heap_oid = index_rel->rd_index->indrelid; index_create_subid = index_rel->rd_createSubid; /* * Read the metapage early so we can validate the magic before * registering shared state. Magic is set at index creation and * never changes, so this check is stable across replay. */ early_metap = tp_get_metapage(index_rel); if (early_metap == NULL) { index_close(index_rel, AccessShareLock); elog(WARNING, "Could not read metapage for index %u", index_oid); return NULL; } if (early_metap->magic != TP_METAPAGE_MAGIC) { uint32 found_magic = early_metap->magic; index_close(index_rel, AccessShareLock); pfree(early_metap); elog(WARNING, "Invalid magic number in metapage for index %u: " "expected 0x%08X, found 0x%08X", index_oid, TP_METAPAGE_MAGIC, found_magic); return NULL; } pfree(early_metap); /* * Register shared state up front. Post-#374, durable state * lives entirely on disk (metapage + chain pages + segment * pages). No docid-page replay or memtable reconstruction * needed — the chain pages are themselves the durable record * of unspilled documents, and standard GenericXLog replay has * already brought them up to date by the time any backend * reaches this path. */ local_state = tp_create_shared_index_state( index_oid, heap_oid, index_create_subid); if (local_state == NULL) { index_close(index_rel, AccessShareLock); return NULL; } /* * Seed the on-disk chain page counter so the auto-spill * threshold isn't blind to pages that pre-existed in the * index when this backend (or this shmem segment) was * started. Corpus statistics (total_docs / total_len) live * entirely on the metapage and need no shmem seed. */ tp_acquire_index_lock(local_state, LW_EXCLUSIVE); reseed_chain_page_count_locked(local_state, index_rel); tp_release_index_lock(local_state); index_close(index_rel, AccessShareLock); return local_state; } /* * Helper function to get memtable from local index state * Canonical implementation used by all modules */ TpMemtable * get_memtable(TpLocalIndexState *local_state) { if (!local_state || !local_state->shared || !local_state->dsa) return NULL; if (!DsaPointerIsValid(local_state->shared->memtable_dp)) return NULL; return (TpMemtable *)dsa_get_address( local_state->dsa, local_state->shared->memtable_dp); } /* * Acquire the per-index lock if not already held by this backend. * Ensures memory consistency on NUMA systems through LWLock's * built-in memory barriers. */ void tp_acquire_index_lock(TpLocalIndexState *local_state, LWLockMode mode) { Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE); /* If we already hold the lock, check mode compatibility */ if (local_state->lock_held) { /* * If we hold exclusive lock, we're good regardless of requested mode. * If we hold shared lock and request shared, we're also good. * But if we hold shared and need exclusive, we must upgrade. */ if (local_state->lock_mode == LW_EXCLUSIVE || (local_state->lock_mode == LW_SHARED && mode == LW_SHARED)) { return; /* Already have sufficient lock */ } /* * Need to upgrade from shared to exclusive. * This is tricky and can deadlock, so we'll release and re-acquire. * In practice, this shouldn't happen as writers should request * exclusive from the start. */ elog(WARNING, "Upgrading index lock from shared to exclusive - " "potential deadlock risk"); LWLockRelease(&local_state->shared->lock); local_state->lock_held = false; } if (mode == LW_SHARED) { ConditionVariablePrepareToSleep( &local_state->shared->exclusive_waiters_cv); while (pg_atomic_read_u32(&local_state->shared->exclusive_waiters) != 0) ConditionVariableSleep( &local_state->shared->exclusive_waiters_cv, PG_WAIT_EXTENSION); ConditionVariableCancelSleep(); } else { pg_atomic_fetch_add_u32(&local_state->shared->exclusive_waiters, 1); TP_INJECTION_POINT(TP_INJECTION_INDEX_LOCK_EXCLUSIVE_WAITER); } PG_TRY(); { LWLockAcquire(&local_state->shared->lock, mode); } PG_FINALLY(); { if (mode == LW_EXCLUSIVE && pg_atomic_sub_fetch_u32( &local_state->shared->exclusive_waiters, 1) == 0) ConditionVariableBroadcast( &local_state->shared->exclusive_waiters_cv); } PG_END_TRY(); local_state->lock_held = true; local_state->lock_mode = mode; /* * The LWLockAcquire provides acquire semantics (memory barrier), * ensuring we see all writes from the previous lock holder. */ } /* * Conditionally acquire the per-index lock. Exclusive attempts * register as waiters for the duration of the attempt so new shared * acquisitions cannot bypass them. */ bool tp_try_acquire_index_lock(TpLocalIndexState *local_state, LWLockMode mode) { bool acquired = false; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE); if (local_state->lock_held) { if (local_state->lock_mode == LW_EXCLUSIVE || (local_state->lock_mode == LW_SHARED && mode == LW_SHARED)) return true; return false; } if (mode == LW_SHARED) { if (pg_atomic_read_u32(&local_state->shared->exclusive_waiters) != 0) return false; acquired = LWLockConditionalAcquire(&local_state->shared->lock, mode); } else { pg_atomic_fetch_add_u32(&local_state->shared->exclusive_waiters, 1); PG_TRY(); { acquired = LWLockConditionalAcquire(&local_state->shared->lock, mode); } PG_FINALLY(); { if (pg_atomic_sub_fetch_u32( &local_state->shared->exclusive_waiters, 1) == 0) ConditionVariableBroadcast( &local_state->shared->exclusive_waiters_cv); } PG_END_TRY(); } if (!acquired) return false; local_state->lock_held = true; local_state->lock_mode = mode; return true; } /* * Release the per-index lock if held */ void tp_release_index_lock(TpLocalIndexState *local_state) { if (!local_state || !local_state->lock_held) return; Assert(local_state->shared != NULL); /* * Double-check that PostgreSQL thinks we hold the lock. * This can prevent crashes if our lock tracking gets out of sync * (e.g., during error recovery). */ if (!LWLockHeldByMe(&local_state->shared->lock)) { /* Our tracking was wrong - fix it and return */ local_state->lock_held = false; local_state->lock_mode = 0; return; } /* ereport(ERROR) resets the holdoff before PG_FINALLY cleanup. */ if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); /* * The LWLockRelease provides release semantics (memory barrier), * ensuring our writes are visible to the next lock acquirer. */ LWLockRelease(&local_state->shared->lock); local_state->lock_held = false; local_state->lock_mode = 0; } void tp_acquire_memtable_write_lock(TpLocalIndexState *local_state, LWLockMode mode) { TpSharedIndexState *shared; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE); shared = local_state->shared; if (mode == LW_SHARED) { ConditionVariablePrepareToSleep( &shared->memtable_write_exclusive_waiters_cv); while (pg_atomic_read_u32(&shared->memtable_write_exclusive_waiters) != 0) ConditionVariableSleep( &shared->memtable_write_exclusive_waiters_cv, PG_WAIT_EXTENSION); ConditionVariableCancelSleep(); } else pg_atomic_fetch_add_u32(&shared->memtable_write_exclusive_waiters, 1); PG_TRY(); { LWLockAcquire(&shared->memtable_write_lock, mode); } PG_FINALLY(); { if (mode == LW_EXCLUSIVE && pg_atomic_sub_fetch_u32( &shared->memtable_write_exclusive_waiters, 1) == 0) ConditionVariableBroadcast( &shared->memtable_write_exclusive_waiters_cv); } PG_END_TRY(); } bool tp_try_acquire_memtable_write_lock( TpLocalIndexState *local_state, LWLockMode mode) { TpSharedIndexState *shared; bool acquired; Assert(local_state != NULL); Assert(local_state->shared != NULL); Assert(mode == LW_SHARED || mode == LW_EXCLUSIVE); shared = local_state->shared; if (mode == LW_SHARED) { if (pg_atomic_read_u32(&shared->memtable_write_exclusive_waiters) != 0) return false; return LWLockConditionalAcquire(&shared->memtable_write_lock, mode); } pg_atomic_fetch_add_u32(&shared->memtable_write_exclusive_waiters, 1); PG_TRY(); { acquired = LWLockConditionalAcquire(&shared->memtable_write_lock, mode); } PG_FINALLY(); { if (pg_atomic_sub_fetch_u32( &shared->memtable_write_exclusive_waiters, 1) == 0) ConditionVariableBroadcast( &shared->memtable_write_exclusive_waiters_cv); } PG_END_TRY(); return acquired; } void tp_release_memtable_write_lock(TpLocalIndexState *local_state) { if (local_state == NULL || local_state->shared == NULL) return; if (!LWLockHeldByMe(&local_state->shared->memtable_write_lock)) return; if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); LWLockRelease(&local_state->shared->memtable_write_lock); } /* * Release all index locks held by this backend. * This is called at transaction end via the transaction callback. */ void tp_release_all_index_locks(void) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; /* Nothing to do if cache not initialized */ if (local_state_cache == NULL) return; /* Iterate through all cached local states */ hash_seq_init(&status, local_state_cache); while ((entry = (LocalStateCacheEntry *)hash_seq_search(&status)) != NULL) { if (entry->local_state && entry->local_state->lock_held) tp_release_index_lock(entry->local_state); } } /* * Check if any index should spill to disk due to bulk load threshold. * Spill is triggered when terms added this transaction exceeds threshold. * * Note: memtable_pages_threshold is checked in real-time via * tp_auto_spill_if_needed() after each document insert. * * This is called at PRE_COMMIT via the transaction callback in mod.c. */ void tp_bulk_load_spill_check(void) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; /* Standby is read-only; spill is primary-only. */ if (RecoveryInProgress()) return; /* Nothing to do if cache not initialized or threshold disabled */ if (local_state_cache == NULL) return; if (tp_bulk_load_threshold <= 0) return; /* Iterate through all cached local states */ hash_seq_init(&status, local_state_cache); while ((entry = (LocalStateCacheEntry *)hash_seq_search(&status)) != NULL) { TpLocalIndexState *local_state = entry->local_state; Relation index_rel; if (!local_state || !local_state->shared) continue; /* Check bulk load threshold */ if (local_state->terms_added_this_xact < tp_bulk_load_threshold) continue; /* * Open the relation before the spill helper takes either the * maintenance lock or the per-index LWLock. No per-index lock is * held on entry because per-operation locking releases after each * insert. */ index_rel = try_index_open( local_state->shared->index_oid, RowExclusiveLock); if (index_rel == NULL) continue; tp_spill_memtable_if_needed(index_rel, local_state, 0); index_close(index_rel, RowExclusiveLock); } } /* * Reset bulk load counters for all cached indexes. * Called at transaction end (COMMIT/ABORT) via the transaction callback. */ void tp_reset_bulk_load_counters(void) { HASH_SEQ_STATUS status; LocalStateCacheEntry *entry; /* Nothing to do if cache not initialized */ if (local_state_cache == NULL) return; /* Iterate through all cached local states and reset counters */ hash_seq_init(&status, local_state_cache); while ((entry = (LocalStateCacheEntry *)hash_seq_search(&status)) != NULL) { if (entry->local_state) { entry->local_state->terms_added_this_xact = 0; /* Note: docs_since_global_check is NOT reset here. * It is an amortization counter that must persist * across transactions so that the global soft limit * check fires for single-row auto-commit INSERTs. */ } } } timescale-pg_textsearch-b63fe58/src/index/state.h000066400000000000000000000260721525762047700221570ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * state.h - Index state management structures */ #pragma once #include #include #include #include #include #include /* * BlockNumber / InvalidBlockNumber are used by the cache's apply * cursor below (and are not transitively pulled in by the other * Postgres headers we include here). */ #include #include /* Forward declarations */ struct TpMemtable; typedef struct TpIndexMetaPageData *TpIndexMetaPage; typedef struct RelationData *Relation; /* * Header of the DSM segment for each index * Contains metadata and space for the DSA area */ typedef struct TpDsmSegmentHeader { dsm_handle dsm_handle; /* DSM segment handle for recovery */ dsa_pointer shared_state_dp; /* DSA pointer to TpSharedIndexState */ /* DSA area space follows immediately after this header */ } TpDsmSegmentHeader; /* * In-memory memtable cache. * * The cache is derived state layered on top of the on-disk memtable * page chain (which remains the source of truth, per #374). This * struct holds the lifecycle handles, apply cursor, and footprint * accounting; the apply protocol, cold build, read source, spill * consumption, and memory cap live in cache.c / cache_source.c. * * Lifetimes: * - string_hash_handle / doc_lengths_handle: * DSHASH_HANDLE_INVALID until cold_build allocates them; reset * back to DSHASH_HANDLE_INVALID by tp_cache_clear. * - apply_lock / lock: * initialized once at TpMemtable allocation, never destroyed * across cache clears (the structs survive; tp_cache_clear only * drops the dshash tables and resets the cursor). * - cursor_*: * (0, InvalidBlockNumber, 0, 0) at init and after tp_cache_clear. * Mutated only under apply_lock EXCL by reader catchup, * cold lazy build, and spill catchup. * - estimated_bytes: * 0 at init. Maintained by the apply paths under apply_lock; * read lock-free by the 3-tier memory cap. * * Lock order: * per-index LWLock (in TpSharedIndexState) → apply_lock → lock * → dshash buckets → TpPostingList.lock */ typedef struct TpMemtable { /* dshash inverted index: term -> TpStringHashEntry. */ dshash_table_handle string_hash_handle; /* dshash doc-length map: ctid -> TpDocLengthEntry. */ dshash_table_handle doc_lengths_handle; /* * Serializes every path that mutates the cache (reader catchup, * cold build, spill catchup, tp_cache_clear). Acquired AFTER the * per-index LWLock. Snapshot readers use SHARED while serving an * exact captured endpoint; mutations use EXCL. */ LWLock apply_lock; /* * Lifetime lock for served TpDataSources. SHARED for the lifetime * of a source (open through close); EXCL only for full drop / * evict. apply_lock holders may take SHARED here concurrently * with served readers. */ LWLock lock; /* * Apply cursor. Identifies the next chain record the cache * expects to consume. Mutated only under apply_lock EXCL by * reader catchup, cold lazy build, and spill catchup. * * cursor_gen_spill_count is the generation token captured from * TpSharedIndexState.spill_generation at cold-build time; a * mismatch on a later apply means a spill raced through and the * cache is stale. cursor_locator identifies the relation file * from which the cache was populated, so transaction or savepoint * rollback after REINDEX cannot reuse a cache built from the * discarded file. * * (cursor_next_blkno, cursor_next_off) is the logical position of * the next chain record to apply; InvalidBlockNumber means "no * cache yet, cold_build is required before any catchup can * advance the cursor". * * cursor_seq is a monotonic apply counter for cheap "did anything * change" checks; read lock-free. */ uint64 cursor_gen_spill_count; BlockNumber cursor_next_blkno; uint16 cursor_next_off; RelFileLocator cursor_locator; bool cursor_locator_valid; pg_atomic_uint64 cursor_seq; /* * Cache footprint, updated by apply paths under apply_lock; * read lock-free for the 3-tier memory cap. */ pg_atomic_uint64 estimated_bytes; } TpMemtable; /* * Shared index state - stored in DSA * This structure is shared across all backends and contains only * data that can be safely stored in dynamic shared memory. * All pointers must be dsa_pointer type. */ typedef struct TpSharedIndexState { /* Index identification */ Oid index_oid; /* OID of this index */ Oid heap_oid; /* OID of the indexed heap relation */ /* * Runtime cache stored in the global DSA. The allocation remains * stable for the lifetime of the registry entry, including REINDEX. */ dsa_pointer memtable_dp; /* DSA pointer to TpMemtable */ /* * Auto-spill heuristic for the on-disk memtable: number of * chain pages currently published in the page chain. * Incremented after each successful page-publish * GenericXLogFinish in src/memtable/log.c; reset to 0 after * tp_spill_finalize() completes. * * A completed CREATE/REINDEX also resets the count and sets * chain_page_count_needs_reseed. The next threshold consumer * recounts the chain in whichever relfilenode PostgreSQL selected * (new file on commit, restored file on abort) under LW_EXCLUSIVE, * records that file's locator, then clears the marker. A locator * mismatch also forces a recount, covering rollback after an * in-transaction reseed. These fields are not WAL-logged. */ pg_atomic_uint32 chain_page_count; pg_atomic_uint32 chain_page_count_needs_reseed; pg_atomic_uint32 chain_page_count_spc_oid; pg_atomic_uint32 chain_page_count_db_oid; pg_atomic_uint32 chain_page_count_rel_number; /* * Cached estimated memtable size in bytes, updated * atomically by writers. Used to maintain the global * estimated_total_bytes counter without scanning. */ pg_atomic_uint64 estimated_bytes; /* * Per-index LWLock for transaction-level serialization. * Writers acquire this in exclusive mode once per transaction. * Readers acquire this in shared mode once per transaction. * This ensures memory consistency on NUMA systems and proper * transaction isolation. */ LWLock lock; /* Per-index lock for this index */ pg_atomic_uint32 exclusive_waiters; ConditionVariable exclusive_waiters_cv; /* * Fence memtable writers while a spill builds from a stable chain. * Readers do not take this lock. */ LWLock memtable_write_lock; pg_atomic_uint32 memtable_write_exclusive_waiters; ConditionVariable memtable_write_exclusive_waiters_cv; /* * Spill generation counter. Bumped under LW_EXCLUSIVE after * the on-disk chain is truncated or a completed index build * replaces the relfilenode. * Acts as the in-memory memtable cache's invalidation * token: a cache built against generation N becomes stale * the moment this counter advances to N+1, even if a new * chain page happens to be allocated at the same block * number as the old head (ABA hole on head_blkno alone). * * Not WAL-logged: standby query paths don't use the cache * (RecoveryInProgress() disables it), so primary-only * generation tracking is sufficient. */ pg_atomic_uint64 spill_generation; } TpSharedIndexState; /* * Local index state - backend-specific * This structure is private to each backend and contains the DSA * attachment and other backend-specific data. */ typedef struct TpLocalIndexState { /* Pointer to shared state in registry */ TpSharedIndexState *shared; /* * DSA identity of shared. This scalar remains safe to compare after a * committed DROP makes shared itself potentially stale. */ dsa_pointer shared_dp; /* DSA attachment for this backend */ dsa_area *dsa; /* Attached DSA area for this index */ /* * True when this wrapper belongs to an index relation created by the * current transaction. It remains set through build finalization so * abort can remove only state owned by that initial CREATE. REINDEX * of a pre-existing relation leaves this false even if the registry * was cold and the build registered the shared state. */ bool build_created_shared_state; /* Transaction-level lock tracking */ bool lock_held; /* True if we hold the lock in this transaction */ LWLockMode lock_mode; /* Mode we're holding (LW_SHARED or LW_EXCLUSIVE) */ /* Bulk load tracking: terms added in current transaction */ int64 terms_added_this_xact; /* Amortization counter for global soft limit check */ int docs_since_global_check; /* * Subtransaction tracking: which subtransaction created this * state. Used to clean up on subtransaction abort (e.g., * ROLLBACK TO SAVEPOINT after CREATE INDEX). * InvalidSubTransactionId means not created in any tracked * subtransaction (e.g., rebuilt from disk on restart). */ SubTransactionId created_in_subxact; } TpLocalIndexState; /* Function declarations for index state management */ extern TpLocalIndexState *tp_get_local_index_state(Oid index_oid); /* * Create or attach the per-index shared runtime state and return a * backend-local wrapper. Existing registry state is always reused. */ extern TpLocalIndexState *tp_create_shared_index_state( Oid index_oid, Oid heap_oid, SubTransactionId index_create_subid); extern TpLocalIndexState *tp_create_build_index_state( Oid index_oid, Oid heap_oid, SubTransactionId index_create_subid); extern void tp_cleanup_index_shared_memory(Oid index_oid); extern void tp_cleanup_database_shared_memory(Oid database_oid); extern void tp_finalize_build_mode(TpLocalIndexState *local_state); extern void tp_cleanup_build_mode_on_abort(void); extern void tp_commit_build_states(void); extern bool tp_has_initial_create_ownership(void); extern TpLocalIndexState *tp_rebuild_index_from_disk(Oid index_oid); /* Helper function for accessing memtable from local state */ extern TpMemtable *get_memtable(TpLocalIndexState *local_state); /* Transaction-level lock management */ extern void tp_acquire_index_lock(TpLocalIndexState *local_state, LWLockMode mode); extern bool tp_try_acquire_index_lock(TpLocalIndexState *local_state, LWLockMode mode); extern void tp_release_index_lock(TpLocalIndexState *local_state); extern void tp_acquire_memtable_write_lock( TpLocalIndexState *local_state, LWLockMode mode); extern bool tp_try_acquire_memtable_write_lock( TpLocalIndexState *local_state, LWLockMode mode); extern void tp_release_memtable_write_lock(TpLocalIndexState *local_state); extern void tp_release_all_index_locks(void); extern void tp_set_chain_page_count_for_relation( TpLocalIndexState *local_state, Relation index_rel, uint32 chain_pages); extern bool tp_reseed_chain_page_count_if_needed( TpLocalIndexState *local_state, Relation index_rel, bool nowait); /* Bulk load auto-spill */ extern void tp_bulk_load_spill_check(void); extern void tp_reset_bulk_load_counters(void); /* Subtransaction cleanup */ extern void tp_cleanup_subxact_abort(SubTransactionId mySubid); extern void tp_promote_subxact_states( SubTransactionId mySubid, SubTransactionId parentSubid); timescale-pg_textsearch-b63fe58/src/layout_check.c000066400000000000000000000135311525762047700223710ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * layout_check.c - Compile-time assertions for layout-sensitive structs * * The dictionary, skip index, CTID map and block posting structs are * stored as raw bytes in segments, so their layout is part of the * on-disk format. TpExpullEntry (memtable arena) and TpSegmentPosting * (postings handed to the scorer) are never written to disk, but the * code depends on their packed stride all the same. * * The reference layout is the one GCC produces on x86-64: a packed * struct is the plain sum of its members, and the two aligned() structs * keep their natural layout because the requested alignment already is * the natural one. The supported 64-bit ABIs agree on these values. * * A compiler that laid the persisted structs out differently would * read and write segment data incorrectly; for the in-memory structs * it would silently lose the compact representation the code is sized * around. Pinning sizeof and offsetof here turns either into a build * failure. The translation unit emits no code. */ #include #include "memtable/expull.h" #include "segment/format.h" #include "segment/segment.h" /* * Alignment probes: offsetof(probe, entry) equals the alignment the * compiler gives the struct, without needing C11 _Alignof. */ struct TpDictEntryV3AlignProbe { char c; TpDictEntryV3 entry; }; struct TpDictEntryAlignProbe { char c; TpDictEntry entry; }; /* Posting entry accumulated in memtable arena blocks: packed, 7 bytes */ StaticAssertDecl(sizeof(TpExpullEntry) == 7, "TpExpullEntry size"); StaticAssertDecl(offsetof(TpExpullEntry, doc_id) == 0, "TpExpullEntry.doc_id"); StaticAssertDecl( offsetof(TpExpullEntry, frequency) == 4, "TpExpullEntry.frequency"); StaticAssertDecl( offsetof(TpExpullEntry, fieldnorm) == 6, "TpExpullEntry.fieldnorm"); /* V3 dictionary entry: aligned(4) is the natural layout, 12 bytes */ StaticAssertDecl(sizeof(TpDictEntryV3) == 12, "TpDictEntryV3 size"); StaticAssertDecl( offsetof(struct TpDictEntryV3AlignProbe, entry) == 4, "TpDictEntryV3 alignment"); StaticAssertDecl( offsetof(TpDictEntryV3, skip_index_offset) == 0, "TpDictEntryV3.skip_index_offset"); StaticAssertDecl( offsetof(TpDictEntryV3, block_count) == 4, "TpDictEntryV3.block_count"); StaticAssertDecl( offsetof(TpDictEntryV3, reserved) == 6, "TpDictEntryV3.reserved"); StaticAssertDecl( offsetof(TpDictEntryV3, doc_freq) == 8, "TpDictEntryV3.doc_freq"); /* Dictionary entry: aligned(8) is the natural layout, 16 bytes */ StaticAssertDecl(sizeof(TpDictEntry) == 16, "TpDictEntry size"); StaticAssertDecl( offsetof(struct TpDictEntryAlignProbe, entry) == 8, "TpDictEntry alignment"); StaticAssertDecl( offsetof(TpDictEntry, skip_index_offset) == 0, "TpDictEntry.skip_index_offset"); StaticAssertDecl( offsetof(TpDictEntry, block_count) == 8, "TpDictEntry.block_count"); StaticAssertDecl( offsetof(TpDictEntry, doc_freq) == 12, "TpDictEntry.doc_freq"); /* V3 skip index entry: packed, 16 bytes */ StaticAssertDecl(sizeof(TpSkipEntryV3) == 16, "TpSkipEntryV3 size"); StaticAssertDecl( offsetof(TpSkipEntryV3, last_doc_id) == 0, "TpSkipEntryV3.last_doc_id"); StaticAssertDecl( offsetof(TpSkipEntryV3, doc_count) == 4, "TpSkipEntryV3.doc_count"); StaticAssertDecl( offsetof(TpSkipEntryV3, block_max_tf) == 5, "TpSkipEntryV3.block_max_tf"); StaticAssertDecl( offsetof(TpSkipEntryV3, block_max_norm) == 7, "TpSkipEntryV3.block_max_norm"); StaticAssertDecl( offsetof(TpSkipEntryV3, posting_offset) == 8, "TpSkipEntryV3.posting_offset"); StaticAssertDecl(offsetof(TpSkipEntryV3, flags) == 12, "TpSkipEntryV3.flags"); StaticAssertDecl( offsetof(TpSkipEntryV3, reserved) == 13, "TpSkipEntryV3.reserved"); /* Skip index entry: packed, 20 bytes */ StaticAssertDecl(sizeof(TpSkipEntry) == 20, "TpSkipEntry size"); StaticAssertDecl( offsetof(TpSkipEntry, last_doc_id) == 0, "TpSkipEntry.last_doc_id"); StaticAssertDecl( offsetof(TpSkipEntry, doc_count) == 4, "TpSkipEntry.doc_count"); StaticAssertDecl( offsetof(TpSkipEntry, block_max_tf) == 5, "TpSkipEntry.block_max_tf"); StaticAssertDecl( offsetof(TpSkipEntry, block_max_norm) == 7, "TpSkipEntry.block_max_norm"); StaticAssertDecl( offsetof(TpSkipEntry, posting_offset) == 8, "TpSkipEntry.posting_offset"); StaticAssertDecl(offsetof(TpSkipEntry, flags) == 16, "TpSkipEntry.flags"); StaticAssertDecl( offsetof(TpSkipEntry, reserved) == 17, "TpSkipEntry.reserved"); /* Doc ID to CTID map entry: packed, 6 bytes */ StaticAssertDecl(sizeof(TpCtidMapEntry) == 6, "TpCtidMapEntry size"); StaticAssertDecl(offsetof(TpCtidMapEntry, ctid) == 0, "TpCtidMapEntry.ctid"); /* Posting handed to the scorer: packed, 14 bytes */ StaticAssertDecl(sizeof(TpSegmentPosting) == 14, "TpSegmentPosting size"); StaticAssertDecl( offsetof(TpSegmentPosting, ctid) == 0, "TpSegmentPosting.ctid"); StaticAssertDecl( offsetof(TpSegmentPosting, doc_id) == 6, "TpSegmentPosting.doc_id"); StaticAssertDecl( offsetof(TpSegmentPosting, frequency) == 10, "TpSegmentPosting.frequency"); StaticAssertDecl( offsetof(TpSegmentPosting, doc_length) == 12, "TpSegmentPosting.doc_length"); /* * Unpacked types the format embeds. They need no attribute to reach * the reference layout, but the layout is just as load-bearing. */ StaticAssertDecl(sizeof(ItemPointerData) == 6, "ItemPointerData size"); StaticAssertDecl( offsetof(ItemPointerData, ip_posid) == 4, "ItemPointerData.ip_posid"); StaticAssertDecl(sizeof(TpBlockPosting) == 8, "TpBlockPosting size"); StaticAssertDecl( offsetof(TpBlockPosting, doc_id) == 0, "TpBlockPosting.doc_id"); StaticAssertDecl( offsetof(TpBlockPosting, frequency) == 4, "TpBlockPosting.frequency"); StaticAssertDecl( offsetof(TpBlockPosting, fieldnorm) == 6, "TpBlockPosting.fieldnorm"); StaticAssertDecl( offsetof(TpBlockPosting, reserved) == 7, "TpBlockPosting.reserved"); timescale-pg_textsearch-b63fe58/src/memtable/000077500000000000000000000000001525762047700213365ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/memtable/arena.c000066400000000000000000000065551525762047700226030ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * arena.c - Page-based arena allocator for index build memtables * * Simple bump allocator using 1MB pages. All allocations are * sequential within a page; when the current page is full, a new * page is allocated. Individual deallocation is not supported — * the entire arena is freed at once via tp_arena_reset() or * tp_arena_destroy(). * * This design is inspired by Tantivy's MemoryArena (stacker crate) * and provides O(1) allocation with zero fragmentation. */ #include #include #include "memtable/arena.h" /* Initial capacity for the pages array */ #define ARENA_INITIAL_PAGES 16 /* * Allocate a new 1MB page and add it to the arena. */ static void arena_add_page(TpArena *arena) { if (arena->num_pages >= ARENA_MAX_PAGES) elog(ERROR, "arena: exceeded maximum capacity (%d pages, %zu MB)", ARENA_MAX_PAGES, (Size)ARENA_MAX_PAGES); /* Grow the pages array if needed */ if (arena->num_pages >= arena->max_pages) { uint32 new_max = arena->max_pages * 2; if (new_max > ARENA_MAX_PAGES) new_max = ARENA_MAX_PAGES; arena->pages = repalloc(arena->pages, new_max * sizeof(char *)); arena->max_pages = new_max; } arena->pages[arena->num_pages] = palloc(ARENA_PAGE_SIZE); arena->num_pages++; arena->current_offset = 0; } /* * Create a new arena. */ TpArena * tp_arena_create(void) { TpArena *arena = palloc(sizeof(TpArena)); arena->pages = palloc(ARENA_INITIAL_PAGES * sizeof(char *)); arena->num_pages = 0; arena->max_pages = ARENA_INITIAL_PAGES; arena->current_offset = 0; arena->total_bytes = 0; /* Allocate the first page */ arena_add_page(arena); return arena; } /* * Allocate `size` bytes from the arena. * * If the current page doesn't have enough space, a new page is * allocated. Allocations larger than ARENA_PAGE_SIZE are not * supported (callers should break large data into smaller chunks). */ ArenaAddr tp_arena_alloc(TpArena *arena, Size size) { ArenaAddr addr; uint32 page_idx; Size aligned_size; Assert(arena != NULL); Assert(size > 0); /* Round up to 4-byte alignment so uint32 stores are safe */ aligned_size = TYPEALIGN(4, size); if (aligned_size > ARENA_PAGE_SIZE) elog(ERROR, "arena: allocation of %zu bytes exceeds page size " "(%d bytes)", size, ARENA_PAGE_SIZE); /* Check if current page has enough space */ if (arena->current_offset + aligned_size > ARENA_PAGE_SIZE) { /* Current page is full, allocate a new one */ arena_add_page(arena); } page_idx = arena->num_pages - 1; addr = arena_addr_make(page_idx, arena->current_offset); arena->current_offset += aligned_size; arena->total_bytes += aligned_size; return addr; } /* * Reset the arena, freeing all pages except the first. * The arena can be reused after reset. */ void tp_arena_reset(TpArena *arena) { uint32 i; Assert(arena != NULL); /* Free all pages except the first */ for (i = 1; i < arena->num_pages; i++) pfree(arena->pages[i]); arena->num_pages = 1; arena->current_offset = 0; arena->total_bytes = 0; } /* * Destroy the arena, freeing all memory. */ void tp_arena_destroy(TpArena *arena) { uint32 i; if (arena == NULL) return; for (i = 0; i < arena->num_pages; i++) pfree(arena->pages[i]); pfree(arena->pages); pfree(arena); } timescale-pg_textsearch-b63fe58/src/memtable/arena.h000066400000000000000000000101471525762047700226000ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * arena.h - Page-based arena allocator for index build memtables * * A simple bump allocator that allocates from 1MB pages. All memory * is freed at once when the arena is destroyed or reset, making it * ideal for batch operations like index builds where individual * deallocation is not needed. * * Addresses are 32-bit (ArenaAddr): upper 12 bits = page index, * lower 20 bits = offset within page. This gives a maximum capacity * of 4096 pages * 1MB = 4GB per arena. */ #pragma once #include /* * Arena address type - compact 32-bit reference into arena memory. * Encodes page index (12 bits) and offset (20 bits). */ typedef uint32 ArenaAddr; #define ARENA_ADDR_INVALID ((ArenaAddr)0xFFFFFFFF) /* Page size: 1MB (2^20 bytes) */ #define ARENA_PAGE_BITS 20 #define ARENA_PAGE_SIZE (1 << ARENA_PAGE_BITS) #define ARENA_OFFSET_MASK ((1 << ARENA_PAGE_BITS) - 1) /* Maximum pages per arena: 4096 (2^12) -> 4GB total */ #define ARENA_MAX_PAGES 4096 /* * Maximum bytes an arena can address: ARENA_MAX_PAGES * ARENA_PAGE_SIZE * = 4096 * 1MB = 4GB = 2^32. Computed in 64-bit on purpose: the product * is exactly 2^32 and wraps to zero in a 32-bit size_t. Used to clamp * the flush budget so a maintenance_work_mem larger than the arena can * still trigger a spill instead of hitting the arena's hard ceiling * (issue #433). */ #define ARENA_MAX_BYTES ((uint64)ARENA_MAX_PAGES * ARENA_PAGE_SIZE) /* * Build an ArenaAddr from page index and offset. */ static inline ArenaAddr arena_addr_make(uint32 page_idx, uint32 offset) { Assert(page_idx < ARENA_MAX_PAGES); Assert(offset < ARENA_PAGE_SIZE); return (page_idx << ARENA_PAGE_BITS) | offset; } /* * Extract page index from an ArenaAddr. */ static inline uint32 arena_addr_page(ArenaAddr addr) { return addr >> ARENA_PAGE_BITS; } /* * Extract offset from an ArenaAddr. */ static inline uint32 arena_addr_offset(ArenaAddr addr) { return addr & ARENA_OFFSET_MASK; } /* * Arena allocator state. */ typedef struct TpArena { char **pages; /* Array of 1MB page pointers */ uint32 num_pages; /* Number of allocated pages */ uint32 max_pages; /* Capacity of pages array */ uint32 current_offset; /* Write position in current page */ Size total_bytes; /* Total bytes allocated (for budget) */ } TpArena; /* * Clamp a requested flush budget to what a single arena can actually * hold. The arena is capped at ARENA_MAX_BYTES by construction, so a * budget at or above that can never fire the flush check and the arena * dies with "exceeded maximum capacity" instead of spilling. We clamp * to 3/4 of capacity, leaving headroom for per-document overshoot * between budget checks (a document may add many terms before the next * check). See issue #433. */ static inline Size tp_arena_clamp_budget(Size budget) { Size cap = (Size)(ARENA_MAX_BYTES - ARENA_MAX_BYTES / 4); if (budget == 0 || budget > cap) return cap; return budget; } /* * Create a new arena. */ extern TpArena *tp_arena_create(void); /* * Allocate `size` bytes from the arena. * Returns an ArenaAddr referencing the allocated memory. * Allocations are rounded up to 4-byte alignment. */ extern ArenaAddr tp_arena_alloc(TpArena *arena, Size size); /* * Resolve an ArenaAddr to a raw pointer. * The pointer is valid until the arena is destroyed or reset. */ static inline void * tp_arena_get_ptr(TpArena *arena, ArenaAddr addr) { Assert(addr != ARENA_ADDR_INVALID); Assert(arena_addr_page(addr) < arena->num_pages); return arena->pages[arena_addr_page(addr)] + arena_addr_offset(addr); } /* * Return total bytes allocated from this arena. * Used for budget-based flushing decisions. */ static inline Size tp_arena_mem_usage(TpArena *arena) { return arena->total_bytes; } /* * Reset the arena, releasing all pages back to palloc. * The arena structure itself remains valid for reuse. */ extern void tp_arena_reset(TpArena *arena); /* * Destroy the arena, freeing all memory including the * arena structure itself. */ extern void tp_arena_destroy(TpArena *arena); timescale-pg_textsearch-b63fe58/src/memtable/cache.c000066400000000000000000001122711525762047700225510ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * cache.c — lifecycle + apply protocol for the in-memory memtable cache. * * See cache.h for the public contract and ARCHITECTURE.md, * "Memtable Cache", for the design. Three entry points: * * tp_cache_cold_build() cache.apply_lock EXCL + * cache.lock EXCL. Allocates dshash * tables, walks the chain from * meta.head_blkno, populates the * cache, seeds the cursor. * * tp_cache_apply_to_tail() cache.apply_lock EXCL + * cache.lock SHARED. Generation- * checks the cursor, walks from * (cursor.next_blkno, cursor.next_off) * to the logical tail, applies each * record, advances the cursor. * * tp_cache_clear() drops the dshash tables, resets the * apply cursor + estimated_bytes, and * drains accounting in lockstep with * the global counter. Caller-supplied * lock contract (see the function * comment). * * Both paths share the same record-apply primitive * (apply_one_record) that decodes the on-disk TpVector, runs * the memory-cap check, calls tp_cache_apply_document(), and * bumps estimated_bytes. * * The walker (src/memtable/chain_walker.c) handles per-page * traversal, fragment reassembly, and structural validation; * this module only owns the cache-side state machine. * * Scaffold SQL functions at the bottom of this file * (bm25_cache_apply_to_tail, bm25_cache_cold_build) provide * unit-level coverage; the SQL harness lives at * test/sql/cache_apply.sql. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "constants.h" #include "index/metapage.h" #include "index/registry.h" #include "index/resolve.h" #include "index/state.h" #include "memtable/cache.h" #include "memtable/chain_walker.h" #include "memtable/posting.h" #include "memtable/stringtable.h" #include "types/vector.h" /* GUC: in mod.c, declared in kB; we work in bytes. */ extern int tp_memory_limit_kb; /* * Per-index soft cap = global memory limit / 8. * * With the default 2 GiB global limit and 8 concurrent indexes, * each gets 256 MiB of cache headroom before catchup starts * returning BUDGET_EXCEEDED. The global-cap eviction protocol * layers on top. */ #define TP_CACHE_INDEX_CAP_DIVISOR 8 /* * Global soft cap = memory_limit / 2; hard cap = memory_limit. * Both are enforced at apply-protocol entry; the per-index soft * cap still drives the per-record short-circuit inside apply. */ #define TP_CACHE_GLOBAL_SOFT_CAP_DIVISOR 2 static bool cache_locator_matches(const TpMemtable *memtable, Relation rel) { const RelFileLocator *cached = &memtable->cursor_locator; const RelFileLocator *current = &rel->rd_locator; return memtable->cursor_locator_valid && cached->spcOid == current->spcOid && cached->dbOid == current->dbOid && cached->relNumber == current->relNumber; } static void cache_locator_set(TpMemtable *memtable, Relation rel) { memtable->cursor_locator = rel->rd_locator; memtable->cursor_locator_valid = true; } /* * Drop cache contents built from a different relation file before * admission checks can reject work and leave those stale bytes charged. * The caller already holds the per-index lock. */ static bool cache_drop_locator_mismatch( TpLocalIndexState *local_state, Relation rel, TpMemtable *memtable) { bool dropped = false; LWLockAcquire(&memtable->apply_lock, LW_EXCLUSIVE); LWLockAcquire(&memtable->lock, LW_SHARED); if (memtable->cursor_locator_valid && !cache_locator_matches(memtable, rel)) { LWLockRelease(&memtable->lock); LWLockAcquire(&memtable->lock, LW_EXCLUSIVE); if (memtable->cursor_locator_valid && !cache_locator_matches(memtable, rel)) { tp_cache_clear(local_state->dsa, memtable); dropped = true; } } LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return dropped; } uint64 tp_cache_per_index_soft_cap_bytes(void) { if (tp_memory_limit_kb <= 0) return 0; /* sentinel: unlimited */ return ((uint64)tp_memory_limit_kb * 1024UL) / TP_CACHE_INDEX_CAP_DIVISOR; } uint64 tp_cache_global_soft_cap_bytes(void) { if (tp_memory_limit_kb <= 0) return 0; /* sentinel: unlimited */ return ((uint64)tp_memory_limit_kb * 1024UL) / TP_CACHE_GLOBAL_SOFT_CAP_DIVISOR; } uint64 tp_cache_global_hard_cap_bytes(void) { if (tp_memory_limit_kb <= 0) return 0; /* sentinel: unlimited */ return (uint64)tp_memory_limit_kb * 1024UL; } /* ---------- memory accounting ---------- */ /* * Add `delta` bytes to both the per-index and the global * estimated_bytes counters, in lockstep. Caller must hold * cache.apply_lock EXCL on the target memtable; the per-index * counter is single-writer under that lock and the global one is * an atomic so concurrent updaters from other indexes compose * safely. * * The two atomics are deliberately NOT mutated under a single * lock — global accounting and therefore hard-cap enforcement * are approximate by design. */ static inline void account_bytes_add(TpMemtable *memtable, uint64 delta) { pg_atomic_fetch_add_u64(&memtable->estimated_bytes, delta); pg_atomic_fetch_add_u64(tp_registry_estimated_total_bytes(), delta); } /* * Drain the per-index counter to zero and subtract the drained * amount from the global counter. Used by tp_cache_clear and * tp_cache_evict_largest. Symmetric with account_bytes_add so * the global counter stays close to Σ per-index counters; a * raw zero-write of the per-index counter (without the global * subtract) would leak budget into the global counter and stall * future evictions. */ void tp_cache_account_bytes_drain(TpMemtable *memtable) { uint64 old; if (memtable == NULL) return; old = pg_atomic_exchange_u64(&memtable->estimated_bytes, 0); if (old > 0) { pg_atomic_uint64 *gp = tp_registry_estimated_total_bytes(); uint64 cur; uint64 sub; /* Clamp to avoid u64 wrap on accounting drift. */ cur = pg_atomic_read_u64(gp); sub = (old > cur) ? cur : old; if (sub > 0) pg_atomic_fetch_sub_u64(gp, sub); } } /* ---------- lifecycle ---------- */ void tp_cache_clear(dsa_area *dsa, TpMemtable *memtable) { if (dsa == NULL || memtable == NULL) return; if (memtable->string_hash_handle != DSHASH_HANDLE_INVALID) { dshash_table *string_table = tp_string_table_attach(dsa, memtable->string_hash_handle); /* * tp_string_table_clear walks the entries, freeing each * interned term string and its posting list back to the * DSA, then deletes the entry. After that, dshash_destroy * frees the dshash table's own internal allocations. */ tp_string_table_clear(dsa, string_table); dshash_destroy(string_table); memtable->string_hash_handle = DSHASH_HANDLE_INVALID; } if (memtable->doc_lengths_handle != DSHASH_HANDLE_INVALID) { dshash_table *doclength_table = tp_doclength_table_attach(dsa, memtable->doc_lengths_handle); /* * Doc-length entries are POD (ItemPointerData + int32) with * no nested DSA allocations, so dshash_destroy alone is * sufficient. */ dshash_destroy(doclength_table); memtable->doc_lengths_handle = DSHASH_HANDLE_INVALID; } memtable->cursor_gen_spill_count = 0; memtable->cursor_next_blkno = InvalidBlockNumber; memtable->cursor_next_off = 0; memset(&memtable->cursor_locator, 0, sizeof(memtable->cursor_locator)); memtable->cursor_locator_valid = false; pg_atomic_write_u64(&memtable->cursor_seq, 0); /* * Drain estimated_bytes and the matching slice of the global * counter in lockstep so eviction accounting doesn't drift. */ tp_cache_account_bytes_drain(memtable); dsa_trim(dsa); } /* ---------- eviction ---------- */ /* * Argmax callback state. We track (registry key, dsa_pointer, * bytes) of the largest non-caller cache observed so far. The * dsa pointer lets the post-walk path resolve the victim's shared * state without re-walking the registry. */ typedef struct EvictCandidate { TpRegistryKey caller_key; TpRegistryKey best_key; dsa_pointer best_shared_dp; uint64 best_bytes; bool found; } EvictCandidate; /* * Per-entry argmax walker. Reads memtable->estimated_bytes * without taking any cache lock — atomic reads are sufficient, * since the only mutators (apply_one_record adds, * tp_cache_account_bytes_drain subtracts) are themselves atomic * and we only need an approximate maximum for the soft-cap * eviction policy. */ static bool evict_walk_cb(TpRegistryKey key, dsa_pointer shared_dp, void *ctx) { EvictCandidate *c = (EvictCandidate *)ctx; dsa_area *dsa; TpSharedIndexState *st; TpMemtable *mt; uint64 bytes; if (key.database_oid == c->caller_key.database_oid && key.index_oid == c->caller_key.index_oid) return false; if (!DsaPointerIsValid(shared_dp)) return false; dsa = tp_registry_get_dsa(); if (dsa == NULL) return false; st = (TpSharedIndexState *)dsa_get_address(dsa, shared_dp); if (st == NULL || !DsaPointerIsValid(st->memtable_dp)) return false; mt = (TpMemtable *)dsa_get_address(dsa, st->memtable_dp); if (mt == NULL) return false; bytes = pg_atomic_read_u64(&mt->estimated_bytes); if (bytes > c->best_bytes) { c->best_key = key; c->best_shared_dp = shared_dp; c->best_bytes = bytes; c->found = true; } return false; /* don't stop early; scan all entries */ } TpCacheEvictResult tp_cache_evict_largest(TpRegistryKey caller_key) { LWLock *mutex = tp_registry_eviction_mutex(); dsa_area *dsa = tp_registry_get_dsa(); EvictCandidate c; TpSharedIndexState *victim; TpMemtable *vt_memtable; if (dsa == NULL || mutex == NULL) return TP_CACHE_EVICT_NOTHING_FOUND; LWLockAcquire(mutex, LW_EXCLUSIVE); c.caller_key = caller_key; c.best_key = tp_registry_key(InvalidOid, InvalidOid); c.best_shared_dp = InvalidDsaPointer; c.best_bytes = 0; c.found = false; tp_registry_walk(evict_walk_cb, &c); if (!c.found || !DsaPointerIsValid(c.best_shared_dp)) { LWLockRelease(mutex); return TP_CACHE_EVICT_NOTHING_FOUND; } /* * Registry replacements do not take the eviction mutex. Confirm * that the qualified victim key still names the DSA allocation * selected by the walk before dereferencing it. */ if (tp_registry_lookup_dsa(c.best_key) != c.best_shared_dp) { LWLockRelease(mutex); return TP_CACHE_EVICT_NOTHING_FOUND; } victim = (TpSharedIndexState *)dsa_get_address(dsa, c.best_shared_dp); if (victim == NULL || !DsaPointerIsValid(victim->memtable_dp)) { LWLockRelease(mutex); return TP_CACHE_EVICT_NOTHING_FOUND; } /* * Deadlock-prevention: conditionally acquire the victim's * per-index LWLock. A reader on the victim already holds * SHARED there; if that reader is also entering apply protocol * entry and has just acquired the eviction_mutex queue (no, * we hold it EXCL), then... actually the only concurrent * holders of `victim->lock` at this moment are readers in * SHARED. Blocking on EXCL until they drain would be safe * for THIS backend, but would create an * eviction-vs-reader-vs-eviction cycle if a reader on the * victim is itself waiting to enter evict_largest (queues * for our mutex while holding victim's SHARED). Conditional * acquire turns that cycle into a benign retry. */ if (!LWLockConditionalAcquire(&victim->lock, LW_EXCLUSIVE)) { LWLockRelease(mutex); return TP_CACHE_EVICT_BUSY; } vt_memtable = (TpMemtable *)dsa_get_address(dsa, victim->memtable_dp); if (vt_memtable == NULL) { LWLockRelease(&victim->lock); LWLockRelease(mutex); return TP_CACHE_EVICT_NOTHING_FOUND; } /* * Re-check after acquiring victim->lock: another evict caller * is impossible (we hold the mutex), but a tp_spill_finalize * on the victim could have just cleared its cache via the * caller's own per-index EXCL path. Skip if drained. */ if (pg_atomic_read_u64(&vt_memtable->estimated_bytes) == 0) { LWLockRelease(&victim->lock); LWLockRelease(mutex); return TP_CACHE_EVICT_NOTHING_FOUND; } LWLockAcquire(&vt_memtable->apply_lock, LW_EXCLUSIVE); LWLockAcquire(&vt_memtable->lock, LW_EXCLUSIVE); tp_cache_clear(dsa, vt_memtable); LWLockRelease(&vt_memtable->lock); LWLockRelease(&vt_memtable->apply_lock); LWLockRelease(&victim->lock); LWLockRelease(mutex); return TP_CACHE_EVICT_EVICTED; } /* * Apply-protocol entry hook: check the global soft cap and * trigger eviction if it's exceeded. Returns true on success * (cap respected, or eviction made room); false on * hard-cap-exceeded. * * Soft-cap eviction is a best-effort step that runs once per * apply-protocol entry — if it returns BUSY (another reader * contends for the victim's lock) we continue with the apply * anyway, because the hard cap below still guards against * uncontrolled growth. * * Called from tp_cache_apply_to_tail and tp_cache_cold_build * BEFORE either acquires cache.apply_lock, so the caller's own * cache locks are out of the chain. */ static bool global_cap_check(TpRegistryKey caller_key) { uint64 soft = tp_cache_global_soft_cap_bytes(); uint64 hard = tp_cache_global_hard_cap_bytes(); uint64 cur; if (soft == 0 && hard == 0) return true; /* unlimited */ cur = pg_atomic_read_u64(tp_registry_estimated_total_bytes()); if (soft > 0 && cur >= soft) { (void)tp_cache_evict_largest(caller_key); cur = pg_atomic_read_u64(tp_registry_estimated_total_bytes()); } if (hard > 0 && cur >= hard) return false; return true; } /* * Decoded view of one chain record's payload, ready to feed * into tp_cache_apply_document(). * * The arrays are palloc'd in CurrentMemoryContext (typically a * short-lived per-record context); the caller frees them via * MemoryContextReset on the parent context. */ typedef struct DecodedDoc { char **terms; /* term_count entries, NUL-terminated */ int32 *frequencies; /* term_count entries */ int term_count; uint64 estimated_size; /* upper bound to charge to estimated_bytes */ } DecodedDoc; /* * Decode an on-disk TpVector payload into (terms[], freqs[]). * * The on-disk byte stream IS a full v2 TpVector (varlena header, * magic, index name, entry stream) — see tp_memtable_append() * + create_tpvector_from_strings(). Fragment payloads are * already bounds-checked by the walker; inline payloads are * trusted (assert-mode validation lives in chain_source.c's * existing ingest_terms and is not duplicated here, since the * walker has tighter per-page validation). * * estimated_size is an UPPER BOUND on the bytes the apply will * push into the DSA arena: * * - sizeof(TpDocLengthEntry) for the doclength insert * - per term: lexeme bytes (with NUL) + sizeof(TpStringHashEntry) * for the dshash key + sizeof(TpPostingEntry) for * the per-doc posting slot. * * We charge the full per-term cost on every record, even though * shared terms reuse the existing TpStringHashEntry (the * dshash already-present case): this matches the design's "soft * cap" semantics (overshoot is acceptable; under-counting would * let pathological workloads bypass the cap) and avoids paying * for a lookup-before-charge that the apply path will repeat * anyway. */ static void decode_record(const char *vector_bytes, uint32 vector_len, DecodedDoc *out) { TpVector *vec; TpVectorEntry *entry; int n; uint64 sz; if (vector_len == 0) { out->terms = NULL; out->frequencies = NULL; out->term_count = 0; out->estimated_size = sizeof(TpDocLengthEntry); return; } vec = (TpVector *)vector_bytes; n = vec->entry_count; out->term_count = n; out->terms = (n > 0) ? palloc(sizeof(char *) * n) : NULL; out->frequencies = (n > 0) ? palloc(sizeof(int32) * n) : NULL; sz = sizeof(TpDocLengthEntry); entry = get_tpvector_first_entry(vec); for (int i = 0; i < n; i++) { TpVectorEntryView v; char *term_cstr; entry = tpvector_entry_decode_advance(entry, &v); term_cstr = palloc(v.lexeme_len + 1); memcpy(term_cstr, v.lexeme, v.lexeme_len); term_cstr[v.lexeme_len] = '\0'; out->terms[i] = term_cstr; out->frequencies[i] = (int32)v.frequency; /* * Per-term charge: lexeme bytes (+ NUL) for the interned * string, plus the dshash key entry, plus the posting * slot. Underestimates the dshash bucket overhead and * the posting list's growth amortisation; that's the * "soft" in soft cap. */ sz += v.lexeme_len + 1; sz += sizeof(TpStringHashEntry); sz += sizeof(TpPostingEntry); } out->estimated_size = sz; } /* * Apply one decoded record to the cache. Returns true on * success, false if the per-index soft cap would be exceeded * (caller must not advance the cursor in that case). * * Assumes the caller holds cache.apply_lock EXCL (so we are the * sole writer of estimated_bytes) and cache.lock at SHARED or * stronger (which protects the dshash table handles from being * reset by tp_cache_clear). * * Memory accounting is done BEFORE the apply. If the cap check * passes, we apply unconditionally and add the upper-bound * estimate to estimated_bytes — even if the underlying * dshash inserts found existing entries. The asymmetry (charge * the full upper bound, only credit on full drop) is intentional * for the soft cap. */ static bool apply_one_record( TpLocalIndexState *local_state, TpMemtable *memtable, ItemPointer ctid, int32 doc_length, const DecodedDoc *doc, uint64 soft_cap) { uint64 cur_bytes; if (soft_cap > 0) { cur_bytes = pg_atomic_read_u64(&memtable->estimated_bytes); if (cur_bytes + doc->estimated_size > soft_cap) return false; } tp_cache_apply_document( local_state, ctid, doc->terms, doc->frequencies, doc->term_count, doc_length); account_bytes_add(memtable, doc->estimated_size); return true; } /* ---------- walker resume position ---------- */ /* * Translate the cursor's "next" field into a walker open() * position. The walker treats start_off=0 as "start at the * page's first record offset", so we don't have to special-case * the cold-build seed (cursor.next_off == 0, cursor.next_blkno * == meta.head_blkno). */ static inline void cursor_to_walker_seed( TpMemtable *memtable, BlockNumber *out_blkno, uint16 *out_off) { *out_blkno = memtable->cursor_next_blkno; *out_off = memtable->cursor_next_off; } /* ---------- catchup ---------- */ TpCacheApplyResult tp_cache_apply_to_tail(TpLocalIndexState *local_state, Relation rel) { TpMemtable *memtable; uint64 cur_gen; uint64 soft_cap; BlockNumber seed_blkno; uint16 seed_off; TpChainWalker *walker = NULL; TpChainWalkerRecord rec; MemoryContext decode_cxt; MemoryContext walker_cxt; MemoryContext oldcxt; bool budget_ok = true; Assert(local_state != NULL); Assert(local_state->lock_held); Assert(rel != NULL); memtable = get_memtable(local_state); if (memtable == NULL) elog(ERROR, "pg_textsearch cache apply: memtable not allocated for " "index oid=%u", local_state->shared->index_oid); if (cache_drop_locator_mismatch(local_state, rel, memtable)) return TP_CACHE_APPLY_DROPPED; /* * Apply-protocol-entry hook: enforce the global cap before * we take cache.apply_lock. If eviction fails to make room * and we're over the hard cap, return BUDGET_EXCEEDED so * the caller falls back to chain_source. */ if (!global_cap_check( tp_registry_key(MyDatabaseId, local_state->shared->index_oid))) return TP_CACHE_APPLY_BUDGET_EXCEEDED; LWLockAcquire(&memtable->apply_lock, LW_EXCLUSIVE); LWLockAcquire(&memtable->lock, LW_SHARED); if (memtable->cursor_locator_valid && !cache_locator_matches(memtable, rel)) { /* * PostgreSQL can restore the previous relfilenode when a * transaction or savepoint containing REINDEX rolls back. * Generation alone cannot detect that switch if this cache * was repopulated after the REINDEX. Drop the discarded-file * cache before any cursor block is read from the restored file. */ LWLockRelease(&memtable->lock); LWLockAcquire(&memtable->lock, LW_EXCLUSIVE); if (memtable->cursor_locator_valid && !cache_locator_matches(memtable, rel)) tp_cache_clear(local_state->dsa, memtable); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return TP_CACHE_APPLY_DROPPED; } if (memtable->cursor_next_blkno == InvalidBlockNumber) { LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return TP_CACHE_APPLY_NOT_INITIALIZED; } cur_gen = pg_atomic_read_u64(&local_state->shared->spill_generation); if (memtable->cursor_gen_spill_count != cur_gen) { /* * Spill raced through under per-index EXCL between our * caller's per-index SHARED acquire and our cache.lock * acquire. Drop the cache: re-acquire cache.lock EXCL * (immediate, since spill held per-index EXCL until done, * so no other reader's source can be alive across the * spill gap), re-check, clear. */ LWLockRelease(&memtable->lock); LWLockAcquire(&memtable->lock, LW_EXCLUSIVE); cur_gen = pg_atomic_read_u64(&local_state->shared->spill_generation); if (memtable->cursor_gen_spill_count != cur_gen) tp_cache_clear(local_state->dsa, memtable); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return TP_CACHE_APPLY_DROPPED; } soft_cap = tp_cache_per_index_soft_cap_bytes(); cursor_to_walker_seed(memtable, &seed_blkno, &seed_off); walker_cxt = AllocSetContextCreate( CurrentMemoryContext, "pg_textsearch cache apply walker", ALLOCSET_DEFAULT_SIZES); decode_cxt = AllocSetContextCreate( CurrentMemoryContext, "pg_textsearch cache apply decode", ALLOCSET_SMALL_SIZES); walker = tp_chain_walker_open(rel, seed_blkno, seed_off, walker_cxt); PG_TRY(); { while (tp_chain_walker_next(walker, &rec)) { DecodedDoc doc; MemoryContextReset(decode_cxt); oldcxt = MemoryContextSwitchTo(decode_cxt); decode_record(rec.vector_bytes, rec.vector_len, &doc); MemoryContextSwitchTo(oldcxt); if (!apply_one_record( local_state, memtable, &rec.ctid, rec.doc_length, &doc, soft_cap)) { budget_ok = false; break; } memtable->cursor_next_blkno = rec.next_blkno; memtable->cursor_next_off = rec.next_off; pg_atomic_fetch_add_u64(&memtable->cursor_seq, 1); CHECK_FOR_INTERRUPTS(); } } PG_CATCH(); { if (walker) tp_chain_walker_close(walker); MemoryContextDelete(decode_cxt); MemoryContextDelete(walker_cxt); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); PG_RE_THROW(); } PG_END_TRY(); tp_chain_walker_close(walker); MemoryContextDelete(decode_cxt); MemoryContextDelete(walker_cxt); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return budget_ok ? TP_CACHE_APPLY_OK : TP_CACHE_APPLY_BUDGET_EXCEEDED; } /* ---------- cold build ---------- */ TpCacheColdBuildResult tp_cache_cold_build(TpLocalIndexState *local_state, Relation rel) { TpMemtable *memtable; TpIndexMetaPage metap; BlockNumber head; uint64 gen; uint64 soft_cap; TpChainWalker *walker = NULL; TpChainWalkerRecord rec; MemoryContext decode_cxt; MemoryContext walker_cxt; MemoryContext oldcxt; TpCacheColdBuildResult result = TP_CACHE_COLD_OK; Assert(local_state != NULL); Assert(local_state->lock_held); Assert(rel != NULL); memtable = get_memtable(local_state); if (memtable == NULL) elog(ERROR, "pg_textsearch cache cold build: memtable not allocated for " "index oid=%u", local_state->shared->index_oid); (void)cache_drop_locator_mismatch(local_state, rel, memtable); /* * Apply-protocol-entry hook: enforce the global cap before * we take cache.apply_lock. Returning ABORT instead of * building lets the caller fall back to chain_source for * this query without dirtying any cache state. */ if (!global_cap_check( tp_registry_key(MyDatabaseId, local_state->shared->index_oid))) return TP_CACHE_COLD_ABORT; LWLockAcquire(&memtable->apply_lock, LW_EXCLUSIVE); LWLockAcquire(&memtable->lock, LW_EXCLUSIVE); if (memtable->cursor_locator_valid && !cache_locator_matches(memtable, rel)) tp_cache_clear(local_state->dsa, memtable); /* * Lost-race detection: another backend cold-built (or * incrementally caught up) while we were waiting on * cache.lock EXCL. RETRY so the caller goes back to * apply_to_tail. */ if (memtable->cursor_next_blkno != InvalidBlockNumber) { LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return TP_CACHE_COLD_RETRY; } /* Capture generation BEFORE walking so any spill that races * after the walk completes is caught by the next catchup's * generation check, not silently absorbed. */ gen = pg_atomic_read_u64(&local_state->shared->spill_generation); metap = tp_get_metapage(rel); head = metap->memtable_head_blkno; pfree(metap); /* * Empty chain: seed the cursor to the metapage head (which * may be InvalidBlockNumber) and return OK. The next * apply_to_tail will be a no-op until a writer publishes a * page. * * Subtle: if head == InvalidBlockNumber here, we DELIBERATELY * leave cursor_next_blkno at InvalidBlockNumber, which means * the next apply_to_tail call returns NOT_INITIALIZED and * forces another cold_build. This is correct: an empty * chain means there's nothing to cache, and a future writer * that publishes the first page bumps the head — at which * point the next cold_build picks up the new head_blkno. * Treating "empty" as a cached state would require either * watching for the head allocation (we don't) or a separate * "chain head was empty" sentinel cursor state (we don't * want one). */ if (head == InvalidBlockNumber) { LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return TP_CACHE_COLD_OK; } soft_cap = tp_cache_per_index_soft_cap_bytes(); /* * Allocate the dshash tables now that we know there's work * to do. We hold cache.lock EXCL, so it is safe to mutate * memtable->{string_hash_handle, doc_lengths_handle} here: * no reader can be holding a SHARED lifetime lock that * would observe a mid-build handle reset. This mirrors the * tp_ensure_string_table_initialized contract documented in * stringtable.c (the spec requires per-index EXCL for that * helper, but cache.lock EXCL is the moral equivalent for * our cache-handle protection). */ tp_ensure_string_table_initialized(local_state); walker_cxt = AllocSetContextCreate( CurrentMemoryContext, "pg_textsearch cache cold-build walker", ALLOCSET_DEFAULT_SIZES); decode_cxt = AllocSetContextCreate( CurrentMemoryContext, "pg_textsearch cache cold-build decode", ALLOCSET_SMALL_SIZES); /* * Seed the cursor. We commit the generation token now so the * apply loop's cursor advance produces a coherent (gen, blkno, * off) tuple if we hit BUDGET_EXCEEDED partway through * (cleared by tp_cache_clear in the abort branch below). */ memtable->cursor_gen_spill_count = gen; memtable->cursor_next_blkno = head; memtable->cursor_next_off = 0; pg_atomic_write_u64(&memtable->cursor_seq, 0); walker = tp_chain_walker_open(rel, head, 0, walker_cxt); PG_TRY(); { while (tp_chain_walker_next(walker, &rec)) { DecodedDoc doc; MemoryContextReset(decode_cxt); oldcxt = MemoryContextSwitchTo(decode_cxt); decode_record(rec.vector_bytes, rec.vector_len, &doc); MemoryContextSwitchTo(oldcxt); if (!apply_one_record( local_state, memtable, &rec.ctid, rec.doc_length, &doc, soft_cap)) { result = TP_CACHE_COLD_ABORT; break; } memtable->cursor_next_blkno = rec.next_blkno; memtable->cursor_next_off = rec.next_off; pg_atomic_fetch_add_u64(&memtable->cursor_seq, 1); CHECK_FOR_INTERRUPTS(); } } PG_CATCH(); { if (walker) tp_chain_walker_close(walker); MemoryContextDelete(decode_cxt); MemoryContextDelete(walker_cxt); tp_cache_clear(local_state->dsa, memtable); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); PG_RE_THROW(); } PG_END_TRY(); tp_chain_walker_close(walker); MemoryContextDelete(decode_cxt); MemoryContextDelete(walker_cxt); if (result == TP_CACHE_COLD_ABORT) { /* * Free the partially-populated cache so the caller's * fallback to chain_source doesn't have stale dshash * tables hanging around consuming budget. tp_cache_clear * resets the cursor to (Invalid, 0) so the next read * path attempt sees NOT_INITIALIZED and re-tries * cold_build (or gives up and serves from chain_source * again, per the read path's two-iteration loop). */ tp_cache_clear(local_state->dsa, memtable); } else cache_locator_set(memtable, rel); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); return result; } /* --------------------------------------------------------------------- * Scaffold SQL functions. Internal-only unit-test entry points, * complementing the end-to-end coverage in the regression suite. * * bm25_cache_cold_build(idx_name) -> (result, records_applied, * cursor_seq, estimated_bytes) * bm25_cache_apply_to_tail(idx_name) -> (result, records_applied, * cursor_seq, estimated_bytes) * * `records_applied` is computed as the cursor_seq delta between * entry and exit; it intentionally does NOT count records that * were rejected by the BUDGET_EXCEEDED path, because the cursor * is not advanced for those. * --------------------------------------------------------------------- */ PG_FUNCTION_INFO_V1(bm25_cache_cold_build); PG_FUNCTION_INFO_V1(bm25_cache_apply_to_tail); PG_FUNCTION_INFO_V1(bm25_cache_bump_spill_generation); static const char * cold_build_result_text(TpCacheColdBuildResult r) { switch (r) { case TP_CACHE_COLD_OK: return "OK"; case TP_CACHE_COLD_RETRY: return "RETRY"; case TP_CACHE_COLD_ABORT: return "ABORT"; } return "UNKNOWN"; } static const char * apply_result_text(TpCacheApplyResult r) { switch (r) { case TP_CACHE_APPLY_OK: return "OK"; case TP_CACHE_APPLY_DROPPED: return "DROPPED"; case TP_CACHE_APPLY_BUDGET_EXCEEDED: return "BUDGET_EXCEEDED"; case TP_CACHE_APPLY_NOT_INITIALIZED: return "NOT_INITIALIZED"; } return "UNKNOWN"; } static Datum build_result_row( FunctionCallInfo fcinfo, const char *result_text, uint64 records_applied, uint64 cursor_seq, uint64 estimated_bytes) { TupleDesc tupdesc; Datum values[4]; bool nulls[4] = {false, false, false, false}; HeapTuple tup; if (get_call_result_type(fcinfo, NULL, &tupdesc) != TYPEFUNC_COMPOSITE) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("function returning record called in context " "that cannot accept type record"))); tupdesc = BlessTupleDesc(tupdesc); values[0] = CStringGetTextDatum(result_text); values[1] = Int64GetDatum((int64)records_applied); values[2] = Int64GetDatum((int64)cursor_seq); values[3] = Int64GetDatum((int64)estimated_bytes); tup = heap_form_tuple(tupdesc, values, nulls); return HeapTupleGetDatum(tup); } Datum bm25_cache_cold_build(PG_FUNCTION_ARGS) { text *idx_text = PG_GETARG_TEXT_PP(0); char *idx_name = text_to_cstring(idx_text); Oid oid = tp_resolve_index_name_shared(idx_name); Relation rel; TpLocalIndexState *state; TpMemtable *memtable; uint64 seq_before; uint64 seq_after; uint64 bytes_after; TpCacheColdBuildResult r; if (!OidIsValid(oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("pg_textsearch index \"%s\" not found", idx_name))); rel = index_open(oid, AccessShareLock); state = tp_get_local_index_state(oid); if (state == NULL) { index_close(rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("local index state not available for \"%s\"", idx_name))); } tp_acquire_index_lock(state, LW_SHARED); memtable = get_memtable(state); seq_before = (memtable != NULL) ? pg_atomic_read_u64(&memtable->cursor_seq) : 0; r = tp_cache_cold_build(state, rel); memtable = get_memtable(state); seq_after = (memtable != NULL) ? pg_atomic_read_u64(&memtable->cursor_seq) : 0; bytes_after = (memtable != NULL) ? pg_atomic_read_u64(&memtable->estimated_bytes) : 0; tp_release_index_lock(state); index_close(rel, AccessShareLock); return build_result_row( fcinfo, cold_build_result_text(r), seq_after - seq_before, seq_after, bytes_after); } Datum bm25_cache_apply_to_tail(PG_FUNCTION_ARGS) { text *idx_text = PG_GETARG_TEXT_PP(0); char *idx_name = text_to_cstring(idx_text); Oid oid = tp_resolve_index_name_shared(idx_name); Relation rel; TpLocalIndexState *state; TpMemtable *memtable; uint64 seq_before; uint64 seq_after; uint64 bytes_after; TpCacheApplyResult r; if (!OidIsValid(oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("pg_textsearch index \"%s\" not found", idx_name))); rel = index_open(oid, AccessShareLock); state = tp_get_local_index_state(oid); if (state == NULL) { index_close(rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("local index state not available for \"%s\"", idx_name))); } tp_acquire_index_lock(state, LW_SHARED); memtable = get_memtable(state); seq_before = (memtable != NULL) ? pg_atomic_read_u64(&memtable->cursor_seq) : 0; r = tp_cache_apply_to_tail(state, rel); memtable = get_memtable(state); seq_after = (memtable != NULL) ? pg_atomic_read_u64(&memtable->cursor_seq) : 0; bytes_after = (memtable != NULL) ? pg_atomic_read_u64(&memtable->estimated_bytes) : 0; tp_release_index_lock(state); index_close(rel, AccessShareLock); return build_result_row( fcinfo, apply_result_text(r), seq_after - seq_before, seq_after, bytes_after); } /* * bm25_cache_bump_spill_generation(index_name) -> bigint * * Permanent test scaffold for the DROPPED defensive branch of * tp_cache_apply_to_tail. Real spill paths (tp_spill_finalize) * bump shared->spill_generation AND immediately drop the cache's * dshash tables via tp_cache_clear, so a subsequent apply_to_tail * lands on the NOT_INITIALIZED branch instead of DROPPED. This * helper increments the generation without touching the tables, * exposing the DROPPED branch to test coverage so the safety net * doesn't bitrot. The returned value is the new generation. * * INTERNAL-only; revoked from PUBLIC in the install/upgrade SQL. */ Datum bm25_cache_bump_spill_generation(PG_FUNCTION_ARGS) { text *idx_text = PG_GETARG_TEXT_PP(0); char *idx_name = text_to_cstring(idx_text); Oid oid = tp_resolve_index_name_shared(idx_name); TpLocalIndexState *state; uint64 new_gen; if (!OidIsValid(oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("pg_textsearch index \"%s\" not found", idx_name))); state = tp_get_local_index_state(oid); if (state == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("local index state not available for \"%s\"", idx_name))); new_gen = pg_atomic_add_fetch_u64(&state->shared->spill_generation, 1); return Int64GetDatum((int64)new_gen); } /* * bm25_cache_global_estimated_bytes() -> bigint * * Returns the registry-wide estimated_total_bytes counter. * Permanent unit-test scaffold for the memory-cap accounting * protocol. * Production callers should not depend on this being precise: * it's a soft accounting tracker, not a resource accounting * source of truth. * * INTERNAL-only; revoked from PUBLIC in the install/upgrade SQL. */ PG_FUNCTION_INFO_V1(bm25_cache_global_estimated_bytes); Datum bm25_cache_global_estimated_bytes(PG_FUNCTION_ARGS) { pg_atomic_uint64 *p = tp_registry_estimated_total_bytes(); return Int64GetDatum((int64)pg_atomic_read_u64(p)); } /* * bm25_cache_evict_largest(index_name) -> text * * Invokes tp_cache_evict_largest, treating `index_name` as the * caller's "own" index (so the argmax skips it). Returns a * one-line summary string: "evicted", "nothing_found", or * "busy". Used by test/sql/cache_memory_cap.sql to exercise * the eviction protocol without needing concurrent SQL * sessions or a low memory_limit GUC. * * INTERNAL-only; revoked from PUBLIC in the install/upgrade SQL. */ PG_FUNCTION_INFO_V1(bm25_cache_evict_largest); Datum bm25_cache_evict_largest(PG_FUNCTION_ARGS) { text *idx_text = PG_GETARG_TEXT_PP(0); char *idx_name = text_to_cstring(idx_text); Oid oid = tp_resolve_index_name_shared(idx_name); TpCacheEvictResult r; const char *txt; if (!OidIsValid(oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("pg_textsearch index \"%s\" not found", idx_name))); r = tp_cache_evict_largest(tp_registry_key(MyDatabaseId, oid)); switch (r) { case TP_CACHE_EVICT_EVICTED: txt = "evicted"; break; case TP_CACHE_EVICT_BUSY: txt = "busy"; break; case TP_CACHE_EVICT_NOTHING_FOUND: default: txt = "nothing_found"; break; } return PointerGetDatum(cstring_to_text(txt)); } timescale-pg_textsearch-b63fe58/src/memtable/cache.h000066400000000000000000000210341525762047700225520ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * cache.h — lifecycle + apply protocol for the in-memory memtable cache. * * The cache is derived state layered on top of the on-disk * memtable page chain (the source of truth). This module owns * three entry points: two that mutate the cache from chain * records, plus a teardown path. * * tp_cache_cold_build() bootstraps an empty cache from the * chain head. Holds cache.apply_lock * EXCL + cache.lock EXCL (because it * allocates the dshash tables). * * tp_cache_apply_to_tail() advances the cache cursor to the * current logical tail. Holds * cache.apply_lock EXCL + cache.lock * SHARED, so concurrent readers can * keep serving from the cache while * catchup runs. * * tp_cache_clear() drops the dshash tables, resets the * cursor + estimated_bytes, and drains * accounting in lockstep with the * global counter. Called on * generation-mismatch drop, spill * finalize, eviction, and index * teardown. See the per-function * comment below for its lock contract. * * Stale-cache detection: every apply call checks both the cursor's * captured spill_generation and the RelFileLocator from which the * cache was populated. A mismatch drops the cache before any cursor * block is read. * * Lock order: * per-index LWLock (SHARED for read, EXCL for spill) * -> cache.apply_lock (EXCL only) * -> cache.lock (SHARED for readers, EXCL for drop/build) * -> dshash buckets * -> TpPostingList.lock * * Both entry points assume the caller already holds the * per-index LWLock at SHARED or stronger; they acquire and * release everything below it themselves. */ #pragma once #include #include "index/registry.h" #include "index/state.h" #include "utils/rel.h" /* * Outcome of tp_cache_apply_to_tail(). * * OK cursor advanced to the current logical * tail; all locks released. Caller can serve * from the cache. * * DROPPED the cursor's captured spill_generation or * RelFileLocator disagreed with the current * index state; the cache has been cleared and * locks released. Caller (read path) decides * whether to cold_build or fall back to * chain_source. * * BUDGET_EXCEEDED applying the next chain record would push * the cache footprint over the per-index soft * cap. The cursor stays at the pre-record * position (so the next catchup attempt * resumes from the same point if eviction has * reclaimed budget); locks are released. * Caller falls back to chain_source. * * NOT_INITIALIZED the cache has never been cold-built (or has * been cleared); cursor.next_blkno is * InvalidBlockNumber. Caller must invoke * tp_cache_cold_build() first. */ typedef enum TpCacheApplyResult { TP_CACHE_APPLY_OK, TP_CACHE_APPLY_DROPPED, TP_CACHE_APPLY_BUDGET_EXCEEDED, TP_CACHE_APPLY_NOT_INITIALIZED, } TpCacheApplyResult; /* * Outcome of tp_cache_cold_build(). * * OK the cache was bootstrapped from the chain head; all * locks released. Caller can serve from the cache. * * RETRY the cache was already populated when we acquired * cache.lock EXCL (another backend cold-built while we * were waiting). Locks released. Caller should * re-attempt tp_cache_apply_to_tail. * * ABORT the build was aborted because applying a record would * push the cache footprint over the per-index soft cap. * Partial state has been freed (tp_cache_clear), the * cursor is back at (Invalid, 0), locks released. * Caller falls back to chain_source. */ typedef enum TpCacheColdBuildResult { TP_CACHE_COLD_OK, TP_CACHE_COLD_RETRY, TP_CACHE_COLD_ABORT, } TpCacheColdBuildResult; extern TpCacheApplyResult tp_cache_apply_to_tail(TpLocalIndexState *local_state, Relation rel); extern TpCacheColdBuildResult tp_cache_cold_build(TpLocalIndexState *local_state, Relation rel); /* * Per-index soft memory cap, in BYTES (the GUC is in kB). * * Returns the threshold that the apply paths compare * estimated_bytes against on every record. 0 means unlimited. * * Exposed for the SQL test scaffold and the eviction policy; * production callers should not gate on this directly. */ extern uint64 tp_cache_per_index_soft_cap_bytes(void); /* * Global soft and hard memory caps, in BYTES (the GUC is in kB). * Global soft cap = memory_limit / 2; hard cap = memory_limit. * Both return 0 to mean "unlimited" when the GUC is disabled. */ extern uint64 tp_cache_global_soft_cap_bytes(void); extern uint64 tp_cache_global_hard_cap_bytes(void); /* * Outcome of tp_cache_evict_largest(). * * EVICTED a victim was found and its cache cleared; the * drained bytes were subtracted from the global * counter. Caller may re-check the global cap. * * NOTHING_FOUND no eligible victim (no registered index other * than the caller's, or the global counter * dropped under the threshold between argmax and * re-check). Caller falls back to chain_source. * * BUSY the victim's per-index LWLock could not be * conditionally acquired (a reader is in flight). * Caller falls back to chain_source; a later * call may succeed. Returning BUSY rather than * blocking is mandatory to avoid the * eviction-vs-reader deadlock documented in * cache.c (caller holds its own per-index * SHARED; blocking on the victim's per-index * EXCL while a concurrent reader on the victim * has already entered evict_largest would form * a cycle). */ typedef enum TpCacheEvictResult { TP_CACHE_EVICT_EVICTED, TP_CACHE_EVICT_NOTHING_FOUND, TP_CACHE_EVICT_BUSY, } TpCacheEvictResult; /* * Pick the largest cache other than `caller_key` and evict it * (clear its dshash tables, subtract its bytes from the global * counter). Caller MUST hold its own per-index LWLock SHARED * (the read path's natural state); MUST NOT hold any cache lock. */ extern TpCacheEvictResult tp_cache_evict_largest(TpRegistryKey caller_key); /* * Drain `memtable->estimated_bytes` to zero and subtract the * drained amount from the global estimated_total_bytes counter. * Called by tp_cache_clear; exposed for the eviction path which * drains symmetrically. Safe to call when the memtable holds no * bytes (no-op). */ extern void tp_cache_account_bytes_drain(TpMemtable *memtable); /* * Drop the in-memory cache state. Frees the dshash inverted index * and doc-length table (returning their DSA memory to the arena), * resets the apply cursor, source locator, and estimated_bytes to * their post-init values, and trims the DSA. Safe to call when the * cache is already empty (handles == DSHASH_HANDLE_INVALID): a no-op * then. * * Does NOT touch: * - apply_lock / lock: these LWLocks live inside the TpMemtable * struct and must survive across clears so subsequent apply * paths reuse them. The struct itself is freed only by the * surrounding dsa_free on memtable_dp. * - TpSharedIndexState.spill_generation: that's the spill path's * invalidation token; tp_cache_clear is a consumer of it, not * a writer. * - corpus stats in TpSharedIndexState: those reflect the on-disk * chain (source of truth), not the cache. * * Lock contract. Callers that may race against readers * (cold_build retry, generation-mismatch drop, spill_finalize) * acquire cache.lock EXCL before calling. The DROP-INDEX / * subxact-abort teardown path runs while no other backend can be * reading the cache (AccessExclusiveLock on the index, or * shared-state already unregistered); no cache.lock acquisition * is needed there. */ extern void tp_cache_clear(dsa_area *dsa, TpMemtable *memtable); timescale-pg_textsearch-b63fe58/src/memtable/cache_source.c000066400000000000000000000517061525762047700241360ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * cache_source.c — TpDataSource over the in-memory memtable * cache and the read-path chain/cache chooser. * * See cache_source.h for the public contract. The constructor * invokes the cache apply protocol (cache.c) to bring the cache * up to date with the on-disk chain, then attaches the dshash * tables for the source lifetime and returns an ops-table-backed * TpDataSource that serves per-term postings, per-ctid doc * lengths, and per-term doc frequencies out of the cache. */ #include #include #include #include #include #include #include #include #include #include #include #include #include "constants.h" #include "index/resolve.h" #include "index/source.h" #include "index/state.h" #include "memtable/cache.h" #include "memtable/cache_source.h" #include "memtable/chain_source.h" #include "memtable/posting.h" #include "memtable/posting_entry.h" #include "memtable/stringtable.h" /* GUCs read here; declared in mod.c. tp_memtable_cache_enabled * gates the chooser; tp_log_cache_state emits LOG-level traces * at decision points for observability. */ extern bool tp_memtable_cache_enabled; extern bool tp_log_cache_state; /* ---------- source struct ---------- */ typedef struct TpMemtableCacheSource { TpDataSource base; /* must be first */ TpLocalIndexState *state; /* used for DSA + lock helpers */ TpMemtable *memtable; /* cached for close() so we don't * silently skip the lock release * if get_memtable() ever returns * NULL again. */ dshash_table *string_table; dshash_table *doclength_table; /* * If non-NULL, this is the state on which *this* source * acquired the per-index LWLock SHARED; close() releases it. * NULL means an outer caller already held the lock before we * were constructed and is responsible for releasing it. * Mirrors the chain_source convention. */ TpLocalIndexState *lock_state; /* * True iff we hold cache.lock SHARED. Release in close(). * Set false when the constructor releases on the error path * before pfree. */ bool holding_cache_lock; bool holding_apply_lock; } TpMemtableCacheSource; /* ---------- TpDataSourceOps implementations ---------- */ static TpPostingData * cache_get_postings(TpDataSource *source, const char *term) { TpMemtableCacheSource *cs = (TpMemtableCacheSource *)source; TpStringHashEntry *entry; TpPostingList *posting_list; TpPostingEntry *entries; TpPostingData *data; size_t term_len; int32 count; int32 doc_freq; int32 i; if (term == NULL || term[0] == '\0') return NULL; term_len = strlen(term); entry = tp_string_table_lookup( cs->state->dsa, cs->string_table, term, term_len); if (entry == NULL) return NULL; /* * tp_string_table_lookup releases the dshash bucket lock * before returning; the entry pointer remains valid for the * lifetime of the source because we hold cache.lock SHARED * (cache_clear is the only path that removes entries and it * requires cache.lock EXCL). */ if (!DsaPointerIsValid(entry->key.posting_list)) return NULL; posting_list = (TpPostingList *) dsa_get_address(cs->state->dsa, entry->key.posting_list); LWLockAcquire(&posting_list->lock, LW_SHARED); count = posting_list->doc_count; doc_freq = posting_list->doc_freq; if (count <= 0 || !DsaPointerIsValid(posting_list->entries_dp)) { LWLockRelease(&posting_list->lock); return NULL; } entries = (TpPostingEntry *) dsa_get_address(cs->state->dsa, posting_list->entries_dp); data = tp_alloc_posting_data(count); data->count = count; data->doc_freq = doc_freq; for (i = 0; i < count; i++) { data->ctids[i] = entries[i].ctid; data->frequencies[i] = entries[i].frequency; } LWLockRelease(&posting_list->lock); return data; } static void cache_free_postings(TpDataSource *source, TpPostingData *data) { (void)source; tp_free_posting_data(data); } static int32 cache_get_doc_length(TpDataSource *source, ItemPointer ctid) { TpMemtableCacheSource *cs = (TpMemtableCacheSource *)source; return tp_get_document_length_attached(cs->doclength_table, ctid); } static uint32 cache_get_doc_freq(TpDataSource *source, const char *term) { TpMemtableCacheSource *cs = (TpMemtableCacheSource *)source; TpStringHashEntry *entry; TpPostingList *posting_list; uint32 doc_freq; size_t term_len; if (term == NULL || term[0] == '\0') return 0; term_len = strlen(term); entry = tp_string_table_lookup( cs->state->dsa, cs->string_table, term, term_len); if (entry == NULL) return 0; if (!DsaPointerIsValid(entry->key.posting_list)) return 0; posting_list = (TpPostingList *) dsa_get_address(cs->state->dsa, entry->key.posting_list); LWLockAcquire(&posting_list->lock, LW_SHARED); doc_freq = (uint32)posting_list->doc_freq; LWLockRelease(&posting_list->lock); return doc_freq; } static void cache_foreach_document( TpDataSource *source, TpDocumentCallback callback, void *arg) { TpMemtableCacheSource *cs = (TpMemtableCacheSource *)source; dshash_seq_status seq; TpDocLengthEntry *entry; dshash_seq_init(&seq, cs->doclength_table, false); while ((entry = (TpDocLengthEntry *)dshash_seq_next(&seq)) != NULL) callback(&entry->ctid, arg); dshash_seq_term(&seq); } static void cache_close(TpDataSource *source) { TpMemtableCacheSource *cs = (TpMemtableCacheSource *)source; if (cs->string_table != NULL) { dshash_detach(cs->string_table); cs->string_table = NULL; } if (cs->doclength_table != NULL) { dshash_detach(cs->doclength_table); cs->doclength_table = NULL; } if (cs->holding_cache_lock) { Assert(cs->memtable != NULL); LWLockRelease(&cs->memtable->lock); cs->holding_cache_lock = false; } if (cs->holding_apply_lock) { LWLockRelease(&cs->memtable->apply_lock); cs->holding_apply_lock = false; } if (cs->lock_state != NULL) { tp_release_index_lock(cs->lock_state); cs->lock_state = NULL; } pfree(cs); } static const TpDataSourceOps cache_source_ops = { .get_postings = cache_get_postings, .free_postings = cache_free_postings, .get_doc_length = cache_get_doc_length, .get_doc_freq = cache_get_doc_freq, .foreach_document = cache_foreach_document, .close = cache_close, }; /* ---------- helpers ---------- */ /* * Walk the doclength dshash once to compute corpus totals * (doc count + sum of lengths). The caller holds cache.lock * SHARED, which keeps the table stable; the per-bucket locks * are taken by dshash_seq itself. */ static void compute_corpus_totals( TpMemtableCacheSource *cs, int32 *out_total_docs, int64 *out_total_len) { dshash_seq_status seq; TpDocLengthEntry *entry; uint32 count = 0; uint64 sum_len = 0; dshash_seq_init(&seq, cs->doclength_table, false); while ((entry = (TpDocLengthEntry *)dshash_seq_next(&seq)) != NULL) { count++; sum_len += (uint32)entry->doc_length; } dshash_seq_term(&seq); *out_total_docs = (count > (uint32)PG_INT32_MAX) ? PG_INT32_MAX : (int32)count; *out_total_len = (int64)(sum_len > (uint64)PG_INT64_MAX ? (uint64)PG_INT64_MAX : sum_len); } /* * Run the apply-protocol dance: bring the cache up to date with * the chain tail, cold-building once if required, with a bounded * number of retries on RETRY or NOT_INITIALIZED races. Returns * true iff the cache is current and serveable after the call. * * Even after `cold_build` returns OK we must call `apply_to_tail` * again to catch records appended to the chain while the cold * walk was running. We `continue` (not return) in that case. * * Caller must hold the per-index LWLock SHARED. */ static bool catchup_cache(TpLocalIndexState *state, Relation rel) { int retries = 0; while (retries++ < 4) { TpCacheApplyResult ar = tp_cache_apply_to_tail(state, rel); if (ar == TP_CACHE_APPLY_OK) { if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: catchup_ok " "(oid=%u, attempt=%d)", state->shared->index_oid, retries); return true; } if (ar == TP_CACHE_APPLY_BUDGET_EXCEEDED) { if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: aborted_budget " "(oid=%u), falling back to chain", state->shared->index_oid); return false; } Assert(ar == TP_CACHE_APPLY_NOT_INITIALIZED || ar == TP_CACHE_APPLY_DROPPED); { TpCacheColdBuildResult cb = tp_cache_cold_build(state, rel); if (cb == TP_CACHE_COLD_OK) { if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: cold_build_ok " "(oid=%u), re-applying", state->shared->index_oid); /* * Cold build returned OK but new records may have * been appended to the chain while we walked. * Re-enter the loop to apply them. */ continue; } if (cb == TP_CACHE_COLD_RETRY) { if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: cold_build_retry " "(oid=%u)", state->shared->index_oid); continue; } Assert(cb == TP_CACHE_COLD_ABORT); if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: cold_build_aborted " "(oid=%u), falling back to chain", state->shared->index_oid); return false; } } if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: catchup_retries_exhausted " "(oid=%u), falling back to chain", state->shared->index_oid); return false; } /* ---------- public constructors ---------- */ static TpDataSource * tp_memtable_cache_source_create_internal( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count, const TpMemtableChainSnapshot *snapshot) { TpMemtableCacheSource *cs; TpMemtable *memtable; TpLocalIndexState *lock_state_to_release; bool matched_snapshot = false; (void)query_terms; (void)query_term_count; Assert(state != NULL); Assert(rel != NULL); Assert(query_term_count >= 0); Assert(query_term_count == 0 || query_terms != NULL); /* * Mirror chain_source's lock-ownership convention: remember * whether *we* acquired the per-index lock, so close() only * releases what we acquired. If an outer caller already * holds it (e.g., the scan layer), we must not release on * close. */ if (state->lock_held) lock_state_to_release = NULL; else lock_state_to_release = state; tp_acquire_index_lock(state, LW_SHARED); if (lock_state_to_release != NULL && !state->lock_held) lock_state_to_release = NULL; memtable = get_memtable(state); if (memtable == NULL) { if (lock_state_to_release != NULL) tp_release_index_lock(lock_state_to_release); return NULL; } if (snapshot != NULL) { bool empty_snapshot = !BlockNumberIsValid(snapshot->head_blkno); bool locator_matches; bool locator_mismatch; LWLockAcquire(&memtable->apply_lock, LW_SHARED); LWLockAcquire(&memtable->lock, LW_SHARED); locator_matches = memtable->cursor_locator_valid && RelFileLocatorEquals( memtable->cursor_locator, rel->rd_locator); locator_mismatch = memtable->cursor_locator_valid && !locator_matches; matched_snapshot = !empty_snapshot && locator_matches && memtable->cursor_next_blkno == snapshot->tail_blkno && memtable->cursor_next_off == snapshot->tail_free_offset && memtable->cursor_gen_spill_count == pg_atomic_read_u64(&state->shared->spill_generation) && memtable->string_hash_handle != DSHASH_HANDLE_INVALID && memtable->doc_lengths_handle != DSHASH_HANDLE_INVALID; if (!matched_snapshot) { LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); } if (locator_mismatch || empty_snapshot) { /* Drain discarded-file cache bytes even for an empty chain. */ if (locator_mismatch) (void)tp_cache_apply_to_tail(state, rel); if (lock_state_to_release != NULL) tp_release_index_lock(lock_state_to_release); return NULL; } } /* * Run the apply protocol with per-index SHARED held. On a * non-OK terminal outcome the cache cannot serve this query; * fall back to chain_source (signalled by NULL return). */ if (!matched_snapshot && !catchup_cache(state, rel)) { if (lock_state_to_release != NULL) tp_release_index_lock(lock_state_to_release); return NULL; } cs = (TpMemtableCacheSource *)palloc0(sizeof(TpMemtableCacheSource)); cs->base.ops = &cache_source_ops; cs->state = state; cs->memtable = memtable; cs->lock_state = lock_state_to_release; cs->holding_cache_lock = matched_snapshot; cs->holding_apply_lock = matched_snapshot; cs->string_table = NULL; cs->doclength_table = NULL; PG_TRY(); { if (snapshot != NULL && !cs->holding_apply_lock) { LWLockAcquire(&memtable->apply_lock, LW_SHARED); cs->holding_apply_lock = true; } if (!cs->holding_cache_lock) { LWLockAcquire(&memtable->lock, LW_SHARED); cs->holding_cache_lock = true; } /* * Re-check handle validity under cache.lock SHARED. * Currently nothing concurrent can invalidate them (per- * index SHARED excludes spill EXCL, and tp_cache_clear * needs cache.lock EXCL which we now hold SHARED against); * but the check is cheap and protects future changes that * may relax that invariant (e.g. cross-index eviction). * We raise an error rather than return NULL here because * we already returned from catchup_cache with OK, so any * invalidation is structural. */ if (memtable->string_hash_handle == DSHASH_HANDLE_INVALID || memtable->doc_lengths_handle == DSHASH_HANDLE_INVALID) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_textsearch cache_source: dshash handles " "invalidated under cache.lock SHARED " "(oid=%u)", state->shared->index_oid))); cs->string_table = tp_string_table_attach( state->dsa, memtable->string_hash_handle); cs->doclength_table = tp_doclength_table_attach( state->dsa, memtable->doc_lengths_handle); compute_corpus_totals(cs, &cs->base.total_docs, &cs->base.total_len); } PG_CATCH(); { if (cs->string_table != NULL) dshash_detach(cs->string_table); if (cs->doclength_table != NULL) dshash_detach(cs->doclength_table); if (cs->holding_cache_lock) LWLockRelease(&memtable->lock); if (cs->holding_apply_lock) LWLockRelease(&memtable->apply_lock); if (lock_state_to_release != NULL) tp_release_index_lock(lock_state_to_release); pfree(cs); PG_RE_THROW(); } PG_END_TRY(); /* * Deferred reclaim prevents endpoint reuse while the executor's heap * snapshot lives. Freeze cache catch-up until this source is closed. */ if (snapshot != NULL && (memtable->cursor_next_blkno != snapshot->tail_blkno || memtable->cursor_next_off != snapshot->tail_free_offset)) { cache_close((TpDataSource *)cs); return NULL; } if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: opened (oid=%u, total_docs=%d, " "total_len=" INT64_FORMAT ")", state->shared->index_oid, cs->base.total_docs, cs->base.total_len); return (TpDataSource *)cs; } TpDataSource * tp_memtable_cache_source_create( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count) { return tp_memtable_cache_source_create_internal( state, rel, query_terms, query_term_count, NULL); } TpDataSource * tp_memtable_source_create_for_snapshot( TpLocalIndexState *state, Relation rel, const TpMemtableChainSnapshot *snapshot, bool recovery, const char *const *query_terms, int query_term_count) { TpDataSource *source; if (!recovery && tp_memtable_cache_enabled) { source = tp_memtable_cache_source_create_internal( state, rel, query_terms, query_term_count, snapshot); if (source != NULL) return source; } if (!BlockNumberIsValid(snapshot->head_blkno)) return NULL; if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: snapshot uses bounded chain " "(oid=%u)", RelationGetRelid(rel)); return tp_memtable_chain_source_create_bounded( rel, snapshot, query_terms, query_term_count); } TpDataSource * tp_memtable_source_create_for_read( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count) { TpDataSource *src; Assert(state != NULL); Assert(rel != NULL); /* * Gate on the GUC and on contexts where the cache cannot be * trusted to keep up with the chain. Standbys never apply * cache mutations (no shared-memory bumps from WAL replay), * so they must always read the chain directly. In these * cases fall through to chain_source unconditionally. */ if (!tp_memtable_cache_enabled) { if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: disabled_by_guc, " "using chain (oid=%u)", state->shared->index_oid); return tp_memtable_chain_source_create( state, rel, query_terms, query_term_count); } if (RecoveryInProgress()) { if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: disabled_by_recovery, " "using chain (oid=%u)", state->shared->index_oid); return tp_memtable_chain_source_create( state, rel, query_terms, query_term_count); } src = tp_memtable_cache_source_create( state, rel, query_terms, query_term_count); if (src != NULL) return src; if (tp_log_cache_state) elog(LOG, "pg_textsearch cache_source: fallback_to_chain " "(oid=%u)", state->shared->index_oid); return tp_memtable_chain_source_create( state, rel, query_terms, query_term_count); } /* --------------------------------------------------------------------- * Scaffold SQL function for unit-level coverage. * * bm25_test_cache_source(idx_name text, case_name text) -> text * * Internal-only. Exercises the cache_source constructor against * an index and reports terse outcome strings the regression test * compares against. * * Cases: * "empty" open a cache source against an empty memtable; * expect NULL (cache empty just like chain). * "open" open a cache source against a populated * memtable; report total_docs / total_len. * "lookup:" open a cache source and look up term ; * report doc_freq / count, or "miss". * "doclen:" open a cache source and look up the doc length * of ctid (n, 1); reports the value or -1. * * Calls tp_memtable_cache_source_create() directly so the GUC * gating does not affect the test; the chooser is exercised by * end-to-end equivalence queries in test/sql/cache_source.sql. */ PG_FUNCTION_INFO_V1(bm25_test_cache_source); Datum bm25_test_cache_source(PG_FUNCTION_ARGS) { text *idx_name_text = PG_GETARG_TEXT_PP(0); text *case_name_text = PG_GETARG_TEXT_PP(1); const char *idx_name = text_to_cstring(idx_name_text); const char *case_name = text_to_cstring(case_name_text); Oid idx_oid; Relation idx_rel; TpLocalIndexState *state; TpDataSource *src; StringInfoData buf; idx_oid = tp_resolve_index_name_shared(idx_name); if (!OidIsValid(idx_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" not found", idx_name))); idx_rel = index_open(idx_oid, AccessShareLock); state = tp_get_local_index_state(idx_oid); if (state == NULL || state->shared == NULL) { index_close(idx_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("local state not initialized for index \"%s\"", idx_name))); } initStringInfo(&buf); src = tp_memtable_cache_source_create(state, idx_rel, NULL, 0); if (strcmp(case_name, "empty") == 0) { appendStringInfoString( &buf, src == NULL ? "ok:null" : "fail:non-null"); } else if (strcmp(case_name, "open") == 0) { if (src == NULL) appendStringInfoString(&buf, "fail:null"); else appendStringInfo( &buf, "ok:total_docs=%d,total_len=" INT64_FORMAT, src->total_docs, src->total_len); } else if (strncmp(case_name, "lookup:", 7) == 0) { const char *term = case_name + 7; if (src == NULL) appendStringInfoString(&buf, "fail:null"); else { TpPostingData *data = tp_source_get_postings(src, term); uint32 df = tp_source_get_doc_freq(src, term); if (data == NULL) appendStringInfo(&buf, "miss:df=%u", df); else { appendStringInfo(&buf, "ok:df=%u,count=%d", df, data->count); tp_source_free_postings(src, data); } } } else if (strncmp(case_name, "doclen:", 7) == 0) { int blk = atoi(case_name + 7); ItemPointerData ctid; if (src == NULL) appendStringInfoString(&buf, "fail:null"); else { ItemPointerSet(&ctid, (BlockNumber)blk, 1); appendStringInfo( &buf, "ok:len=%d", tp_source_get_doc_length(src, &ctid)); } } else appendStringInfo(&buf, "fail:unknown_case=%s", case_name); if (src != NULL) tp_source_close(src); index_close(idx_rel, AccessShareLock); PG_RETURN_TEXT_P(cstring_to_text(buf.data)); } timescale-pg_textsearch-b63fe58/src/memtable/cache_source.h000066400000000000000000000102621525762047700241330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * cache_source.h — TpDataSource over the in-memory memtable * cache, plus the read-path chooser. * * The cache source serves get_postings / get_doc_length out of * the per-index dshash tables that the apply protocol (cache.c) * keeps in sync with the on-disk chain. It is API-compatible * with `tp_memtable_chain_source_create` so call sites can swap * one for the other transparently. * * Concurrency contract (mirrors chain_source where possible): * * per-index LWLock SHARED (lifetime) * └─► cache.apply_lock (EXCL for apply/cold; * SHARED for bounded reads) * └─► cache.lock SHARED (lifetime, held by served source) * └─► dshash bucket lock (per-lookup) * └─► TpPostingList.lock (per-lookup) * * The cache source holds the per-index LWLock and cache.lock * SHARED for its entire lifetime so concurrent spills (which * require both at EXCL) are excluded. Apply/cold_build are * invoked once at construction with their preconditions * satisfied; the dshash attachments are then kept open until * close(). * * Memory ownership: the source struct is palloc'd in * CurrentMemoryContext. No private MemoryContext is created — * get_postings copies entries out into its own palloc'd * TpPostingData, which the caller frees via free_postings(). */ #pragma once #include #include #include "index/source.h" #include "index/state.h" #include "memtable/chain_walker.h" /* Serve exactly the captured endpoint, falling back to the bounded chain. */ extern TpDataSource *tp_memtable_source_create_for_snapshot( TpLocalIndexState *state, Relation rel, const TpMemtableChainSnapshot *snapshot, bool recovery, const char *const *query_terms, int query_term_count); /* * Construct a cache-backed data source for `rel`. * * On success the returned source holds: * - the per-index LWLock at SHARED (acquired here if not already held) * - cache.lock at SHARED (acquired here, released in close) * - dshash attachments for the string and doc-length tables * * Returns NULL when the cache cannot serve the query — either * because the GUC is disabled, the per-index cache state is * unavailable, the apply protocol returned BUDGET_EXCEEDED, or * a second cold_build/apply attempt still failed. Callers that * need a guaranteed source should use * tp_memtable_source_create_for_read() which falls back to the * chain source on NULL. * * `query_terms` and `query_term_count` are accepted for API * symmetry with the chain source but are currently ignored: the * cache already holds every term, so there is no per-query HTAB * pre-population analog. * * Caller must free via tp_source_close(). */ extern TpDataSource *tp_memtable_cache_source_create( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count); /* * Read-path chooser. * * Returns a TpDataSource that the caller can use without caring * whether the cache or the chain is the backing store. When the * cache GUC is on and the cache successfully catches up to the * chain tail, returns a cache source; otherwise falls through to * tp_memtable_chain_source_create(). Returns NULL only when the * chain itself is empty (matching chain_source's empty-chain * NULL contract). * * Callers should treat the returned source identically to a * chain source: total_docs / total_len are populated, the * per-index LWLock is held SHARED for the source lifetime, * close() releases any locks acquired on behalf of the caller. * * Use this from scoring / standalone query paths. Do NOT use it * from the spill path, debug dumps, or anything that needs the * chain accumulators directly (those continue to call * tp_memtable_chain_source_create explicitly). */ extern TpDataSource *tp_memtable_source_create_for_read( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count); timescale-pg_textsearch-b63fe58/src/memtable/chain_source.c000066400000000000000000001150411525762047700241460ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * chain_source.c - TpDataSource over the on-disk memtable page * chain (issue #374). * * See chain_source.h for the concurrency contract and high-level * semantics. * * A scaffold SQL function is exposed at the bottom of this file * (bm25_test_chain_source) for unit-level coverage, complementing * the end-to-end coverage in the regression suite. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "constants.h" #include "index/metapage.h" #include "index/resolve.h" #include "index/source.h" #include "index/state.h" #include "memtable/chain_source.h" #include "memtable/chain_walker.h" #include "memtable/page.h" #include "types/vector.h" /* * One row in the per-term accumulator hash table. The key is a * NUL-terminated lexeme C string copied into the source memory * context; the value carries parallel (ctid, freq) arrays plus * the rolling doc_freq. Arrays grow geometrically. */ typedef struct ChainTermEntry { const char *term; /* hash key; pointer into source mcxt */ ItemPointerData *ctids; /* count slots; in source mcxt */ int32 *frequencies; /* count slots; in source mcxt */ int32 count; /* records in this term */ int32 capacity; /* allocated array capacity */ int32 doc_freq; /* documents containing the term */ } ChainTermEntry; /* * One row in the per-ctid doc-length hash table. Keyed by raw * ItemPointerData (HASH_BLOBS). Records (ctid → doc_length); * duplicate ctids are an invariant violation (each heap tuple * writes exactly one chain record) and we surface them as * DATA_CORRUPTED at open time. */ typedef struct ChainDocLenEntry { ItemPointerData ctid; int32 doc_length; } ChainDocLenEntry; typedef struct TpMemtableChainSource { TpDataSource base; /* must be first */ MemoryContext mcxt; /* private context, deleted in close() */ HTAB *term_ht; /* char* → ChainTermEntry */ HTAB *doclen_ht; /* ItemPointerData → ChainDocLenEntry */ uint32 chain_pages; /* total chain pages walked * (regular + continuation); used to * seed shared->chain_page_count when * a backend reopens an index whose * counter atomic was reset to 0 by * shmem init. */ bool filter_terms; /* true when constructor was passed * a non-NULL query_terms[]: the * term HTAB is pre-populated with * those terms (and only those) at * construction time, and ingest_terms * uses HASH_FIND (not HASH_ENTER) so * non-query terms are skipped. See * issue #374, query-term filtering. */ TpLocalIndexState *lock_state; /* state we acquired the per-index * LWLock on (NULL if the lock was * already held by an outer caller * before this source was created). */ } TpMemtableChainSource; /* ---------- hash helpers ---------- */ static uint32 chain_term_hash(const void *key, Size keysize) { const char *term = *(const char *const *)key; (void)keysize; return DatumGetUInt32(hash_any((const unsigned char *)term, strlen(term))); } static int chain_term_match(const void *key1, const void *key2, Size keysize) { const char *t1 = *(const char *const *)key1; const char *t2 = *(const char *const *)key2; (void)keysize; return strcmp(t1, t2); } /* ---------- term-table append helpers ---------- */ static ChainTermEntry * lookup_or_create_term( TpMemtableChainSource *src, const char *term, int term_len) { bool found; const char *key; ChainTermEntry *entry; /* * The HTAB key is `const char *` stored in the entry's * first field. At HASH_ENTER, dynahash memcpy's `*keyPtr` * (i.e. the pointer value `term`) into entry->term. Since * `term` here may be a transient stack buffer we must * overwrite entry->term with a durable copy living in * src->mcxt before any subsequent lookup. Overwriting the * first field is safe because the hash bucket is determined * by the *bytes* the pointer references, not the pointer * address, and the bytes are unchanged after we copy them. * * Filter mode (set when the constructor was passed a * non-NULL query_terms[]): query terms have already been * pre-inserted into the HTAB, so HASH_FIND is sufficient and * any miss means the lexeme is not a query term — return * NULL so the caller skips term_entry_append. This is the * key win: for an N-term query against a memtable with M * unique lexemes, we do O(N) HASH_ENTERs at construction + * O(records × terms_per_record) HASH_FINDs during the walk, * vs. O(records × terms_per_record) HASH_ENTERs in the * legacy unfiltered path. HASH_FIND is materially cheaper * (no allocator traffic, no bucket-split rehash). */ key = term; if (src->filter_terms) { entry = (ChainTermEntry *) hash_search(src->term_ht, &key, HASH_FIND, &found); return found ? entry : NULL; } entry = (ChainTermEntry *) hash_search(src->term_ht, &key, HASH_ENTER, &found); if (!found) { MemoryContext old = MemoryContextSwitchTo(src->mcxt); char *copy; copy = palloc(term_len + 1); memcpy(copy, term, term_len); copy[term_len] = '\0'; entry->term = copy; /* overwrites the HTAB key slot */ entry->ctids = NULL; entry->frequencies = NULL; entry->count = 0; entry->capacity = 0; entry->doc_freq = 0; MemoryContextSwitchTo(old); } return entry; } static void term_entry_append( TpMemtableChainSource *src, ChainTermEntry *entry, ItemPointer ctid, int32 frequency) { if (entry->count == entry->capacity) { MemoryContext old = MemoryContextSwitchTo(src->mcxt); int32 new_cap = entry->capacity == 0 ? 4 : entry->capacity * 2; if (entry->ctids == NULL) { entry->ctids = palloc(new_cap * sizeof(ItemPointerData)); entry->frequencies = palloc(new_cap * sizeof(int32)); } else { entry->ctids = repalloc(entry->ctids, new_cap * sizeof(ItemPointerData)); entry->frequencies = repalloc(entry->frequencies, new_cap * sizeof(int32)); } entry->capacity = new_cap; MemoryContextSwitchTo(old); } entry->ctids[entry->count] = *ctid; entry->frequencies[entry->count] = frequency; entry->count++; entry->doc_freq++; } /* ---------- record ingestion ---------- */ /* * Record one (ctid → doc_length) row in the source's * doc-length HTAB. Duplicate ctids in the chain would indicate * a corrupt write (each heap tuple writes exactly one chain * record); ereport on collision. */ static void ingest_doclen(TpMemtableChainSource *src, ItemPointer ctid, int32 doc_length) { bool found; ChainDocLenEntry *dle; dle = (ChainDocLenEntry *) hash_search(src->doclen_ht, ctid, HASH_ENTER, &found); if (found) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch chain has duplicate ctid (%u,%u)", ItemPointerGetBlockNumber(ctid), ItemPointerGetOffsetNumber(ctid)))); dle->doc_length = doc_length; } /* * Decode a v2 TpVector payload and append (ctid, freq) entries * to per-term accumulators. Payload may be inline on a record * or reassembled across continuation pages by the caller. */ static void ingest_terms( TpMemtableChainSource *src, ItemPointer ctid, const char *vector_bytes, uint32 vector_len) { TpVector *vec; TpVectorEntry *entry; if (vector_len == 0) return; /* * The bytes here come from chain pages we wrote ourselves * via `tp_add_document_terms` → `create_tpvector_from_strings`, * which always emits a well-formed v2 buffer. Disk-level * corruption is caught by the page CRC in the buffer * manager, and structural corruption of an in-process write * would point at our own bug rather than untrusted input. * Run a full bounds-check only in assert-enabled builds so * we don't pay for re-walking every varint on the read path. */ #ifdef USE_ASSERT_CHECKING tpvector_validate_v2_buffer(vector_bytes, vector_len); #endif vec = (TpVector *)vector_bytes; entry = get_tpvector_first_entry(vec); for (int i = 0; i < vec->entry_count; i++) { TpVectorEntryView v; ChainTermEntry *te; char stackbuf[256]; char *term_cstr; /* * Decode-and-advance in a single pass: returns the next * entry pointer in addition to populating `v`, so we * don't pay for a second varint decode of (freq, lex_len) * just to advance the cursor. The legacy pattern * (`tpvector_entry_decode` + `get_tpvector_next_entry`) * decoded both varints twice per entry, which became the * dominant per-record cost on read-heavy workloads * against the on-disk memtable chain (issue #374). */ entry = tpvector_entry_decode_advance(entry, &v); /* * Stack-allocate the lookup key for short lexemes * (the common case in English / European text after * stemming). Fall back to a palloc for unusually * long ones. The stack copy is reused per record * and not retained. */ if (v.lexeme_len < sizeof(stackbuf)) term_cstr = stackbuf; else term_cstr = palloc(v.lexeme_len + 1); memcpy(term_cstr, v.lexeme, v.lexeme_len); term_cstr[v.lexeme_len] = '\0'; te = lookup_or_create_term(src, term_cstr, (int)v.lexeme_len); if (te != NULL) term_entry_append(src, te, ctid, (int32)v.frequency); if (term_cstr != stackbuf) pfree(term_cstr); } } /* ---------- chain walk ---------- */ /* * Walk every record in the on-disk chain and ingest it into * src->doclen_ht and src->term_ht. Delegates page-level * traversal (buffer locking, fragment reassembly, structural * validation) to the shared TpChainWalker primitive in * chain_walker.c so the apply protocol (cache.c) can reuse it. */ static void walk_chain( TpMemtableChainSource *src, Relation rel, const TpMemtableChainSnapshot *snapshot) { TpChainWalker *walker; TpChainWalkerRecord rec; if (snapshot != NULL) walker = tp_chain_walker_open_bounded(rel, snapshot, src->mcxt, false); else { TpIndexMetaPage metap = tp_get_metapage(rel); BlockNumber start = metap->memtable_head_blkno; pfree(metap); walker = tp_chain_walker_open(rel, start, 0, src->mcxt); } while (tp_chain_walker_next(walker, &rec)) { ingest_doclen(src, &rec.ctid, rec.doc_length); ingest_terms(src, &rec.ctid, rec.vector_bytes, rec.vector_len); /* * Fragment payloads are palloc'd in src->mcxt by the * walker; the buffer survives until src is closed. We * could pfree() here to bound peak usage, but the source * mcxt is short-lived (per-query) and fragment payloads * are rare enough that the bookkeeping isn't worth it. */ } src->chain_pages = tp_chain_walker_pages_visited(walker); tp_chain_walker_close(walker); } /* ---------- TpDataSourceOps implementations ---------- */ static TpPostingData * chain_get_postings(TpDataSource *source, const char *term) { TpMemtableChainSource *src = (TpMemtableChainSource *)source; ChainTermEntry *entry; const char *key; bool found; TpPostingData *data; if (term == NULL || term[0] == '\0') return NULL; key = term; entry = (ChainTermEntry *) hash_search(src->term_ht, &key, HASH_FIND, &found); if (!found || entry->count == 0) return NULL; data = tp_alloc_posting_data(entry->count); data->count = entry->count; data->doc_freq = entry->doc_freq; memcpy(data->ctids, entry->ctids, entry->count * sizeof(ItemPointerData)); memcpy(data->frequencies, entry->frequencies, entry->count * sizeof(int32)); return data; } static void chain_free_postings(TpDataSource *source, TpPostingData *data) { (void)source; tp_free_posting_data(data); } static int32 chain_get_doc_length(TpDataSource *source, ItemPointer ctid) { TpMemtableChainSource *src = (TpMemtableChainSource *)source; ChainDocLenEntry *dle; bool found; dle = (ChainDocLenEntry *) hash_search(src->doclen_ht, ctid, HASH_FIND, &found); if (!found) return -1; return dle->doc_length; } static uint32 chain_get_doc_freq(TpDataSource *source, const char *term) { TpMemtableChainSource *src = (TpMemtableChainSource *)source; ChainTermEntry *entry; const char *key; bool found; if (term == NULL || term[0] == '\0') return 0; key = term; entry = (ChainTermEntry *) hash_search(src->term_ht, &key, HASH_FIND, &found); if (!found) return 0; return (uint32)entry->doc_freq; } static void chain_foreach_document( TpDataSource *source, TpDocumentCallback callback, void *arg) { TpMemtableChainSource *src = (TpMemtableChainSource *)source; HASH_SEQ_STATUS sequence; ChainDocLenEntry *entry; hash_seq_init(&sequence, src->doclen_ht); while ((entry = hash_seq_search(&sequence)) != NULL) callback(&entry->ctid, arg); } static void chain_close(TpDataSource *source) { TpMemtableChainSource *src = (TpMemtableChainSource *)source; if (src->mcxt != NULL) MemoryContextDelete(src->mcxt); if (src->lock_state != NULL) tp_release_index_lock(src->lock_state); pfree(src); } static const TpDataSourceOps chain_source_ops = { .get_postings = chain_get_postings, .free_postings = chain_free_postings, .get_doc_length = chain_get_doc_length, .get_doc_freq = chain_get_doc_freq, .foreach_document = chain_foreach_document, .close = chain_close, }; /* ---------- shared constructor ---------- */ static TpDataSource * tp_memtable_chain_source_create_internal( TpLocalIndexState *state, Relation rel, const TpMemtableChainSnapshot *snapshot, bool acquire_index_lock, const char *const *query_terms, int query_term_count) { TpMemtableChainSource *src; MemoryContext mcxt; HASHCTL info; TpLocalIndexState *lock_state_to_release; Assert(state != NULL || !acquire_index_lock); Assert(rel != NULL); /* * A bounded (snapshot) source never acquires the lock, so the * empty-snapshot early return below has nothing to release. */ Assert(snapshot == NULL || !acquire_index_lock); Assert(query_term_count >= 0); Assert(query_term_count == 0 || query_terms != NULL); /* * The ordinary path acquires the per-index LWLock in SHARED mode for * the whole scan. The bounded path instead relies on its * copied endpoint and deferred reclaim, because WAL replay does not * acquire this extension lock. * * Lock ownership: if the caller already held the lock (e.g., * scan.c acquires SHARED for the whole scan and we're being * created from inside that scope), tp_acquire_index_lock is a * no-op and *we* must not release on close — otherwise the * outer caller would suddenly be operating without the lock. * We remember whether we ourselves grabbed the lock; close() * releases only when we own the acquisition. Releasing * eagerly in close() (instead of deferring to the xact-end * callback) is required because the LWLock memory lives in * the shared DSA region, which is freed by * tp_cleanup_index_shared_memory() when an index is dropped * (including ON COMMIT DROP on a temp table); leaving the * lock held past the source's lifetime risks the * LWLockReleaseAll() at xact-end dereferencing freed memory. */ lock_state_to_release = NULL; if (acquire_index_lock) { if (!state->lock_held) lock_state_to_release = state; tp_acquire_index_lock(state, LW_SHARED); /* * Defensive: if our acquire was a no-op because the caller * already held an exclusive lock (e.g., spill path), still * don't release on close(). */ if (lock_state_to_release != NULL && !state->lock_held) lock_state_to_release = NULL; } /* * A supplied snapshot already contains the empty-chain result. The * ordinary constructor retains its quick metapage check under the * per-index lock. */ if (snapshot != NULL && !BlockNumberIsValid(snapshot->head_blkno)) return NULL; if (snapshot == NULL) { Buffer metabuf; bool empty; metabuf = ReadBuffer(rel, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_SHARE); /* * Use the version-tolerant helper (issue #383): on a v6 * page the on-disk field bytes are zero (= block 0, * the metapage itself), not InvalidBlockNumber. */ empty = (tp_metapage_read_memtable_head(BufferGetPage(metabuf)) == InvalidBlockNumber); UnlockReleaseBuffer(metabuf); if (empty) { /* * Release the lock we just acquired (if any) before * returning NULL — the chain_close path doesn't run * for a NULL source. */ if (lock_state_to_release != NULL) tp_release_index_lock(lock_state_to_release); return NULL; } } /* * Allocate the source struct in CurrentMemoryContext so it * survives our private mcxt deletion in close(). All * accumulator memory lives inside mcxt. */ src = (TpMemtableChainSource *)palloc0(sizeof(TpMemtableChainSource)); mcxt = AllocSetContextCreate( CurrentMemoryContext, "pg_textsearch chain source", ALLOCSET_DEFAULT_SIZES); src->mcxt = mcxt; src->base.ops = &chain_source_ops; src->lock_state = lock_state_to_release; src->filter_terms = (query_terms != NULL); { MemoryContext old = MemoryContextSwitchTo(mcxt); long initial_term_buckets; /* * Filter mode: pre-size the term HTAB to exactly the * number of query terms (rounded up to dynahash's * minimum). Unfiltered mode keeps the legacy 1024 * default since it must accommodate the entire term * dictionary in the chain (used by the spill path). */ initial_term_buckets = src->filter_terms ? (long)query_term_count + 1 : 1024L; memset(&info, 0, sizeof(info)); info.keysize = sizeof(char *); info.entrysize = sizeof(ChainTermEntry); info.hash = chain_term_hash; info.match = chain_term_match; info.hcxt = mcxt; src->term_ht = hash_create( "pg_textsearch chain source: terms", initial_term_buckets, &info, HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT); memset(&info, 0, sizeof(info)); info.keysize = sizeof(ItemPointerData); info.entrysize = sizeof(ChainDocLenEntry); info.hcxt = mcxt; src->doclen_ht = hash_create( "pg_textsearch chain source: doc lengths", 1024, &info, HASH_ELEM | HASH_BLOBS | HASH_CONTEXT); MemoryContextSwitchTo(old); } /* * Filter mode: pre-insert query terms into the term HTAB * with empty postings. The chain walk's ingest_terms uses * HASH_FIND-only against this HTAB, so any lexeme not * pre-inserted is skipped (no posting list materialised, no * dynahash growth, no per-term memcpy). This is the * dominant read-path speedup for queries against a sizeable * memtable: instead of building an HTAB with every unique * lexeme in the memtable (and then reading only N of them), * we build an HTAB with exactly the N terms the caller is * going to read. Issue #374. */ if (src->filter_terms) { MemoryContext old = MemoryContextSwitchTo(mcxt); for (int q = 0; q < query_term_count; q++) { const char *qt = query_terms[q]; Size qt_len; char *copy; ChainTermEntry *entry; const char *key; bool found; if (qt == NULL || qt[0] == '\0') continue; qt_len = strlen(qt); copy = palloc(qt_len + 1); memcpy(copy, qt, qt_len + 1); /* * Briefly flip filter_terms off so this HASH_ENTER * goes through the normal create path. We re-enable * filtering after pre-insertion completes; the walk * uses HASH_FIND-only. */ src->filter_terms = false; key = copy; entry = (ChainTermEntry *) hash_search(src->term_ht, &key, HASH_ENTER, &found); src->filter_terms = true; if (!found) { entry->term = copy; entry->ctids = NULL; entry->frequencies = NULL; entry->count = 0; entry->capacity = 0; entry->doc_freq = 0; } else { /* * Duplicate query terms (e.g. "the the") collapse * into a single HTAB entry — already present from * an earlier iteration, nothing more to do. */ pfree(copy); } } MemoryContextSwitchTo(old); } walk_chain(src, rel, snapshot); /* * Corpus-level counters: report just the in-flight memtable * contribution (derived from this chain). Scoring callers * (bm25.c, types/query.c) layer this on top of the * persisted-segment totals stored on the metapage to obtain * whole-index totals. */ { long doc_count; int64 total_len = 0; HASH_SEQ_STATUS seq; ChainDocLenEntry *dle; doc_count = hash_get_num_entries(src->doclen_ht); hash_seq_init(&seq, src->doclen_ht); while ((dle = (ChainDocLenEntry *)hash_seq_search(&seq)) != NULL) total_len += dle->doc_length; src->base.total_docs = (int32)doc_count; src->base.total_len = total_len; } return (TpDataSource *)src; } /* ---------- public constructors ---------- */ TpDataSource * tp_memtable_chain_source_create( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count) { return tp_memtable_chain_source_create_internal( state, rel, NULL, true, query_terms, query_term_count); } TpDataSource * tp_memtable_chain_source_create_bounded( Relation rel, const TpMemtableChainSnapshot *snapshot, const char *const *query_terms, int query_term_count) { Assert(snapshot != NULL); return tp_memtable_chain_source_create_internal( NULL, rel, snapshot, false, query_terms, query_term_count); } /* * Return the total number of chain pages walked by a chain source. * Includes both regular memtable pages and continuation pages of * any fragments encountered. Matches the writer's * shared->chain_page_count accounting (which adds +1 per regular * page and +(2 + n_cont) per fragment publish). Used by * tp_rebuild_index_from_disk to seed the atomic counter when a * backend reopens an index whose shmem state was just (re-)created. */ uint32 tp_memtable_chain_source_page_count(TpDataSource *base) { TpMemtableChainSource *src; if (base == NULL) return 0; Assert(base->ops == &chain_source_ops); src = (TpMemtableChainSource *)base; return src->chain_pages; } /* ---------- public extractor ---------- */ #include "segment/dictionary.h" #include "segment/docmap.h" static int term_info_cmp(const void *a, const void *b) { const TermInfo *ta = (const TermInfo *)a; const TermInfo *tb = (const TermInfo *)b; int r = strcmp(ta->term, tb->term); return r; } void tp_memtable_chain_source_extract( TpDataSource *base, MemoryContext dest_mcxt, TermInfo **out_terms, uint32 *out_num_terms, TpDocMapBuilder **out_docmap) { TpMemtableChainSource *src = (TpMemtableChainSource *)base; MemoryContext old; HASH_SEQ_STATUS seq; ChainTermEntry *te; ChainDocLenEntry *dle; uint32 num_terms; uint32 i; TermInfo *terms; TpDocMapBuilder *docmap; Assert(base != NULL); Assert(base->ops == &chain_source_ops); Assert(out_terms != NULL && out_num_terms != NULL && out_docmap != NULL); old = MemoryContextSwitchTo(dest_mcxt); /* Build docmap from the doclen HTAB. */ docmap = tp_docmap_create(); hash_seq_init(&seq, src->doclen_ht); while ((dle = (ChainDocLenEntry *)hash_seq_search(&seq)) != NULL) tp_docmap_add(docmap, &dle->ctid, (uint32)dle->doc_length); tp_docmap_finalize(docmap); /* Materialize the term dictionary. */ num_terms = (uint32)hash_get_num_entries(src->term_ht); if (num_terms == 0) { *out_terms = NULL; *out_num_terms = 0; *out_docmap = docmap; MemoryContextSwitchTo(old); return; } terms = (TermInfo *)palloc_extended( num_terms * sizeof(TermInfo), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); i = 0; hash_seq_init(&seq, src->term_ht); while ((te = (ChainTermEntry *)hash_seq_search(&seq)) != NULL) { uint32 c = (uint32)te->count; terms[i].term_len = (uint32)strlen(te->term); terms[i].term = (char *)palloc(terms[i].term_len + 1); memcpy(terms[i].term, te->term, terms[i].term_len + 1); terms[i].count = c; terms[i].doc_freq = (uint32)te->doc_freq; if (c > 0) { terms[i].ctids = (ItemPointerData *)palloc( c * sizeof(ItemPointerData)); memcpy(terms[i].ctids, te->ctids, c * sizeof(ItemPointerData)); terms[i].freqs = (int32 *)palloc(c * sizeof(int32)); memcpy(terms[i].freqs, te->frequencies, c * sizeof(int32)); } i++; } Assert(i == num_terms); /* Sort lexicographically for the segment writer's dict layout. */ qsort(terms, num_terms, sizeof(TermInfo), term_info_cmp); for (i = 0; i < num_terms; i++) terms[i].dict_entry_idx = i; *out_terms = terms; *out_num_terms = num_terms; *out_docmap = docmap; MemoryContextSwitchTo(old); } /* --------------------------------------------------------------------- * Scaffold SQL function for unit-level coverage. * * bm25_test_chain_source(idx_name text, case_name text) -> text * * Each case writes one or more records via tp_memtable_append() * with real TpVector payloads, then opens a chain source over * the same index and verifies the read-back through the * TpDataSource interface. Returns 'OK' or 'FAIL: '. * * The harness drops and re-creates the host index between test * cases (in the SQL harness) so each case starts on an empty * chain. * --------------------------------------------------------------------- */ #include "memtable/log.h" PG_FUNCTION_INFO_V1(bm25_test_chain_source); #define TEST_FAIL(fmt, ...) \ do \ { \ char *_buf = psprintf("FAIL: " fmt, ##__VA_ARGS__); \ if (rel) \ index_close(rel, RowExclusiveLock); \ if (src) \ tp_source_close(src); \ PG_RETURN_TEXT_P(cstring_to_text(_buf)); \ } while (0) #define TEST_OK() \ do \ { \ if (src) \ tp_source_close(src); \ index_close(rel, RowExclusiveLock); \ PG_RETURN_TEXT_P(cstring_to_text("OK")); \ } while (0) static TpVector * test_make_tpvector( const char *index_name, int n_terms, const char **terms, const int32 *freqs) { return create_tpvector_from_strings(index_name, n_terms, terms, freqs); } static int32 test_append_terms( Relation rel, const char *idx_name, ItemPointer ctid, int n_terms, const char **terms, const int32 *freqs) { TpVector *v = test_make_tpvector(idx_name, n_terms, terms, freqs); int32 doc_length = 0; for (int i = 0; i < n_terms; i++) doc_length += freqs[i]; tp_memtable_append(rel, ctid, doc_length, (const char *)v, VARSIZE(v)); pfree(v); return doc_length; } Datum bm25_test_chain_source(PG_FUNCTION_ARGS) { text *idx_text = PG_GETARG_TEXT_PP(0); text *case_text = PG_GETARG_TEXT_PP(1); char *idx_name = text_to_cstring(idx_text); char *case_name = text_to_cstring(case_text); Oid oid = tp_resolve_index_name_shared(idx_name); Relation rel = NULL; TpDataSource *src = NULL; TpLocalIndexState *state; if (!OidIsValid(oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("pg_textsearch index \"%s\" not found", idx_name))); rel = index_open(oid, RowExclusiveLock); state = tp_get_local_index_state(oid); if (state == NULL) TEST_FAIL("local index state not available"); if (strcmp(case_name, "empty_chain") == 0) { ItemPointerData any_ctid; src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src != NULL) TEST_FAIL("chain source over empty index is non-NULL"); ItemPointerSet(&any_ctid, 1, 1); /* Without a source we cannot exercise the API, so just OK. */ TEST_OK(); } else if (strcmp(case_name, "one_doc_one_term") == 0) { ItemPointerData ctid; const char *terms[] = {"alpha"}; int32 freqs[] = {3}; int32 expected_doc_length; TpPostingData *post; ItemPointerSet(&ctid, 100, 1); expected_doc_length = test_append_terms(rel, idx_name, &ctid, 1, terms, freqs); src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL after one append"); if (src->total_docs != 1) TEST_FAIL("total_docs=%d, expected 1", src->total_docs); if (src->total_len != expected_doc_length) TEST_FAIL( "total_len=%lld, expected %d", (long long)src->total_len, expected_doc_length); post = tp_source_get_postings(src, "alpha"); if (post == NULL) TEST_FAIL("get_postings(alpha) returned NULL"); if (post->count != 1) TEST_FAIL("count=%d, expected 1", post->count); if (post->doc_freq != 1) TEST_FAIL("doc_freq=%d, expected 1", post->doc_freq); if (!ItemPointerEquals(&post->ctids[0], &ctid)) TEST_FAIL("ctid roundtrip mismatch"); if (post->frequencies[0] != 3) TEST_FAIL("freq=%d, expected 3", post->frequencies[0]); tp_source_free_postings(src, post); if (tp_source_get_doc_length(src, &ctid) != expected_doc_length) TEST_FAIL("get_doc_length mismatch"); TEST_OK(); } else if (strcmp(case_name, "one_doc_no_terms") == 0) { ItemPointerData ctid; TpVector *v; ItemPointerSet(&ctid, 800, 1); v = test_make_tpvector(idx_name, 0, NULL, NULL); tp_memtable_append(rel, &ctid, 7, (const char *)v, VARSIZE(v)); pfree(v); src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL after termless append"); if (src->total_docs != 1) TEST_FAIL("total_docs=%d, expected 1", src->total_docs); if (src->total_len != 7) TEST_FAIL("total_len=%lld, expected 7", (long long)src->total_len); TEST_OK(); } else if (strcmp(case_name, "multi_doc_shared_term") == 0) { const char *terms[] = {"shared"}; int32 freqs[] = {1}; const int N = 5; TpPostingData *post; ItemPointerData expected_ctids[5]; for (int i = 0; i < N; i++) { ItemPointerSet( &expected_ctids[i], (BlockNumber)(200 + i), (OffsetNumber)(i + 1)); test_append_terms( rel, idx_name, &expected_ctids[i], 1, terms, freqs); } src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL"); post = tp_source_get_postings(src, "shared"); if (post == NULL) TEST_FAIL("get_postings(shared) NULL"); if (post->count != N) TEST_FAIL("count=%d, expected %d", post->count, N); if (post->doc_freq != N) TEST_FAIL("doc_freq=%d, expected %d", post->doc_freq, N); for (int i = 0; i < N; i++) { if (!ItemPointerEquals(&post->ctids[i], &expected_ctids[i])) TEST_FAIL("ctid mismatch at i=%d", i); if (post->frequencies[i] != 1) TEST_FAIL( "freq mismatch at i=%d: got %d", i, post->frequencies[i]); } tp_source_free_postings(src, post); TEST_OK(); } else if (strcmp(case_name, "multi_term") == 0) { ItemPointerData ctid; const char *terms[] = {"red", "green", "blue"}; int32 freqs[] = {2, 1, 4}; TpPostingData *post; ItemPointerSet(&ctid, 300, 1); test_append_terms(rel, idx_name, &ctid, 3, terms, freqs); src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL"); for (int i = 0; i < 3; i++) { post = tp_source_get_postings(src, terms[i]); if (post == NULL) TEST_FAIL("get_postings(%s) NULL", terms[i]); if (post->count != 1) TEST_FAIL("count for %s = %d", terms[i], post->count); if (post->frequencies[0] != freqs[i]) TEST_FAIL( "freq for %s = %d, expected %d", terms[i], post->frequencies[0], freqs[i]); if (!ItemPointerEquals(&post->ctids[0], &ctid)) TEST_FAIL("ctid mismatch for %s", terms[i]); tp_source_free_postings(src, post); } TEST_OK(); } else if (strcmp(case_name, "documents_only") == 0) { ItemPointerData ctid; const char *terms[] = {"alpha"}; const char *documents_only = NULL; int32 freqs[] = {2}; ItemPointerSet(&ctid, 301, 1); test_append_terms(rel, idx_name, &ctid, 1, terms, freqs); src = tp_memtable_chain_source_create(state, rel, &documents_only, 0); if (src == NULL) TEST_FAIL("chain source NULL"); if (!((TpMemtableChainSource *)src)->filter_terms) TEST_FAIL("documents-only source did not enable term filtering"); if (hash_get_num_entries(((TpMemtableChainSource *)src)->term_ht) != 0) TEST_FAIL( "documents-only source retained %ld terms", hash_get_num_entries( ((TpMemtableChainSource *)src)->term_ht)); if (src->total_docs != 1) TEST_FAIL("total_docs=%d, expected 1", src->total_docs); if (tp_source_get_doc_length(src, &ctid) != 2) TEST_FAIL("get_doc_length mismatch"); if (tp_source_get_postings(src, "alpha") != NULL) TEST_FAIL("documents-only source materialized postings"); TEST_OK(); } else if (strcmp(case_name, "multi_page_chain") == 0) { const int N = 30; /* * Use a fixed lexeme so the per-record TpVector is * relatively large from filler frequencies and forces * the chain past one page. */ char big_term[512]; const char *terms[1]; int32 freqs[1]; TpPostingData *post; for (size_t i = 0; i < sizeof(big_term) - 1; i++) big_term[i] = 'a' + (char)(i % 26); big_term[sizeof(big_term) - 1] = '\0'; terms[0] = big_term; freqs[0] = 1; for (int i = 0; i < N; i++) { ItemPointerData ctid; ItemPointerSet( &ctid, (BlockNumber)(400 + i), (OffsetNumber)(i + 1)); test_append_terms(rel, idx_name, &ctid, 1, terms, freqs); } src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL"); if (src->total_docs != N) TEST_FAIL("total_docs=%d, expected %d", src->total_docs, N); post = tp_source_get_postings(src, big_term); if (post == NULL) TEST_FAIL("get_postings(big_term) NULL"); if (post->count != N) TEST_FAIL("count=%d, expected %d", post->count, N); tp_source_free_postings(src, post); TEST_OK(); } else if (strcmp(case_name, "term_not_found") == 0) { ItemPointerData ctid; const char *terms[] = {"present"}; int32 freqs[] = {1}; TpPostingData *post; ItemPointerSet(&ctid, 500, 1); test_append_terms(rel, idx_name, &ctid, 1, terms, freqs); src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL"); post = tp_source_get_postings(src, "absent"); if (post != NULL) TEST_FAIL("get_postings(absent) returned non-NULL"); TEST_OK(); } else if (strcmp(case_name, "doc_length_lookup") == 0) { ItemPointerData ctid_a, ctid_b, ctid_missing; const char *t_a[] = {"x", "y"}; int32 f_a[] = {3, 4}; const char *t_b[] = {"z"}; int32 f_b[] = {7}; ItemPointerSet(&ctid_a, 600, 1); ItemPointerSet(&ctid_b, 600, 2); ItemPointerSet(&ctid_missing, 9999, 1); test_append_terms(rel, idx_name, &ctid_a, 2, t_a, f_a); test_append_terms(rel, idx_name, &ctid_b, 1, t_b, f_b); src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL"); if (tp_source_get_doc_length(src, &ctid_a) != 7) TEST_FAIL( "doc_length(ctid_a)=%d, expected 7", tp_source_get_doc_length(src, &ctid_a)); if (tp_source_get_doc_length(src, &ctid_b) != 7) TEST_FAIL( "doc_length(ctid_b)=%d, expected 7", tp_source_get_doc_length(src, &ctid_b)); if (tp_source_get_doc_length(src, &ctid_missing) != -1) TEST_FAIL("doc_length(missing) != -1"); TEST_OK(); } else if (strcmp(case_name, "fragment_roundtrip") == 0) { /* * Construct a TpVector whose payload exceeds * TP_MEMTABLE_PAGE_MAX_VECTOR_LEN so the writer takes * the fragment branch (head page + N continuations * + fresh tail). Then verify the chain source * reassembles the payload and round-trips per-term * postings and the doc-length entry. */ const int N_TERMS = 240; const int LEX_LEN = 60; const char **terms = palloc(N_TERMS * sizeof(char *)); int32 *freqs = palloc(N_TERMS * sizeof(int32)); int32 expected = 0; ItemPointerData ctid; TpPostingData *post; uint32 vec_len; for (int i = 0; i < N_TERMS; i++) { char *t = palloc(LEX_LEN + 1); snprintf(t, LEX_LEN + 1, "frag_%05d_", i); /* * Pad with deterministic bytes so each lexeme is * exactly LEX_LEN characters; the lookup later * needs to use the same encoding. */ for (int j = (int)strlen(t); j < LEX_LEN; j++) t[j] = 'a' + (j % 26); t[LEX_LEN] = '\0'; terms[i] = t; freqs[i] = i + 1; expected += freqs[i]; } /* * Sanity: confirm the resulting TpVector exceeds the * single-page cap so we're actually exercising the * fragment path. */ { TpVector *v = test_make_tpvector(idx_name, N_TERMS, terms, freqs); vec_len = VARSIZE(v); pfree(v); } if (vec_len <= TP_MEMTABLE_PAGE_MAX_VECTOR_LEN) TEST_FAIL( "test built TpVector with len=%u (<= max %u); " "increase N_TERMS/LEX_LEN to exercise fragmentation", vec_len, (unsigned)TP_MEMTABLE_PAGE_MAX_VECTOR_LEN); ItemPointerSet(&ctid, 700, 1); test_append_terms(rel, idx_name, &ctid, N_TERMS, terms, freqs); src = tp_memtable_chain_source_create(state, rel, NULL, 0); if (src == NULL) TEST_FAIL("chain source NULL after fragment append"); if (src->total_docs != 1) TEST_FAIL("total_docs=%d, expected 1", src->total_docs); if (src->total_len != expected) TEST_FAIL( "total_len=%lld, expected %d", (long long)src->total_len, expected); if (tp_source_get_doc_length(src, &ctid) != expected) TEST_FAIL( "doc_length=%d, expected %d", tp_source_get_doc_length(src, &ctid), expected); /* Spot-check a handful of terms across the payload. */ { int probe_idx[] = {0, 1, N_TERMS / 4, N_TERMS / 2, N_TERMS - 1}; for (size_t k = 0; k < sizeof(probe_idx) / sizeof(probe_idx[0]); k++) { int i = probe_idx[k]; const char *t = terms[i]; post = tp_source_get_postings(src, t); if (post == NULL) TEST_FAIL("get_postings(%s) returned NULL", t); if (post->count != 1) TEST_FAIL("count for %s = %d, expected 1", t, post->count); if (post->doc_freq != 1) TEST_FAIL( "doc_freq for %s = %d, expected 1", t, post->doc_freq); if (!ItemPointerEquals(&post->ctids[0], &ctid)) TEST_FAIL("ctid mismatch for %s", t); if (post->frequencies[0] != freqs[i]) TEST_FAIL( "freq for %s = %d, expected %d", t, post->frequencies[0], freqs[i]); tp_source_free_postings(src, post); } } for (int i = 0; i < N_TERMS; i++) pfree((void *)terms[i]); pfree(terms); pfree(freqs); TEST_OK(); } TEST_FAIL("unknown case '%s'", case_name); } timescale-pg_textsearch-b63fe58/src/memtable/chain_source.h000066400000000000000000000116471525762047700241620ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * chain_source.h - TpDataSource over the on-disk memtable page * chain (issue #374). * * Reads document records from the chain rooted at * meta.memtable_head_blkno, aggregates them into in-memory * per-term posting lists and a per-ctid doc-length map, and * serves them through the standard TpDataSourceOps interface. * * Ordinary-constructor concurrency contract: * * per-index LWLock SHARED * └─► metapage buffer SHARED (held briefly, released) * └─► chain page buffer SHARED (one at a time) * * The per-index LWLock SHARED acquisition (idempotent within an * xact via tp_acquire_index_lock) excludes spill (which holds * LW_EXCLUSIVE) from racing the eager open-walk. Concurrent * writers also hold the per-index lock SHARED so they coexist * with us; their tail-buffer EXCL serializes against our SHARED * read of the same page. * * Memory ownership: the constructor creates a child * MemoryContext under CurrentMemoryContext; all accumulators, * key copies, and HTAB internals live inside it. `close()` * deletes the child context in one shot (no per-entry frees). * * The bounded constructor consumes an endpoint * captured with the segment roots under the metapage buffer lock, never * rereads the metapage, and needs no extension LWLock. It serves standby * readers and primary readers retaining an earlier scoring generation. */ #pragma once #include #include #include "index/source.h" #include "index/state.h" #include "memtable/chain_walker.h" /* * Construct a chain-backed data source for `rel`. * * Eagerly walks the entire chain in the constructor (one pass, * one SHARED buffer lock at a time). The returned TpDataSource * serves get_postings / get_doc_length from in-memory * accumulators thereafter. * * When `query_terms != NULL`, the term HTAB is pre-populated * with `query_terms[0..query_term_count-1]` (NUL-terminated * lexeme bytes; UTF-8 / index text_config's encoding) and the * chain walk only materialises postings for those terms. This is * the typical scoring path: a 2–5 term query against a memtable * with thousands of unique lexemes avoids building an HTAB of * every lexeme just to read N of them. A non-NULL pointer with * `query_term_count=0` collects document metadata without any * postings. Pass `query_terms=NULL, query_term_count=0` to * disable the filter and accumulate every lexeme — this is the * contract required by the spill path. * * Returns NULL if the chain is empty (mirrors the existing * tp_memtable_source_create() contract that an absent memtable * is signalled by a NULL source). * * Caller must free via tp_source_close(). */ extern TpDataSource *tp_memtable_chain_source_create( TpLocalIndexState *state, Relation rel, const char *const *query_terms, int query_term_count); /* * Construct a chain source from a previously captured bounded endpoint. * * This path never rereads the metapage and does not acquire the per-index * LWLock. The common index read snapshot must have atomically copied * segment roots and the memtable endpoint. * The shared constructor and ingestion path are otherwise identical to * tp_memtable_chain_source_create(). */ extern TpDataSource *tp_memtable_chain_source_create_bounded( Relation rel, const TpMemtableChainSnapshot *snapshot, const char *const *query_terms, int query_term_count); /* * Return the total number of memtable chain pages walked by a * chain source (regular pages + fragment continuation pages). * * Used by recovery / fresh-backend paths to seed * shared->chain_page_count after shmem init has reset the atomic * to zero, so the auto-spill heuristic isn't blind to a chain * that already exists on disk. Returns 0 when src is NULL. * * Asserts that `src` is a chain source (not any other * TpDataSource implementation). */ extern uint32 tp_memtable_chain_source_page_count(TpDataSource *src); /* * Extract a sorted term dictionary + finalized doc map from a * chain source. Used by the spill path (`tp_do_spill`) to feed * `tp_write_segment` without re-walking the chain pages. * * The output arrays live in `dest_mcxt` and survive the source's * `close()`. Each TermInfo's `term`, `ctids`, and `freqs` are * deep-copied out of the source's private mcxt so the caller can * safely close the source before consuming the dictionary. The * TpDocMapBuilder is built fresh and `tp_docmap_finalize`-ed; the * caller frees it via `tp_docmap_destroy`. * * Asserts that `src` is a chain source (not any other TpDataSource * impl). */ struct TermInfo; struct TpDocMapBuilder; extern void tp_memtable_chain_source_extract( TpDataSource *src, MemoryContext dest_mcxt, struct TermInfo **out_terms, uint32 *out_num_terms, struct TpDocMapBuilder **out_docmap); timescale-pg_textsearch-b63fe58/src/memtable/chain_walker.c000066400000000000000000000411431525762047700241340ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * chain_walker.c - resumable iterator over the on-disk memtable * page chain. See chain_walker.h for the API * contract, concurrency assumptions, and cursor * semantics. * * Implementation notes. * * - The walker holds at most one chain page buffer at a time * (SHARED). Multiple records on the same page are read under * one buffer-lock acquisition; the buffer is released at the * page boundary, before opening the next page. * * - FRAGMENT reassembly: when the walker hits a FRAGMENT head * record, it walks the continuation chain in-line (each * continuation page is opened SHARED, its chunk memcpy'd into * the caller's mcxt, then released) and returns the reassembled * payload as the record's vector_bytes. After yielding, the * walker's pending next-page-to-open is the post-fragment * regular page (Tnew in the writer's terminology — always a * regular page, never InvalidBlockNumber, established by * memtable_append_fragment in src/memtable/log.c). * * - Validation: every page validates magic, free_offset range, * self-loop, and (for fragment heads) n_records == 1. Every * inline record validates that its tail fits within the page's * free_offset. These match the defenses in the predecessor * walk_chain() in chain_source.c, where they have been * stable in production. */ #include #include #include #include #include #include #include #include "memtable/chain_walker.h" #include "memtable/page.h" #include "types/vector.h" struct TpChainWalker { Relation rel; MemoryContext mcxt; /* Currently-held page (Invalid means no page held). */ Buffer cur_buf; /* * State of the currently-held page. Only meaningful while * cur_buf != InvalidBuffer; cached up front so we don't have * to re-read the page header on every record. */ BlockNumber cur_blkno; uint16 cur_off; /* next record offset to read */ uint16 cur_free_offset; /* page free_offset (stop here) */ BlockNumber cur_next_block; /* page's hdr->next_block */ /* * Next page to open when cur_buf is released (or on the * first call). InvalidBlockNumber means "walker is done". */ BlockNumber pending_blkno; /* * Pending start offset for the next page we open. Honored * only on the first page we open (i.e. when the walker was * given a non-zero start_off at _open() time). After that, * every fresh page starts at TP_MEMTABLE_PAGE_FIRST_RECORD_ * OFFSET. */ uint16 pending_off; bool bounded; bool endpoint_reached; BlockNumber endpoint_blkno; uint16 endpoint_off; bool copy_records; uint32 pages_visited; }; static void validate_page_layout(Page page, BlockNumber blkno) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); if (hdr->free_offset < TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET || hdr->free_offset > BLCKSZ) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable page block %u has " "out-of-range free_offset %u", blkno, hdr->free_offset))); if (hdr->next_block == blkno) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable page block %u links to " "itself", blkno))); } /* * Open `blkno` as the walker's current page. Acquires * BUFFER_LOCK_SHARE and caches header fields into walker state. * `start_off` is the byte offset to begin reading at on this * page (clamped up to FIRST_RECORD_OFFSET). */ static void open_page(TpChainWalker *w, BlockNumber blkno, uint16 start_off) { Buffer buf; Page page; TpMemtablePageHeader *hdr; Assert(w->cur_buf == InvalidBuffer); Assert(BlockNumberIsValid(blkno)); CHECK_FOR_INTERRUPTS(); buf = ReadBuffer(w->rel, blkno); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable page at block %u in " "index \"%s\" has invalid magic", blkno, RelationGetRelationName(w->rel)))); } /* * The outer chain walk only ever opens regular pages — * cursors are never planted on continuation pages by any * code path (the writer's fragment protocol leaves a fresh * regular Tnew as the post-fragment tail, and that's what * the walker stores into pending_blkno after a fragment). * If we land on a continuation page here it indicates either * a corrupt next_block link or a caller-supplied cursor * pointing into a fragment's interior; both are errors. */ if (tp_memtable_page_is_continuation(page)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable outer walk reached " "continuation page at block %u in index " "\"%s\"", blkno, RelationGetRelationName(w->rel)))); } validate_page_layout(page, blkno); hdr = tp_memtable_page_header(page); w->cur_buf = buf; w->cur_blkno = blkno; w->cur_free_offset = hdr->free_offset; w->cur_next_block = hdr->next_block; w->cur_off = Max(start_off, TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET); if (w->bounded && blkno == w->endpoint_blkno) { if (w->endpoint_off < TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET || w->endpoint_off > hdr->free_offset) { UnlockReleaseBuffer(w->cur_buf); w->cur_buf = InvalidBuffer; ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable snapshot endpoint at " "block %u has invalid offset %u", blkno, w->endpoint_off))); } w->cur_free_offset = w->endpoint_off; w->cur_next_block = InvalidBlockNumber; w->endpoint_reached = true; } w->pages_visited++; } static void release_cur_page(TpChainWalker *w) { if (w->cur_buf != InvalidBuffer) { UnlockReleaseBuffer(w->cur_buf); w->cur_buf = InvalidBuffer; } } /* * Reassemble a FRAGMENT record's payload from continuation * pages into a palloc'd buffer in walker mcxt. Returns the * BlockNumber of the post-fragment page (Cn->next_block), which * the writer guarantees to be a freshly-initialized regular * page (Tnew, never Invalid). * * On entry, w->cur_buf holds the fragment-head page SHARED and * `head_rec` points at the fragment head record on that page. * On exit, the fragment-head page is still held SHARED; the * caller releases it. */ static BlockNumber reassemble_fragment( TpChainWalker *w, TpMemtableRecord *head_rec, char **out_buf) { Page head_page = BufferGetPage(w->cur_buf); TpMemtablePageHeader *hdr = tp_memtable_page_header(head_page); uint32 full_len = head_rec->vector_len; char *vec_start = (char *)head_rec->vector_bytes; char *page_end = (char *)head_page + hdr->free_offset; uint32 inline_len; char *full; uint32 copied; BlockNumber cont_blkno; MemoryContext old; if (page_end <= vec_start) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch fragment head record has no inline " "payload"))); inline_len = (uint32)(page_end - vec_start); if (inline_len >= full_len) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch fragment record has inline_len=%u " ">= full_len=%u (should have been a regular record)", inline_len, full_len))); old = MemoryContextSwitchTo(w->mcxt); full = palloc(full_len); MemoryContextSwitchTo(old); memcpy(full, head_rec->vector_bytes, inline_len); copied = inline_len; cont_blkno = hdr->next_block; while (copied < full_len) { Buffer cbuf; Page cpage; const char *chunk; uint32 chunk_len; BlockNumber next_blk; CHECK_FOR_INTERRUPTS(); if (cont_blkno == InvalidBlockNumber) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch fragment continuation chain " "truncated: copied=%u, expected=%u", copied, full_len))); cbuf = ReadBuffer(w->rel, cont_blkno); LockBuffer(cbuf, BUFFER_LOCK_SHARE); cpage = BufferGetPage(cbuf); if (!tp_memtable_page_is_continuation(cpage)) { UnlockReleaseBuffer(cbuf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch fragment continuation at block " "%u is not a continuation page", cont_blkno))); } chunk = tp_memtable_continuation_page_chunk(cpage, &chunk_len); if (chunk_len == 0 || copied + chunk_len > full_len) { UnlockReleaseBuffer(cbuf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch fragment continuation at block " "%u has invalid chunk_len=%u (copied=%u, " "full_len=%u)", cont_blkno, chunk_len, copied, full_len))); } memcpy(full + copied, chunk, chunk_len); copied += chunk_len; next_blk = tp_memtable_page_get_next(cpage); UnlockReleaseBuffer(cbuf); cont_blkno = next_blk; w->pages_visited++; } *out_buf = full; /* * cont_blkno is now the LAST continuation's next_block — * the writer (memtable_append_fragment) initializes this as * Tnew, a fresh regular page that becomes the new chain * tail. Assert in debug builds that it is in fact valid; * production data corruption surfaces at the next open_page * via validate_page_layout / is_valid checks. */ Assert(BlockNumberIsValid(cont_blkno)); return cont_blkno; } TpChainWalker * tp_chain_walker_open( Relation rel, BlockNumber start_blkno, uint16 start_off, MemoryContext mcxt) { TpChainWalker *w; Assert(rel != NULL); Assert(mcxt != NULL); w = (TpChainWalker *)palloc0(sizeof(TpChainWalker)); w->rel = rel; w->mcxt = mcxt; w->cur_buf = InvalidBuffer; w->pending_blkno = start_blkno; w->pending_off = start_off; return w; } bool tp_memtable_chain_snapshot_capture_locked( Buffer tail_buffer, BlockNumber head_blkno, BlockNumber tail_blkno, TpMemtableChainSnapshot *snapshot) { Page page; TpMemtablePageHeader *hdr; Assert(snapshot != NULL); if (!BlockNumberIsValid(head_blkno)) { if (BlockNumberIsValid(tail_blkno) || BufferIsValid(tail_buffer)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 memtable has a tail page but no head " "page"))); memset(snapshot, 0, sizeof(*snapshot)); snapshot->head_blkno = InvalidBlockNumber; snapshot->tail_blkno = InvalidBlockNumber; return false; } if (!BlockNumberIsValid(tail_blkno)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 memtable has a head page but no tail page"))); if (!BufferIsValid(tail_buffer) || BufferGetBlockNumber(tail_buffer) != tail_blkno) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("BM25 memtable tail snapshot buffer mismatch"))); page = BufferGetPage(tail_buffer); if (!tp_memtable_page_is_valid(page) || tp_memtable_page_is_continuation(page)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 memtable tail page %u is invalid", tail_blkno))); validate_page_layout(page, tail_blkno); hdr = tp_memtable_page_header(page); if (hdr->next_block != InvalidBlockNumber) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 memtable tail page %u has a successor", tail_blkno))); snapshot->head_blkno = head_blkno; snapshot->tail_blkno = tail_blkno; snapshot->tail_free_offset = hdr->free_offset; return true; } TpChainWalker * tp_chain_walker_open_bounded( Relation rel, const TpMemtableChainSnapshot *snapshot, MemoryContext mcxt, bool copy_records) { TpChainWalker *w; Assert(snapshot != NULL); Assert(BlockNumberIsValid(snapshot->head_blkno)); Assert(BlockNumberIsValid(snapshot->tail_blkno)); w = tp_chain_walker_open(rel, snapshot->head_blkno, 0, mcxt); w->bounded = true; w->endpoint_blkno = snapshot->tail_blkno; w->endpoint_off = snapshot->tail_free_offset; w->copy_records = copy_records; return w; } void tp_chain_walker_close(TpChainWalker *w) { if (w == NULL) return; release_cur_page(w); pfree(w); } uint32 tp_chain_walker_pages_visited(const TpChainWalker *w) { return w == NULL ? 0 : w->pages_visited; } bool tp_chain_walker_next(TpChainWalker *w, TpChainWalkerRecord *out) { Assert(w != NULL); Assert(out != NULL); for (;;) { Page page; TpMemtableRecord *rec; /* * If no page is currently held, try to open the pending * one; if there's no pending page either, walker is * exhausted. */ if (w->cur_buf == InvalidBuffer) { BlockNumber to_open = w->pending_blkno; uint16 off; if (!BlockNumberIsValid(to_open)) { if (w->bounded && !w->endpoint_reached) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 memtable chain ended before " "snapshot tail page %u", w->endpoint_blkno))); return false; } w->pending_blkno = InvalidBlockNumber; off = w->pending_off; w->pending_off = 0; open_page(w, to_open, off); } /* No more records on this page? Release and follow next_block. */ if (w->cur_off >= w->cur_free_offset) { w->pending_blkno = w->cur_next_block; w->pending_off = 0; release_cur_page(w); continue; } page = BufferGetPage(w->cur_buf); rec = (TpMemtableRecord *)((char *)page + w->cur_off); if ((rec->flags & TP_MEMTABLE_RECORD_FLAG_FRAGMENT) != 0) { BlockNumber post_cont; char *full; TpMemtablePageHeader *hdr = tp_memtable_page_header(page); if (hdr->n_records != 1) { release_cur_page(w); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch fragment head page %u " "has n_records=%u (must be 1)", w->cur_blkno, hdr->n_records))); } post_cont = reassemble_fragment(w, rec, &full); /* * A multi-page fragment payload has just been * stitched together across multiple buffer-locked * page reads. Run a full bounds-check * unconditionally — the defensive value is much * higher than for an inline record where we can * lean on the page CRC and the same-backend write * rationale; the cost is amortised over a payload * that is by definition larger than a single page. * Without this check, a structurally-malformed * `full` buffer would silently propagate into the * caller's downstream decoder. */ tpvector_validate_v2_buffer(full, rec->vector_len); out->ctid = rec->ctid; out->doc_length = rec->doc_length; out->vector_bytes = full; out->vector_len = rec->vector_len; out->is_fragment = true; out->owns_vector = true; out->next_blkno = post_cont; out->next_off = TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET; /* * Release the fragment-head page and stage the * post-fragment regular page for the next call. */ release_cur_page(w); w->pending_blkno = post_cont; w->pending_off = 0; return true; } /* Inline record: validate that its tail fits in the page. */ { char *rec_end = (char *)rec + tp_memtable_record_size(rec->vector_len); char *page_end_bytes = (char *)page + w->cur_free_offset; if (rec_end > page_end_bytes) { release_cur_page(w); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable record at block " "%u, offset %u extends past free_offset", w->cur_blkno, w->cur_off))); } } out->ctid = rec->ctid; out->doc_length = rec->doc_length; out->vector_bytes = rec->vector_bytes; out->vector_len = rec->vector_len; out->is_fragment = false; out->owns_vector = false; /* Advance our internal cur_off. */ w->cur_off += tp_memtable_record_size(rec->vector_len); /* * Report the cursor: if there are more records on this * page (or the page is the tail and may grow), anchor at * the same page. If this page is sealed (next_block is * valid) and we just consumed its last record, advance * to the start of the next page so resume doesn't pay * an unnecessary ReadBuffer on the sealed page. */ if (w->cur_off >= w->cur_free_offset && BlockNumberIsValid(w->cur_next_block)) { out->next_blkno = w->cur_next_block; out->next_off = TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET; } else { out->next_blkno = w->cur_blkno; out->next_off = w->cur_off; } if (w->copy_records) { char *copy; copy = MemoryContextAlloc(w->mcxt, rec->vector_len); memcpy(copy, rec->vector_bytes, rec->vector_len); out->vector_bytes = copy; out->owns_vector = true; w->pending_blkno = out->next_blkno; w->pending_off = out->next_off; release_cur_page(w); } return true; } } timescale-pg_textsearch-b63fe58/src/memtable/chain_walker.h000066400000000000000000000142001525762047700241330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * chain_walker.h - resumable iterator over the on-disk memtable * page chain. * * The walker reads document records from the chain page-by-page * (acquiring buffer SHARED, parsing records, transparently * reassembling FRAGMENT payloads across continuation pages, then * releasing the buffer at page boundaries). It is the shared * primitive used by both: * * - chain_source.c: an eager-walk-the-whole-chain * TpDataSource (today's read/spill code path), and * - cache.c (memtable in-memory cache, #374 follow-up): * incrementally catches the cache up to the current chain * tail by resuming from a stored cursor position. * * Concurrency: ordinary walkers match chain_source's contract, * so the caller MUST hold the per-index LWLock SHARED (or * stronger). A bounded walker may run after that lock is * released because its endpoint was captured while the lock was * held and retired pages remain protected by the caller's scan * snapshot. Bounded walkers can optionally copy each inline vector and * release the page buffer before returning it. * * Cursor semantics. Each call to tp_chain_walker_next() that * returns true populates out->next_blkno / out->next_off with * the LOGICAL position of the next record to read. Saving * (next_blkno, next_off) and reopening a fresh walker at that * position picks up exactly where the previous walker left off * — even across concurrent appends that extend the same tail * page or chain a new page. Fragment-page allocation order can * produce BlockNumber(Tnew) < BlockNumber(Thead), but the cursor * stays synchronized because the walker follows logical links, * never block-number comparisons. * * Lifetime of out->vector_bytes: for ordinary walkers, inline * records point into the buffer page held SHARED until the next * call or close. Bounded walkers with copy_records enabled copy inline * records and release the page before returning. Fragment payloads are always * reassembled into caller-owned memory. In both allocated cases * owns_vector is true and the caller is responsible for pfree. */ #pragma once #include #include #include #include #include #include typedef struct TpChainWalker TpChainWalker; typedef struct TpMemtableChainSnapshot { BlockNumber head_blkno; BlockNumber tail_blkno; uint16 tail_free_offset; } TpMemtableChainSnapshot; /* * One record yielded by the walker. * * (ctid, doc_length, vector_bytes, vector_len) are decoded from * the on-disk record; (next_blkno, next_off) is the cursor * position the next call to tp_chain_walker_next() (in this or a * subsequent walker) should resume from. * * When the LAST record on cur_page is read and cur_page has a * non-Invalid next_block (i.e. the page is sealed), next_(blkno, * off) advances to (next_block, FIRST_RECORD_OFFSET) — there's * no reason to anchor the cursor on a sealed page, and advancing * saves an unnecessary ReadBuffer next time. When cur_page is * the chain tail (next_block == Invalid), the cursor anchors at * (cur_blkno, free_offset) so future writes that extend the * tail page in place are picked up on resume. */ typedef struct TpChainWalkerRecord { ItemPointerData ctid; int32 doc_length; const char *vector_bytes; uint32 vector_len; /* * True iff this record's payload was reassembled from one * or more continuation pages (FRAGMENT). The walker has * already run tpvector_validate_v2_buffer() on the * reassembled buffer in this case, so callers do not need * to re-validate on the fragment path. For inline records * (is_fragment == false), the caller may want to run * assert-mode validation via USE_ASSERT_CHECKING. */ bool is_fragment; bool owns_vector; BlockNumber next_blkno; uint16 next_off; } TpChainWalkerRecord; /* * Open a walker positioned at (start_blkno, start_off). * * start_blkno == InvalidBlockNumber yields an immediately-empty * walker (the first _next() returns false). start_off is * clamped up to TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET — passing 0 * is the natural "start at beginning of page" sentinel used by * cold-build callers seeding the cursor from meta.head_blkno. * * `mcxt` is the MemoryContext used for fragment-payload * reassembly buffers. The walker does not allocate from `mcxt` * for inline records (those are zero-copy into the buffer * page). * * Returns a heap-allocated walker; close with * tp_chain_walker_close() to release any held buffer SHARED lock. */ extern TpChainWalker *tp_chain_walker_open( Relation rel, BlockNumber start_blkno, uint16 start_off, MemoryContext mcxt); /* * Capture an endpoint from a tail buffer the caller already holds locked. * Passing InvalidBuffer captures an empty chain. This helper never acquires * or releases a buffer lock. */ extern bool tp_memtable_chain_snapshot_capture_locked( Buffer tail_buffer, BlockNumber head_blkno, BlockNumber tail_blkno, TpMemtableChainSnapshot *snapshot); extern TpChainWalker *tp_chain_walker_open_bounded( Relation rel, const TpMemtableChainSnapshot *snapshot, MemoryContext mcxt, bool copy_records); /* * Advance the walker by one record. On true, *out is populated * with the record and the next cursor position. On false, the * walker has reached the end of the chain at the time of the * call; it is safe to call again later (the chain may have * grown). However, the simpler pattern is to close and reopen * the walker at the last reported (next_blkno, next_off). */ extern bool tp_chain_walker_next(TpChainWalker *w, TpChainWalkerRecord *out); /* * Total chain pages visited so far (regular + continuation). * Used by chain_source.c to seed shared->chain_page_count. * Continuation pages are counted in the page where they are * read during fragment reassembly; the head page is counted * exactly once. */ extern uint32 tp_chain_walker_pages_visited(const TpChainWalker *w); extern void tp_chain_walker_close(TpChainWalker *w); timescale-pg_textsearch-b63fe58/src/memtable/expull.c000066400000000000000000000130131525762047700230110ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * expull.c - Exponential unrolled linked list for posting lists * * See expull.h for design overview. */ #include #include "memtable/arena.h" #include "memtable/expull.h" /* * Allocate a new block and link it into the EXPULL chain. */ static void expull_alloc_block(TpArena *arena, TpExpull *expull) { uint32 block_size; uint32 data_cap; ArenaAddr block_addr; ArenaAddr *next_ptr; block_size = tp_expull_block_size(expull->block_num); data_cap = block_size - TP_EXPULL_LINK_SIZE; block_addr = tp_arena_alloc(arena, block_size); /* Initialize the next-pointer at the start of the new block */ next_ptr = (ArenaAddr *)tp_arena_get_ptr(arena, block_addr); *next_ptr = ARENA_ADDR_INVALID; /* Link previous block to this new block */ if (expull->last_block != ARENA_ADDR_INVALID) { ArenaAddr *prev_next = (ArenaAddr *) tp_arena_get_ptr(arena, expull->last_block); *prev_next = block_addr; } else { /* First block ever */ expull->head = block_addr; } /* Update state to point to the new block */ expull->last_block = block_addr; { uint32 page = arena_addr_page(block_addr); uint32 off = arena_addr_offset(block_addr); expull->tail = arena_addr_make(page, off + TP_EXPULL_LINK_SIZE); } expull->remaining_cap = data_cap; expull->block_num++; } /* * Append a posting entry to the EXPULL list. */ void tp_expull_append( TpArena *arena, TpExpull *expull, uint32 doc_id, uint16 frequency, uint8 fieldnorm) { TpExpullEntry *entry_ptr; Assert(arena != NULL); Assert(expull != NULL); /* Allocate a new block if needed */ if (expull->remaining_cap < TP_EXPULL_ENTRY_SIZE) expull_alloc_block(arena, expull); Assert(expull->remaining_cap >= TP_EXPULL_ENTRY_SIZE); /* Write the entry at the tail position */ entry_ptr = (TpExpullEntry *)tp_arena_get_ptr(arena, expull->tail); entry_ptr->doc_id = doc_id; entry_ptr->frequency = frequency; entry_ptr->fieldnorm = fieldnorm; /* * Advance tail pointer and decrement remaining capacity. * * When remaining_cap would become 0 (block exhausted), we must avoid * creating a potentially invalid ArenaAddr. This can happen when a * block is placed at the very end of an arena page and filled * completely -- the "next" tail offset would equal ARENA_PAGE_SIZE, * violating the arena addressing invariant. Instead, set tail to * ARENA_ADDR_INVALID to indicate the block is exhausted. The next * append will allocate a fresh block and reset the tail. */ expull->remaining_cap -= TP_EXPULL_ENTRY_SIZE; if (expull->remaining_cap < TP_EXPULL_ENTRY_SIZE) { expull->tail = ARENA_ADDR_INVALID; } else { uint32 page = arena_addr_page(expull->tail); uint32 off = arena_addr_offset(expull->tail); expull->tail = arena_addr_make(page, off + TP_EXPULL_ENTRY_SIZE); } expull->num_entries++; } /* * Initialize a reader for iterating over an EXPULL list. */ void tp_expull_reader_init(TpExpullReader *reader, TpArena *arena, TpExpull *expull) { uint32 first_block_size; uint32 first_data_cap; Assert(reader != NULL); Assert(arena != NULL); Assert(expull != NULL); reader->arena = arena; reader->entries_remaining = expull->num_entries; reader->block_num = 0; if (expull->num_entries == 0 || expull->head == ARENA_ADDR_INVALID) { reader->current_block = ARENA_ADDR_INVALID; reader->block_offset = 0; reader->block_data_size = 0; return; } reader->current_block = expull->head; reader->block_offset = 0; /* First block data capacity */ first_block_size = tp_expull_block_size(0); first_data_cap = first_block_size - TP_EXPULL_LINK_SIZE; reader->block_data_size = first_data_cap; } /* * Read up to max_entries postings from the EXPULL list. * Returns number of entries read (0 when exhausted). */ uint32 tp_expull_reader_read( TpExpullReader *reader, TpExpullEntry *out, uint32 max_entries) { uint32 count = 0; Assert(reader != NULL); Assert(out != NULL); while (count < max_entries && reader->entries_remaining > 0) { uint32 avail_bytes; uint32 avail_entries; uint32 to_read; char *block_base; char *data_start; if (reader->current_block == ARENA_ADDR_INVALID) break; /* Bytes available in current block from current offset */ avail_bytes = reader->block_data_size - reader->block_offset; avail_entries = avail_bytes / TP_EXPULL_ENTRY_SIZE; if (avail_entries == 0) { /* * Current block exhausted. Follow the next pointer. */ ArenaAddr *next_ptr; ArenaAddr next_block; block_base = (char *) tp_arena_get_ptr(reader->arena, reader->current_block); next_ptr = (ArenaAddr *)block_base; next_block = *next_ptr; if (next_block == ARENA_ADDR_INVALID) { reader->current_block = ARENA_ADDR_INVALID; break; } reader->block_num++; reader->current_block = next_block; reader->block_offset = 0; reader->block_data_size = tp_expull_block_size(reader->block_num) - TP_EXPULL_LINK_SIZE; continue; } /* Read entries from current block */ to_read = Min(avail_entries, max_entries - count); to_read = Min(to_read, reader->entries_remaining); block_base = (char *) tp_arena_get_ptr(reader->arena, reader->current_block); data_start = block_base + TP_EXPULL_LINK_SIZE + reader->block_offset; memcpy(&out[count], data_start, to_read * TP_EXPULL_ENTRY_SIZE); count += to_read; reader->block_offset += to_read * TP_EXPULL_ENTRY_SIZE; reader->entries_remaining -= to_read; } return count; } timescale-pg_textsearch-b63fe58/src/memtable/expull.h000066400000000000000000000077351525762047700230340ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * expull.h - Exponential unrolled linked list for posting lists * * A memory-efficient append-only data structure for accumulating * posting list entries during index builds. Blocks grow * exponentially (32B, 64B, ..., 32KB) to minimize waste for rare * terms while providing amortized O(1) appends for frequent terms. * * All memory is allocated from a TpArena, so individual frees are * not needed — the arena handles bulk deallocation. * * Inspired by Tantivy's ExpUnrolledLinkedList (stacker/src/expull.rs). */ #pragma once #include #include "compat.h" #include "memtable/arena.h" /* * Single posting entry stored in EXPULL blocks. * 7 bytes of data; packed to avoid waste in blocks. */ #ifdef _MSC_VER #pragma pack(push, 1) /* MSVC equivalent of TP_PACKED below */ #endif typedef struct TpExpullEntry { uint32 doc_id; /* Segment-local document ID */ uint16 frequency; /* Term frequency in document */ uint8 fieldnorm; /* Quantized document length */ } TP_PACKED TpExpullEntry; #ifdef _MSC_VER #pragma pack(pop) #endif #define TP_EXPULL_ENTRY_SIZE sizeof(TpExpullEntry) /* 7 bytes */ /* * Block layout in arena memory: * [ArenaAddr next] -- 4 bytes: link to next block * [TpExpullEntry entries...] -- remaining bytes: posting data * * Usable capacity per block = block_size - sizeof(ArenaAddr) */ #define TP_EXPULL_LINK_SIZE sizeof(ArenaAddr) /* * Block sizing: first block is 32 bytes (FIRST_BLOCK = 5, 2^5), * each subsequent block doubles up to 32KB (MAX_BLOCK = 15, 2^15). */ #define TP_EXPULL_FIRST_BLOCK_LOG2 5 /* 32 bytes */ #define TP_EXPULL_MAX_BLOCK_LOG2 15 /* 32768 bytes */ static inline uint32 tp_expull_block_size(uint32 block_num) { uint32 log2 = TP_EXPULL_FIRST_BLOCK_LOG2 + block_num; if (log2 > TP_EXPULL_MAX_BLOCK_LOG2) log2 = TP_EXPULL_MAX_BLOCK_LOG2; return 1u << log2; } /* * EXPULL header — stored in the hash table entry, not in the arena. * Tracks the linked list state for one term's posting list. */ typedef struct TpExpull { ArenaAddr head; /* First block in the chain */ ArenaAddr last_block; /* Start of the current (last) block */ ArenaAddr tail; /* Current write position (byte addr) */ uint16 remaining_cap; /* Bytes remaining in current block */ uint16 block_num; /* Number of blocks allocated (for sizing) */ uint32 num_entries; /* Total posting entries stored */ } TpExpull; /* * Initialize a new EXPULL (does not allocate any blocks yet). */ static inline void tp_expull_init(TpExpull *expull) { expull->head = ARENA_ADDR_INVALID; expull->last_block = ARENA_ADDR_INVALID; expull->tail = ARENA_ADDR_INVALID; expull->remaining_cap = 0; expull->block_num = 0; expull->num_entries = 0; } /* * Append a posting entry to the EXPULL list. * Allocates a new block from the arena if needed. */ extern void tp_expull_append( TpArena *arena, TpExpull *expull, uint32 doc_id, uint16 frequency, uint8 fieldnorm); /* * Reader state for iterating over EXPULL entries. */ typedef struct TpExpullReader { TpArena *arena; ArenaAddr current_block; /* Current block being read */ uint32 block_offset; /* Byte offset within current block */ uint32 block_data_size; /* Usable bytes in current block */ uint32 block_num; /* Block index (for sizing next block) */ uint32 entries_remaining; /* Total entries left to read */ } TpExpullReader; /* * Initialize a reader for iterating over an EXPULL list. */ extern void tp_expull_reader_init( TpExpullReader *reader, TpArena *arena, TpExpull *expull); /* * Read up to `max_entries` postings into `out` array. * Returns the number of entries actually read (0 when exhausted). * * Output is written as TpExpullEntry structs. The caller can * convert to TpBlockPosting if needed for the segment writer. */ extern uint32 tp_expull_reader_read( TpExpullReader *reader, TpExpullEntry *out, uint32 max_entries); timescale-pg_textsearch-b63fe58/src/memtable/log.c000066400000000000000000001434641525762047700222770ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * log.c - On-disk memtable write path. * * Memtable v2 (issue #374). See log.h for the concurrency * contract and high-level semantics. * * Scaffold SQL functions are exposed at the bottom of this file: * * bm25_test_memtable_append(index_name text, case_name text) * → text scenario harness, returns 'OK' or 'FAIL: …' * * bm25_memtable_chain(index_name text) * → SETOF (blkno bigint, n_records int, free_offset int, * next_block bigint, flags int) * * bm25_memtable_dead_pages(index_name text) * → SETOF (blkno bigint, flags int, dead_fxid bigint, * n_records int) * * These functions are internal-only unit-test entry points, * complementing the end-to-end coverage in the regression suite. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "compat.h" #include "constants.h" #include "debug/injection.h" #include "index/freepage.h" #include "index/metapage.h" #include "index/resolve.h" #include "index/state.h" #include "memtable/log.h" #include "memtable/page.h" /* * Read meta.memtable_tail_blkno under SHARED lock on the * metapage buffer, then release. Returns InvalidBlockNumber if * the chain is empty. The release-before-page-lock pattern is * the reader half of the lock-order contract documented in * log.h. */ static BlockNumber memtable_read_tail_blkno(Relation rel) { Buffer buf; BlockNumber tail; buf = ReadBuffer(rel, TP_METAPAGE_BLKNO); LockBuffer(buf, BUFFER_LOCK_SHARE); tail = tp_metapage_read_memtable_tail(BufferGetPage(buf)); UnlockReleaseBuffer(buf); return tail; } /* * Allocate a memtable chain page: reuse a recyclable FSM page, else * extend the relation. Returns a pinned buffer with * BUFFER_LOCK_EXCLUSIVE (same as ExtendBufferedRel EB_LOCK_FIRST). * Caller must UnlockReleaseBuffer when done. * * tp_fsm_claim_free_buffer holds the returned buffer's lock from the * recyclability check onward, so the caller's page_init (under that * same lock) atomically claims the block: a concurrent insert handed * the same block by the non-atomic FSM blocks on the lock, then sees * the now-live memtable page and skips it (issue #426). It also * refuses any block the FSM offers that is still a live structure * page, so a stale/crash FSM entry can't clobber the tombstone chain * or another memtable page (issues #380, #427). */ static Buffer tp_memtable_alloc_page(Relation rel) { Buffer buf = tp_fsm_claim_free_buffer(rel); if (BufferIsValid(buf)) return buf; return ExtendBufferedRel(BMR_REL(rel), MAIN_FORKNUM, NULL, EB_LOCK_FIRST); } /* * Bootstrap path: the chain is empty. Allocate the first page, * append the record, and update meta.{head,tail}_blkno — all * atomically inside one GenericXLog covering {meta, new page}. * * Re-checks meta.tail_blkno under EXCLUSIVE meta lock to detect * the lost-race case where another backend bootstrapped first. * On re-check failure we release everything and return * InvalidBlockNumber so the caller can retry via the normal * append path. */ static BlockNumber memtable_bootstrap_and_append( Relation rel, TpSharedIndexState *shared, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len) { Buffer metabuf; Buffer newbuf; BlockNumber newblk; Page metapage_local; Page newpage_local; TpIndexMetaPage metap; GenericXLogState *xlog_state; metabuf = ReadBuffer(rel, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); if (tp_metapage_read_memtable_tail(BufferGetPage(metabuf)) != InvalidBlockNumber) { /* Lost the race: another backend bootstrapped first. */ UnlockReleaseBuffer(metabuf); return InvalidBlockNumber; } newbuf = tp_memtable_alloc_page(rel); newblk = BufferGetBlockNumber(newbuf); xlog_state = GenericXLogStart(rel); metapage_local = GenericXLogRegisterBuffer(xlog_state, metabuf, 0); newpage_local = GenericXLogRegisterBuffer( xlog_state, newbuf, GENERIC_XLOG_FULL_IMAGE); tp_memtable_page_init(newpage_local); tp_memtable_page_append( newpage_local, ctid, doc_length, vector_bytes, vector_len); /* * v6 -> v7 in-place upgrade (issue #383): no-op on v7 pages; * on v6 pages flips version + initializes new chain fields * before we overwrite them below, all within this GenericXLog. */ tp_metapage_upgrade_to_current(rel, metapage_local); metap = (TpIndexMetaPage)PageGetContents(metapage_local); metap->memtable_head_blkno = newblk; metap->memtable_tail_blkno = newblk; GenericXLogFinish(xlog_state); UnlockReleaseBuffer(newbuf); UnlockReleaseBuffer(metabuf); /* * Bump chain page count after successful publish. No lock * needed: chain_page_count is an unsynchronized heuristic * for tp_auto_spill_if_needed; the spill path (which resets * it to 0) holds LW_EXCLUSIVE so concurrent inserts are * fenced. */ if (shared != NULL) pg_atomic_fetch_add_u32(&shared->chain_page_count, 1); return newblk; } /* * Extend path: tail is full. Allocate a new tail page, link * old tail → new, append the record on the new page, and * update meta.tail_blkno. All inside one GenericXLog covering * {old tail, new page, meta} (3 of 4 GenericXLog buffer slots). * * Caller passes the old tail buffer already locked EXCLUSIVE * and verified valid + full. */ static BlockNumber memtable_extend_and_append( Relation rel, TpSharedIndexState *shared, Buffer tailbuf, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len) { Buffer newbuf; Buffer metabuf; BlockNumber newblk; Page tailpage_local; Page newpage_local; Page metapage_local; TpIndexMetaPage metap; GenericXLogState *xlog_state; TP_INJECTION_POINT(TP_INJECTION_MEMTABLE_EXTEND); newbuf = tp_memtable_alloc_page(rel); newblk = BufferGetBlockNumber(newbuf); metabuf = ReadBuffer(rel, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(rel); tailpage_local = GenericXLogRegisterBuffer(xlog_state, tailbuf, 0); newpage_local = GenericXLogRegisterBuffer( xlog_state, newbuf, GENERIC_XLOG_FULL_IMAGE); metapage_local = GenericXLogRegisterBuffer(xlog_state, metabuf, 0); tp_memtable_page_init(newpage_local); tp_memtable_page_append( newpage_local, ctid, doc_length, vector_bytes, vector_len); tp_memtable_page_set_next(tailpage_local, newblk); /* v6 -> v7 in-place upgrade (issue #383); see bootstrap path. */ tp_metapage_upgrade_to_current(rel, metapage_local); metap = (TpIndexMetaPage)PageGetContents(metapage_local); metap->memtable_tail_blkno = newblk; GenericXLogFinish(xlog_state); UnlockReleaseBuffer(metabuf); UnlockReleaseBuffer(newbuf); if (shared != NULL) pg_atomic_fetch_add_u32(&shared->chain_page_count, 1); return newblk; } /* * Multi-page (fragment) write path. Called when vector_len * exceeds TP_MEMTABLE_PAGE_MAX_VECTOR_LEN. * * Pre-conditions: * - rel is open and the per-index LWLock is held SHARED. * - old_tailbuf is either: * (a) InvalidBuffer -- bootstrap path (chain was empty * when we last read the metapage). We hold no * buffer lock on entry. * (b) Locked EXCLUSIVE by the caller and verified valid * + tail (next_block==Invalid). We keep it locked * across the sequence and release it on return. * * Layout of the new pages we create: * * Thead (regular memtable page; fragment head record) * ─► C1 (continuation page; payload chunk #1) * ─► C2 ─► ... ─► Cn ─► Tnew (regular memtable page; * initially empty, becomes * the new tail) * * Extensions are done in reverse order (Tnew first, then * Cn..C1, then Thead) so each newly-initialized page already * knows the block number of its successor. Each new page gets * its own GenericXLog record (FULL_IMAGE). * * Publish is one final GenericXLog covering at most 2 buffers: * - {meta} for bootstrap; sets head=Thead, tail=Tnew. * - {old_tail, meta} for extend; sets old_tail.next=Thead * and meta.tail=Tnew. * * Returns the block number of Thead on success, or * InvalidBlockNumber on a lost bootstrap race (caller should * retry — the already-extended pages become orphans without a * DEAD stamp; FSM reclaim is Step 2 in amvacuumcleanup). * * Crash safety: a crash anywhere up to and including the publish * GenericXLog leaves orphan pages with valid magic but * unreachable via meta.head → chain. No code may sequentially * scan the relation without validating each page via * tp_memtable_page_is_valid(). */ static BlockNumber memtable_append_fragment( Relation rel, TpSharedIndexState *shared, Buffer old_tailbuf, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len) { uint32 inline_cap = tp_memtable_fragment_inline_capacity_max(); uint32 cont_cap = tp_memtable_continuation_max_chunk(); uint32 inline_len = inline_cap; uint32 remaining; uint32 n_cont; BlockNumber *cont_blknos; BlockNumber tnew_blkno; BlockNumber thead_blkno; Buffer metabuf; GenericXLogState *xlog_state; bool bootstrap = (old_tailbuf == InvalidBuffer); Assert(vector_len > inline_cap); Assert(cont_cap > 0); remaining = vector_len - inline_len; n_cont = (remaining + cont_cap - 1) / cont_cap; Assert(n_cont >= 1); cont_blknos = palloc(n_cont * sizeof(BlockNumber)); /* Step 1: extend Tnew (the post-fragment tail page). */ { Buffer buf; Page page_local; GenericXLogState *x; buf = tp_memtable_alloc_page(rel); tnew_blkno = BufferGetBlockNumber(buf); x = GenericXLogStart(rel); page_local = GenericXLogRegisterBuffer(x, buf, GENERIC_XLOG_FULL_IMAGE); tp_memtable_page_init(page_local); /* next_block remains InvalidBlockNumber (tail) */ GenericXLogFinish(x); UnlockReleaseBuffer(buf); } /* * Step 2: extend Cn..C1 in reverse, each linking to its * already-allocated successor. */ for (int i = (int)n_cont - 1; i >= 0; i--) { Buffer buf; Page page_local; GenericXLogState *x; BlockNumber next_blk; uint32 chunk_off; uint32 chunk_len; CHECK_FOR_INTERRUPTS(); next_blk = (i == (int)n_cont - 1) ? tnew_blkno : cont_blknos[i + 1]; chunk_off = inline_len + (uint32)i * cont_cap; Assert(chunk_off < vector_len); chunk_len = vector_len - chunk_off; if (chunk_len > cont_cap) chunk_len = cont_cap; buf = tp_memtable_alloc_page(rel); cont_blknos[i] = BufferGetBlockNumber(buf); x = GenericXLogStart(rel); page_local = GenericXLogRegisterBuffer(x, buf, GENERIC_XLOG_FULL_IMAGE); tp_memtable_continuation_page_init( page_local, vector_bytes + chunk_off, chunk_len, next_blk); GenericXLogFinish(x); UnlockReleaseBuffer(buf); } /* Step 3: extend Thead and write the FRAGMENT head record. */ { Buffer buf; Page page_local; GenericXLogState *x; buf = tp_memtable_alloc_page(rel); thead_blkno = BufferGetBlockNumber(buf); x = GenericXLogStart(rel); page_local = GenericXLogRegisterBuffer(x, buf, GENERIC_XLOG_FULL_IMAGE); tp_memtable_page_init(page_local); tp_memtable_page_append_fragment_head( page_local, ctid, doc_length, vector_bytes, inline_len, vector_len); tp_memtable_page_set_next(page_local, cont_blknos[0]); GenericXLogFinish(x); UnlockReleaseBuffer(buf); } /* Step 4-5: publish. */ metabuf = ReadBuffer(rel, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); if (bootstrap) { if (tp_metapage_read_memtable_tail(BufferGetPage(metabuf)) != InvalidBlockNumber) { /* * Lost bootstrap race: another backend populated the * chain while we were extending pages. Mark our orphaned * pages DEAD so they can be reclaimed by vacuum. */ FullTransactionId horizon = ReadNextFullTransactionId(); UnlockReleaseBuffer(metabuf); tp_memtable_mark_chain_dead(rel, thead_blkno, horizon); pfree(cont_blknos); return InvalidBlockNumber; } { Page meta_local; TpIndexMetaPage metap; xlog_state = GenericXLogStart(rel); meta_local = GenericXLogRegisterBuffer(xlog_state, metabuf, 0); /* v6 -> v7 in-place upgrade (issue #383). */ tp_metapage_upgrade_to_current(rel, meta_local); metap = (TpIndexMetaPage)PageGetContents(meta_local); metap->memtable_head_blkno = thead_blkno; metap->memtable_tail_blkno = tnew_blkno; GenericXLogFinish(xlog_state); } UnlockReleaseBuffer(metabuf); } else { Page old_tail_local; Page meta_local; TpIndexMetaPage metap; /* * We still hold old_tailbuf EXCL. Caller already * verified old_tail.next_block == Invalid; concurrent * writers are blocked by that lock until we publish. */ xlog_state = GenericXLogStart(rel); old_tail_local = GenericXLogRegisterBuffer(xlog_state, old_tailbuf, 0); meta_local = GenericXLogRegisterBuffer(xlog_state, metabuf, 0); tp_memtable_page_set_next(old_tail_local, thead_blkno); /* v6 -> v7 in-place upgrade (issue #383). */ tp_metapage_upgrade_to_current(rel, meta_local); metap = (TpIndexMetaPage)PageGetContents(meta_local); metap->memtable_tail_blkno = tnew_blkno; GenericXLogFinish(xlog_state); UnlockReleaseBuffer(metabuf); UnlockReleaseBuffer(old_tailbuf); } pfree(cont_blknos); /* * Bump chain page count by the number of pages we just * published: Thead + n_cont continuation pages + Tnew. * Done after both publish branches finish so we never * count pages on the lost-bootstrap-race return path * above. */ if (shared != NULL) pg_atomic_fetch_add_u32(&shared->chain_page_count, 2 + n_cont); return thead_blkno; } BlockNumber tp_memtable_append( Relation rel, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len) { TpLocalIndexState *index_state; Oid index_oid; BlockNumber stale_tail_blkno = InvalidBlockNumber; BlockNumber stale_next_blkno = InvalidBlockNumber; Assert(rel != NULL); Assert(ctid != NULL); Assert(vector_len == 0 || vector_bytes != NULL); /* * Acquire the per-index LWLock in SHARED mode. Spill takes * it EXCLUSIVE (see build.c's tp_spill_*) so this excludes * concurrent spill, while permitting other writers and * readers (which both take SHARED). */ index_oid = RelationGetRelid(rel); index_state = tp_get_local_index_state(index_oid); if (index_state == NULL) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_textsearch index state not available for \"%s\"", RelationGetRelationName(rel)))); tp_acquire_index_lock(index_state, LW_SHARED); /* * Fast / extend / bootstrap / fragment retry loop. Each * retry is caused by another backend either bootstrapping * or extending the chain; under bounded concurrency the * loop terminates. We bound the iteration count * defensively with INT_MAX/2 just to surface any * pathological live-lock during development. */ for (int iter = 0; iter < INT_MAX / 2; iter++) { BlockNumber tail_blkno; Buffer tailbuf; Page tailpage; BlockNumber result; CHECK_FOR_INTERRUPTS(); tail_blkno = memtable_read_tail_blkno(rel); TP_INJECTION_POINT(TP_INJECTION_MEMTABLE_TAIL_READ); /* * An extend publishes old_tail.next and meta.tail under both * buffer locks in one WAL record. With spill excluded, a stale * tail cannot remain current after re-reading the metapage. */ if (tail_blkno != InvalidBlockNumber && tail_blkno == stale_tail_blkno) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("pg_textsearch index \"%s\" is corrupted: " "metapage memtable tail is block %u, but that " "page already links to block %u", RelationGetRelationName(rel), tail_blkno, stale_next_blkno), errhint("REINDEX the index."))); /* * Oversized record path. Routes both the bootstrap * (tail_blkno==Invalid) and extend cases through the * shared fragment writer. */ if (vector_len > TP_MEMTABLE_PAGE_MAX_VECTOR_LEN) { Buffer fragment_old_tail = InvalidBuffer; if (tail_blkno != InvalidBlockNumber) { tailbuf = ReadBuffer(rel, tail_blkno); LockBuffer(tailbuf, BUFFER_LOCK_EXCLUSIVE); tailpage = BufferGetPage(tailbuf); if (!tp_memtable_page_is_valid(tailpage)) { UnlockReleaseBuffer(tailbuf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable tail page at " "block %u in index \"%s\" is not a valid " "memtable page (magic mismatch)", tail_blkno, RelationGetRelationName(rel)))); } if (tp_memtable_page_get_next(tailpage) != InvalidBlockNumber) { stale_tail_blkno = tail_blkno; stale_next_blkno = tp_memtable_page_get_next(tailpage); UnlockReleaseBuffer(tailbuf); continue; } fragment_old_tail = tailbuf; } result = memtable_append_fragment( rel, index_state->shared, fragment_old_tail, ctid, doc_length, vector_bytes, vector_len); if (result != InvalidBlockNumber) return result; /* Lost bootstrap race; retry. */ continue; } if (tail_blkno == InvalidBlockNumber) { result = memtable_bootstrap_and_append( rel, index_state->shared, ctid, doc_length, vector_bytes, vector_len); if (result != InvalidBlockNumber) return result; /* Lost the bootstrap race; retry as an appender. */ continue; } tailbuf = ReadBuffer(rel, tail_blkno); LockBuffer(tailbuf, BUFFER_LOCK_EXCLUSIVE); tailpage = BufferGetPage(tailbuf); if (!tp_memtable_page_is_valid(tailpage)) { UnlockReleaseBuffer(tailbuf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable tail page at block " "%u in index \"%s\" is not a valid memtable " "page (magic mismatch)", tail_blkno, RelationGetRelationName(rel)))); } if (tp_memtable_page_get_next(tailpage) != InvalidBlockNumber) { /* * Stale tail: another backend extended the chain * after our metapage read but before our tail * lock. Release and re-read metapage to find the * new tail. Each retry observes one completed * extend by another backend. */ stale_tail_blkno = tail_blkno; stale_next_blkno = tp_memtable_page_get_next(tailpage); UnlockReleaseBuffer(tailbuf); continue; } if (tp_memtable_page_can_fit(tailpage, vector_len)) { GenericXLogState *xlog_state; Page tailpage_local; xlog_state = GenericXLogStart(rel); tailpage_local = GenericXLogRegisterBuffer(xlog_state, tailbuf, 0); tp_memtable_page_append( tailpage_local, ctid, doc_length, vector_bytes, vector_len); GenericXLogFinish(xlog_state); UnlockReleaseBuffer(tailbuf); return tail_blkno; } /* Tail is full; allocate a new page and append there. */ result = memtable_extend_and_append( rel, index_state->shared, tailbuf, ctid, doc_length, vector_bytes, vector_len); UnlockReleaseBuffer(tailbuf); return result; } /* Unreachable under bounded concurrency. */ ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("pg_textsearch memtable append did not converge for " "index \"%s\"", RelationGetRelationName(rel)))); return InvalidBlockNumber; /* keep compilers happy */ } /* * tp_spill_finalize -- atomic spill publication. * * Updates the metapage to point at the new segment, resets the * memtable chain head/tail, and bumps total_docs/total_len, all * inside a single GenericXLog record. The caller WAL-stamps the unlinked * chain pages DEAD only after this publication record is inserted. * * Caller must hold the per-index LWLock in EXCLUSIVE mode. * * If the new segment block is valid, we also splice its * next_segment to the previous L0 head, adding it to the same * GenericXLog record. GenericXLog allows up to 4 buffers; we * use at most 2 (meta + segment header). * * In-memory memtable cache integration: * * 1. Before touching the metapage we bump * `state->shared->spill_generation` so that any reader * acquiring per-index SHARED after we release EXCL observes * a generation mismatch against any cursor that was captured * before this spill. The bump goes through a single atomic * add; standbys never see it because spill is primary-only. * * 2. After the metapage is published we drop the cache's dshash * tables via `tp_cache_clear` so the next read does a clean * cold-build and we don't carry dead memory across the spill * boundary. The DROPPED defensive path in * `tp_cache_apply_to_tail` remains as a belt-and-suspenders * check for any cursor that captured the old generation * between (1) and (2). * * Both steps are skipped when `state` is NULL (e.g. spill paths * that never wired up a local index state). */ #include "memtable/cache.h" #include "segment/format.h" #include "utils/dsa.h" void tp_spill_finalize( TpLocalIndexState *local_state, Relation rel, BlockNumber new_segment_root, uint64 docs_delta, uint64 len_delta, uint32 segment_capacity, bool *published) { Buffer metabuf; Buffer seg_buf = InvalidBuffer; GenericXLogState *state; Page metapage; TpIndexMetaPage metap; BlockNumber old_l0_head; TpMemtable *memtable = NULL; Assert(rel != NULL); Assert(segment_capacity > 0 && segment_capacity <= PG_UINT16_MAX); /* * Step 1: bump the cache's spill-generation token BEFORE we * publish the new metapage. Cache readers that miss this * happens-before edge will still detect the staleness via the * cursor's captured generation on the next apply_to_tail call, * but we want the common case to be "new reader sees new * generation immediately and cold-builds from a clean cache". */ if (local_state != NULL && local_state->shared != NULL) pg_atomic_add_fetch_u64(&local_state->shared->spill_generation, 1); metabuf = ReadBuffer(rel, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(rel); metapage = GenericXLogRegisterBuffer(state, metabuf, 0); /* * v6 -> v7 in-place upgrade (issue #383). Must run before * we touch any v7-only field below. On a v6 page this * piggybacks on the spill's GenericXLog record; on v7 it's * a no-op. Note: spill cannot be entered on a v6 index * with unspilled chain data (v6 had no chain pages), so the * helper only ever flips version + zero-inits the new * fields here. */ tp_metapage_upgrade_to_current(rel, metapage); metap = (TpIndexMetaPage)PageGetContents(metapage); old_l0_head = metap->level_heads[0]; if (new_segment_root != InvalidBlockNumber) { Page seg_page; TpSegmentHeader *seg_header; if (metap->level_counts[0] >= segment_capacity) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("bm25 segment count limit reached at level 0"))); seg_buf = ReadBuffer(rel, new_segment_root); LockBuffer(seg_buf, BUFFER_LOCK_EXCLUSIVE); seg_page = GenericXLogRegisterBuffer(state, seg_buf, 0); /* Ensure pd_lower covers content for GenericXLog */ ((PageHeader)seg_page)->pd_lower = BLCKSZ; seg_header = (TpSegmentHeader *)PageGetContents(seg_page); seg_header->next_segment = old_l0_head; metap->level_heads[0] = new_segment_root; metap->level_counts[0]++; } metap->memtable_head_blkno = InvalidBlockNumber; metap->memtable_tail_blkno = InvalidBlockNumber; metap->total_docs += docs_delta; metap->total_len += len_delta; GenericXLogFinish(state); *published = true; if (local_state != NULL && local_state->shared != NULL) tp_set_chain_page_count_for_relation(local_state, rel, 0); if (BufferIsValid(seg_buf)) UnlockReleaseBuffer(seg_buf); UnlockReleaseBuffer(metabuf); TP_INJECTION_POINT(TP_INJECTION_AFTER_SPILL_FINALIZE); /* * Step 2: drop the in-memory cache's dshash tables. * * Lock order: * per-index LWLock EXCL (held by the caller, blocks all * readers / catchup paths) * -> cache.apply_lock (no other holder is possible, since * every apply path takes per-index * SHARED first; taken EXCL here * strictly to honor the documented * order with respect to cache.lock) * -> cache.lock (EXCL for drop) * * We resolve the TpMemtable via the local index state's DSA * attachment, not by reaching into the global registry, to * keep this call backend-local and avoid any chance of * deadlocking against the global state lock. */ if (local_state != NULL && local_state->dsa != NULL && local_state->shared != NULL && DsaPointerIsValid(local_state->shared->memtable_dp)) { memtable = (TpMemtable *)dsa_get_address( local_state->dsa, local_state->shared->memtable_dp); if (memtable != NULL) { LWLockAcquire(&memtable->apply_lock, LW_EXCLUSIVE); LWLockAcquire(&memtable->lock, LW_EXCLUSIVE); tp_cache_clear(local_state->dsa, memtable); LWLockRelease(&memtable->lock); LWLockRelease(&memtable->apply_lock); } } } /* * WAL-log DEAD + dead_fxid on one memtable page (delta record). * Caller must hold an exclusive lock on buf. */ static void tp_memtable_wal_mark_page_dead_buf( Relation rel, Buffer buf, FullTransactionId horizon) { Page page; GenericXLogState *state; Assert(BufferIsValid(buf)); state = GenericXLogStart(rel); page = GenericXLogRegisterBuffer(state, buf, 0); if (!tp_memtable_page_is_dead(page)) tp_memtable_page_mark_dead(page, horizon); GenericXLogFinish(state); } /* * Mark continuation pages for one FRAGMENT record. Returns the * first block that is not a continuation page (resume outer walk). */ static BlockNumber tp_memtable_mark_continuation_chain_dead( Relation rel, BlockNumber blk, FullTransactionId horizon) { BlockNumber cur = blk; while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpMemtablePageHeader *hdr; BlockNumber next; CHECK_FOR_INTERRUPTS(); buf = ReadBuffer(rel, cur); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); page = BufferGetPage(buf); hdr = tp_memtable_page_header(page); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable marking dead chain page " "inner walk at block %u in index \"%s\" has " "invalid magic", cur, RelationGetRelationName(rel)))); } if (!tp_memtable_page_is_continuation(page)) { UnlockReleaseBuffer(buf); return cur; } next = hdr->next_block; tp_memtable_wal_mark_page_dead_buf(rel, buf, horizon); UnlockReleaseBuffer(buf); cur = next; } return cur; } /* * Mark every page in the spilled memtable chain DEAD (outer walk + * fragment continuation sub-chains) after spill publication. Does * not free blocks; amvacuumcleanup recycles them later. See * ARCHITECTURE.md, "Spill and Compaction". */ void tp_memtable_mark_chain_dead( Relation rel, BlockNumber head, FullTransactionId horizon) { BlockNumber cur; if (head == InvalidBlockNumber) return; cur = head; while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpMemtablePageHeader *hdr; TpMemtableRecord *rec; BlockNumber next; CHECK_FOR_INTERRUPTS(); buf = ReadBuffer(rel, cur); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable marking dead chain page " "outer walk at block %u in index \"%s\" has " "invalid magic", cur, RelationGetRelationName(rel)))); } if (tp_memtable_page_is_continuation(page)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch memtable marking dead page outer " "walk reached invalid continuation page at " "block %u in index \"%s\"", cur, RelationGetRelationName(rel)))); } hdr = tp_memtable_page_header(page); next = hdr->next_block; rec = tp_memtable_page_first(page); if (rec != NULL && (rec->flags & TP_MEMTABLE_RECORD_FLAG_FRAGMENT) != 0) { /* * Inner walk marks C1..Cn; returns the first * non-continuation block (usually the post-fragment * tail), same resume rule as chain_source.c. */ next = tp_memtable_mark_continuation_chain_dead( rel, next, horizon); } tp_memtable_wal_mark_page_dead_buf(rel, buf, horizon); UnlockReleaseBuffer(buf); cur = next; } } /* * Append a single document to the on-disk memtable chain and * bump the per-index counters used by auto-spill heuristics. * * Memtable v2 (issue #374): the previous DSA write path * (dshash inserts + tp_store_document_length + * tp_add_document_to_posting_list + idempotency gate) is * replaced by an unconditional tp_memtable_append() into the * on-disk chain. Crash idempotency is provided by GenericXLog * LSN-checked redo at the page level, not by application-level * deduplication. * * The `vector_bytes` payload is an in-memory v2 TpVector; only * its byte image is appended to the chain. `term_count` is * used to bump the bulk-load detection counter; the chain * source reconstructs per-term postings on read. */ void tp_add_document_terms( TpLocalIndexState *local_state, Relation rel, ItemPointer ctid, const char *vector_bytes, uint32 vector_len, int term_count, int32 doc_length) { tp_memtable_append(rel, ctid, doc_length, vector_bytes, vector_len); /* Track terms added in this transaction for bulk load detection. */ local_state->terms_added_this_xact += term_count; } /* --------------------------------------------------------------------- * Scaffold SQL functions. Internal-only unit-test entry points, * complementing the end-to-end coverage in the regression suite. * --------------------------------------------------------------------- */ /* * Open the named pg_textsearch index by name, taking * RowExclusiveLock so concurrent DDL is excluded but normal * inserts/scans are not. Errors if the name does not resolve. */ static Relation test_open_index(const char *index_name) { Oid oid; oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("pg_textsearch index \"%s\" not found", index_name))); return index_open(oid, RowExclusiveLock); } /* * Deterministic synthetic vector payload, generated from the * caller's seed so two calls with the same arguments produce * the same bytes. Used by the scaffold cases below. */ static void test_fill_vector(char *buf, uint32 len, uint32 seed) { for (uint32 i = 0; i < len; i++) buf[i] = (char)((seed * 1664525u + i * 1013904223u) & 0xff); } #define TEST_FAIL(fmt, ...) \ do \ { \ char *_buf = psprintf("FAIL: " fmt, ##__VA_ARGS__); \ index_close(rel, RowExclusiveLock); \ PG_RETURN_TEXT_P(cstring_to_text(_buf)); \ } while (0) #define TEST_OK() \ do \ { \ index_close(rel, RowExclusiveLock); \ PG_RETURN_TEXT_P(cstring_to_text("OK")); \ } while (0) static uint32 test_chain_length(Relation rel) { TpIndexMetaPage metap = tp_get_metapage(rel); BlockNumber cur = metap->memtable_head_blkno; uint32 count = 0; pfree(metap); while (cur != InvalidBlockNumber) { Buffer buf; Page page; BlockNumber next; buf = ReadBuffer(rel, cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); elog(ERROR, "test_chain_length: invalid memtable page at block %u", cur); } next = tp_memtable_page_get_next(page); UnlockReleaseBuffer(buf); count++; cur = next; } return count; } PG_FUNCTION_INFO_V1(bm25_test_memtable_append); Datum bm25_test_memtable_append(PG_FUNCTION_ARGS) { text *idx_text = PG_GETARG_TEXT_PP(0); text *case_text = PG_GETARG_TEXT_PP(1); char *idx_name = text_to_cstring(idx_text); char *case_name = text_to_cstring(case_text); Relation rel = test_open_index(idx_name); if (strcmp(case_name, "empty_index") == 0) { TpIndexMetaPage metap = tp_get_metapage(rel); if (metap->memtable_head_blkno != InvalidBlockNumber) TEST_FAIL( "empty index has head_blkno=%u, expected " "InvalidBlockNumber", metap->memtable_head_blkno); if (metap->memtable_tail_blkno != InvalidBlockNumber) TEST_FAIL( "empty index has tail_blkno=%u, expected " "InvalidBlockNumber", metap->memtable_tail_blkno); pfree(metap); TEST_OK(); } else if (strcmp(case_name, "single_append") == 0) { ItemPointerData ctid; char vec[64]; TpIndexMetaPage metap; BlockNumber blk; ItemPointerSet(&ctid, 100, 1); test_fill_vector(vec, sizeof(vec), 1); blk = tp_memtable_append(rel, &ctid, 12345, vec, sizeof(vec)); if (blk == InvalidBlockNumber) TEST_FAIL("single append returned InvalidBlockNumber"); metap = tp_get_metapage(rel); if (metap->memtable_head_blkno != blk) TEST_FAIL( "head_blkno=%u, expected %u", metap->memtable_head_blkno, blk); if (metap->memtable_tail_blkno != blk) TEST_FAIL( "tail_blkno=%u, expected %u", metap->memtable_tail_blkno, blk); pfree(metap); if (test_chain_length(rel) != 1) TEST_FAIL("chain length != 1 after single append"); TEST_OK(); } else if (strcmp(case_name, "fits_on_one_page") == 0) { TpIndexMetaPage metap; const int N = 10; char vec[32]; for (int i = 0; i < N; i++) { ItemPointerData ctid; ItemPointerSet( &ctid, (BlockNumber)(200 + i), (OffsetNumber)(i + 1)); test_fill_vector(vec, sizeof(vec), (uint32)(i + 7)); tp_memtable_append(rel, &ctid, 100 + i, vec, sizeof(vec)); } metap = tp_get_metapage(rel); if (metap->memtable_head_blkno != metap->memtable_tail_blkno) TEST_FAIL( "head=%u tail=%u differ after small inserts", metap->memtable_head_blkno, metap->memtable_tail_blkno); pfree(metap); if (test_chain_length(rel) != 1) TEST_FAIL("chain length != 1 after %d small inserts", N); TEST_OK(); } else if (strcmp(case_name, "extends_chain") == 0) { /* * Each record is large enough that ~5 fill one page. * Insert enough to force at least two pages. */ char *vec; uint32 vec_len = 1500; const int N = 20; uint32 chain_len; BlockNumber head_blkno; vec = palloc(vec_len); test_fill_vector(vec, vec_len, 42); for (int i = 0; i < N; i++) { ItemPointerData ctid; ItemPointerSet( &ctid, (BlockNumber)(300 + i), (OffsetNumber)(i + 1)); tp_memtable_append(rel, &ctid, 500 + i, vec, vec_len); } pfree(vec); chain_len = test_chain_length(rel); if (chain_len < 2) TEST_FAIL("chain length %u, expected >= 2", chain_len); { TpIndexMetaPage metap = tp_get_metapage(rel); head_blkno = metap->memtable_head_blkno; if (metap->memtable_head_blkno == metap->memtable_tail_blkno) TEST_FAIL( "head_blkno=tail_blkno=%u after %d big inserts", metap->memtable_head_blkno, N); pfree(metap); } /* One more insert: head_blkno should not change. */ { ItemPointerData ctid; char small_vec[16]; TpIndexMetaPage metap; ItemPointerSet(&ctid, 999, 1); test_fill_vector(small_vec, sizeof(small_vec), 99); tp_memtable_append(rel, &ctid, 777, small_vec, sizeof(small_vec)); metap = tp_get_metapage(rel); if (metap->memtable_head_blkno != head_blkno) TEST_FAIL( "head_blkno changed from %u to %u", head_blkno, metap->memtable_head_blkno); pfree(metap); } TEST_OK(); } else if (strcmp(case_name, "roundtrip") == 0) { /* * Append a handful of records and read them back via a * chain scan, verifying ctid/doc_length/vector bytes. */ const int N = 7; char vec[64]; uint32 vec_len = 64; TpIndexMetaPage metap; BlockNumber cur; int idx; for (int i = 0; i < N; i++) { ItemPointerData ctid; ItemPointerSet( &ctid, (BlockNumber)(400 + i), (OffsetNumber)(i + 1)); test_fill_vector(vec, vec_len, (uint32)(i * 17 + 3)); tp_memtable_append(rel, &ctid, 800 + i, vec, vec_len); } metap = tp_get_metapage(rel); cur = metap->memtable_head_blkno; pfree(metap); idx = 0; while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpMemtableRecord *r; BlockNumber next_blk; buf = ReadBuffer(rel, cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); TEST_FAIL("invalid page at block %u", cur); } next_blk = tp_memtable_page_get_next(page); for (r = tp_memtable_page_first(page); r != NULL; r = tp_memtable_page_next(page, r)) { ItemPointerData want_ctid; char want_vec[64]; if (idx >= N) { UnlockReleaseBuffer(buf); TEST_FAIL("more records than appended (idx=%d)", idx); } ItemPointerSet( &want_ctid, (BlockNumber)(400 + idx), (OffsetNumber)(idx + 1)); if (!ItemPointerEquals(&r->ctid, &want_ctid)) { UnlockReleaseBuffer(buf); TEST_FAIL( "ctid mismatch at idx=%d: got (%u,%u)", idx, ItemPointerGetBlockNumber(&r->ctid), ItemPointerGetOffsetNumber(&r->ctid)); } if (r->doc_length != 800 + idx) { UnlockReleaseBuffer(buf); TEST_FAIL( "doc_length mismatch at idx=%d: got %d", idx, r->doc_length); } if (r->vector_len != vec_len) { UnlockReleaseBuffer(buf); TEST_FAIL( "vector_len mismatch at idx=%d: got %u", idx, r->vector_len); } test_fill_vector(want_vec, vec_len, (uint32)(idx * 17 + 3)); if (memcmp(r->vector_bytes, want_vec, vec_len) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("vector_bytes mismatch at idx=%d", idx); } idx++; } UnlockReleaseBuffer(buf); cur = next_blk; } if (idx != N) TEST_FAIL("found %d records, expected %d", idx, N); TEST_OK(); } else if (strcmp(case_name, "multi_page_fragment") == 0) { /* * Append three records in sequence: * 1. A small record on a fresh page (ctid1). * 2. A record whose vector_len exceeds * TP_MEMTABLE_PAGE_MAX_VECTOR_LEN (ctid2). This * forces the fragment branch: head page + N * continuation pages + a fresh tail page. * 3. Another small record (ctid3), which should * land on the just-extended fresh tail. * * Verify the resulting chain shape page-by-page and * that the fragment head record's metadata * round-trips (full vector_len, FRAGMENT flag, inline * prefix bytes match the input). */ ItemPointerData ctid1, ctid2, ctid3; uint32 small_len = 64; uint32 large_len; char *small_vec; char *large_vec; uint32 inline_cap; uint32 cont_cap; uint32 expected_n_cont; uint32 expected_total_pages; uint32 observed_pages = 0; BlockNumber head_blk; BlockNumber tail_blk; BlockNumber cur; TpIndexMetaPage metap; inline_cap = tp_memtable_fragment_inline_capacity_max(); cont_cap = tp_memtable_continuation_max_chunk(); large_len = inline_cap + (uint32)(2.5 * (double)cont_cap); expected_n_cont = (large_len - inline_cap + cont_cap - 1) / cont_cap; /* small_page + Thead + Cn + Tnew (the post-fragment tail). */ expected_total_pages = 1 + 1 + expected_n_cont + 1; small_vec = palloc(small_len); large_vec = palloc(large_len); test_fill_vector(small_vec, small_len, 11); test_fill_vector(large_vec, large_len, 22); ItemPointerSet(&ctid1, 1, 1); tp_memtable_append(rel, &ctid1, 10, small_vec, small_len); ItemPointerSet(&ctid2, 2, 1); tp_memtable_append(rel, &ctid2, 20, large_vec, large_len); ItemPointerSet(&ctid3, 3, 1); tp_memtable_append(rel, &ctid3, 30, small_vec, small_len); metap = tp_get_metapage(rel); head_blk = metap->memtable_head_blkno; tail_blk = metap->memtable_tail_blkno; pfree(metap); if (head_blk == InvalidBlockNumber) TEST_FAIL("head_blk Invalid after appends"); if (tail_blk == InvalidBlockNumber) TEST_FAIL("tail_blk Invalid after appends"); cur = head_blk; while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpMemtablePageHeader *hdr; BlockNumber next_blk; buf = ReadBuffer(rel, cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); TEST_FAIL("invalid page at block %u", cur); } hdr = tp_memtable_page_header(page); next_blk = hdr->next_block; if (observed_pages == 0) { /* Original small_record page (ctid1). */ TpMemtableRecord *r; if ((hdr->flags & TP_MEMTABLE_PAGE_FLAG_CONTINUATION) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("first page has CONTINUATION flag"); } if (hdr->n_records != 1) { UnlockReleaseBuffer(buf); TEST_FAIL( "first page n_records=%u, expected 1", hdr->n_records); } r = tp_memtable_page_first(page); if ((r->flags & TP_MEMTABLE_RECORD_FLAG_FRAGMENT) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("first page record has FRAGMENT flag"); } if (!ItemPointerEquals(&r->ctid, &ctid1)) { UnlockReleaseBuffer(buf); TEST_FAIL("first page record ctid mismatch"); } } else if (observed_pages == 1) { /* Fragment head page. */ TpMemtableRecord *r; if ((hdr->flags & TP_MEMTABLE_PAGE_FLAG_CONTINUATION) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("fragment head has CONTINUATION flag"); } if (hdr->n_records != 1) { UnlockReleaseBuffer(buf); TEST_FAIL( "fragment head n_records=%u, expected 1", hdr->n_records); } r = tp_memtable_page_first(page); if ((r->flags & TP_MEMTABLE_RECORD_FLAG_FRAGMENT) == 0) { UnlockReleaseBuffer(buf); TEST_FAIL("fragment head record missing FRAGMENT flag"); } if (r->vector_len != large_len) { UnlockReleaseBuffer(buf); TEST_FAIL( "fragment vector_len=%u, expected %u", r->vector_len, large_len); } if (!ItemPointerEquals(&r->ctid, &ctid2)) { UnlockReleaseBuffer(buf); TEST_FAIL("fragment head ctid mismatch"); } /* Spot-check inline prefix bytes. */ if (memcmp(r->vector_bytes, large_vec, 32) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("fragment inline prefix bytes mismatch"); } } else if (observed_pages < 2 + expected_n_cont) { /* Continuation page. */ if ((hdr->flags & TP_MEMTABLE_PAGE_FLAG_CONTINUATION) == 0) { UnlockReleaseBuffer(buf); TEST_FAIL( "expected continuation flag at page %u", observed_pages); } if (hdr->n_records != 0) { UnlockReleaseBuffer(buf); TEST_FAIL( "continuation n_records=%u, expected 0", hdr->n_records); } } else { /* Tnew tail page containing ctid3. */ TpMemtableRecord *r; if (cur != tail_blk) { UnlockReleaseBuffer(buf); TEST_FAIL( "last page is block %u, but meta.tail=%u", cur, tail_blk); } if ((hdr->flags & TP_MEMTABLE_PAGE_FLAG_CONTINUATION) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("tail page has CONTINUATION flag"); } if (hdr->n_records != 1) { UnlockReleaseBuffer(buf); TEST_FAIL( "tail page n_records=%u, expected 1", hdr->n_records); } r = tp_memtable_page_first(page); if ((r->flags & TP_MEMTABLE_RECORD_FLAG_FRAGMENT) != 0) { UnlockReleaseBuffer(buf); TEST_FAIL("tail page record has FRAGMENT flag"); } if (!ItemPointerEquals(&r->ctid, &ctid3)) { UnlockReleaseBuffer(buf); TEST_FAIL("tail page ctid mismatch"); } } UnlockReleaseBuffer(buf); observed_pages++; cur = next_blk; } if (observed_pages != expected_total_pages) TEST_FAIL( "observed %u pages, expected %u (n_cont=%u)", observed_pages, expected_total_pages, expected_n_cont); pfree(small_vec); pfree(large_vec); TEST_OK(); } TEST_FAIL("unknown case '%s'", case_name); } /* * Pages-info SRF. Walks the chain from meta.head_blkno, * returning one row per page. */ PG_FUNCTION_INFO_V1(bm25_memtable_chain); typedef struct ChainScanState { Relation rel; BlockNumber cur; } ChainScanState; Datum bm25_memtable_chain(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; ChainScanState *state; if (SRF_IS_FIRSTCALL()) { MemoryContext oldcontext; text *idx_text = PG_GETARG_TEXT_PP(0); char *idx_name = text_to_cstring(idx_text); TupleDesc tupdesc; TpIndexMetaPage metap; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); state = (ChainScanState *)palloc(sizeof(ChainScanState)); state->rel = test_open_index(idx_name); metap = tp_get_metapage(state->rel); state->cur = metap->memtable_head_blkno; pfree(metap); tupdesc = CreateTemplateTupleDesc(5); TupleDescInitEntry(tupdesc, 1, "blkno", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, 2, "n_records", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, 3, "free_offset", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, 4, "next_block", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, 5, "flags", INT4OID, -1, 0); /* Hand-built TupleDescs must be finalized before use (PG19+). */ TupleDescFinalize(tupdesc); funcctx->tuple_desc = BlessTupleDesc(tupdesc); funcctx->user_fctx = state; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); state = (ChainScanState *)funcctx->user_fctx; if (state->cur == InvalidBlockNumber) { index_close(state->rel, RowExclusiveLock); SRF_RETURN_DONE(funcctx); } { Buffer buf; Page page; TpMemtablePageHeader *hdr; Datum values[5]; bool nulls[5] = {false, false, false, false, false}; BlockNumber next_blk; HeapTuple tup; buf = ReadBuffer(state->rel, state->cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page)) { UnlockReleaseBuffer(buf); index_close(state->rel, RowExclusiveLock); elog(ERROR, "bm25_memtable_chain: invalid memtable page at block %u", state->cur); } hdr = tp_memtable_page_header(page); next_blk = hdr->next_block; values[0] = Int64GetDatum((int64)state->cur); values[1] = Int32GetDatum((int32)hdr->n_records); values[2] = Int32GetDatum((int32)hdr->free_offset); if (next_blk == InvalidBlockNumber) { values[3] = (Datum)0; nulls[3] = true; } else { values[3] = Int64GetDatum((int64)next_blk); } values[4] = Int32GetDatum((int32)hdr->flags); UnlockReleaseBuffer(buf); state->cur = next_blk; tup = heap_form_tuple(funcctx->tuple_desc, values, nulls); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tup)); } } /* * Dead-orphan SRF. Scans index blocks after the metapage and * returns one row per memtable page stamped DEAD after spill * publication. */ PG_FUNCTION_INFO_V1(bm25_memtable_dead_pages); typedef struct DeadScanState { Relation rel; BlockNumber cur; BlockNumber nblocks; } DeadScanState; Datum bm25_memtable_dead_pages(PG_FUNCTION_ARGS) { FuncCallContext *funcctx; DeadScanState *state; if (SRF_IS_FIRSTCALL()) { MemoryContext oldcontext; text *idx_text = PG_GETARG_TEXT_PP(0); char *idx_name = text_to_cstring(idx_text); TupleDesc tupdesc; funcctx = SRF_FIRSTCALL_INIT(); oldcontext = MemoryContextSwitchTo(funcctx->multi_call_memory_ctx); state = (DeadScanState *)palloc(sizeof(DeadScanState)); state->rel = test_open_index(idx_name); state->nblocks = RelationGetNumberOfBlocks(state->rel); state->cur = TP_METAPAGE_BLKNO + 1; tupdesc = CreateTemplateTupleDesc(4); TupleDescInitEntry(tupdesc, 1, "blkno", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, 2, "flags", INT4OID, -1, 0); TupleDescInitEntry(tupdesc, 3, "dead_fxid", INT8OID, -1, 0); TupleDescInitEntry(tupdesc, 4, "n_records", INT4OID, -1, 0); /* Hand-built TupleDescs must be finalized before use (PG19+). */ TupleDescFinalize(tupdesc); funcctx->tuple_desc = BlessTupleDesc(tupdesc); funcctx->user_fctx = state; MemoryContextSwitchTo(oldcontext); } funcctx = SRF_PERCALL_SETUP(); state = (DeadScanState *)funcctx->user_fctx; while (state->cur < state->nblocks) { Buffer buf; Page page; TpMemtablePageHeader *hdr; BlockNumber blk = state->cur; Datum values[4]; bool nulls[4] = {false, false, false, false}; HeapTuple tup; state->cur++; buf = ReadBuffer(state->rel, blk); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_memtable_page_is_valid(page) || !tp_memtable_page_is_dead(page)) { UnlockReleaseBuffer(buf); continue; } hdr = tp_memtable_page_header(page); values[0] = Int64GetDatum((int64)blk); values[1] = Int32GetDatum((int32)hdr->flags); values[2] = Int64GetDatum( (int64)U64FromFullTransactionId(hdr->dead_fxid)); values[3] = Int32GetDatum((int32)hdr->n_records); UnlockReleaseBuffer(buf); tup = heap_form_tuple(funcctx->tuple_desc, values, nulls); SRF_RETURN_NEXT(funcctx, HeapTupleGetDatum(tup)); } index_close(state->rel, RowExclusiveLock); SRF_RETURN_DONE(funcctx); } timescale-pg_textsearch-b63fe58/src/memtable/log.h000066400000000000000000000161211525762047700222710ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * log.h - On-disk memtable write path. * * Appends document records to the on-disk memtable page chain * rooted at the metapage's (memtable_head_blkno, * memtable_tail_blkno) fields. See ARCHITECTURE.md, "Storage * and WAL", for the design. * * Concurrency contract: * * per-index LWLock SHARED/EXCL * └─► tail buffer EXCL * └─► new buffer EXCL (just-extended) * └─► metapage buffer EXCL * * Reverse-order acquisitions are forbidden. Ordinary chain readers read * the head from the metapage and release it before acquiring a chain page. * The common read snapshot reads a candidate head/tail and releases the * metapage, then locks the candidate tail SHARED before reacquiring the * metapage SHARED and validating the candidate. It never nests * metapage -> tail. Spill (build.c) holds the per-index LWLock EXCLUSIVE, * which excludes all writers above; that is the path by which spill gains * uncontended access to the chain pages. * * Crash safety: a crash between page allocation * (ExtendBufferedRel or FSM reuse via tp_fsm_claim_free_buffer) and * GenericXLogFinish() leaves a block on disk that is unreachable * via meta.head → next chain and usually without a DEAD stamp. * No code may sequentially scan the relation without first * validating each page via tp_memtable_page_is_valid(). Orphans * without DEAD may be recycled later by amvacuumcleanup once * horizon rules exist for never-linked blocks. */ #pragma once #include #include #include /* * Append one document record to the on-disk memtable chain. * * Caller responsibilities: * - `rel` is open with at least RowExclusiveLock (heavyweight) * and writes to it are permitted in the current xact * - `vector_bytes` points to `vector_len` bytes of opaque * payload (the v2 TpVector wire format; scaffold callers * in log.c may pass synthetic bytes for unit-test coverage) * * On entry, this function: * - Acquires the per-index LWLock in SHARED mode (idempotent * within an xact via tp_acquire_index_lock). * - Bootstraps the chain (first append) under one GenericXLog * record covering {metapage, new page}; OR * - Appends to the existing tail page under one GenericXLog * record covering {tail}; OR * - Extends the chain under one GenericXLog record covering * {tail, new page, metapage}, then appends to new tail; OR * - For oversized payloads (vector_len greater than * TP_MEMTABLE_PAGE_MAX_VECTOR_LEN): writes a head fragment * record on a fresh page followed by N continuation pages * and a fresh tail; each new page gets its own GenericXLog * record, and a final atomic GenericXLog publishes the * pages by linking the old tail (or meta head) to the head * fragment and advancing the meta tail. * * Returns the block number of the page that received the new * record. The per-index LWLock is left held in SHARED mode on * return; the AM-level caller (`tp_insert`) releases it before * the auto-spill check, and the standard xact-end callback is a * safety net for any path that forgets to release. */ extern BlockNumber tp_memtable_append( Relation rel, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len); /* * Atomically publish a spill. Used by the spill path after * tp_write_segment has produced a new L0 segment from the chain. * * Performs, in a single GenericXLog record: * - Resets memtable_head_blkno / memtable_tail_blkno to * InvalidBlockNumber on the metapage. * - Splices the new segment in at the head of the L0 chain, * setting its `next_segment` to the previous L0 head and * bumping the metapage's level_heads[0] / level_counts[0]. * - Adds `docs_delta` / `len_delta` to total_docs / total_len * (these are caller-supplied: the chain source's totals). * * Does not modify the old memtable chain pages. After this call * the chain blocks are orphans (no metapage pointer); the caller * should run `tp_memtable_mark_chain_dead` to WAL-stamp each page * `DEAD` + `dead_fxid` for later reclaim (crash-safe ordering: * finalize first, then mark dead). In-flight scans walking the * old chain via a metapage snapshot they already latched complete * safely; new scans see the new metapage and skip the orphans. * `tp_vacuumcleanup` returns them to the index FSM via * `tp_record_free_index_page`; `tp_memtable_alloc_page` reuses them. * * Caller MUST already hold the per-index LWLock in EXCLUSIVE mode. * * `state` is required when the in-memory memtable cache is active * (runtime mode); it provides the DSA attachment used to drop the * cache's dshash tables after the metapage is published, and the * pg_atomic counter that the cache's apply protocol uses as a * stale-detection token. Callers that legitimately have no cache * (e.g. spill-from-build before the local state is wired up) may * pass NULL; in that case the spill_generation bump and the * tp_cache_clear call are both skipped (the build path constructs * an empty cache anyway). * * `segment_capacity` is the maximum representable L0 count for this * operation. Normal spills pass the configured test limit; force compaction * passes the persisted uint16 limit so lowering the test limit cannot block * its mandatory spill. * `published` is set at the WAL publication boundary, before cache cleanup * or other fallible work, so error cleanup cannot discard the live segment. */ extern void tp_spill_finalize( TpLocalIndexState *state, Relation rel, BlockNumber new_segment_root, uint64 docs_delta, uint64 len_delta, uint32 segment_capacity, bool *published); /* * WAL-stamp every page in the memtable chain rooted at `head` as * dead (including fragment continuation pages). Called from * `tp_do_spill` under LW_EXCLUSIVE after tp_spill_finalize has * disconnected the chain (crash-safe ordering: finalize first, * then mark dead). `horizon` is stored in each page's `dead_fxid` * for later reclaim once globally safe. */ extern void tp_memtable_mark_chain_dead( Relation rel, BlockNumber head, FullTransactionId horizon); /* Forward declaration to avoid heavy header include. */ typedef struct TpLocalIndexState TpLocalIndexState; /* * Append one document to the on-disk memtable chain and update * the per-transaction bulk-load counter. * * Memtable v2 (issue #374): the previous DSA-based posting and * doc-length tables are gone; this is a thin wrapper around * tp_memtable_append() that also bumps terms_added_this_xact, * used by tp_bulk_load_spill_check. * * `vector_bytes` points to `vector_len` bytes of opaque payload * (the in-memory v2 TpVector wire format). The chain source * reconstructs per-term postings on read. */ extern void tp_add_document_terms( TpLocalIndexState *local_state, Relation rel, ItemPointer ctid, const char *vector_bytes, uint32 vector_len, int term_count, int32 doc_length); timescale-pg_textsearch-b63fe58/src/memtable/page.c000066400000000000000000000475341525762047700224330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * page.c - On-disk memtable page format helpers. * * See page.h for the layout and conventions. This file is * intentionally callable from anywhere that has a Page pointer; * it does no buffer locking, WAL emission, or chain management. * Those concerns live one layer up (log.c). * * Also exposed here is bm25_test_memtable_page(text), a scaffold * SQL function used to exercise the helpers from the SQL * regression suite for unit-level coverage. */ #include #include #include #include #include #include #include #include #include "constants.h" #include "memtable/page.h" /* * Page header is placed at PageGetContents(page), which is * MAXALIGN(SizeOfPageHeaderData) bytes from the start of the * page. TP_MEMTABLE_PAGE_HEADER_OFFSET captures that. */ TpMemtablePageHeader * tp_memtable_page_header(Page page) { return (TpMemtablePageHeader *)((char *)page + TP_MEMTABLE_PAGE_HEADER_OFFSET); } void tp_memtable_page_init(Page page) { TpMemtablePageHeader *hdr; PageInit(page, BLCKSZ, 0); hdr = tp_memtable_page_header(page); hdr->magic = TP_MEMTABLE_PAGE_MAGIC; hdr->version = TP_MEMTABLE_PAGE_VERSION; hdr->flags = 0; hdr->n_records = 0; hdr->free_offset = (uint16)TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET; hdr->next_block = InvalidBlockNumber; hdr->dead_fxid = InvalidFullTransactionId; /* * Mark the entire page as "in-use" so GenericXLog does not treat * the area we use for records as the standard PD_LOWER..PD_UPPER * hole. GenericXLogFinish() zeroes that hole when applying the * scratch image to the buffer, which would silently wipe out our * header and records. Setting pd_lower=BLCKSZ collapses the hole * to zero bytes so the full page is included in the delta and * the buffer write-back. The standard PageHeader.pd_special and * pd_upper remain at BLCKSZ from PageInit, which is fine: we do * not use line pointers or PageAddItem on this page format, so * the invariants those macros rely on are not exercised. */ ((PageHeader)page)->pd_lower = BLCKSZ; } bool tp_memtable_page_is_valid(Page page) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); return hdr->magic == TP_MEMTABLE_PAGE_MAGIC && hdr->version == TP_MEMTABLE_PAGE_VERSION; } uint32 tp_memtable_record_size(uint32 vector_len) { return (uint32)MAXALIGN(TP_MEMTABLE_RECORD_BASE_SIZE + vector_len); } bool tp_memtable_page_can_fit(Page page, uint32 vector_len) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); uint32 need = tp_memtable_record_size(vector_len); /* * free_offset is always MAXALIGN'd; BLCKSZ is naturally * MAXALIGN'd; therefore the comparison is exact (no rounding). */ return (uint32)hdr->free_offset + need <= (uint32)BLCKSZ; } TpMemtableRecord * tp_memtable_page_append( Page page, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len) { TpMemtablePageHeader *hdr; TpMemtableRecord *rec; uint32 record_size; hdr = tp_memtable_page_header(page); if (!tp_memtable_page_can_fit(page, vector_len)) elog(ERROR, "tp_memtable_page_append: record (vector_len=%u, total=%u) does " "not fit on page (free_offset=%u, BLCKSZ=%d)", vector_len, tp_memtable_record_size(vector_len), hdr->free_offset, BLCKSZ); rec = (TpMemtableRecord *)((char *)page + hdr->free_offset); ItemPointerCopy(ctid, &rec->ctid); rec->flags = 0; rec->doc_length = doc_length; rec->vector_len = vector_len; if (vector_len > 0) memcpy(rec->vector_bytes, vector_bytes, vector_len); record_size = tp_memtable_record_size(vector_len); hdr->free_offset += (uint16)record_size; hdr->n_records++; return rec; } TpMemtableRecord * tp_memtable_page_first(Page page) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); if (hdr->n_records == 0) return NULL; return (TpMemtableRecord *)((char *)page + TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET); } TpMemtableRecord * tp_memtable_page_next(Page page, TpMemtableRecord *cur) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); char *cur_end; char *next; cur_end = (char *)cur + tp_memtable_record_size(cur->vector_len); next = (char *)page + hdr->free_offset; if (cur_end >= next) return NULL; return (TpMemtableRecord *)cur_end; } BlockNumber tp_memtable_page_get_next(Page page) { return tp_memtable_page_header(page)->next_block; } void tp_memtable_page_set_next(Page page, BlockNumber blk) { tp_memtable_page_header(page)->next_block = blk; } uint16 tp_memtable_page_n_records(Page page) { return tp_memtable_page_header(page)->n_records; } /* --------------------------------------------------------------------- * Multi-page (fragment) record support. * * Layout of a FRAGMENT head page: * header (flags=0, n_records=1, next_block=C1) * one TpMemtableRecord with flags=FRAGMENT, vector_len=FULL, * inline prefix bytes that fill the rest of the page. * The page's free_offset advances to BLCKSZ; nothing else * appends to this page. * * Layout of a continuation page: * header (flags=CONTINUATION, n_records=0, next_block=Cnext|Tnew|Invalid) * raw payload bytes at [first_record_offset, free_offset). * No record framing on this page. * * The chain walker, on encountering a FRAGMENT head, reassembles * the full payload by reading inline bytes + each continuation's * chunk in order, then skips past the last continuation when * resuming the outer walk. * --------------------------------------------------------------------- */ uint32 tp_memtable_fragment_inline_capacity_max(void) { /* * The head page starts empty (free_offset = * TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET). After appending a * FRAGMENT head record with inline payload `inline_len`, the * page's free_offset must reach BLCKSZ exactly (under * MAXALIGN). The record total size is * MAXALIGN(base + inline_len), so we want the largest * `inline_len` such that * first_record_offset + MAXALIGN(base + inline_len) <= BLCKSZ * * Since first_record_offset and BLCKSZ are MAXALIGN-multiples, * MAXALIGN(base + inline_len) <= BLCKSZ - first_record_offset. * Solving: inline_len <= BLCKSZ - first_record_offset - base * once we account for the MAXALIGN. The largest aligned * record total that fits is BLCKSZ - first_record_offset, so: */ return (uint32)((BLCKSZ - TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET) - TP_MEMTABLE_RECORD_BASE_SIZE); } TpMemtableRecord * tp_memtable_page_append_fragment_head( Page page, ItemPointer ctid, int32 doc_length, const char *prefix, uint32 prefix_len, uint32 total_vector_len) { TpMemtablePageHeader *hdr; TpMemtableRecord *rec; uint32 expected_prefix; hdr = tp_memtable_page_header(page); expected_prefix = tp_memtable_fragment_inline_capacity_max(); if (prefix_len != expected_prefix) elog(ERROR, "tp_memtable_page_append_fragment_head: prefix_len=%u " "must equal inline capacity %u", prefix_len, expected_prefix); if (total_vector_len <= prefix_len) elog(ERROR, "tp_memtable_page_append_fragment_head: total_vector_len=%u " "must exceed prefix_len=%u", total_vector_len, prefix_len); if (hdr->n_records != 0 || hdr->free_offset != TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET) elog(ERROR, "tp_memtable_page_append_fragment_head: head page must be " "empty (n_records=%u, free_offset=%u)", hdr->n_records, hdr->free_offset); rec = (TpMemtableRecord *)((char *)page + hdr->free_offset); ItemPointerCopy(ctid, &rec->ctid); rec->flags = TP_MEMTABLE_RECORD_FLAG_FRAGMENT; rec->doc_length = doc_length; rec->vector_len = total_vector_len; if (prefix_len > 0) memcpy(rec->vector_bytes, prefix, prefix_len); /* * Advance free_offset to BLCKSZ. The inline payload * occupies bytes [vector_bytes_offset, BLCKSZ); since * prefix_len == BLCKSZ - first_record_offset - base (the * formula above), free_offset = first_record_offset + base * + prefix_len = BLCKSZ exactly. Sanity-asserted via the * arithmetic. */ hdr->free_offset = (uint16)BLCKSZ; hdr->n_records = 1; return rec; } uint32 tp_memtable_continuation_max_chunk(void) { /* * Continuation pages hold a single contiguous chunk of * payload bytes at [first_record_offset, BLCKSZ). No * record framing; the chunk's length is implicit in the * page's free_offset. */ return (uint32)(BLCKSZ - TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET); } void tp_memtable_continuation_page_init( Page page, const char *chunk, uint32 chunk_len, BlockNumber next_block) { TpMemtablePageHeader *hdr; uint32 cap = tp_memtable_continuation_max_chunk(); if (chunk_len > cap) elog(ERROR, "tp_memtable_continuation_page_init: chunk_len %u exceeds " "capacity %u", chunk_len, cap); PageInit(page, BLCKSZ, 0); hdr = tp_memtable_page_header(page); hdr->magic = TP_MEMTABLE_PAGE_MAGIC; hdr->version = TP_MEMTABLE_PAGE_VERSION; hdr->flags = TP_MEMTABLE_PAGE_FLAG_CONTINUATION; hdr->n_records = 0; hdr->free_offset = (uint16)(TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET + chunk_len); hdr->next_block = next_block; hdr->dead_fxid = InvalidFullTransactionId; if (chunk_len > 0) memcpy((char *)page + TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET, chunk, chunk_len); /* * Suppress the GenericXLog hole between pd_lower and pd_upper * for the same reason as regular memtable pages — see * tp_memtable_page_init(). */ ((PageHeader)page)->pd_lower = BLCKSZ; } bool tp_memtable_page_is_continuation(Page page) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); return tp_memtable_page_is_valid(page) && (hdr->flags & TP_MEMTABLE_PAGE_FLAG_CONTINUATION) != 0; } const char * tp_memtable_continuation_page_chunk(Page page, uint32 *out_len) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); Assert((hdr->flags & TP_MEMTABLE_PAGE_FLAG_CONTINUATION) != 0); *out_len = (uint32)hdr->free_offset - TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET; return (const char *)page + TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET; } /* * Set DEAD + dead_fxid on a GenericXLog working copy. Idempotent: * no-op if the page is already marked dead (first horizon wins). */ void tp_memtable_page_mark_dead(Page page, FullTransactionId horizon) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); if (!tp_memtable_page_is_valid(page)) return; if (tp_memtable_page_is_dead(page)) return; hdr->flags |= TP_MEMTABLE_PAGE_FLAG_DEAD; hdr->dead_fxid = horizon; } bool tp_memtable_page_is_dead(Page page) { TpMemtablePageHeader *hdr = tp_memtable_page_header(page); return tp_memtable_page_is_valid(page) && (hdr->flags & TP_MEMTABLE_PAGE_FLAG_DEAD) != 0; } /* --------------------------------------------------------------------- * Scaffold SQL function for unit-level coverage. * * bm25_test_memtable_page(case_name text) -> text * * Returns 'OK' if `case_name` passes, or 'FAIL: ' otherwise. * Cases exercise the helpers above against a scratch BLCKSZ buffer; * no shared buffers, no WAL, no relation involvement. See * test/sql/memtable_page.sql for the per-case asserts. * --------------------------------------------------------------------- */ PG_FUNCTION_INFO_V1(bm25_test_memtable_page); #define TEST_FAIL(fmt, ...) \ do \ { \ char *_buf = psprintf("FAIL: " fmt, ##__VA_ARGS__); \ PG_RETURN_TEXT_P(cstring_to_text(_buf)); \ } while (0) #define TEST_OK() PG_RETURN_TEXT_P(cstring_to_text("OK")) static Page test_alloc_page(void) { /* * BLCKSZ is at most 32 KB and palloc returns MAXALIGN'd memory. * Zero-fill so the "reject_uninitialized" case sees a page that * deliberately does not carry our magic. */ Page p = (Page)palloc0(BLCKSZ); return p; } static ItemPointerData test_make_ctid(BlockNumber blk, OffsetNumber off) { ItemPointerData itp; ItemPointerSet(&itp, blk, off); return itp; } Datum bm25_test_memtable_page(PG_FUNCTION_ARGS) { text *case_text = PG_GETARG_TEXT_PP(0); char *case_name = text_to_cstring(case_text); if (strcmp(case_name, "reject_uninitialized") == 0) { Page p = test_alloc_page(); if (tp_memtable_page_is_valid(p)) TEST_FAIL("uninitialized (zero) page reported valid"); TEST_OK(); } else if (strcmp(case_name, "init_empty") == 0) { Page p = test_alloc_page(); TpMemtablePageHeader *hdr; tp_memtable_page_init(p); if (!tp_memtable_page_is_valid(p)) TEST_FAIL("init produced an invalid page"); hdr = tp_memtable_page_header(p); if (hdr->magic != TP_MEMTABLE_PAGE_MAGIC) TEST_FAIL("unexpected magic 0x%x", hdr->magic); if (hdr->version != TP_MEMTABLE_PAGE_VERSION) TEST_FAIL("unexpected version %u", hdr->version); if (hdr->n_records != 0) TEST_FAIL("n_records=%u, expected 0", hdr->n_records); if (hdr->flags != 0) TEST_FAIL("flags=%u, expected 0", hdr->flags); if (hdr->free_offset != TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET) TEST_FAIL( "free_offset=%u, expected %u", hdr->free_offset, (unsigned)TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET); if (hdr->next_block != InvalidBlockNumber) TEST_FAIL( "next_block=%u, expected InvalidBlockNumber", hdr->next_block); if (tp_memtable_page_n_records(p) != 0) TEST_FAIL( "n_records accessor returned %u, expected 0", tp_memtable_page_n_records(p)); TEST_OK(); } else if (strcmp(case_name, "iterate_empty") == 0) { Page p = test_alloc_page(); tp_memtable_page_init(p); if (tp_memtable_page_first(p) != NULL) TEST_FAIL("first(empty) returned non-NULL"); TEST_OK(); } else if (strcmp(case_name, "one_record_roundtrip") == 0) { Page p = test_alloc_page(); ItemPointerData ctid = test_make_ctid(42, 7); const char *vec = "\x01\x02\x03\x04\x05\x06\x07"; uint32 vec_len = 7; TpMemtableRecord *rec; TpMemtableRecord *iter; tp_memtable_page_init(p); if (!tp_memtable_page_can_fit(p, vec_len)) TEST_FAIL("fresh page cannot fit a 7-byte record"); rec = tp_memtable_page_append(p, &ctid, 12345, vec, vec_len); if (rec == NULL) TEST_FAIL("append returned NULL"); if (tp_memtable_page_n_records(p) != 1) TEST_FAIL( "n_records=%u after one append", tp_memtable_page_n_records(p)); iter = tp_memtable_page_first(p); if (iter == NULL) TEST_FAIL("first() returned NULL after one append"); if (iter != rec) TEST_FAIL("first() differs from append() pointer"); if (!ItemPointerEquals(&iter->ctid, &ctid)) TEST_FAIL( "ctid roundtrip mismatch: got (%u,%u)", ItemPointerGetBlockNumber(&iter->ctid), ItemPointerGetOffsetNumber(&iter->ctid)); if (iter->doc_length != 12345) TEST_FAIL("doc_length=%d, expected 12345", iter->doc_length); if (iter->vector_len != vec_len) TEST_FAIL("vector_len=%u, expected %u", iter->vector_len, vec_len); if (memcmp(iter->vector_bytes, vec, vec_len) != 0) TEST_FAIL("vector_bytes mismatch"); if (tp_memtable_page_next(p, iter) != NULL) TEST_FAIL("next() of last record returned non-NULL"); TEST_OK(); } else if (strcmp(case_name, "many_records_roundtrip") == 0) { Page p = test_alloc_page(); const int N = 10; int i; TpMemtableRecord *iter; char vec[64]; tp_memtable_page_init(p); for (i = 0; i < N; i++) { ItemPointerData ctid = test_make_ctid( (BlockNumber)(1000 + i), (OffsetNumber)(i + 1)); uint32 vec_len = (uint32)(1 + (i % 17)); int j; for (j = 0; j < (int)vec_len; j++) vec[j] = (char)((i * 31 + j) & 0xff); if (!tp_memtable_page_can_fit(p, vec_len)) TEST_FAIL( "can_fit returned false at i=%d, vec_len=%u", i, vec_len); tp_memtable_page_append(p, &ctid, 100 + i, vec, vec_len); } if (tp_memtable_page_n_records(p) != N) TEST_FAIL( "n_records=%u after %d appends", tp_memtable_page_n_records(p), N); iter = tp_memtable_page_first(p); for (i = 0; i < N; i++) { ItemPointerData expected_ctid = test_make_ctid( (BlockNumber)(1000 + i), (OffsetNumber)(i + 1)); uint32 expected_vec_len = (uint32)(1 + (i % 17)); int j; if (iter == NULL) TEST_FAIL( "iterator exhausted at i=%d, expected %d records", i, N); if (!ItemPointerEquals(&iter->ctid, &expected_ctid)) TEST_FAIL( "ctid mismatch at i=%d: got (%u,%u)", i, ItemPointerGetBlockNumber(&iter->ctid), ItemPointerGetOffsetNumber(&iter->ctid)); if (iter->doc_length != 100 + i) TEST_FAIL( "doc_length mismatch at i=%d: got %d", i, iter->doc_length); if (iter->vector_len != expected_vec_len) TEST_FAIL( "vector_len mismatch at i=%d: got %u", i, iter->vector_len); for (j = 0; j < (int)expected_vec_len; j++) { char expected = (char)((i * 31 + j) & 0xff); if (iter->vector_bytes[j] != expected) TEST_FAIL("vector_bytes mismatch at i=%d, j=%d", i, j); } iter = tp_memtable_page_next(p, iter); } if (iter != NULL) TEST_FAIL("iterator did not terminate after %d records", N); TEST_OK(); } else if (strcmp(case_name, "can_fit_boundary") == 0) { Page p = test_alloc_page(); TpMemtablePageHeader *hdr; uint32 remaining; tp_memtable_page_init(p); hdr = tp_memtable_page_header(p); remaining = BLCKSZ - hdr->free_offset; /* * A record requiring `remaining` bytes total (header + * payload + padding) must fit; one byte more must not. */ if (remaining < TP_MEMTABLE_RECORD_BASE_SIZE) TEST_FAIL("remaining=%u less than record base size", remaining); /* * Largest vector_len that yields total <= remaining. * Total = MAXALIGN(base + vec). Find the largest vec that * still fits. */ { uint32 max_vec = remaining - TP_MEMTABLE_RECORD_BASE_SIZE; uint32 too_big = max_vec + (uint32)MAXIMUM_ALIGNOF + 1; if (!tp_memtable_page_can_fit(p, max_vec)) TEST_FAIL("can_fit(max_vec=%u) returned false", max_vec); if (tp_memtable_page_can_fit(p, too_big)) TEST_FAIL("can_fit(too_big=%u) returned true", too_big); } TEST_OK(); } else if (strcmp(case_name, "fill_until_full") == 0) { Page p = test_alloc_page(); int appended = 0; uint32 vec_len = 32; char vec[64]; TpMemtableRecord *iter; int i; for (i = 0; i < (int)vec_len; i++) vec[i] = (char)i; tp_memtable_page_init(p); while (tp_memtable_page_can_fit(p, vec_len)) { ItemPointerData ctid = test_make_ctid( (BlockNumber)appended, (OffsetNumber)(appended + 1)); tp_memtable_page_append(p, &ctid, appended, vec, vec_len); appended++; } if (appended == 0) TEST_FAIL("appended zero records (page too small?)"); if (tp_memtable_page_n_records(p) != appended) TEST_FAIL( "n_records=%u, appended=%d", tp_memtable_page_n_records(p), appended); iter = tp_memtable_page_first(p); for (i = 0; i < appended; i++) { if (iter == NULL) TEST_FAIL("iterator NULL at i=%d", i); if (iter->doc_length != i) TEST_FAIL("doc_length=%d, expected %d", iter->doc_length, i); iter = tp_memtable_page_next(p, iter); } if (iter != NULL) TEST_FAIL("iterator did not terminate"); TEST_OK(); } else if (strcmp(case_name, "next_block_roundtrip") == 0) { Page p = test_alloc_page(); tp_memtable_page_init(p); if (tp_memtable_page_get_next(p) != InvalidBlockNumber) TEST_FAIL( "fresh page next_block=%u, expected Invalid", tp_memtable_page_get_next(p)); tp_memtable_page_set_next(p, 12345); if (tp_memtable_page_get_next(p) != 12345) TEST_FAIL( "after set, next_block=%u, expected 12345", tp_memtable_page_get_next(p)); tp_memtable_page_set_next(p, InvalidBlockNumber); if (tp_memtable_page_get_next(p) != InvalidBlockNumber) TEST_FAIL( "after clear, next_block=%u, expected Invalid", tp_memtable_page_get_next(p)); TEST_OK(); } TEST_FAIL("unknown case '%s'", case_name); } timescale-pg_textsearch-b63fe58/src/memtable/page.h000066400000000000000000000230071525762047700224250ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * page.h - On-disk memtable page format. * * Memtable v2 (see issue #374). This module defines the layout * of a memtable log page and exposes a minimal set of helpers * that operate on a single in-memory page buffer. Concurrency, * WAL emission, and chain management live one layer up in log.c; * everything here is mechanical bytes-on-a-page code. * * Layout summary (offsets shown for BLCKSZ=8192, MAXALIGN=8): * * offset 0..23 PageHeaderData (managed by PageInit) * offset 24..47 TpMemtablePageHeader (this file) * offset 48..free_offset Packed TpMemtableRecord stream * offset free_offset.. Free space * * Each record is laid out as: * * ItemPointerData ctid (6 bytes, 2-byte aligned) * <2 bytes padding> * int32 doc_length (4 bytes) * uint32 vector_len (4 bytes) * char vector_bytes[vector_len] * * * The vector_bytes payload is an opaque blob from this layer's * point of view; the write path will populate it with the same * bytes as the v2 TpVector wire format so that the read path can * walk it without an intermediate palloc. */ #pragma once #include #include #include #include #include #include "constants.h" /* Flags occupying the `flags` field of TpMemtablePageHeader. */ #define TP_MEMTABLE_PAGE_FLAG_DEAD \ 0x0001 /* unlinked from metapage; dead_fxid \ * is spill horizon for FSM recycle */ /* * Page contains continuation bytes of a previous record's * oversized payload. n_records is 0 and the on-page bytes * between [first_record_offset, free_offset) form an opaque * chunk of payload. See tp_memtable_continuation_page_init(). */ #define TP_MEMTABLE_PAGE_FLAG_CONTINUATION 0x0002 /* Flags occupying the `flags` field of TpMemtableRecord. */ /* * Record's vector payload spans this page + one or more * following continuation pages. vector_len stores the FULL * payload length; the inline byte count on this page is * implicit (page_free_offset - record_offset - base_size). * The record is always the sole record on its head page; the * head page's next_block points to the first continuation. */ #define TP_MEMTABLE_RECORD_FLAG_FRAGMENT 0x0001 /* * Tapir memtable page header. * * Placed at PageGetContents(page). Total size is 24 bytes and is * naturally aligned. Future flag bits or version bumps can extend * the unused space without breaking the read/write helpers. */ typedef struct TpMemtablePageHeader { uint32 magic; /* TP_MEMTABLE_PAGE_MAGIC */ uint16 version; /* TP_MEMTABLE_PAGE_VERSION */ uint16 flags; /* see TP_MEMTABLE_PAGE_FLAG_* */ uint16 n_records; /* number of records on the page */ uint16 free_offset; /* byte offset (from start of page) of the * next record write position; always * MAXALIGN'd */ BlockNumber next_block; /* next page in the chain, or * InvalidBlockNumber */ FullTransactionId dead_fxid; /* valid only when flags & DEAD: the xid * horizon after which the page may be * FSM-recycled */ } TpMemtablePageHeader; /* * Tapir memtable record. * * One record per indexed document. Variable length; the trailing * vector_bytes payload is the v2 TpVector wire format (write path * will use TPVECTOR_VARSIZE(...) bytes copied verbatim). */ typedef struct TpMemtableRecord { ItemPointerData ctid; /* heap tuple id of the indexed doc */ uint16 flags; /* see TP_MEMTABLE_RECORD_FLAG_* */ int32 doc_length; /* token count of the indexed doc */ uint32 vector_len; /* byte length of full vector payload * (may exceed the inline byte count * when FRAGMENT flag is set) */ char vector_bytes[FLEXIBLE_ARRAY_MEMBER]; } TpMemtableRecord; #define TP_MEMTABLE_RECORD_BASE_SIZE (offsetof(TpMemtableRecord, vector_bytes)) #define TP_MEMTABLE_PAGE_HEADER_OFFSET (MAXALIGN(SizeOfPageHeaderData)) #define TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET \ (TP_MEMTABLE_PAGE_HEADER_OFFSET + MAXALIGN(sizeof(TpMemtablePageHeader))) /* * Largest vector payload that can fit on an otherwise-empty * memtable page as a single (non-fragmented) record. Records * whose vector_len exceeds this are written via the FRAGMENT * path: a head record on its own page followed by one or more * continuation pages. * * Derived constant so callers can sanity-check inputs before * touching the writer; the writer also re-checks. */ #define TP_MEMTABLE_PAGE_MAX_VECTOR_LEN \ ((uint32)((BLCKSZ - TP_MEMTABLE_PAGE_FIRST_RECORD_OFFSET) - \ TP_MEMTABLE_RECORD_BASE_SIZE)) /* Return the on-page header. Caller must hold the buffer pinned. */ extern TpMemtablePageHeader *tp_memtable_page_header(Page page); /* * Initialize an empty memtable page in `page`. The page must * already have been zeroed and laid out by PageInit(...) with * specialSize==0. After this call the page reports n_records=0, * next_block=InvalidBlockNumber, flags=0, and is recognized as * valid by tp_memtable_page_is_valid(). */ extern void tp_memtable_page_init(Page page); /* * Returns true iff `page` carries a recognizable memtable page * header (magic + version match). Used to detect uninitialized * or recycled pages. */ extern bool tp_memtable_page_is_valid(Page page); /* * Total on-page space (including padding) consumed by a record * whose vector payload is `vector_len` bytes. */ extern uint32 tp_memtable_record_size(uint32 vector_len); /* * Returns true iff appending a record with vector payload * `vector_len` would fit on the page. */ extern bool tp_memtable_page_can_fit(Page page, uint32 vector_len); /* * Append a record to the page. Caller must have established * via tp_memtable_page_can_fit() that the record fits; this * function will ereport(ERROR) if it does not. Returns a * pointer into the page at the newly written record. */ extern TpMemtableRecord *tp_memtable_page_append( Page page, ItemPointer ctid, int32 doc_length, const char *vector_bytes, uint32 vector_len); /* * Iteration: returns a pointer to the first record on the page, * or NULL if the page is empty. */ extern TpMemtableRecord *tp_memtable_page_first(Page page); /* * Returns the next record after `cur`, or NULL if `cur` is the * last record on the page. */ extern TpMemtableRecord * tp_memtable_page_next(Page page, TpMemtableRecord *cur); /* * Chain-pointer accessors. Storing InvalidBlockNumber means "tail * of chain". These touch only the header. */ extern BlockNumber tp_memtable_page_get_next(Page page); extern void tp_memtable_page_set_next(Page page, BlockNumber blk); /* Number of records currently on the page. */ extern uint16 tp_memtable_page_n_records(Page page); /* ---------- multi-page (fragment) record support ---------- */ /* * Inline payload bytes that a FRAGMENT head record can store on * an otherwise-empty page. The remainder of the payload lives * on continuation pages (chained via the head page's next_block, * each continuation's own next_block). * * The fragment writer always sizes the inline prefix to this * maximum so the head page contains exactly one (FRAGMENT) * record and has no remaining free space — the page's * free_offset advances to BLCKSZ. */ extern uint32 tp_memtable_fragment_inline_capacity_max(void); /* * Append a FRAGMENT head record onto an otherwise-empty page. * `total_vector_len` is the full payload length (must exceed * TP_MEMTABLE_PAGE_MAX_VECTOR_LEN — caller ensures this). * `prefix_len` bytes from `prefix` are copied inline; the * remainder lives on continuation pages the caller will * populate separately. * * `prefix_len` must equal tp_memtable_fragment_inline_capacity_max() * (i.e. fill the page) — the caller is responsible for slicing * the input payload accordingly. * * Returns a pointer into the page at the newly written record. */ extern TpMemtableRecord *tp_memtable_page_append_fragment_head( Page page, ItemPointer ctid, int32 doc_length, const char *prefix, uint32 prefix_len, uint32 total_vector_len); /* * Maximum payload chunk size, in bytes, that fits on a single * continuation page. Continuation pages carry no records; * they are pure payload immediately following the page header. */ extern uint32 tp_memtable_continuation_max_chunk(void); /* * Initialize `page` as a continuation page carrying `chunk_len` * bytes of payload (`chunk` may be NULL only when chunk_len==0) * and linking forward to `next_block`. The page must already * have been zeroed and laid out by PageInit(). Caller is * responsible for ensuring chunk_len <= continuation_max_chunk(). */ extern void tp_memtable_continuation_page_init( Page page, const char *chunk, uint32 chunk_len, BlockNumber next_block); /* * Returns true iff `page` carries a valid memtable page header * with the CONTINUATION flag set. */ extern bool tp_memtable_page_is_continuation(Page page); /* * Return a pointer to the chunk bytes stored on a continuation * page; writes the chunk length to *out_len. Caller holds the * buffer pinned (and at least SHARED-locked) for the bytes' * lifetime. */ extern const char * tp_memtable_continuation_page_chunk(Page page, uint32 *out_len); /* * DEAD flag helpers (page buffer / GenericXLog working copy). * Idempotent: first horizon wins when marking dead. */ extern void tp_memtable_page_mark_dead(Page page, FullTransactionId horizon); extern bool tp_memtable_page_is_dead(Page page); timescale-pg_textsearch-b63fe58/src/memtable/posting.c000066400000000000000000000224411525762047700231700ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * posting.c — posting list and doclength table management for * the in-memory memtable cache. */ #include #include #include #include #include #include #include #include #include "common/hashfn.h" #include "constants.h" #include "index/metapage.h" #include "index/registry.h" #include "index/state.h" #include "memtable/posting.h" #include "memtable/stringtable.h" /* Configuration parameters */ int tp_posting_list_growth_factor = TP_POSTING_LIST_GROWTH_FACTOR; /* * Free a posting list and its entries array */ void tp_free_posting_list(dsa_area *area, dsa_pointer posting_list_dp) { TpPostingList *posting_list; if (!DsaPointerIsValid(posting_list_dp)) return; posting_list = (TpPostingList *)dsa_get_address(area, posting_list_dp); /* Free entries array if it exists */ if (DsaPointerIsValid(posting_list->entries_dp)) dsa_free(area, posting_list->entries_dp); /* Free the posting list structure itself */ dsa_free(area, posting_list_dp); } /* Helper function to get entries array from posting list */ TpPostingEntry * tp_get_posting_entries(dsa_area *area, TpPostingList *posting_list) { TpPostingEntry *entries; if (!posting_list || !DsaPointerIsValid(posting_list->entries_dp)) return NULL; if (!area) return NULL; entries = dsa_get_address(area, posting_list->entries_dp); #ifdef USE_ASSERT_CHECKING /* * In debug builds, check if we're accessing freed memory. * If memory was freed by tp_dsa_free, it will be filled with * 0xDD sentinel pattern. Detecting this indicates use-after-free. */ if (entries && posting_list->doc_count > 0) { unsigned char *check = (unsigned char *)entries; bool looks_freed = (check[0] == 0xDD && check[1] == 0xDD && check[2] == 0xDD && check[3] == 0xDD); Assert(!looks_freed); if (looks_freed) elog(ERROR, "use-after-free detected: accessing freed posting " "list entries"); } #endif return entries; } /* * Allocate and initialize a new posting list in DSA * Returns the DSA pointer to the allocated posting list */ dsa_pointer tp_alloc_posting_list(dsa_area *dsa) { dsa_pointer posting_list_dp; TpPostingList *posting_list; Assert(dsa != NULL); posting_list_dp = dsa_allocate(dsa, sizeof(TpPostingList)); if (!DsaPointerIsValid(posting_list_dp)) elog(ERROR, "Failed to allocate posting list in DSA"); posting_list = dsa_get_address(dsa, posting_list_dp); /* Initialize posting list */ memset(posting_list, 0, sizeof(TpPostingList)); LWLockInitialize( &posting_list->lock, tp_tranche_id(TP_TRANCHE_POSTING_LOCK)); posting_list->doc_count = 0; posting_list->capacity = 0; posting_list->is_sorted = false; posting_list->doc_freq = 0; posting_list->entries_dp = InvalidDsaPointer; return posting_list_dp; } /* * Add a document entry to a posting list - simplified */ void tp_add_document_to_posting_list( TpLocalIndexState *local_state, TpPostingList *posting_list, ItemPointer ctid, int32 frequency) { TpPostingEntry *entries; TpPostingEntry *new_entry; dsa_pointer new_entries_dp; Assert(local_state != NULL); Assert(posting_list != NULL); Assert(ItemPointerIsValid(ctid)); LWLockAcquire(&posting_list->lock, LW_EXCLUSIVE); /* Expand array if needed */ if (posting_list->doc_count >= posting_list->capacity) { int32 new_capacity; Size new_size; if (posting_list->capacity == 0) new_capacity = TP_INITIAL_POSTING_LIST_CAPACITY; else { /* Check for int32 overflow before multiplying */ if (posting_list->capacity > INT32_MAX / tp_posting_list_growth_factor) elog(ERROR, "posting list capacity overflow: %d * %d", posting_list->capacity, tp_posting_list_growth_factor); new_capacity = posting_list->capacity * tp_posting_list_growth_factor; } new_size = (Size)new_capacity * sizeof(TpPostingEntry); new_entries_dp = dsa_allocate(local_state->dsa, new_size); if (!DsaPointerIsValid(new_entries_dp)) elog(ERROR, "Failed to allocate posting entries in DSA"); /* Copy existing entries if any */ if (posting_list->doc_count > 0 && DsaPointerIsValid(posting_list->entries_dp)) { TpPostingEntry *old_entries = tp_get_posting_entries(local_state->dsa, posting_list); TpPostingEntry *new_entries = dsa_get_address(local_state->dsa, new_entries_dp); memcpy(new_entries, old_entries, posting_list->doc_count * sizeof(TpPostingEntry)); dsa_free(local_state->dsa, posting_list->entries_dp); } posting_list->entries_dp = new_entries_dp; posting_list->capacity = new_capacity; } /* Add new document entry */ entries = tp_get_posting_entries(local_state->dsa, posting_list); new_entry = &entries[posting_list->doc_count]; new_entry->ctid = *ctid; new_entry->frequency = frequency; posting_list->doc_count++; posting_list->doc_freq = posting_list->doc_count; posting_list->is_sorted = false; /* New entry may break sort order */ LWLockRelease(&posting_list->lock); } /* * Hash function for document length entries (CTID-based) */ static dshash_hash tp_doclength_hash_function(const void *key, size_t keysize, void *arg) { const ItemPointer ctid = (const ItemPointer)key; Assert(keysize == sizeof(ItemPointerData)); (void)keysize; (void)arg; /* Hash both block number and offset */ return hash_bytes((const unsigned char *)ctid, sizeof(ItemPointerData)); } /* * Compare function for document length entries (CTID comparison) */ static int tp_doclength_compare_function( const void *a, const void *b, size_t keysize, void *arg) { const ItemPointer ctid_a = (const ItemPointer)a; const ItemPointer ctid_b = (const ItemPointer)b; Assert(keysize == sizeof(ItemPointerData)); (void)keysize; (void)arg; return ItemPointerCompare(ctid_a, ctid_b); } /* * Copy function for document length entries (CTID copy) */ static void tp_doclength_copy_function( void *dest, const void *src, size_t keysize, void *arg) { Assert(keysize == sizeof(ItemPointerData)); (void)keysize; (void)arg; *((ItemPointer)dest) = *((const ItemPointer)src); } /* * Create document length hash table */ dshash_table * tp_doclength_table_create(dsa_area *area) { dshash_parameters params; params.key_size = sizeof(ItemPointerData); params.entry_size = sizeof(TpDocLengthEntry); params.hash_function = tp_doclength_hash_function; params.compare_function = tp_doclength_compare_function; params.copy_function = tp_doclength_copy_function; params.tranche_id = tp_tranche_id(TP_DOCLENGTH_HASH_TRANCHE_ID); return dshash_create(area, ¶ms, area); } /* * Attach to existing document length hash table */ dshash_table * tp_doclength_table_attach(dsa_area *area, dshash_table_handle handle) { dshash_parameters params; params.key_size = sizeof(ItemPointerData); params.entry_size = sizeof(TpDocLengthEntry); params.hash_function = tp_doclength_hash_function; params.compare_function = tp_doclength_compare_function; params.copy_function = tp_doclength_copy_function; params.tranche_id = tp_tranche_id(TP_DOCLENGTH_HASH_TRANCHE_ID); return dshash_attach(area, ¶ms, handle, area); } /* * Reserve the doc-length slot for `ctid` in the document length hash * table. Returns true if the entry is newly inserted (caller now * owns the addition of this doc to the cache), false if an entry * for `ctid` was already present. * * Used as the idempotency gate in tp_cache_apply_document so that * concurrent cache appliers cannot double-add the same CTID into * the cache. The doc-length table is the gate (rather than a * separate "seen" set) so we get insert + dedup in a single * dshash op under the partition lock. */ bool tp_store_document_length( TpLocalIndexState *local_state, ItemPointer ctid, int32 doc_length) { TpMemtable *memtable; dshash_table *doclength_table; TpDocLengthEntry *entry; bool found; Assert(local_state != NULL); Assert(ctid != NULL); memtable = get_memtable(local_state); if (!memtable) elog(ERROR, "Cannot get memtable - index state corrupted"); /* Initialize document length table if needed */ if (memtable->doc_lengths_handle == DSHASH_HANDLE_INVALID) { doclength_table = tp_doclength_table_create(local_state->dsa); memtable->doc_lengths_handle = dshash_get_hash_table_handle( doclength_table); } else { doclength_table = tp_doclength_table_attach( local_state->dsa, memtable->doc_lengths_handle); } entry = (TpDocLengthEntry *) dshash_find_or_insert(doclength_table, ctid, &found); if (!found) { entry->ctid = *ctid; entry->doc_length = doc_length; } dshash_release_lock(doclength_table, entry); dshash_detach(doclength_table); return !found; } /* * Get document length using a pre-attached doclength table. * This is the optimized version for bulk lookups - avoids repeated * dshash_attach/detach overhead. */ int32 tp_get_document_length_attached( dshash_table *doclength_table, ItemPointer ctid) { TpDocLengthEntry *entry; Assert(doclength_table != NULL); Assert(ctid != NULL); entry = (TpDocLengthEntry *)dshash_find(doclength_table, ctid, false); if (entry) { int32 doc_length = entry->doc_length; dshash_release_lock(doclength_table, entry); return doc_length; } return -1; /* Not found */ }timescale-pg_textsearch-b63fe58/src/memtable/posting.h000066400000000000000000000064061525762047700232000ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * posting.h — in-memory posting list + doclength entry types * * The cache holds DSA-resident posting lists keyed by term, each * a sorted (after finalization) array of (ctid, frequency) * entries. Mutated by the cache apply protocol from chain doc * records; queried by the cache TpDataSource. The doclength * hash entry (TpDocLengthEntry) is colocated here because the * doclength table API + DSA storage are owned by this module. */ #pragma once #include #include #include #include #include #include "constants.h" #include "index/state.h" #include "memtable/posting_entry.h" /* * Posting list for a single term * Uses dynamic arrays with O(1) amortized inserts during building, * then sorts once at finalization for optimal query performance */ typedef struct TpPostingList { LWLock lock; /* Per-posting-list concurrency */ int32 doc_count; /* Length of the entries array */ int32 capacity; /* Allocated array capacity */ bool is_sorted; /* True after final sort for queries */ int32 doc_freq; /* Document frequency (for IDF calculation) */ dsa_pointer entries_dp; /* DSA pointer to TpPostingEntry array */ } TpPostingList; /* * Document length entry for the dshash hash table. * Maps document CTID to its length for BM25 calculations. */ typedef struct TpDocLengthEntry { ItemPointerData ctid; /* Key: Document heap tuple ID */ int32 doc_length; /* Value: Document length (sum of term * frequencies) */ } TpDocLengthEntry; /* Array growth multiplier */ extern int tp_posting_list_growth_factor; /* Posting list memory management */ extern dsa_pointer tp_alloc_posting_list(dsa_area *dsa); extern void tp_free_posting_list(dsa_area *area, dsa_pointer posting_list_dp); extern TpPostingEntry * tp_get_posting_entries(dsa_area *area, TpPostingList *posting_list); extern void tp_add_document_to_posting_list( TpLocalIndexState *local_state, TpPostingList *posting_list, ItemPointer ctid, int32 frequency); /* Document length hash table tranche ID (part of the fixed block) */ #define TP_DOCLENGTH_HASH_TRANCHE_ID TP_TRANCHE_DOC_LENGTHS /* * Reserve a doc-length slot for `ctid`. Returns true if newly * inserted, false if an entry for `ctid` was already present. * Acts as the idempotency gate in tp_cache_apply_document so * that concurrent appliers cannot double-add the same CTID. */ extern bool tp_store_document_length( TpLocalIndexState *local_state, ItemPointer ctid, int32 doc_length); /* * Get document length using a pre-attached doclength table. * This avoids repeated dshash_attach/detach overhead when looking up * multiple document lengths. */ extern int32 tp_get_document_length_attached( dshash_table *doclength_table, ItemPointer ctid); extern dshash_table *tp_doclength_table_create(dsa_area *area); extern dshash_table * tp_doclength_table_attach(dsa_area *area, dshash_table_handle handle); /* tp_calculate_idf lives in src/scoring/bm25.c; declared in * src/scoring/bm25.h. Not re-declared here. */ /* tp_cleanup_index_shared_memory lives in src/index/state.c; declared * in src/index/state.h. Not re-declared here. */ timescale-pg_textsearch-b63fe58/src/memtable/posting_entry.h000066400000000000000000000012401525762047700244100ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * posting_entry.h — common posting entry type. * * Shared between the in-memory memtable cache (DSA-allocated * posting lists) and the palloc-based local memtable used in * parallel builds. */ #pragma once #include #include /* * Individual document occurrence within a posting list. * Doc IDs are assigned at segment write time via docmap lookup. */ typedef struct TpPostingEntry { ItemPointerData ctid; /* Document heap tuple ID */ int32 frequency; /* Term frequency in document */ } TpPostingEntry; timescale-pg_textsearch-b63fe58/src/memtable/scan.c000066400000000000000000000061711525762047700224330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * scan.c - Memtable scan operations * * This module provides the search interface for scanning the memtable * (and segments) during index scans. */ #include #include #include #include "index/limit.h" #include "index/metapage.h" #include "index/state.h" #include "memtable/scan.h" #include "scoring/bm25.h" #include "types/vector.h" /* * Search the memtable (and segments) for documents matching the query vector. * Returns true on success (results stored in scan opaque), false on failure. * * This is the main entry point called from am/scan.c during tp_gettuple. */ bool tp_memtable_search( IndexScanDesc scan, TpLocalIndexState *index_state, TpVector *query_vector) { TpScanOpaque so = (TpScanOpaque)scan->opaque; int max_results; int result_count = 0; MemoryContext oldcontext; /* Extract terms and frequencies from query vector */ char **query_terms; int32 *query_frequencies; TpVectorEntry *entries_ptr; int entry_count; char *ptr; if (!so) return false; /* Use limit from scan state, fallback to GUC parameter */ if (so->limit > 0) max_results = so->limit; else max_results = tp_default_limit; if ((Size)max_results > MaxAllocSize / sizeof(ItemPointerData)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("BM25 result batch is too large"))); entry_count = query_vector->entry_count; query_terms = palloc(entry_count * sizeof(char *)); query_frequencies = palloc(entry_count * sizeof(int32)); entries_ptr = TPVECTOR_ENTRIES_PTR(query_vector); /* Parse the query vector entries */ ptr = (char *)entries_ptr; for (int i = 0; i < entry_count; i++) { TpVectorEntry *entry = (TpVectorEntry *)ptr; TpVectorEntryView v; char *term_str; ptr = (char *)tpvector_entry_decode_advance(entry, &v); term_str = palloc(v.lexeme_len + 1); memcpy(term_str, v.lexeme, v.lexeme_len); term_str[v.lexeme_len] = '\0'; query_terms[i] = term_str; query_frequencies[i] = (int32)v.frequency; } /* Allocate result arrays in scan context */ oldcontext = MemoryContextSwitchTo(so->scan_context); so->result_ctids = palloc(max_results * sizeof(ItemPointerData)); /* Initialize to invalid TIDs for safety */ memset(so->result_ctids, 0, max_results * sizeof(ItemPointerData)); MemoryContextSwitchTo(oldcontext); Assert(index_state != NULL); Assert(query_terms != NULL); Assert(query_frequencies != NULL); Assert(so->result_ctids != NULL); /* Score documents using the unified scoring function */ result_count = tp_score_documents( index_state, scan->indexRelation, query_terms, query_frequencies, entry_count, max_results, so->result_ctids, &so->result_scores); so->result_count = result_count; so->current_pos = 0; so->max_results_used = max_results; /* Free the query terms array and individual term strings */ for (int i = 0; i < entry_count; i++) pfree(query_terms[i]); pfree(query_terms); pfree(query_frequencies); return result_count > 0; } timescale-pg_textsearch-b63fe58/src/memtable/scan.h000066400000000000000000000012071525762047700224330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * scan.h - Memtable scan operations */ #pragma once #include #include #include "access/am.h" #include "index/metapage.h" #include "index/state.h" #include "types/vector.h" /* * Search the memtable and segments for documents matching the query vector. * Results are stored in the scan opaque structure. * * Returns true on success (at least one result), false otherwise. */ bool tp_memtable_search( IndexScanDesc scan, TpLocalIndexState *index_state, TpVector *query_vector); timescale-pg_textsearch-b63fe58/src/memtable/stringtable.c000066400000000000000000000265321525762047700240300ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * stringtable.c — string interning hash table using dshash, for * the in-memory memtable cache. * * Strings are stored in DSA memory and referenced by dsa_pointer * keys. Provides concurrent term lookup with a variant key that * compares char* (for lookup) and dsa_pointer (for storage) under * the same hash function. */ #include #include #include #include #include "common/hashfn.h" #include "common/hashfn_unstable.h" #include "index/state.h" #include "memtable/posting.h" #include "memtable/stringtable.h" /* * Get string length from key. * For lookup keys (posting_list == Invalid), use the stored len field. * For stored keys, use strlen on the null-terminated DSA string. */ static inline size_t tp_get_key_len(dsa_area *area, const TpStringKey *key) { if (key->posting_list == InvalidDsaPointer) return key->len; else return strlen((const char *)dsa_get_address(area, key->term.dp)); } /* * Hash function for variant string keys (char* or dsa_pointer) */ static dshash_hash tp_string_hash_function(const void *key, size_t keysize, void *arg) { const TpStringKey *string_key = (const TpStringKey *)key; dsa_area *area = (dsa_area *)arg; const char *str; size_t len; dshash_hash hash_result; Assert(keysize == sizeof(TpStringKey)); (void)keysize; str = tp_get_key_str(area, string_key); len = tp_get_key_len(area, string_key); /* Hash the string content using explicit length (no strlen needed) */ hash_result = (dshash_hash)hash_bytes((const unsigned char *)str, len); return hash_result; } const char * tp_get_key_str(dsa_area *area, const TpStringKey *key) { if (key->posting_list == InvalidDsaPointer) return key->term.str; else return (const char *)dsa_get_address(area, key->term.dp); } /* * Compare function for variant string keys (char* or dsa_pointer) * Uses explicit lengths to avoid requiring null-terminated strings for * lookups. */ static int tp_string_compare_function( const void *a, const void *b, size_t keysize, void *arg) { const TpStringKey *key_a = (const TpStringKey *)a; const TpStringKey *key_b = (const TpStringKey *)b; dsa_area *area = (dsa_area *)arg; const char *str_a = tp_get_key_str(area, key_a); const char *str_b = tp_get_key_str(area, key_b); size_t len_a = tp_get_key_len(area, key_a); size_t len_b = tp_get_key_len(area, key_b); int result; Assert(keysize == sizeof(TpStringKey)); (void)keysize; /* Compare by length first for efficiency */ if (len_a != len_b) return (len_a < len_b) ? -1 : 1; /* Same length - compare contents */ result = memcmp(str_a, str_b, len_a); return result; } /* * Copy function for variant string keys * Simple structure copy */ static void tp_string_copy_function( void *dest, const void *src, size_t keysize, void *arg pg_attribute_unused()) { Assert(keysize == sizeof(TpStringKey)); (void)keysize; *((TpStringKey *)dest) = *((TpStringKey *)src); } /* * Create and initialize a new string hash table using dshash. The hash table * contents live in the DSA area, but the returned handle is allocated from * backend memory using the current memory context. */ static dshash_table * tp_string_table_create(dsa_area *area) { dshash_parameters params; /* Set up dshash parameters */ params.key_size = sizeof(TpStringKey); params.entry_size = sizeof(TpStringHashEntry); params.hash_function = tp_string_hash_function; params.compare_function = tp_string_compare_function; params.copy_function = tp_string_copy_function; params.tranche_id = tp_tranche_id(TP_STRING_HASH_TRANCHE_ID); /* Create the dshash table */ return dshash_create(area, ¶ms, area); } /* * Attach to an existing string hash table using its handle. Returns an object * allocated from backend memory using the current memory context to can be * used to access the hash table stored in the DSA area. */ dshash_table * tp_string_table_attach(dsa_area *area, dshash_table_handle handle) { dshash_parameters params; /* Set up dshash parameters */ params.key_size = sizeof(TpStringKey); params.entry_size = sizeof(TpStringHashEntry); params.hash_function = tp_string_hash_function; params.compare_function = tp_string_compare_function; params.copy_function = tp_string_copy_function; params.tranche_id = tp_tranche_id(TP_STRING_HASH_TRANCHE_ID); /* Attach to the dshash table */ return dshash_attach(area, ¶ms, handle, area); } /* * Allocate a null-terminated string in DSA memory * Returns the dsa_pointer to the allocated string */ static dsa_pointer tp_alloc_string_dsa(dsa_area *area, const char *str, size_t len) { dsa_pointer string_dp; char *string_data; string_dp = dsa_allocate(area, len + 1); if (!DsaPointerIsValid(string_dp)) elog(ERROR, "Failed to allocate string in DSA"); string_data = (char *)dsa_get_address(area, string_dp); /* Copy string data and null terminate */ memcpy(string_data, str, len); string_data[len] = '\0'; return string_dp; } /* * Look up a string in the hash table * Returns NULL if not found */ TpStringHashEntry * tp_string_table_lookup( dsa_area *area, dshash_table *ht, const char *str, size_t len) { TpStringKey lookup_key; TpStringHashEntry *entry; Assert(area != NULL); (void)area; Assert(ht != NULL); Assert(str != NULL); if (len == 0) return NULL; /* Build lookup key with explicit length (no null termination required) */ lookup_key.term.str = str; lookup_key.posting_list = InvalidDsaPointer; lookup_key.len = len; /* Look up using the stack-allocated key */ entry = (TpStringHashEntry *)dshash_find(ht, &lookup_key, false); if (entry) { /* Release the lock acquired by dshash_find. * * SAFETY: The per-index LWLock ensures exclusive access during writes * and prevents concurrent destruction of the hash table. */ dshash_release_lock(ht, entry); } return entry; } /* * Clear the hash table, removing all entries * Frees all DSA string allocations */ void tp_string_table_clear(dsa_area *area, dshash_table *ht) { dshash_seq_status status; TpStringHashEntry *entry; Assert(area != NULL); Assert(ht != NULL); /* Iterate through all entries and delete them */ dshash_seq_init(&status, ht, true); /* exclusive for deletion */ while ((entry = (TpStringHashEntry *)dshash_seq_next(&status)) != NULL) { /* Free the string */ dsa_free(area, entry->key.term.dp); /* Free posting list */ tp_free_posting_list(area, entry->key.posting_list); /* Delete current entry */ dshash_delete_current(&status); } dshash_seq_term(&status); } /* * Ensure the memtable's string hash table is initialized. * * Must be called under LW_EXCLUSIVE on the per-index lock. * This is the cold path — only needed once per memtable * lifecycle (after creation or spill). Separated from * tp_get_or_create_posting_list so the hot path (which * runs under LW_SHARED) never races on hash table creation. */ void tp_ensure_string_table_initialized(TpLocalIndexState *local_state) { TpMemtable *memtable; dshash_table *table; Assert(local_state != NULL); memtable = get_memtable(local_state); if (!memtable) return; /* Initialize string hash table if needed */ if (memtable->string_hash_handle == DSHASH_HANDLE_INVALID) { table = tp_string_table_create(local_state->dsa); if (!table) elog(ERROR, "Failed to create string hash table"); memtable->string_hash_handle = dshash_get_hash_table_handle(table); dshash_detach(table); } /* Initialize doc lengths hash table if needed */ if (memtable->doc_lengths_handle == DSHASH_HANDLE_INVALID) { table = tp_doclength_table_create(local_state->dsa); if (!table) elog(ERROR, "Failed to create doc length table"); memtable->doc_lengths_handle = dshash_get_hash_table_handle(table); dshash_detach(table); } } /* * Get or create a posting list for a term. * * Uses a single dshash_find_or_insert call to atomically find or * create the entry, eliminating the lookup-then-insert race that * exists when two backends see "not found" for the same term. * * Caller must have ensured the string hash table is initialized * (via tp_ensure_string_table_initialized under LW_EXCLUSIVE). */ static TpPostingList * tp_get_or_create_posting_list(TpLocalIndexState *local_state, const char *term) { TpMemtable *memtable; dshash_table *string_table; TpStringHashEntry *entry; TpStringKey lookup_key; TpPostingList *posting_list; size_t term_len; bool found; Assert(local_state != NULL); Assert(term != NULL); term_len = strlen(term); memtable = get_memtable(local_state); if (!memtable) { elog(ERROR, "Cannot get memtable"); return NULL; } /* * The string hash table must already be initialized * by tp_ensure_string_table_initialized (called under * LW_EXCLUSIVE). */ if (memtable->string_hash_handle == DSHASH_HANDLE_INVALID) elog(ERROR, "String hash table not initialized " "(call tp_ensure_string_table_initialized first)"); else { string_table = tp_string_table_attach( local_state->dsa, memtable->string_hash_handle); if (!string_table) elog(ERROR, "Failed to attach to string hash " "table"); } /* Build lookup key */ lookup_key.term.str = term; lookup_key.posting_list = InvalidDsaPointer; lookup_key.len = term_len; /* Atomic find-or-insert: holds exclusive partition lock */ entry = (TpStringHashEntry *) dshash_find_or_insert(string_table, &lookup_key, &found); if (!found) { /* New term -- initialize under partition lock */ entry->key.term.dp = tp_alloc_string_dsa(local_state->dsa, term, term_len); entry->key.posting_list = tp_alloc_posting_list(local_state->dsa); } posting_list = dsa_get_address(local_state->dsa, entry->key.posting_list); /* Release partition lock */ dshash_release_lock(string_table, entry); dshash_detach(string_table); return posting_list; } /* * tp_cache_apply_document — apply a single doc record into the * in-memory cache structures (string table, posting lists, doc- * length table). * * Used by the cache apply protocol to bring the cache up to * date with the on-disk chain. The caller has already parsed a * (ctid, terms[], frequencies[], doc_length) tuple out of a * chain doc record. * * Idempotent by CTID: tp_store_document_length is the single * check-and-add gate. If the CTID is already in the doclength * table (e.g., another applier raced us), we return without * touching the posting lists. Corpus statistics * (TpSharedIndexState.total_docs / total_len) are NOT updated * here — those are owned by the on-disk write path * (tp_add_document_terms in src/memtable/log.c) and reflect the * chain, not the cache. Bulk-load counter is also untouched * for the same reason. */ void tp_cache_apply_document( TpLocalIndexState *local_state, ItemPointer ctid, char **terms, int32 *frequencies, int term_count, int32 doc_length) { int i; if (!tp_store_document_length(local_state, ctid, doc_length)) return; for (i = 0; i < term_count; i++) { TpPostingList *posting_list; int32 frequency = frequencies[i]; /* Get or create posting list for this term */ posting_list = tp_get_or_create_posting_list(local_state, terms[i]); /* Add document entry to posting list */ tp_add_document_to_posting_list( local_state, posting_list, ctid, frequency); } } timescale-pg_textsearch-b63fe58/src/memtable/stringtable.h000066400000000000000000000055631525762047700240360ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * stringtable.h — string interning hash table for the in-memory * memtable cache. Maps interned terms to their DSA-resident * posting lists via dshash. */ #pragma once #include #include #include #include #include #include "memtable/posting.h" typedef struct TpStringHashEntry TpStringHashEntry; /* * Key structure that supports both lookup via char* and storage via * dsa_pointer. posting_list is set to InvalidDsaPointer for lookup keys and * non-InvalidDsaPointer for table entry keys. Tricky but avoids allocations * for lookups while saving space in the hash table. * * For lookup keys: term.str points to (possibly non-null-terminated) string, * len contains the string length. * For stored keys: term.dp points to null-terminated DSA string, len is * unused. */ typedef struct TpStringKey { union { const char *str; dsa_pointer dp; } term; dsa_pointer posting_list; uint32 len; /* String length for lookup keys (avoids strlen) */ } TpStringKey; /* Helper function to get the string from a key */ extern const char *tp_get_key_str(dsa_area *area, const TpStringKey *key); /* * dshash entry structure for string interning and posting list mapping * Key distinguishes between local char* (for lookups) and DSA strings (for * storage) */ struct TpStringHashEntry { /* Term plus a pointer to the posting list */ TpStringKey key; }; /* String table creation and initialization */ extern dshash_table * tp_string_table_attach(dsa_area *area, dshash_table_handle handle); /* Core hash table operations */ extern TpStringHashEntry *tp_string_table_lookup( dsa_area *area, dshash_table *ht, const char *str, size_t len); extern void tp_string_table_clear(dsa_area *area, dshash_table *ht); /* Ensure the string hash table is initialized (call under EXCLUSIVE) */ extern void tp_ensure_string_table_initialized(TpLocalIndexState *local_state); /* Document term management functions * * tp_cache_apply_document is the cache-side analogue of v2's * existing tp_add_document_terms (src/memtable/log.c). The * on-disk version appends a doc record to the on-disk chain * (source of truth). * This version applies an already-parsed record (terms + * frequencies + doc_length for a CTID) into the in-memory cache * structures (string interning table, posting lists, doclength * table). Used by the cache apply protocol to bring the cache * up to date with the chain. */ extern void tp_cache_apply_document( TpLocalIndexState *local_state, ItemPointer ctid, char **terms, int32 *frequencies, int term_count, int32 doc_length); /* LWLock tranche for string table locking (part of the fixed block) */ #define TP_STRING_HASH_TRANCHE_ID TP_TRANCHE_STRING timescale-pg_textsearch-b63fe58/src/mod.c000066400000000000000000004144011525762047700204770ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * mod.c - Extension initialization and GUC registration */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "access/rls.h" #include "constants.h" #include "index/compaction_job.h" #include "index/compaction_request.h" #include "index/metapage.h" #include "index/registry.h" #include "index/state.h" #include "memtable/log.h" #include "planner/hooks.h" #include "scoring/bm25.h" #include "scoring/snapshot.h" #include "segment/compaction.h" #include "segment/graph_snapshot.h" #if PG_VERSION_NUM >= 180000 PG_MODULE_MAGIC_EXT(.name = "pg_textsearch", .version = "1.5.1"); #else PG_MODULE_MAGIC; #endif /* Relation options for Tapir indexes */ relopt_kind tp_relopt_kind; /* External variable from limits module */ extern int tp_default_limit; /* Library version string for stale binary detection */ static char *tp_library_version = NULL; /* Global variable for score logging */ bool tp_log_scores = false; /* Global variable for BMW stats logging - declared in query/score.c */ bool tp_log_bmw_stats = false; /* Global variable for bulk load spill threshold (0 = disabled) */ int tp_bulk_load_threshold = TP_DEFAULT_BULK_LOAD_THRESHOLD; /* * Memtable v2 (issue #374) auto-spill: chain pages per index * before the next insert triggers a spill (0 = disabled). */ int tp_memtable_pages_threshold = TP_DEFAULT_MEMTABLE_PAGES_THRESHOLD; /* Global variable for segments per level before compaction */ int tp_segments_per_level = TP_DEFAULT_SEGMENTS_PER_LEVEL; /* Conservative size budget for newly merged multi-source segments. */ int tp_max_segment_size_mb = TP_DEFAULT_SEGMENT_SIZE_MB; /* Allow BM25 indexes on relations protected by row-level security. */ bool tp_allow_rls = true; static const relopt_enum_elt_def compaction_mode_options[] = {{"inline", TP_COMPACTION_INLINE}, {"background", TP_COMPACTION_BACKGROUND}, {"manual", TP_COMPACTION_MANUAL}, {"off", TP_COMPACTION_MANUAL}, {(const char *)NULL, 0}}; /* Global variable for segment compression (on by default - benchmarks show * compression improves both size and query performance) */ bool tp_compress_segments = true; /* * Selectivity-seeded top-K for filtered BM25 search. * tp_filtered_seed gates expansion when the scan reads its query hint; * tp_filtered_seed_margin scales the seed * (ceil(margin * user_limit / filter_selectivity)). */ bool tp_filtered_seed = true; double tp_filtered_seed_margin = TP_DEFAULT_FILTERED_SEED_MARGIN; /* * Memtable shared-memory cache enable flag. Gates the read-path * chooser (tp_memtable_source_create_for_read) on the cache vs * the on-disk chain. Defaults to on: tp_spill_finalize bumps * the per-index spill_generation and calls tp_cache_clear, * closing the cross-spill staleness window that would otherwise * force opt-in. Standbys still bypass the cache via * RecoveryInProgress() in the chooser. */ bool tp_memtable_cache_enabled = true; /* * Debug GUC for the in-memory memtable cache. When enabled the * read-path chooser logs cache state transitions (apply OK / * BUDGET_EXCEEDED / cold_build OK / RETRY / ABORT / fall back to * chain). Off by default; intended for development and tests * that need to assert which path served a query. */ bool tp_log_cache_state = false; /* * Soft+hard memory budget for the in-memory memtable cache, in * kilobytes. Restored from v1; scaffolding only in this build — * no consumer reads this yet. Default and bounds live in * constants.h (TP_DEFAULT_MEMORY_LIMIT_KB) so the GUC registration * and the backing-variable initializer cannot drift. */ int tp_memory_limit_kb = TP_DEFAULT_MEMORY_LIMIT_KB; /* Previous object access hook */ static object_access_hook_type prev_object_access_hook = NULL; /* Previous shared memory startup hook */ static shmem_startup_hook_type prev_shmem_startup_hook = NULL; /* Shared memory request hook */ static shmem_request_hook_type prev_shmem_request_hook = NULL; /* Previous ProcessUtility hook */ static ProcessUtility_hook_type prev_process_utility_hook = NULL; typedef struct TpProcessUtilityContext { struct TpProcessUtilityContext *previous; bool extension_lifecycle; bool track_index_build; bool check_rls_enable; bool check_hierarchy_change; bool track_relation_create; bool serialize_rls_ddl; bool retain_rls_ddl_lock; bool allow_rls; bool rls_ddl_lock_acquired; LOCKMODE rls_ddl_lock_mode; bool build_progress_started; Oid rls_ddl_lock_object; List *altered_relids; List *hierarchy_relids; List *concurrent_index_drops; } TpProcessUtilityContext; static TpProcessUtilityContext *current_utility_context = NULL; bool tp_rls_allowed_for_current_utility(void) { if (current_utility_context != NULL) return current_utility_context->allow_rls; return tp_allow_rls; } void tp_rls_note_bm25_build(void) { if (current_utility_context != NULL) current_utility_context->retain_rls_ddl_lock = true; } /* * Commands allowed to create an RLS/BM25 combination take a shared lock; * commands enforcing the restriction and extension lifecycle commands take * an exclusive lock. The stable lock serializes across extension OID * replacement, while the current extension-object lock preserves core lock * ordering. Session ownership survives internal commits; transaction * ownership covers completed protected changes through commit. * * Nested acquisition without an inherited lock, or an exclusive request while * this backend holds only a shared lock, must not wait. Either case can * invert lock order with an outer command or deadlock with another backend * upgrading the same lock. */ static bool acquire_rls_ddl_lock( LOCKMODE lockmode, bool nested, bool extension_lifecycle, Oid *extension_oid_out) { LOCKTAG stable_tag; LOCKTAG extension_tag; LockAcquireResult result; Oid extension_oid = InvalidOid; bool dont_wait; if (!extension_lifecycle && !OidIsValid(get_extension_oid("pg_textsearch", true))) return false; SET_LOCKTAG_OBJECT( stable_tag, MyDatabaseId, ExtensionRelationId, InvalidOid, 0); dont_wait = (lockmode == ExclusiveLock && LockHeldByMe(&stable_tag, ShareLock, false) && !LockHeldByMe(&stable_tag, ExclusiveLock, true)) || (nested && !LockHeldByMe(&stable_tag, lockmode, true)); result = LockAcquire(&stable_tag, lockmode, true, dont_wait); if (result == LOCKACQUIRE_NOT_AVAIL) ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not acquire the pg_textsearch RLS DDL lock"), errdetail( "Protected DDL was invoked from a nested utility " "command whose outer command does not hold the " "pg_textsearch DDL lock."), errhint("Retry the outer command."))); extension_oid = get_extension_oid("pg_textsearch", true); if (OidIsValid(extension_oid)) { SET_LOCKTAG_OBJECT( extension_tag, MyDatabaseId, ExtensionRelationId, extension_oid, 0); result = LockAcquire(&extension_tag, lockmode, true, dont_wait); if (result == LOCKACQUIRE_NOT_AVAIL) { (void)LockRelease(&stable_tag, lockmode, true); ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not acquire the pg_textsearch RLS DDL " "lock"), errdetail( "Protected DDL was invoked from a nested utility " "command whose outer command does not hold the " "pg_textsearch DDL lock."), errhint("Retry the outer command."))); } } *extension_oid_out = extension_oid; return true; } static void release_rls_ddl_lock( Oid extension_oid, LOCKMODE lockmode, bool keep_transaction_lock) { LOCKTAG stable_tag; LOCKTAG extension_tag; SET_LOCKTAG_OBJECT( stable_tag, MyDatabaseId, ExtensionRelationId, InvalidOid, 0); if (OidIsValid(extension_oid)) SET_LOCKTAG_OBJECT( extension_tag, MyDatabaseId, ExtensionRelationId, extension_oid, 0); if (keep_transaction_lock) { (void)LockAcquire(&stable_tag, lockmode, false, false); if (OidIsValid(extension_oid)) (void)LockAcquire(&extension_tag, lockmode, false, false); } if (OidIsValid(extension_oid)) (void)LockRelease(&extension_tag, lockmode, true); (void)LockRelease(&stable_tag, lockmode, true); } typedef struct TpReindexTarget { TpCompactionJobIdentity identity; Oid current_index_oid; Oid current_namespace_oid; char *current_index_name; Oid pending_index_oid; bool pending; bool completed; } TpReindexTarget; typedef struct TpReindexState { MemoryContext context; List *targets; List *intermediate_targets; ReindexObjectType scope_kind; Oid scope_oid; bool scope_refresh_once; bool reconciling; bool defer_reconciliation; bool post_publication; bool nowait; struct TpReindexState *previous; } TpReindexState; typedef struct TpCreateIndexState { MemoryContext context; Oid heap_oid; List *created_indexes; int utility_depth; struct TpCreateIndexState *previous; } TpCreateIndexState; typedef struct TpReindexCandidate { TpReindexState *state; TpReindexTarget *target; Oid index_oid; } TpReindexCandidate; static void tp_reindex_add_targets(TpReindexState *state, List *indexoids); typedef TpCompactionJobIdentity TpOwnerChangeTarget; /* * Concurrent and partitioned REINDEX commit inside ProcessUtility. Keep a * stack of invocation-owned targets across those commits so each replacement * can be reconciled in the transaction that publishes it. */ static TpReindexState *tp_reindex_states = NULL; /* * CREATE INDEX CONCURRENTLY also commits inside ProcessUtility. The object * access hook records only indexes created by the matching utility invocation. */ static TpCreateIndexState *tp_create_index_states = NULL; static int tp_process_utility_depth = 0; /* * Managed lifecycle hooks collect transaction-local desired state here. * Reconciliation is delayed until a terminal barrier so every target can be * admitted before any pg_durable dependency or SPI work begins. */ static MemoryContext tp_managed_intent_context = NULL; static List *tp_managed_intents = NIL; static List *tp_reconciled_indexoids = NIL; static bool tp_managed_reconciling = false; static bool tp_post_publication_reconciliation = false; typedef struct TpDropDatabaseContext { struct TpDropDatabaseContext *previous; Oid database_oid; } TpDropDatabaseContext; /* Active DROP DATABASE invocation, including nested utility hooks. */ static TpDropDatabaseContext *active_drop_database_context = NULL; typedef struct TpPendingIndexDrop { struct TpPendingIndexDrop *next; Oid index_oid; SubTransactionId subid; } TpPendingIndexDrop; /* Allocated in TopTransactionContext; shared state survives until commit. */ static TpPendingIndexDrop *tp_pending_index_drops = NULL; /* Shared memory size calculation */ static void tp_shmem_request(void); /* Shared memory startup hook */ static void tp_shmem_startup(void); /* Object access hook for catalog-object validation and DROP INDEX cleanup */ static void tp_object_access( ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg); /* Transaction callback to release index locks */ static void tp_xact_callback(XactEvent event, void *arg); /* Subtransaction callback for savepoint rollback cleanup */ static void tp_subxact_callback( SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg); /* ProcessUtility hook for nestable DDL validation and build tracking */ static void tp_process_utility( PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); static void tp_process_utility_impl( PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc); static void tp_collect_reindex_state_intents( TpReindexState *only_state, bool include_deferred, bool final_pass); static bool tp_reconcile_managed_intents(void); static bool tp_reconcile_managed_intents_at_precommit(void); static void tp_finish_reindex_pending(bool reconciled); static void tp_validate_compaction_lineage(const char *lineage) { if (lineage == NULL) return; if (strlen(lineage) != TP_COMPACTION_LINEAGE_BYTES * 2) goto invalid; for (Size i = 0; lineage[i] != '\0'; i++) if (!((lineage[i] >= '0' && lineage[i] <= '9') || (lineage[i] >= 'a' && lineage[i] <= 'f'))) goto invalid; return; invalid: ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid background compaction lineage"))); } /* * Extension entry point - called when the extension is loaded */ void _PG_init(void) { if (!process_shared_preload_libraries_in_progress) ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("pg_textsearch must be loaded via " "shared_preload_libraries"), errhint("Add '%s' to shared_preload_libraries " "in postgresql.conf and restart the " "server.", "pg_textsearch"))); /* * Define GUC parameters */ DefineCustomStringVariable( "pg_textsearch.library_version", "Version of the loaded pg_textsearch shared library", NULL, &tp_library_version, PG_TEXTSEARCH_VERSION, PGC_INTERNAL, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.default_limit", "Default limit for BM25 queries when no LIMIT is detected", "Controls the maximum number of documents to process when no " "LIMIT clause is present", &tp_default_limit, TP_DEFAULT_QUERY_LIMIT, /* default 1000 */ 1, /* min 1 */ TP_MAX_INITIAL_QUERY_LIMIT, /* max initial batch 100k */ PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.log_scores", "Log BM25 scores during index scans", "When enabled, logs the BM25 score for each document returned " "during index scans. Useful for debugging score calculation.", &tp_log_scores, false, /* default off */ PGC_SUSET, /* superuser-only: prevents GUC persistence * in connection-pooled environments */ 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.log_bmw_stats", "Log Block-Max WAND statistics during queries", "When enabled, logs blocks scanned/skipped and documents scored " "for each query. Useful for understanding BMW optimization.", &tp_log_bmw_stats, false, /* default off */ PGC_SUSET, /* superuser-only: prevents GUC persistence * in connection-pooled environments */ 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.bulk_load_threshold", "Terms per transaction to trigger memtable spill", "When this many terms are added in a single transaction, spill to " "disk at transaction end. Set to 0 to disable.", &tp_bulk_load_threshold, TP_DEFAULT_BULK_LOAD_THRESHOLD, /* default 100K */ 0, /* min 0 (disabled) */ INT_MAX, /* max INT_MAX */ PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.memtable_pages_threshold", "Chain pages to trigger memtable spill", "When the on-disk memtable chain reaches this many pages, " "spill to an L0 segment at the next insert. Each page is " "8 KiB. Set to 0 to disable auto-spill (chain grows until " "VACUUM or manual bm25_spill_index()).", &tp_memtable_pages_threshold, TP_DEFAULT_MEMTABLE_PAGES_THRESHOLD, /* default 64 */ 0, /* min 0 (disabled) */ INT_MAX, /* max */ PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.segments_per_level", "Segments per level before compaction", "When a level reaches this many segments, they are merged into " "a single segment at the next level.", &tp_segments_per_level, TP_DEFAULT_SEGMENTS_PER_LEVEL, /* default 8 */ 2, /* min 2 */ 64, /* max 64 */ PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.max_segment_size", "Maximum conservative size of a merged segment.", "Bounds newly merged multi-source segments. A larger existing " "segment remains an uncombinable singleton.", &tp_max_segment_size_mb, TP_DEFAULT_SEGMENT_SIZE_MB, TP_MIN_SEGMENT_SIZE_MB, TP_MAX_SEGMENT_SIZE_MB, PGC_SUSET, GUC_UNIT_MB, NULL, NULL, NULL); DefineCustomStringVariable( "pg_textsearch.background_compaction_schedule", "Default schedule for managed background compaction.", NULL, &tp_background_compaction_schedule, "*/5 * * * *", PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.compress_segments", "Enable compression for new segment blocks", "When enabled, posting blocks in new segments are compressed " "using delta encoding and bitpacking. Reduces index size and " "improves query performance by reducing I/O.", &tp_compress_segments, true, /* default on - benchmarks show net benefit */ PGC_USERSET, /* Can be changed per session */ 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.filtered_seed", "Seed the BM25 top-K from estimated filter selectivity.", "When a filter (e.g. a facet WHERE clause) sits above a BM25 " "top-k index scan, seed the scan's internal top-K to " "ceil(margin * LIMIT / selectivity) so a single scoring pass " "usually surfaces enough matching rows, avoiding the executor's " "backoff re-drives. Results are identical either way; the " "backoff remains the correctness safety net.", &tp_filtered_seed, true, /* default on */ PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomRealVariable( "pg_textsearch.filtered_seed_margin", "Margin applied when seeding the BM25 top-K from selectivity.", "Seed = ceil(margin * LIMIT / selectivity). A higher margin " "captures the true top-k matching rows in one scoring pass more " "often, at the cost of scoring deeper. Only used when " "pg_textsearch.filtered_seed is on.", &tp_filtered_seed_margin, TP_DEFAULT_FILTERED_SEED_MARGIN, /* default 3.0 */ TP_MIN_FILTERED_SEED_MARGIN, /* min 1.0 */ TP_MAX_FILTERED_SEED_MARGIN, /* max 1000.0 */ PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.memtable_cache_enabled", "Enable the in-memory memtable cache for queries.", "When enabled, queries are served from a derived " "shared-memory cache rather than walking the on-disk " "memtable chain. The chain remains the source of " "truth. Default is on: queries against the chain are " "served by a derived cache so per-term posting lookups " "don't pay the O(records) chain walk. Disable to " "force every query through the on-disk chain. " "Standbys always use the chain regardless of this " "setting (RecoveryInProgress disables the cache).", &tp_memtable_cache_enabled, true, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.log_cache_state", "Log in-memory memtable cache state transitions.", "When enabled, the read-path chooser emits LOG messages " "for each cache apply outcome (OK / BUDGET_EXCEEDED / " "cold_build / RETRY / ABORT / fall back to chain). " "Intended for development and observability.", &tp_log_cache_state, false, PGC_USERSET, 0, NULL, NULL, NULL); DefineCustomBoolVariable( "pg_textsearch.allow_rls", "Allow BM25 indexes on row-level security tables.", "When disabled, BM25 indexes cannot be created or rebuilt on " "RLS-protected relations, and RLS cannot be enabled on relations " "that have BM25 indexes.", &tp_allow_rls, true, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.debug_segment_graph_snapshot_pause_before_lock_ms", "Pause before capturing a segment and memtable graph.", "Testing-only interruptible pause before acquiring the metapage " "buffer share lock for a common read snapshot.", &tp_debug_segment_graph_snapshot_pause_before_lock_ms, 0, 0, 60000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch." "debug_segment_graph_snapshot_pause_before_unlock_ms", "Pause before releasing a completed index read snapshot.", "Testing-only interruptible pause after copying every segment " "root and the bounded memtable endpoint while retaining the " "metapage buffer share lock.", &tp_debug_segment_graph_snapshot_pause_before_unlock_ms, 0, 0, 60000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.debug_segment_graph_snapshot_pause_ms", "Pause after copying a complete index read snapshot.", "Testing-only interruptible pause after releasing the metapage " "buffer lock and before consuming the copied segment roots or " "bounded memtable endpoint.", &tp_debug_segment_graph_snapshot_pause_ms, 0, 0, 60000, PGC_SUSET, 0, NULL, NULL, NULL); DefineCustomIntVariable( "pg_textsearch.memory_limit", "Approximate shared-memory budget for the in-memory memtable " "cache.", "Applied as a three-tier budget: the per-index limit/8 " "per-record growth guard returns BUDGET_EXCEEDED and " "falls back to the chain; the global limit/2 threshold " "tries to evict the largest non-caller cache; and limit " "is an approximate admission threshold for catch-up " "and cold builds. Admitted or concurrent work may " "increase estimated usage past the limit. A value of " "0 means unlimited.", &tp_memory_limit_kb, TP_DEFAULT_MEMORY_LIMIT_KB, 0, INT_MAX, PGC_SIGHUP, GUC_UNIT_KB, NULL, NULL, NULL); /* * Reserve the pg_textsearch.* GUC prefix so unknown settings * (typos, or GUCs removed in a future release) produce a * warning at server start rather than being silently ignored. */ MarkGUCPrefixReserved("pg_textsearch"); /* * Initialize index access method options */ tp_relopt_kind = add_reloption_kind(); add_string_reloption( tp_relopt_kind, "text_config", "Text search configuration for Tapir index", NULL, NULL, NoLock); add_real_reloption( tp_relopt_kind, "k1", "BM25 k1 parameter", TP_DEFAULT_K1, 0.1, 10.0, NoLock); add_real_reloption( tp_relopt_kind, "b", "BM25 b parameter", TP_DEFAULT_B, 0.0, 1.0, NoLock); /* * ShareUpdateExclusiveLock: the value is read after a spill and * changes no on-disk structure, so ALTER INDEX need not block. */ add_enum_reloption( tp_relopt_kind, "compaction", "Spill-time segment compaction policy", (relopt_enum_elt_def *)compaction_mode_options, TP_COMPACTION_INLINE, "Valid values are \"inline\", \"background\" and \"manual\".", ShareUpdateExclusiveLock); add_string_reloption( tp_relopt_kind, "compaction_schedule", "Managed background compaction schedule", NULL, NULL, ShareUpdateExclusiveLock); add_string_reloption( tp_relopt_kind, "compaction_lineage", "Internal managed background compaction lineage", NULL, tp_validate_compaction_lineage, ShareUpdateExclusiveLock); /* * Install shared memory hooks (needed for registry) */ prev_shmem_request_hook = shmem_request_hook; shmem_request_hook = tp_shmem_request; prev_shmem_startup_hook = shmem_startup_hook; shmem_startup_hook = tp_shmem_startup; /* Install object access hook for DROP INDEX detection */ prev_object_access_hook = object_access_hook; object_access_hook = tp_object_access; /* Register transaction callback to release index locks at transaction end */ RegisterXactCallback(tp_xact_callback, NULL); /* Register subtransaction callback for savepoint rollback cleanup */ RegisterSubXactCallback(tp_subxact_callback, NULL); /* Install planner hook for implicit index resolution */ tp_planner_hook_init(); tp_scoring_snapshot_init(); /* Install ProcessUtility hook for partitioned build tracking */ prev_process_utility_hook = ProcessUtility_hook; ProcessUtility_hook = tp_process_utility; } static TpCreateIndexState * tp_create_index_tracking_begin(Oid heap_oid) { MemoryContext caller_context = CurrentMemoryContext; MemoryContext context; TpCreateIndexState *state; context = AllocSetContextCreate( TopMemoryContext, "pg_textsearch create index", ALLOCSET_SMALL_SIZES); PG_TRY(); { state = MemoryContextAllocZero(context, sizeof(*state)); state->context = context; state->heap_oid = heap_oid; state->utility_depth = tp_process_utility_depth; state->previous = tp_create_index_states; tp_create_index_states = state; } PG_CATCH(); { MemoryContextSwitchTo(caller_context); MemoryContextDelete(context); PG_RE_THROW(); } PG_END_TRY(); return state; } static void tp_create_index_tracking_end(TpCreateIndexState *state) { MemoryContext context; if (state == NULL) return; Assert(tp_create_index_states == state); context = state->context; tp_create_index_states = state->previous; MemoryContextDelete(context); } static void tp_copy_compaction_identity( TpCompactionJobIdentity *destination, const TpCompactionJobIdentity *source) { memcpy(destination, source, sizeof(*destination)); destination->index_name = source->index_name == NULL ? NULL : pstrdup(source->index_name); destination->lineage = source->lineage == NULL ? NULL : pstrdup(source->lineage); destination->schedule = source->schedule == NULL ? NULL : pstrdup(source->schedule); } static void tp_reset_compaction_identity(TpCompactionJobIdentity *identity) { if (identity->index_name != NULL) pfree(identity->index_name); if (identity->lineage != NULL) pfree(identity->lineage); if (identity->schedule != NULL) pfree(identity->schedule); memset(identity, 0, sizeof(*identity)); } static void tp_replace_managed_string(char **destination, const char *source) { if (*destination != NULL) pfree(*destination); *destination = source == NULL ? NULL : pstrdup(source); } static void tp_update_managed_intent( TpManagedIndexIntent *intent, const TpCompactionJobIdentity *source, const char *schedule, const char *lineage, int flags) { const int mode_flags = TP_MANAGED_INTENT_REFRESH_DEFAULT | TP_MANAGED_INTENT_RECONCILE_OPTIONS | TP_MANAGED_INTENT_RECONCILE_LINEAGE | TP_MANAGED_INTENT_PRESERVE_SCHEDULE | TP_MANAGED_INTENT_DISABLE | TP_MANAGED_INTENT_POST_PUBLICATION; const int action_flags = mode_flags & ~TP_MANAGED_INTENT_POST_PUBLICATION; bool current_options_authoritative = (intent->flags & (TP_MANAGED_INTENT_REFRESH_DEFAULT | TP_MANAGED_INTENT_RECONCILE_OPTIONS)) != 0; bool pending_default_refresh = (intent->flags & TP_MANAGED_INTENT_REFRESH_DEFAULT) != 0; bool pending_option_reconciliation = (intent->flags & TP_MANAGED_INTENT_RECONCILE_OPTIONS) != 0; int persistent_flags = (intent->flags | flags) & TP_MANAGED_INTENT_LINEAGE_SUPPLIED; int new_mode_flags = flags & mode_flags; if ((new_mode_flags & action_flags) == 0) new_mode_flags |= intent->flags & action_flags; if ((flags & TP_MANAGED_INTENT_PRESERVE_SCHEDULE) != 0 && pending_option_reconciliation) new_mode_flags |= TP_MANAGED_INTENT_RECONCILE_OPTIONS; if ((flags & TP_MANAGED_INTENT_PRESERVE_SCHEDULE) != 0 && pending_default_refresh) { new_mode_flags &= ~TP_MANAGED_INTENT_PRESERVE_SCHEDULE; new_mode_flags |= TP_MANAGED_INTENT_REFRESH_DEFAULT; } intent->flags = persistent_flags | new_mode_flags; if ((flags & (TP_MANAGED_INTENT_RECONCILE_OPTIONS | TP_MANAGED_INTENT_RECONCILE_LINEAGE)) != 0) { tp_replace_managed_string(&intent->schedule, schedule); tp_replace_managed_string(&intent->lineage, lineage); } else if ( (flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) != 0 && intent->lineage == NULL && lineage != NULL) intent->lineage = pstrdup(lineage); if ((flags & TP_MANAGED_INTENT_PRESERVE_SCHEDULE) != 0 && !pending_default_refresh) { tp_reset_compaction_identity(&intent->source); if (source != NULL) { tp_copy_compaction_identity(&intent->source, source); if (pending_option_reconciliation) tp_replace_managed_string( &intent->source.schedule, intent->schedule); if (current_options_authoritative) intent->source.schedule_resolved = true; } } } static void tp_collect_managed_intent_internal( Oid indexoid, const TpCompactionJobIdentity *source, const char *schedule, const char *lineage, int flags, bool validate_relation) { MemoryContext old_context; SubTransactionId subid = GetCurrentSubTransactionId(); TpManagedIndexIntent *intent = NULL; ListCell *lc; Relation index_rel; if (!OidIsValid(indexoid)) return; if (validate_relation) { index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) return; if (!RelationUsesLocalBuffers(index_rel)) { relation_close(index_rel, AccessShareLock); index_rel = NULL; } } else index_rel = NULL; if (index_rel != NULL) { relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("background compaction is not supported for " "temporary indexes"))); } if (tp_managed_intent_context == NULL) tp_managed_intent_context = AllocSetContextCreate( TopMemoryContext, "pg_textsearch managed intents", ALLOCSET_SMALL_SIZES); foreach (lc, tp_managed_intents) { TpManagedIndexIntent *candidate = lfirst(lc); if (candidate->index_oid == indexoid && candidate->subid == subid) { intent = candidate; break; } } old_context = MemoryContextSwitchTo(tp_managed_intent_context); if (intent == NULL) { intent = palloc0(sizeof(*intent)); intent->index_oid = indexoid; intent->subid = subid; tp_managed_intents = lappend(tp_managed_intents, intent); } tp_update_managed_intent(intent, source, schedule, lineage, flags); MemoryContextSwitchTo(old_context); } static void tp_collect_managed_intent( Oid indexoid, const TpCompactionJobIdentity *source, const char *schedule, const char *lineage, int flags) { tp_collect_managed_intent_internal( indexoid, source, schedule, lineage, flags, true); } static void tp_collect_prevalidated_managed_intent( Oid indexoid, const TpCompactionJobIdentity *source, const char *schedule, const char *lineage, int flags) { tp_collect_managed_intent_internal( indexoid, source, schedule, lineage, flags, false); } static void tp_reset_managed_intents(void) { if (tp_managed_intent_context != NULL) MemoryContextDelete(tp_managed_intent_context); tp_managed_intent_context = NULL; tp_managed_intents = NIL; list_free(tp_reconciled_indexoids); tp_reconciled_indexoids = NIL; tp_managed_reconciling = false; tp_post_publication_reconciliation = false; } static void tp_free_managed_intent(TpManagedIndexIntent *intent) { tp_reset_compaction_identity(&intent->source); if (intent->schedule != NULL) pfree(intent->schedule); if (intent->lineage != NULL) pfree(intent->lineage); pfree(intent); } static void tp_abort_managed_intents(SubTransactionId subid) { ListCell *lc; foreach (lc, tp_managed_intents) { TpManagedIndexIntent *intent = lfirst(lc); if (intent->subid == subid) { tp_managed_intents = foreach_delete_current(tp_managed_intents, lc); tp_free_managed_intent(intent); } } } static void tp_merge_managed_intent( TpManagedIndexIntent *destination, const TpManagedIndexIntent *source) { tp_update_managed_intent( destination, OidIsValid(source->source.index_oid) ? &source->source : NULL, source->schedule, source->lineage, source->flags); } static void tp_promote_managed_intents( SubTransactionId subid, SubTransactionId parent_subid) { MemoryContext old_context; List *merged = NIL; ListCell *lc; if (tp_managed_intents == NIL) return; old_context = MemoryContextSwitchTo(tp_managed_intent_context); foreach (lc, tp_managed_intents) { TpManagedIndexIntent *intent = lfirst(lc); TpManagedIndexIntent *existing = NULL; ListCell *merged_lc; if (intent->subid == subid) intent->subid = parent_subid; foreach (merged_lc, merged) { TpManagedIndexIntent *candidate = lfirst(merged_lc); if (candidate->index_oid == intent->index_oid && candidate->subid == intent->subid) { existing = candidate; break; } } if (existing != NULL) { tp_merge_managed_intent(existing, intent); tp_free_managed_intent(intent); } else merged = lappend(merged, intent); } list_free(tp_managed_intents); tp_managed_intents = merged; MemoryContextSwitchTo(old_context); } static void tp_schedule_index_drop(Oid index_oid) { TpPendingIndexDrop *drop = MemoryContextAlloc( TopTransactionContext, sizeof(TpPendingIndexDrop)); drop->index_oid = index_oid; drop->subid = GetCurrentSubTransactionId(); drop->next = tp_pending_index_drops; tp_pending_index_drops = drop; } /* * Object access hook - enforce RLS checks, record CREATE INDEX objects, and * handle DROP INDEX. */ static void tp_object_access( ObjectAccessType access, Oid classId, Oid objectId, int subId, void *arg) { /* Call previous hook if exists */ if (prev_object_access_hook) prev_object_access_hook(access, classId, objectId, subId, arg); if (access == OAT_POST_CREATE && classId == RelationRelationId && subId == 0) { bool is_bm25 = tp_check_bm25_index_create_allowed(objectId); if (is_bm25 && current_utility_context != NULL && current_utility_context->track_index_build && !current_utility_context->build_progress_started) { tp_build_progress_begin(); current_utility_context->build_progress_started = true; } if (current_utility_context != NULL && (is_bm25 || current_utility_context->track_relation_create)) current_utility_context->retain_rls_ddl_lock = true; if (tp_create_index_states != NULL && tp_create_index_states->utility_depth == tp_process_utility_depth) { MemoryContext old_context; old_context = MemoryContextSwitchTo( tp_create_index_states->context); tp_create_index_states->created_indexes = list_append_unique_oid( tp_create_index_states->created_indexes, objectId); MemoryContextSwitchTo(old_context); } } if (access == OAT_POST_ALTER && current_utility_context != NULL && subId == 0) { if (classId == RelationRelationId && current_utility_context->check_rls_enable && !list_member_oid( current_utility_context->altered_relids, objectId)) { current_utility_context->altered_relids = lappend_oid( current_utility_context->altered_relids, objectId); current_utility_context->retain_rls_ddl_lock = true; } else if ( classId == InheritsRelationId && current_utility_context->check_hierarchy_change && !list_member_oid( current_utility_context->hierarchy_relids, objectId)) { current_utility_context->hierarchy_relids = lappend_oid( current_utility_context->hierarchy_relids, objectId); current_utility_context->retain_rls_ddl_lock = true; } } if (access == OAT_DROP && classId == DatabaseRelationId && subId == 0 && active_drop_database_context != NULL) active_drop_database_context->database_oid = objectId; /* Clean up DROP events on relations (indexes are relations). */ if (access == OAT_DROP && classId == RelationRelationId && subId == 0) { /* * Always cleanup our indexes regardless of drop flags. * PERFORM_DELETION_INTERNAL is set for cascade drops (e.g., DROP * TABLE dropping its indexes) but we still need to free registry * entries and shared memory in those cases. */ /* Check if this is one of our indexes */ if (!tp_registry_is_registered( tp_registry_key(MyDatabaseId, objectId))) return; if (arg != NULL && (((ObjectAccessDrop *)arg)->dropflags & PERFORM_DELETION_CONCURRENTLY)) { MemoryContext old_context; /* * Concurrent DROP commits before draining readers. Keep its * cleanup outside those transactions until core DROP returns. */ if (current_utility_context == NULL) elog(ERROR, "cannot track concurrent pg_textsearch index drop " "outside a utility command"); old_context = MemoryContextSwitchTo(TopMemoryContext); current_utility_context->concurrent_index_drops = lappend_oid( current_utility_context->concurrent_index_drops, objectId); MemoryContextSwitchTo(old_context); } else tp_schedule_index_drop(objectId); } } /* * Shared memory request hook - calculate and request shared memory */ static void tp_shmem_request(void) { /* Call previous hook first if it exists */ if (prev_shmem_request_hook) prev_shmem_request_hook(); /* Request shared memory for registry (includes DSA control) */ tp_registry_init(); } /* * Shared memory startup hook - initialize the registry */ static void tp_shmem_startup(void) { /* Call previous hook first if it exists */ if (prev_shmem_startup_hook) prev_shmem_startup_hook(); /* Initialize the registry in shared memory (includes DSA control) */ tp_registry_shmem_startup(); } /* * Transaction callback - reconcile REINDEX replacements, release index * locks at transaction end, check for bulk load auto-spill at pre-commit, * and dispatch any compaction requests those spills registered. * * The spill check runs first: a spill can register a request, so * flushing afterwards is what lets that request reach the callback in * the same transaction. Parallel workers deliberately do not flush; * dispatch runs subtransactions and SPI, which a worker must not do. */ static void tp_xact_callback(XactEvent event, void *arg pg_attribute_unused()) { bool reconciled; switch (event) { case XACT_EVENT_PRE_COMMIT: if (tp_reindex_states != NULL) tp_collect_reindex_state_intents(NULL, true, false); /* * Check for bulk load auto-spill before commit. * If any index had a large number of terms added this transaction, * spill to disk to prevent unbounded memory growth. */ tp_bulk_load_spill_check(); reconciled = tp_reconcile_managed_intents_at_precommit(); if (tp_reindex_states != NULL) tp_finish_reindex_pending(reconciled); tp_compaction_flush_requests(); break; case XACT_EVENT_PARALLEL_PRE_COMMIT: tp_bulk_load_spill_check(); break; case XACT_EVENT_COMMIT: case XACT_EVENT_PARALLEL_COMMIT: tp_commit_build_states(); /* Release all index locks held by this backend */ tp_release_all_index_locks(); /* Reset bulk load counters for next transaction */ tp_reset_bulk_load_counters(); tp_reset_managed_intents(); /* Commit is irrevocable, but relation locks are still held. */ while (tp_pending_index_drops != NULL) { TpPendingIndexDrop *drop = tp_pending_index_drops; tp_pending_index_drops = drop->next; tp_cleanup_index_shared_memory(drop->index_oid); } break; case XACT_EVENT_ABORT: case XACT_EVENT_PARALLEL_ABORT: tp_pending_index_drops = NULL; /* Remove shared state owned by an aborted initial CREATE INDEX. */ tp_cleanup_build_mode_on_abort(); /* Release all index locks held by this backend */ tp_release_all_index_locks(); /* Reset bulk load counters for next transaction */ tp_reset_bulk_load_counters(); tp_reset_managed_intents(); break; case XACT_EVENT_PRE_PREPARE: if (tp_pending_index_drops != NULL) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot prepare a transaction that dropped a " "pg_textsearch index"), errdetail( "Pending index shared-state cleanup is " "backend-local and cannot be serialized " "for two-phase commit."), errhint("Commit or roll back the DROP INDEX before " "preparing the transaction."))); if (tp_has_initial_create_ownership()) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot prepare a transaction that created a " "pg_textsearch index"), errdetail( "Initial CREATE INDEX shared-state ownership " "is backend-local and cannot be serialized " "for two-phase commit."), errhint("Commit or roll back the CREATE INDEX before " "preparing the transaction."))); /* * Spill while PREPARE can still fail. Once PostgreSQL records the * prepared transaction, a later backend cannot safely reproduce * this backend-local threshold decision. */ tp_bulk_load_spill_check(); break; case XACT_EVENT_PREPARE: /* * PREPARE is terminal for this backend's transaction just like * COMMIT or ABORT. Do not let extension lock tracking or bulk * counters bleed into the next transaction on this connection. */ tp_release_all_index_locks(); tp_reset_bulk_load_counters(); break; } } /* * Subtransaction callback - clean up index state on savepoint rollback * * When a subtransaction aborts (ROLLBACK TO SAVEPOINT), we must: * 1. Clean up registry/shared memory for indexes created in that * subtransaction (OAT_DROP doesn't fire for subtransaction abort) * 2. Reset lock tracking (LWLockReleaseAll releases all locks) * * When a subtransaction commits (RELEASE SAVEPOINT), we promote * states to the parent subtransaction so they get cleaned up if the * parent later aborts. */ static void tp_subxact_callback( SubXactEvent event, SubTransactionId mySubid, SubTransactionId parentSubid, void *arg pg_attribute_unused()) { TpPendingIndexDrop **link; switch (event) { case SUBXACT_EVENT_ABORT_SUB: for (link = &tp_pending_index_drops; *link != NULL;) { TpPendingIndexDrop *drop = *link; if (drop->subid == mySubid) { *link = drop->next; pfree(drop); } else link = &drop->next; } tp_cleanup_subxact_abort(mySubid); tp_abort_managed_intents(mySubid); break; case SUBXACT_EVENT_COMMIT_SUB: for (TpPendingIndexDrop *drop = tp_pending_index_drops; drop != NULL; drop = drop->next) { if (drop->subid == mySubid) drop->subid = parentSubid; } tp_promote_subxact_states(mySubid, parentSubid); tp_promote_managed_intents(mySubid, parentSubid); break; case SUBXACT_EVENT_START_SUB: case SUBXACT_EVENT_PRE_COMMIT_SUB: /* Nothing to do */ break; } } static const char * tp_index_stmt_option(IndexStmt *stmt, const char *option_name) { ListCell *lc; foreach (lc, stmt->options) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, option_name) == 0) return defGetString(option); } return NULL; } static int tp_index_stmt_option_count(IndexStmt *stmt, const char *option_name) { ListCell *lc; int count = 0; foreach (lc, stmt->options) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, option_name) == 0) count++; } return count; } static void tp_index_stmt_remove_option(IndexStmt *stmt, const char *option_name) { ListCell *lc; foreach (lc, stmt->options) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, option_name) == 0) { stmt->options = foreach_delete_current(stmt->options, lc); return; } } } static void tp_index_stmt_ensure_lineage(IndexStmt *stmt, Oid heap_oid, Oid owner_oid) { char *lineage; if (tp_index_stmt_option(stmt, "compaction_lineage") != NULL) return; (void)heap_oid; (void)owner_oid; lineage = tp_new_compaction_lineage(); stmt->options = lappend( stmt->options, makeDefElem( "compaction_lineage", (Node *)makeString(lineage), -1)); } static void tp_index_stmt_validate_supplied_lineage( IndexStmt *stmt, Oid heap_oid, Oid owner_oid) { const char *lineage; int count; count = tp_index_stmt_option_count(stmt, "compaction_lineage"); if (count == 0) return; if (count > 1) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("parameter \"compaction_lineage\" specified more " "than once"))); lineage = tp_index_stmt_option(stmt, "compaction_lineage"); tp_validate_compaction_lineage(lineage); if (!object_ownercheck(RelationRelationId, heap_oid, GetUserId())) return; if (tp_compaction_lineage_in_use(lineage, heap_oid, owner_oid)) { /* * PostgreSQL copies index reloptions while expanding CREATE TABLE * LIKE. A nested clone onto an unrelated heap needs a new lineage; * top-level supplied values still retain strict collision checks. */ if (tp_process_utility_depth > 1) { tp_index_stmt_remove_option(stmt, "compaction_lineage"); return; } ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("background compaction lineage is already in use"))); } } static bool tp_alter_index_mentions_lineage(AlterTableStmt *stmt) { ListCell *lc; if (stmt->objtype != OBJECT_INDEX) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); ListCell *option_lc; if (cmd->subtype != AT_SetRelOptions && cmd->subtype != AT_ResetRelOptions && cmd->subtype != AT_ReplaceRelOptions) continue; foreach (option_lc, castNode(List, cmd->def)) { DefElem *option = lfirst_node(DefElem, option_lc); if (strcmp(option->defname, "compaction_lineage") == 0) return true; } } return false; } static void tp_reject_user_lineage_alter(AlterTableStmt *stmt) { LOCKMODE lockmode; Oid indexoid; Relation index_rel; bool is_bm25; if (!tp_alter_index_mentions_lineage(stmt)) return; lockmode = AlterTableGetLockLevel(stmt->cmds); indexoid = RangeVarGetRelidExtended( stmt->relation, lockmode, stmt->missing_ok ? RVR_MISSING_OK : 0, RangeVarCallbackOwnsRelation, NULL); if (!OidIsValid(indexoid)) return; index_rel = try_relation_open(indexoid, NoLock); if (index_rel == NULL) return; is_bm25 = (index_rel->rd_rel->relkind == RELKIND_INDEX || index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) && index_rel->rd_indam != NULL && index_rel->rd_indam->ambuild == tp_build; relation_close(index_rel, NoLock); if (is_bm25) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("cannot alter internal background compaction " "lineage"))); } static bool tp_background_cic_needs_preflight(IndexStmt *stmt, Relation heap_rel) { const char *mode; if (!stmt->concurrent) return false; mode = tp_index_stmt_option(stmt, "compaction"); if (mode == NULL || pg_strcasecmp(mode, "background") != 0) return false; if (stmt->if_not_exists && stmt->idxname != NULL && OidIsValid(get_relname_relid( stmt->idxname, RelationGetNamespace(heap_rel)))) return false; return true; } static bool tp_alter_index_refreshes_background(AlterTableStmt *stmt) { ListCell *lc; if (stmt->objtype != OBJECT_INDEX) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); ListCell *option_lc; if (cmd->subtype != AT_SetRelOptions && cmd->subtype != AT_ResetRelOptions && cmd->subtype != AT_ReplaceRelOptions) continue; foreach (option_lc, castNode(List, cmd->def)) { DefElem *option = lfirst_node(DefElem, option_lc); if (strcmp(option->defname, "compaction") == 0 || strcmp(option->defname, "compaction_schedule") == 0) return true; } } return false; } static bool tp_is_physical_bm25_index_relation(Relation index_rel) { return index_rel->rd_rel->relkind == RELKIND_INDEX && index_rel->rd_indam != NULL && index_rel->rd_indam->ambuild == tp_build && index_rel->rd_index != NULL && index_rel->rd_index->indisvalid && index_rel->rd_index->indisready && index_rel->rd_index->indislive; } static bool tp_is_bm25_index_node_relation(Relation index_rel) { return (index_rel->rd_rel->relkind == RELKIND_INDEX || index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) && index_rel->rd_indam != NULL && index_rel->rd_indam->ambuild == tp_build && index_rel->rd_index != NULL; } static bool tp_is_background_physical_index_relation(Relation index_rel) { return tp_is_physical_bm25_index_relation(index_rel) && tp_index_compaction_mode(index_rel) == TP_COMPACTION_BACKGROUND; } static bool tp_is_background_physical_index(Oid indexoid) { Relation index_rel; bool background; index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) return false; background = tp_is_background_physical_index_relation(index_rel); relation_close(index_rel, AccessShareLock); return background; } static List * tp_physical_bm25_indexes(List *indexoids, bool background_only) { List *result = NIL; ListCell *lc; foreach (lc, indexoids) { Oid indexoid = lfirst_oid(lc); Relation index_rel; bool matches; index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) continue; matches = tp_is_physical_bm25_index_relation(index_rel) && (!background_only || tp_index_compaction_mode(index_rel) == TP_COMPACTION_BACKGROUND); relation_close(index_rel, AccessShareLock); if (matches) result = list_append_unique_oid(result, indexoid); } return result; } static void tp_report_nowait_relation_lock_error(Oid relation_oid) { char *relation_name = get_rel_name(relation_oid); if (relation_name != NULL) ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("could not obtain lock on relation \"%s\"", relation_name))); ereport(ERROR, (errcode(ERRCODE_OBJECT_IN_USE), errmsg("could not obtain lock on relation with OID %u", relation_oid))); } static List * tp_physical_bm25_indexes_for_rewrite(List *indexoids, bool nowait) { List *result = NIL; ListCell *lc; if (!nowait) return tp_physical_bm25_indexes(indexoids, false); foreach (lc, indexoids) { Oid indexoid = lfirst_oid(lc); Relation index_rel; bool matches; if (!ConditionalLockRelationOid(indexoid, AccessShareLock)) tp_report_nowait_relation_lock_error(indexoid); index_rel = try_relation_open(indexoid, NoLock); if (index_rel == NULL) { UnlockRelationOid(indexoid, AccessShareLock); continue; } matches = tp_is_physical_bm25_index_relation(index_rel); relation_close(index_rel, NoLock); UnlockRelationOid(indexoid, AccessShareLock); if (matches) result = list_append_unique_oid(result, indexoid); } return result; } static List * tp_bm25_index_nodes(List *indexoids) { List *result = NIL; ListCell *lc; foreach (lc, indexoids) { Oid indexoid = lfirst_oid(lc); Relation index_rel; bool matches; index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) continue; matches = tp_is_bm25_index_node_relation(index_rel); relation_close(index_rel, AccessShareLock); if (matches) result = list_append_unique_oid(result, indexoid); } return result; } static List * tp_all_bm25_indexes( Oid tablespace_oid, bool filter_tablespace, List *owner_oids, bool require_current_owner) { Relation class_rel; SysScanDesc scan; HeapTuple tuple; Oid bm25_am_oid = get_index_am_oid("bm25", true); List *indexoids = NIL; /* The library may be preloaded in databases without the extension. */ if (!OidIsValid(bm25_am_oid)) return NIL; class_rel = table_open(RelationRelationId, AccessShareLock); scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 0, NULL); while ((tuple = systable_getnext(scan)) != NULL) { Form_pg_class class_form = (Form_pg_class)GETSTRUCT(tuple); if (class_form->relkind != RELKIND_INDEX || class_form->relam != bm25_am_oid || (filter_tablespace && class_form->reltablespace != tablespace_oid) || (owner_oids != NIL && !list_member_oid(owner_oids, class_form->relowner)) || (require_current_owner && !object_ownercheck( RelationRelationId, class_form->oid, GetUserId()))) continue; indexoids = lappend_oid(indexoids, class_form->oid); } systable_endscan(scan); table_close(class_rel, AccessShareLock); return indexoids; } static Oid tp_catalog_tablespace_oid(Oid tablespace_oid) { HeapTuple tuple; Form_pg_database database; Oid catalog_oid = tablespace_oid; tuple = SearchSysCache1(DATABASEOID, ObjectIdGetDatum(MyDatabaseId)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for database %u", MyDatabaseId); database = (Form_pg_database)GETSTRUCT(tuple); if (tablespace_oid == database->dattablespace) catalog_oid = InvalidOid; ReleaseSysCache(tuple); return catalog_oid; } static List * tp_role_oids(List *roles) { List *owner_oids = NIL; ListCell *lc; foreach (lc, roles) { RoleSpec *role = lfirst_node(RoleSpec, lc); Oid owner_oid = get_rolespec_oid(role, false); owner_oids = list_append_unique_oid(owner_oids, owner_oid); } return owner_oids; } static bool tp_bulk_move_preflight(AlterTableMoveAllStmt *stmt, Oid *source_tablespace) { AclResult aclresult; Oid destination_tablespace; *source_tablespace = get_tablespace_oid(stmt->orig_tablespacename, false); destination_tablespace = get_tablespace_oid(stmt->new_tablespacename, false); if (*source_tablespace == GLOBALTABLESPACE_OID || destination_tablespace == GLOBALTABLESPACE_OID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("cannot move relations in to or out of pg_global " "tablespace"))); if (OidIsValid(destination_tablespace) && destination_tablespace != MyDatabaseTableSpace) { aclresult = object_aclcheck( TableSpaceRelationId, destination_tablespace, GetUserId(), ACL_CREATE); if (aclresult != ACLCHECK_OK) aclcheck_error( aclresult, OBJECT_TABLESPACE, get_tablespace_name(destination_tablespace)); } if (*source_tablespace == MyDatabaseTableSpace) *source_tablespace = InvalidOid; if (destination_tablespace == MyDatabaseTableSpace) destination_tablespace = InvalidOid; return *source_tablespace != destination_tablespace; } static bool tp_bulk_move_has_unauthorized_index(Oid tablespace_oid, List *owner_oids) { Relation class_rel; SysScanDesc scan; HeapTuple tuple; bool unauthorized = false; class_rel = table_open(RelationRelationId, AccessShareLock); scan = systable_beginscan(class_rel, InvalidOid, false, NULL, 0, NULL); while ((tuple = systable_getnext(scan)) != NULL) { Form_pg_class class_form = (Form_pg_class)GETSTRUCT(tuple); if (class_form->reltablespace != tablespace_oid || (class_form->relkind != RELKIND_INDEX && class_form->relkind != RELKIND_PARTITIONED_INDEX) || IsCatalogNamespace(class_form->relnamespace) || class_form->relisshared || isAnyTempNamespace(class_form->relnamespace) || IsToastNamespace(class_form->relnamespace) || (owner_oids != NIL && !list_member_oid(owner_oids, class_form->relowner))) continue; if (!object_ownercheck( RelationRelationId, class_form->oid, GetUserId())) { unauthorized = true; break; } } systable_endscan(scan); table_close(class_rel, AccessShareLock); return unauthorized; } static bool tp_can_maintain_relation(Oid relation_oid) { return pg_class_aclcheck(relation_oid, GetUserId(), ACL_MAINTAIN) == ACLCHECK_OK; } static List * tp_maintainable_indexes(List *indexoids, bool maintain_privilege) { List *eligible = NIL; ListCell *lc; foreach (lc, indexoids) { Oid indexoid = lfirst_oid(lc); Oid heap_oid = IndexGetRelation(indexoid, true); if (!OidIsValid(heap_oid)) continue; if (maintain_privilege ? tp_can_maintain_relation(heap_oid) : object_ownercheck( RelationRelationId, heap_oid, GetUserId())) eligible = lappend_oid(eligible, indexoid); } list_free(indexoids); return eligible; } static List * tp_namespace_bm25_indexes(Oid namespace_oid) { List *all_indexes = tp_all_bm25_indexes(InvalidOid, false, NIL, false); List *indexoids = NIL; ListCell *lc; foreach (lc, all_indexes) { Oid indexoid = lfirst_oid(lc); if (get_rel_namespace(indexoid) == namespace_oid) indexoids = lappend_oid(indexoids, indexoid); } list_free(all_indexes); return indexoids; } static List * tp_reassign_owned_indexes(ReassignOwnedStmt *stmt) { List *owner_oids = NIL; List *indexoids; ListCell *lc; Oid new_owner_oid = get_rolespec_oid(stmt->newrole, false); if (!has_privs_of_role(GetUserId(), new_owner_oid)) return NIL; foreach (lc, stmt->roles) { RoleSpec *role = lfirst_node(RoleSpec, lc); Oid owner_oid = get_rolespec_oid(role, false); if (!has_privs_of_role(GetUserId(), owner_oid)) { list_free(owner_oids); return NIL; } owner_oids = list_append_unique_oid(owner_oids, owner_oid); } indexoids = tp_all_bm25_indexes(InvalidOid, false, owner_oids, false); list_free(owner_oids); return indexoids; } static List * tp_relation_indexes_locked(Oid relation_oid, LOCKMODE lockmode) { Relation relation; List *indexes; relation = relation_open(relation_oid, lockmode); indexes = list_copy(RelationGetIndexList(relation)); relation_close(relation, NoLock); return indexes; } static List * tp_relation_tree_indexes_locked(Oid relation_oid, LOCKMODE lockmode) { Relation root; List *relations; List *indexes = NIL; ListCell *lc; root = relation_open(relation_oid, NoLock); if (root->rd_rel->relkind != RELKIND_PARTITIONED_TABLE) { indexes = list_copy(RelationGetIndexList(root)); relation_close(root, NoLock); return indexes; } relation_close(root, NoLock); relations = find_all_inheritors(relation_oid, lockmode, NULL); foreach (lc, relations) { Oid child_oid = lfirst_oid(lc); Relation child; List *child_indexes; child = relation_open(child_oid, NoLock); child_indexes = list_copy(RelationGetIndexList(child)); relation_close(child, NoLock); indexes = list_concat_unique_oid(indexes, child_indexes); } list_free(relations); return indexes; } static List * tp_index_tree_locked(Oid indexoid, LOCKMODE lockmode) { Relation index_rel; List *indexes; index_rel = index_open(indexoid, NoLock); if (index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) indexes = find_all_inheritors(indexoid, lockmode, NULL); else indexes = list_make1_oid(indexoid); index_close(index_rel, NoLock); return indexes; } static bool tp_reindex_concurrently(ReindexStmt *stmt) { ListCell *lc; foreach (lc, stmt->params) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, "concurrently") == 0) return defGetBoolean(option); } return false; } typedef struct TpReindexIndexLookupState { LOCKMODE table_lockmode; Oid locked_table_oid; } TpReindexIndexLookupState; static void tp_reindex_index_lookup_callback( const RangeVar *relation, Oid relid, Oid old_relid, void *arg) { TpReindexIndexLookupState *state = arg; Oid table_oid; char relkind; (void)relation; if (relid != old_relid && OidIsValid(old_relid) && OidIsValid(state->locked_table_oid)) { UnlockRelationOid(state->locked_table_oid, state->table_lockmode); state->locked_table_oid = InvalidOid; } if (!OidIsValid(relid)) return; relkind = get_rel_relkind(relid); if (!relkind) return; if (relkind != RELKIND_INDEX && relkind != RELKIND_PARTITIONED_INDEX) ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not an index", relation->relname))); table_oid = IndexGetRelation(relid, true); if (OidIsValid(table_oid)) { AclResult aclresult; aclresult = pg_class_aclcheck(table_oid, GetUserId(), ACL_MAINTAIN); if (aclresult != ACLCHECK_OK) aclcheck_error(aclresult, OBJECT_INDEX, relation->relname); } if (relid != old_relid && OidIsValid(table_oid)) { LockRelationOid(table_oid, state->table_lockmode); state->locked_table_oid = table_oid; } } static List * tp_reindex_initial_indexes( ReindexStmt *stmt, bool *tracks_commits, bool is_top_level, Oid *scope_oid) { TpReindexIndexLookupState lookup_state; LOCKMODE relation_lockmode; Oid relation_oid; char relkind; bool concurrently = tp_reindex_concurrently(stmt); List *indexoids; if (stmt->kind == REINDEX_OBJECT_SCHEMA) { Oid namespace_oid = get_namespace_oid(stmt->name, false); if (!object_ownercheck( NamespaceRelationId, namespace_oid, GetUserId()) && !has_privs_of_role(GetUserId(), ROLE_PG_MAINTAIN)) { *tracks_commits = false; return NIL; } *tracks_commits = true; *scope_oid = namespace_oid; return tp_namespace_bm25_indexes(namespace_oid); } if (stmt->kind == REINDEX_OBJECT_DATABASE) { char *database_name = get_database_name(MyDatabaseId); if (database_name == NULL || (stmt->name != NULL && strcmp(stmt->name, database_name) != 0)) { if (database_name != NULL) pfree(database_name); *tracks_commits = false; return NIL; } pfree(database_name); if (!object_ownercheck( DatabaseRelationId, MyDatabaseId, GetUserId()) && !has_privs_of_role(GetUserId(), ROLE_PG_MAINTAIN)) { *tracks_commits = false; return NIL; } *tracks_commits = true; *scope_oid = MyDatabaseId; return tp_all_bm25_indexes(InvalidOid, false, NIL, false); } if (stmt->kind != REINDEX_OBJECT_INDEX && stmt->kind != REINDEX_OBJECT_TABLE) { *tracks_commits = false; return NIL; } if (stmt->kind == REINDEX_OBJECT_INDEX) { lookup_state.table_lockmode = concurrently ? ShareUpdateExclusiveLock : ShareLock; lookup_state.locked_table_oid = InvalidOid; relation_lockmode = concurrently ? ShareUpdateExclusiveLock : AccessExclusiveLock; relation_oid = RangeVarGetRelidExtended( stmt->relation, relation_lockmode, 0, tp_reindex_index_lookup_callback, &lookup_state); relkind = get_rel_relkind(relation_oid); *tracks_commits = concurrently || relkind == RELKIND_PARTITIONED_INDEX; if (relkind == RELKIND_PARTITIONED_INDEX) { PreventInTransactionBlock(is_top_level, "REINDEX INDEX"); return tp_index_tree_locked(relation_oid, ShareLock); } return list_make1_oid(relation_oid); } relation_lockmode = concurrently ? ShareUpdateExclusiveLock : ShareLock; relation_oid = RangeVarGetRelidExtended( stmt->relation, relation_lockmode, 0, RangeVarCallbackMaintainsTable, NULL); relkind = get_rel_relkind(relation_oid); *tracks_commits = concurrently || relkind == RELKIND_PARTITIONED_TABLE; if (relkind == RELKIND_PARTITIONED_TABLE) { PreventInTransactionBlock(is_top_level, "REINDEX TABLE"); return tp_relation_tree_indexes_locked(relation_oid, ShareLock); } indexoids = tp_relation_indexes_locked(relation_oid, NoLock); return indexoids; } static void tp_reindex_preflight(ReindexStmt *stmt, bool is_top_level) { ListCell *lc; bool concurrently = false; char *tablespace_name = NULL; foreach (lc, stmt->params) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, "verbose") == 0) (void)defGetBoolean(option); else if (strcmp(option->defname, "concurrently") == 0) concurrently = defGetBoolean(option); else if (strcmp(option->defname, "tablespace") == 0) tablespace_name = defGetString(option); else ereport(ERROR, (errcode(ERRCODE_SYNTAX_ERROR), errmsg("unrecognized REINDEX option \"%s\"", option->defname))); } if (concurrently) PreventInTransactionBlock(is_top_level, "REINDEX CONCURRENTLY"); if (tablespace_name != NULL) { AclResult aclresult; Oid tablespace_oid = get_tablespace_oid(tablespace_name, false); if (OidIsValid(tablespace_oid) && tablespace_oid != MyDatabaseTableSpace) { aclresult = object_aclcheck( TableSpaceRelationId, tablespace_oid, GetUserId(), ACL_CREATE); if (aclresult != ACLCHECK_OK) aclcheck_error( aclresult, OBJECT_TABLESPACE, get_tablespace_name(tablespace_oid)); } } } static void tp_collect_background_indexes(List *indexoids, bool refresh_default) { List *candidates; ListCell *lc; candidates = tp_physical_bm25_indexes(indexoids, true); foreach (lc, candidates) tp_collect_managed_intent( lfirst_oid(lc), NULL, NULL, NULL, refresh_default ? TP_MANAGED_INTENT_REFRESH_DEFAULT : 0); list_free(candidates); } static int tp_lineage_cmp(const ListCell *left, const ListCell *right) { return strcmp((const char *)lfirst(left), (const char *)lfirst(right)); } static bool tp_reconcile_managed_intents(void) { List *frozen; List *indexoids = NIL; List *lineages = NIL; List *locked; ListCell *lc; TpCompactionJobObjects *objects = NULL; bool deferred = false; bool post_publication = false; bool snapshot_pushed = false; bool strict_lineage = false; bool activation_needed = false; if (tp_managed_reconciling || tp_managed_intents == NIL) return true; frozen = tp_managed_intents; tp_managed_intents = NIL; foreach (lc, frozen) { TpManagedIndexIntent *intent = lfirst(lc); if ((intent->flags & TP_MANAGED_INTENT_DISABLE) == 0 || (intent->flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) != 0) indexoids = list_append_unique_oid(indexoids, intent->index_oid); if ((intent->flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) != 0 && SearchSysCacheExists1(RELOID, ObjectIdGetDatum(intent->index_oid))) strict_lineage = true; if ((intent->flags & TP_MANAGED_INTENT_POST_PUBLICATION) != 0) post_publication = true; } tp_managed_reconciling = true; PG_TRY(); { if (!ActiveSnapshotSet()) { PushActiveSnapshot(GetLatestSnapshot()); snapshot_pushed = true; } locked = tp_try_prelock_compaction_indexes(indexoids); if (list_length(locked) < list_length(indexoids)) deferred = true; if (strict_lineage && list_length(locked) < list_length(indexoids)) ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not validate background compaction " "lineage"), errdetail("A required index is concurrently locked."), errhint("Retry the command after the conflicting " "transaction completes."))); foreach (lc, frozen) { TpManagedIndexIntent *intent = lfirst(lc); if ((intent->flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) != 0 && list_member_oid(locked, intent->index_oid)) lineages = lappend(lineages, intent->lineage); } list_sort(lineages, tp_lineage_cmp); foreach (lc, lineages) { if (!tp_try_lock_compaction_lineage(lfirst(lc))) ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not validate background compaction " "lineage"), errdetail( "A requested lineage is concurrently " "locked."), errhint("Retry the command after the conflicting " "transaction completes."))); } foreach (lc, locked) { char *lineage = tp_ensure_index_compaction_lineage(lfirst_oid(lc), NULL); if (lineage != NULL) pfree(lineage); } foreach (lc, frozen) { TpManagedIndexIntent *intent = lfirst(lc); if (!list_member_oid(locked, intent->index_oid) || (intent->flags & TP_MANAGED_INTENT_DISABLE) != 0 || (intent->flags & (TP_MANAGED_INTENT_RECONCILE_OPTIONS | TP_MANAGED_INTENT_RECONCILE_LINEAGE)) == 0 || (intent->flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) != 0) continue; if ((intent->flags & TP_MANAGED_INTENT_RECONCILE_LINEAGE) != 0) tp_reconcile_index_compaction_lineage( intent->index_oid, intent->lineage); else tp_reconcile_index_compaction_options( intent->index_oid, intent->schedule, intent->lineage); } foreach (lc, frozen) { TpManagedIndexIntent *intent = lfirst(lc); if (list_member_oid(locked, intent->index_oid) && (intent->flags & TP_MANAGED_INTENT_DISABLE) == 0 && tp_is_background_physical_index(intent->index_oid)) { activation_needed = true; break; } } if (activation_needed && (objects = tp_compaction_job_try_lock_objects( !post_publication)) == NULL) { if (strict_lineage) ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not validate background compaction " "lineage"), errdetail( "A required extension object is concurrently " "locked."), errhint("Retry the command after the conflicting " "transaction completes."))); list_free(locked); locked = NIL; deferred = true; } foreach (lc, frozen) { TpManagedIndexIntent *intent = lfirst(lc); Relation index_rel; Oid heap_oid; Oid owner_oid; if ((intent->flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) == 0 || !list_member_oid(locked, intent->index_oid)) continue; index_rel = try_relation_open(intent->index_oid, NoLock); if (index_rel == NULL || index_rel->rd_index == NULL) { if (index_rel != NULL) relation_close(index_rel, NoLock); continue; } heap_oid = index_rel->rd_index->indrelid; owner_oid = index_rel->rd_rel->relowner; relation_close(index_rel, NoLock); if (tp_compaction_lineage_in_use_by_other( intent->lineage, intent->index_oid, heap_oid) || (objects != NULL && tp_compaction_job_lineage_exists( objects, intent->lineage, heap_oid, owner_oid))) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("background compaction lineage is already " "in use"))); } foreach (lc, frozen) { TpManagedIndexIntent *intent = lfirst(lc); if (!list_member_oid(locked, intent->index_oid) || (intent->flags & TP_MANAGED_INTENT_DISABLE) != 0) continue; if ((intent->flags & TP_MANAGED_INTENT_LINEAGE_SUPPLIED) != 0) { if ((intent->flags & TP_MANAGED_INTENT_RECONCILE_LINEAGE) != 0) tp_reconcile_index_compaction_lineage( intent->index_oid, intent->lineage); else if ( (intent->flags & TP_MANAGED_INTENT_RECONCILE_OPTIONS) != 0) tp_reconcile_index_compaction_options( intent->index_oid, intent->schedule, intent->lineage); } if (!tp_is_background_physical_index(intent->index_oid)) { tp_reconciled_indexoids = list_append_unique_oid( tp_reconciled_indexoids, intent->index_oid); continue; } if ((intent->flags & TP_MANAGED_INTENT_PRESERVE_SCHEDULE) != 0) { tp_compaction_job_resolve_schedule( objects, intent->index_oid, &intent->source, tp_managed_intent_context); tp_compaction_job_activate_with_schedule( objects, intent->index_oid, intent->source.schedule); } else tp_compaction_job_activate( objects, intent->index_oid, (intent->flags & TP_MANAGED_INTENT_REFRESH_DEFAULT) != 0); tp_reconciled_indexoids = list_append_unique_oid( tp_reconciled_indexoids, intent->index_oid); } list_free(locked); if (tp_managed_intents != NIL) ereport(ERROR, (errmsg("cannot execute DDL during background compaction " "reconciliation"))); if (deferred) ereport(WARNING, (errmsg("background compaction lifecycle reconciliation " "was deferred"), errdetail( "A required relation or extension object was " "concurrently locked."), errhint("Repeat the managed DDL after the conflicting " "transaction completes."))); if (snapshot_pushed) { PopActiveSnapshot(); snapshot_pushed = false; } } PG_FINALLY(); { if (snapshot_pushed) PopActiveSnapshot(); tp_managed_reconciling = false; list_free(indexoids); list_free(lineages); list_free(frozen); } PG_END_TRY(); return !deferred; } static bool tp_reconcile_managed_intents_at_precommit(void) { MemoryContext old_context = CurrentMemoryContext; ResourceOwner old_owner = CurrentResourceOwner; ListCell *lc; bool post_publication = tp_post_publication_reconciliation; bool reconciled = true; foreach (lc, tp_managed_intents) { TpManagedIndexIntent *intent = lfirst(lc); if ((intent->flags & TP_MANAGED_INTENT_POST_PUBLICATION) != 0) { post_publication = true; break; } } if (!post_publication) return tp_reconcile_managed_intents(); BeginInternalSubTransaction(NULL); PG_TRY(); { MemoryContextSwitchTo(old_context); reconciled = tp_reconcile_managed_intents(); tp_post_publication_reconciliation = false; ReleaseCurrentSubTransaction(); MemoryContextSwitchTo(old_context); CurrentResourceOwner = old_owner; } PG_CATCH(); { ErrorData *edata; MemoryContextSwitchTo(old_context); edata = CopyErrorData(); FlushErrorState(); RollbackAndReleaseCurrentSubTransaction(); MemoryContextSwitchTo(old_context); CurrentResourceOwner = old_owner; tp_reset_managed_intents(); if (edata->elevel >= FATAL || edata->sqlerrcode == ERRCODE_QUERY_CANCELED || edata->sqlerrcode == ERRCODE_ADMIN_SHUTDOWN || edata->sqlerrcode == ERRCODE_CRASH_SHUTDOWN || edata->sqlerrcode == ERRCODE_CANNOT_CONNECT_NOW) ReThrowError(edata); ereport(WARNING, (errmsg("background compaction lifecycle reconciliation was " "deferred"), errdetail( "Managed reconciliation failed after core " "publication: %s.", edata->message), errhint("Repeat the managed DDL after correcting the " "reported condition."))); FreeErrorData(edata); reconciled = false; } PG_END_TRY(); return reconciled; } static void tp_prelock_owner_change_heaps(List *indexoids) { List *heap_oids = NIL; ListCell *lc; Oid previous = InvalidOid; foreach (lc, indexoids) { Oid heap_oid = IndexGetRelation(lfirst_oid(lc), true); if (OidIsValid(heap_oid)) heap_oids = list_append_unique_oid(heap_oids, heap_oid); } list_sort(heap_oids, list_oid_cmp); foreach (lc, heap_oids) { Oid heap_oid = lfirst_oid(lc); if (heap_oid == previous) continue; LockRelationOid(heap_oid, AccessExclusiveLock); previous = heap_oid; } list_free(heap_oids); } static List * tp_capture_owner_change_targets(List *indexoids) { List *targets = NIL; ListCell *lc; foreach (lc, indexoids) { Oid indexoid = lfirst_oid(lc); TpOwnerChangeTarget *target; if (!tp_is_background_physical_index(indexoid)) continue; target = palloc0(sizeof(*target)); tp_compaction_job_capture(indexoid, target); targets = lappend(targets, target); } return targets; } static void tp_collect_owner_change_targets(List *targets) { ListCell *lc; foreach (lc, targets) { TpOwnerChangeTarget *target = lfirst(lc); if (tp_is_background_physical_index(target->index_oid)) tp_collect_managed_intent( target->index_oid, target, NULL, NULL, TP_MANAGED_INTENT_PRESERVE_SCHEDULE); } } static void tp_free_owner_change_targets(List *targets) { ListCell *lc; foreach (lc, targets) { TpOwnerChangeTarget *target = lfirst(lc); pfree(target->index_name); pfree(target->lineage); pfree(target->schedule); pfree(target); } list_free(targets); } static List * tp_created_index_tree_locked(List *created_indexes, Oid heap_oid) { List *result = NIL; ListCell *lc; foreach (lc, created_indexes) { Oid indexoid = lfirst_oid(lc); Relation index_rel; List *index_tree; List *ancestors = NIL; Oid index_heap_oid; bool belongs; index_rel = try_relation_open(indexoid, AccessShareLock); if (index_rel == NULL) continue; if ((index_rel->rd_rel->relkind != RELKIND_INDEX && index_rel->rd_rel->relkind != RELKIND_PARTITIONED_INDEX) || index_rel->rd_index == NULL) { relation_close(index_rel, AccessShareLock); continue; } index_heap_oid = index_rel->rd_index->indrelid; belongs = index_heap_oid == heap_oid; if (!belongs && get_rel_relispartition(index_heap_oid)) { ancestors = get_partition_ancestors(index_heap_oid); belongs = list_member_oid(ancestors, heap_oid); list_free(ancestors); } if (!belongs) { relation_close(index_rel, AccessShareLock); continue; } if (index_rel->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) index_tree = find_all_inheritors(indexoid, AccessShareLock, NULL); else if (index_rel->rd_rel->relkind == RELKIND_INDEX) index_tree = list_make1_oid(indexoid); else { char *index_name = pstrdup(RelationGetRelationName(index_rel)); relation_close(index_rel, AccessShareLock); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not an index", index_name))); } relation_close(index_rel, NoLock); result = list_concat_unique_oid(result, index_tree); } return result; } static void tp_collect_created_background_indexes( List *created_indexes, Oid heap_oid, const char *schedule, const char *lineage, bool lineage_supplied) { List *index_tree; List *bm25_indexes; ListCell *lc; if (lineage == NULL) elog(ERROR, "background index has no compaction lineage"); index_tree = tp_created_index_tree_locked(created_indexes, heap_oid); bm25_indexes = tp_bm25_index_nodes(index_tree); list_free(index_tree); foreach (lc, bm25_indexes) { Oid indexoid = lfirst_oid(lc); bool physical = get_rel_relkind(indexoid) == RELKIND_INDEX; tp_collect_managed_intent( indexoid, NULL, schedule, lineage, TP_MANAGED_INTENT_REFRESH_DEFAULT | TP_MANAGED_INTENT_RECONCILE_OPTIONS | (lineage_supplied && physical ? TP_MANAGED_INTENT_LINEAGE_SUPPLIED : 0)); } list_free(bm25_indexes); } static void tp_reconcile_partition_index_options(Oid relation_oid) { List *parent_indexes = tp_relation_indexes_locked(relation_oid, NoLock); ListCell *lc; foreach (lc, parent_indexes) { Oid parent_index_oid = lfirst_oid(lc); Relation parent_index; List *index_tree; char *lineage; char *schedule = NULL; parent_index = try_relation_open(parent_index_oid, AccessShareLock); if (parent_index == NULL) continue; if (parent_index->rd_rel->relkind != RELKIND_PARTITIONED_INDEX || tp_index_compaction_mode(parent_index) != TP_COMPACTION_BACKGROUND) { relation_close(parent_index, AccessShareLock); continue; } if (tp_index_compaction_lineage(parent_index) == NULL) { relation_close(parent_index, AccessShareLock); continue; } lineage = pstrdup(tp_index_compaction_lineage(parent_index)); if (tp_index_compaction_schedule(parent_index) != NULL) schedule = pstrdup(tp_index_compaction_schedule(parent_index)); index_tree = find_all_inheritors(parent_index_oid, AccessShareLock, NULL); relation_close(parent_index, AccessShareLock); tp_collect_created_background_indexes( index_tree, relation_oid, schedule, lineage, false); list_free(index_tree); if (schedule != NULL) pfree(schedule); pfree(lineage); } list_free(parent_indexes); } static void tp_reconcile_attached_index_options(Oid parent_index_oid) { Relation parent_index; List *index_tree; Oid heap_oid; char *lineage; char *schedule = NULL; parent_index = relation_open(parent_index_oid, AccessShareLock); if (parent_index->rd_rel->relkind != RELKIND_PARTITIONED_INDEX || parent_index->rd_index == NULL || tp_index_compaction_mode(parent_index) != TP_COMPACTION_BACKGROUND || tp_index_compaction_lineage(parent_index) == NULL) { relation_close(parent_index, AccessShareLock); return; } heap_oid = parent_index->rd_index->indrelid; lineage = pstrdup(tp_index_compaction_lineage(parent_index)); if (tp_index_compaction_schedule(parent_index) != NULL) schedule = pstrdup(tp_index_compaction_schedule(parent_index)); index_tree = find_all_inheritors(parent_index_oid, AccessShareLock, NULL); relation_close(parent_index, AccessShareLock); tp_collect_created_background_indexes( index_tree, heap_oid, schedule, lineage, false); list_free(index_tree); if (schedule != NULL) pfree(schedule); pfree(lineage); } static void tp_reconcile_detached_partition_lineages(Oid relation_oid) { List *root_indexes; ListCell *root_lc; root_indexes = tp_relation_indexes_locked(relation_oid, AccessShareLock); foreach (root_lc, root_indexes) { Oid root_index_oid = lfirst_oid(root_lc); Relation root_index; List *index_tree; List *bm25_indexes; ListCell *index_lc; char *new_lineage; root_index = try_relation_open(root_index_oid, AccessShareLock); if (root_index == NULL) continue; if (!tp_is_bm25_index_node_relation(root_index) || tp_index_compaction_lineage(root_index) == NULL) { relation_close(root_index, AccessShareLock); continue; } if (root_index->rd_rel->relkind == RELKIND_PARTITIONED_INDEX) index_tree = find_all_inheritors(root_index_oid, AccessShareLock, NULL); else index_tree = list_make1_oid(root_index_oid); relation_close(root_index, AccessShareLock); new_lineage = tp_new_compaction_lineage(); bm25_indexes = tp_bm25_index_nodes(index_tree); list_free(index_tree); foreach (index_lc, bm25_indexes) { Oid indexoid = lfirst_oid(index_lc); Relation index_rel; TpCompactionJobIdentity source; bool physical; bool background; index_rel = relation_open(indexoid, AccessShareLock); physical = index_rel->rd_rel->relkind == RELKIND_INDEX; background = tp_index_compaction_mode(index_rel) == TP_COMPACTION_BACKGROUND; relation_close(index_rel, AccessShareLock); if (physical && background) { tp_compaction_job_capture(indexoid, &source); tp_collect_managed_intent( indexoid, &source, NULL, new_lineage, TP_MANAGED_INTENT_RECONCILE_LINEAGE | TP_MANAGED_INTENT_LINEAGE_SUPPLIED | TP_MANAGED_INTENT_PRESERVE_SCHEDULE); tp_reset_compaction_identity(&source); } else tp_collect_managed_intent( indexoid, NULL, NULL, new_lineage, TP_MANAGED_INTENT_RECONCILE_LINEAGE | TP_MANAGED_INTENT_LINEAGE_SUPPLIED); } list_free(bm25_indexes); pfree(new_lineage); } list_free(root_indexes); } static TpReindexState * tp_reindex_tracking_begin( List *indexoids, ReindexObjectType scope_kind, Oid scope_oid, bool defer_reconciliation, bool post_publication, bool nowait) { MemoryContext caller_context = CurrentMemoryContext; MemoryContext context; TpReindexState *state = NULL; context = AllocSetContextCreate( TopMemoryContext, "pg_textsearch reindex", ALLOCSET_SMALL_SIZES); PG_TRY(); { state = MemoryContextAllocZero(context, sizeof(*state)); state->context = context; state->previous = tp_reindex_states; state->scope_kind = scope_kind; state->scope_oid = scope_oid; state->defer_reconciliation = defer_reconciliation; state->post_publication = post_publication; state->nowait = nowait; tp_reindex_add_targets(state, indexoids); state->scope_refresh_once = OidIsValid(state->scope_oid); } PG_CATCH(); { MemoryContextSwitchTo(caller_context); MemoryContextDelete(context); PG_RE_THROW(); } PG_END_TRY(); if (state->targets == NIL && !OidIsValid(state->scope_oid)) { MemoryContextDelete(context); return NULL; } tp_reindex_states = state; return state; } static bool tp_reindex_target_equals( const TpReindexTarget *left, const TpReindexTarget *right) { if (left->identity.index_oid == right->identity.index_oid) return true; return left->identity.heap_oid == right->identity.heap_oid && left->identity.lineage != NULL && right->identity.lineage != NULL && strcmp(left->identity.lineage, right->identity.lineage) == 0; } static bool tp_reindex_target_capture( TpReindexState *state, Oid indexoid, TpReindexTarget *target) { TpCompactionJobIdentity *identity = &target->identity; Relation index_rel; const char *lineage; const char *schedule; LOCKMODE lockmode = state->nowait ? NoLock : AccessShareLock; if (state->nowait && !ConditionalLockRelationOid(indexoid, AccessShareLock)) tp_report_nowait_relation_lock_error(indexoid); index_rel = try_relation_open(indexoid, lockmode); if (index_rel == NULL) { if (state->nowait) UnlockRelationOid(indexoid, AccessShareLock); return false; } if (!tp_is_physical_bm25_index_relation(index_rel)) { relation_close(index_rel, lockmode); if (state->nowait) UnlockRelationOid(indexoid, AccessShareLock); return false; } memset(target, 0, sizeof(*target)); identity->heap_oid = index_rel->rd_index->indrelid; identity->namespace_oid = RelationGetNamespace(index_rel); identity->index_name = pstrdup(RelationGetRelationName(index_rel)); identity->index_oid = indexoid; identity->tablespace_oid = index_rel->rd_locator.spcOid; identity->relfilenumber = index_rel->rd_locator.relNumber; identity->owner_oid = index_rel->rd_rel->relowner; target->current_index_oid = identity->index_oid; target->current_namespace_oid = identity->namespace_oid; target->current_index_name = pstrdup(identity->index_name); if (tp_index_compaction_mode(index_rel) == TP_COMPACTION_BACKGROUND) { lineage = tp_index_compaction_lineage(index_rel); if (lineage == NULL) { identity->lineage = tp_new_compaction_lineage(); identity->lineage_backfilled = true; } else identity->lineage = pstrdup(lineage); schedule = tp_index_compaction_schedule(index_rel); if (schedule == NULL) schedule = tp_background_compaction_schedule; identity->schedule = pstrdup(schedule); } relation_close(index_rel, lockmode); if (state->nowait) UnlockRelationOid(indexoid, AccessShareLock); return true; } static void tp_reindex_add_targets(TpReindexState *state, List *indexoids) { ListCell *lc; foreach (lc, indexoids) { Oid indexoid = lfirst_oid(lc); TpReindexTarget *target; MemoryContext old_context; ListCell *target_cell; bool duplicate = false; foreach (target_cell, state->targets) { TpReindexTarget *existing = lfirst(target_cell); if (existing->current_index_oid == indexoid) { duplicate = true; break; } } if (duplicate) continue; old_context = MemoryContextSwitchTo(state->context); target = palloc0(sizeof(*target)); if (!tp_reindex_target_capture(state, indexoid, target)) { pfree(target); MemoryContextSwitchTo(old_context); continue; } foreach (target_cell, state->targets) { if (tp_reindex_target_equals(lfirst(target_cell), target)) { duplicate = true; break; } } if (duplicate) { pfree(target->current_index_name); tp_reset_compaction_identity(&target->identity); pfree(target); } else { state->targets = lappend(state->targets, target); if (target->identity.lineage != NULL) state->intermediate_targets = lappend(state->intermediate_targets, target); } MemoryContextSwitchTo(old_context); } } static void tp_reindex_refresh_scope_targets(TpReindexState *state) { List *indexoids; List *candidates; if (!OidIsValid(state->scope_oid)) return; if (state->scope_kind == REINDEX_OBJECT_SCHEMA) indexoids = tp_namespace_bm25_indexes(state->scope_oid); else { Assert(state->scope_kind == REINDEX_OBJECT_DATABASE); indexoids = tp_all_bm25_indexes(InvalidOid, false, NIL, false); } candidates = tp_physical_bm25_indexes(indexoids, false); tp_reindex_add_targets(state, candidates); if (state->scope_refresh_once) { state->scope_oid = InvalidOid; state->scope_refresh_once = false; } list_free(candidates); list_free(indexoids); } static Relation tp_reindex_try_relation_open(Oid relation_oid, bool *contended) { Relation relation; if (!ConditionalLockRelationOid(relation_oid, AccessShareLock)) { *contended = true; return NULL; } relation = try_relation_open(relation_oid, NoLock); if (relation == NULL) UnlockRelationOid(relation_oid, AccessShareLock); return relation; } static void tp_reindex_relation_close(Relation relation) { Oid relation_oid = RelationGetRelid(relation); relation_close(relation, NoLock); UnlockRelationOid(relation_oid, AccessShareLock); } static void tp_reindex_target_refresh_identity( TpReindexState *state, TpReindexTarget *target, Relation index_rel) { MemoryContext old_context; old_context = MemoryContextSwitchTo(state->context); pfree(target->current_index_name); target->current_index_oid = RelationGetRelid(index_rel); target->current_namespace_oid = RelationGetNamespace(index_rel); target->current_index_name = pstrdup(RelationGetRelationName(index_rel)); MemoryContextSwitchTo(old_context); } static bool tp_reindex_target_matches(const TpReindexTarget *target, Relation index_rel) { const TpCompactionJobIdentity *identity = &target->identity; const char *lineage = tp_index_compaction_lineage(index_rel); return index_rel->rd_index != NULL && index_rel->rd_index->indrelid == identity->heap_oid && (identity->lineage == NULL || identity->lineage_backfilled || (lineage != NULL && strcmp(lineage, identity->lineage) == 0)); } static bool tp_reindex_target_live_original( TpReindexState *state, TpReindexTarget *target, Oid *indexoid, Oid *tablespace_oid, RelFileNumber *relfilenumber, bool *contended) { Relation index_rel; *indexoid = InvalidOid; index_rel = tp_reindex_try_relation_open(target->current_index_oid, contended); if (index_rel == NULL) return false; if (index_rel->rd_rel->relkind != RELKIND_INDEX || !tp_reindex_target_matches(target, index_rel)) { tp_reindex_relation_close(index_rel); return false; } /* * A concurrent replacement invalidates the old OID before publishing the * replacement under its original name. Only that state permits fallback * to a name lookup; a still-live index remains the authoritative target. */ if (!index_rel->rd_index->indisvalid || !index_rel->rd_index->indisready || !index_rel->rd_index->indislive) { tp_reindex_relation_close(index_rel); return false; } if (tp_is_physical_bm25_index_relation(index_rel)) { tp_reindex_target_refresh_identity(state, target, index_rel); *indexoid = target->current_index_oid; *tablespace_oid = index_rel->rd_locator.spcOid; *relfilenumber = index_rel->rd_locator.relNumber; } tp_reindex_relation_close(index_rel); return true; } static bool tp_reindex_collect_candidates( TpReindexState *state, List **candidates, List **indexoids, bool final_pass) { List *scan_targets = final_pass ? state->targets : state->intermediate_targets; ListCell *lc; bool contended = false; foreach (lc, scan_targets) { TpReindexTarget *target = lfirst(lc); TpReindexCandidate *candidate; Relation index_rel; Oid indexoid; Oid tablespace_oid = InvalidOid; RelFileNumber relfilenumber = InvalidRelFileNumber; if (target->pending || (!final_pass && target->completed)) continue; if (!tp_reindex_target_live_original( state, target, &indexoid, &tablespace_oid, &relfilenumber, &contended)) { indexoid = get_relname_relid( target->current_index_name, target->current_namespace_oid); if (!OidIsValid(indexoid)) continue; index_rel = tp_reindex_try_relation_open(indexoid, &contended); if (index_rel == NULL) continue; if (!tp_is_physical_bm25_index_relation(index_rel) || !tp_reindex_target_matches(target, index_rel)) { tp_reindex_relation_close(index_rel); continue; } tp_reindex_target_refresh_identity(state, target, index_rel); tablespace_oid = index_rel->rd_locator.spcOid; relfilenumber = index_rel->rd_locator.relNumber; tp_reindex_relation_close(index_rel); } if (!OidIsValid(indexoid)) continue; if (indexoid == target->identity.index_oid && tablespace_oid == target->identity.tablespace_oid && relfilenumber == target->identity.relfilenumber) continue; candidate = palloc(sizeof(*candidate)); candidate->state = state; candidate->target = target; candidate->index_oid = indexoid; *candidates = lappend(*candidates, candidate); *indexoids = lappend_oid(*indexoids, indexoid); target->pending = true; target->pending_index_oid = indexoid; } return contended; } static void tp_collect_reindex_state_intents( TpReindexState *only_state, bool include_deferred, bool final_pass) { List *candidates = NIL; List *indexoids = NIL; ListCell *lc; TpReindexState *state; bool deferred = false; for (state = tp_reindex_states; state != NULL; state = state->previous) { if (only_state != NULL && state != only_state) continue; if (state->reconciling || (state->defer_reconciliation && !include_deferred)) continue; tp_reindex_refresh_scope_targets(state); if (state->post_publication) tp_post_publication_reconciliation = true; if (tp_reindex_collect_candidates( state, &candidates, &indexoids, final_pass)) { if (!state->post_publication) ereport(ERROR, (errcode(ERRCODE_LOCK_NOT_AVAILABLE), errmsg("could not inspect a managed index after " "physical rewrite"), errhint("Retry the command after the conflicting " "transaction completes."))); deferred = true; } } foreach (lc, candidates) { TpReindexCandidate *candidate = lfirst(lc); int flags = TP_MANAGED_INTENT_POST_PUBLICATION; if (candidate->target->identity.lineage != NULL) flags |= TP_MANAGED_INTENT_PRESERVE_SCHEDULE; tp_collect_prevalidated_managed_intent( candidate->index_oid, &candidate->target->identity, NULL, NULL, flags); } list_free(indexoids); list_free_deep(candidates); if (deferred) ereport(WARNING, (errmsg("background compaction lifecycle reconciliation was " "deferred"), errdetail( "A published managed index could not be inspected " "without waiting."), errhint("Repeat the managed DDL after the conflicting " "transaction completes."))); } static void tp_finish_reindex_pending(bool reconciled) { TpReindexState *state; for (state = tp_reindex_states; state != NULL; state = state->previous) { ListCell *lc; foreach (lc, state->targets) { TpReindexTarget *target = lfirst(lc); if (!target->pending) continue; if (reconciled || list_member_oid( tp_reconciled_indexoids, target->pending_index_oid)) target->completed = true; target->pending = false; target->pending_index_oid = InvalidOid; } } list_free(tp_reconciled_indexoids); tp_reconciled_indexoids = NIL; } static void tp_reindex_tracking_end(TpReindexState *state) { MemoryContext context; if (state == NULL) return; Assert(tp_reindex_states == state); context = state->context; tp_reindex_states = state->previous; MemoryContextDelete(context); } static List * tp_reindex_current_indexes(ReindexStmt *stmt) { Oid relation_oid; relation_oid = RangeVarGetRelid(stmt->relation, AccessShareLock, false); if (stmt->kind == REINDEX_OBJECT_INDEX) return list_make1_oid(relation_oid); return tp_relation_indexes_locked(relation_oid, NoLock); } static bool tp_vacuum_option_enabled(VacuumStmt *stmt, const char *name) { ListCell *lc; foreach (lc, stmt->options) { DefElem *option = lfirst_node(DefElem, lc); if (strcmp(option->defname, name) == 0) return defGetBoolean(option); } return false; } static bool tp_vacuum_rewrites_storage(VacuumStmt *stmt) { return stmt->is_vacuumcmd && tp_vacuum_option_enabled(stmt, "full"); } static bool tp_vacuum_can_maintain_relation(Oid relation_oid) { return object_ownercheck(DatabaseRelationId, MyDatabaseId, GetUserId()) || tp_can_maintain_relation(relation_oid); } static List * tp_vacuum_skip_locked_relation_indexes(Oid relation_oid, bool include_children) { List *relation_oids; List *indexoids = NIL; ListCell *lc; if (include_children) relation_oids = find_all_inheritors(relation_oid, NoLock, NULL); else relation_oids = list_make1_oid(relation_oid); foreach (lc, relation_oids) { Oid child_oid = lfirst_oid(lc); Relation child; List *child_indexes; if (child_oid != relation_oid && !ConditionalLockRelationOid(child_oid, AccessShareLock)) continue; if (!tp_vacuum_can_maintain_relation(child_oid)) continue; child = try_relation_open(child_oid, NoLock); if (child == NULL) continue; child_indexes = list_copy(RelationGetIndexList(child)); relation_close(child, NoLock); indexoids = list_concat_unique_oid(indexoids, child_indexes); } list_free(relation_oids); return indexoids; } static List * tp_vacuum_relation_indexes_locked(Oid relation_oid, bool include_children) { List *relation_oids; List *indexoids = NIL; ListCell *lc; if (include_children) relation_oids = find_all_inheritors(relation_oid, AccessExclusiveLock, NULL); else relation_oids = list_make1_oid(relation_oid); foreach (lc, relation_oids) { Oid child_oid = lfirst_oid(lc); Relation child; List *child_indexes; if (!tp_vacuum_can_maintain_relation(child_oid)) continue; child = try_relation_open(child_oid, NoLock); if (child == NULL) continue; child_indexes = list_copy(RelationGetIndexList(child)); relation_close(child, NoLock); indexoids = list_concat_unique_oid(indexoids, child_indexes); } list_free(relation_oids); return indexoids; } static List * tp_vacuum_skip_locked_all_indexes(void) { List *all_indexes; List *indexoids = NIL; List *locked_heaps = NIL; List *skipped_heaps = NIL; ListCell *lc; all_indexes = tp_all_bm25_indexes(InvalidOid, false, NIL, false); foreach (lc, all_indexes) { Oid index_oid = lfirst_oid(lc); Oid heap_oid = IndexGetRelation(index_oid, true); if (!OidIsValid(heap_oid) || list_member_oid(skipped_heaps, heap_oid)) continue; if (!list_member_oid(locked_heaps, heap_oid)) { if (!ConditionalLockRelationOid(heap_oid, AccessShareLock)) { skipped_heaps = lappend_oid(skipped_heaps, heap_oid); continue; } locked_heaps = lappend_oid(locked_heaps, heap_oid); } if (tp_vacuum_can_maintain_relation(heap_oid)) indexoids = lappend_oid(indexoids, index_oid); } list_free(skipped_heaps); list_free(locked_heaps); list_free(all_indexes); return indexoids; } static List * tp_vacuum_rewrite_indexes(VacuumStmt *stmt) { List *indexoids = NIL; ListCell *lc; bool skip_locked = tp_vacuum_option_enabled(stmt, "skip_locked"); if (stmt->rels == NIL) { if (skip_locked) return tp_vacuum_skip_locked_all_indexes(); return tp_maintainable_indexes( tp_all_bm25_indexes(InvalidOid, false, NIL, false), true); } foreach (lc, stmt->rels) { VacuumRelation *vacuum_rel = lfirst_node(VacuumRelation, lc); Oid relation_oid = vacuum_rel->oid; List *relation_indexes; if (!OidIsValid(relation_oid) && vacuum_rel->relation != NULL) relation_oid = RangeVarGetRelidExtended( vacuum_rel->relation, AccessShareLock, RVR_MISSING_OK | (skip_locked ? RVR_SKIP_LOCKED : 0), NULL, NULL); else if ( skip_locked && OidIsValid(relation_oid) && !ConditionalLockRelationOid(relation_oid, AccessShareLock)) continue; if (!OidIsValid(relation_oid)) continue; if (skip_locked) relation_indexes = tp_vacuum_skip_locked_relation_indexes( relation_oid, vacuum_rel->relation != NULL && vacuum_rel->relation->inh); else relation_indexes = tp_vacuum_relation_indexes_locked( relation_oid, vacuum_rel->relation != NULL && vacuum_rel->relation->inh); indexoids = list_concat_unique_oid(indexoids, relation_indexes); } return indexoids; } static List * tp_global_cluster_bm25_indexes(void) { Relation index_rel; SysScanDesc scan; HeapTuple tuple; List *bm25_indexes; List *clustered_heaps = NIL; List *indexoids = NIL; ListCell *lc; index_rel = table_open(IndexRelationId, AccessShareLock); scan = systable_beginscan(index_rel, InvalidOid, false, NULL, 0, NULL); while ((tuple = systable_getnext(scan)) != NULL) { Form_pg_index index_form = (Form_pg_index)GETSTRUCT(tuple); if (!index_form->indisclustered || !tp_can_maintain_relation(index_form->indrelid)) continue; clustered_heaps = list_append_unique_oid(clustered_heaps, index_form->indrelid); } systable_endscan(scan); table_close(index_rel, AccessShareLock); bm25_indexes = tp_all_bm25_indexes(InvalidOid, false, NIL, false); foreach (lc, bm25_indexes) { Oid indexoid = lfirst_oid(lc); Oid heap_oid = IndexGetRelation(indexoid, true); if (OidIsValid(heap_oid) && list_member_oid(clustered_heaps, heap_oid)) indexoids = lappend_oid(indexoids, indexoid); } list_free(bm25_indexes); list_free(clustered_heaps); return indexoids; } static List * tp_cluster_rewrite_indexes( #if PG_VERSION_NUM >= 190000 RepackStmt *stmt, #else ClusterStmt *stmt, #endif bool *multi_transaction) { RangeVar *relation; Oid relation_oid; #if PG_VERSION_NUM >= 190000 relation = stmt->relation == NULL ? NULL : stmt->relation->relation; #else relation = stmt->relation; #endif if (relation == NULL) { *multi_transaction = true; return tp_global_cluster_bm25_indexes(); } relation_oid = RangeVarGetRelidExtended( relation, AccessExclusiveLock, 0, RangeVarCallbackMaintainsTable, NULL); *multi_transaction = get_rel_relkind(relation_oid) == RELKIND_PARTITIONED_TABLE; return tp_relation_tree_indexes_locked(relation_oid, NoLock); } static bool tp_alter_index_sets_tablespace(AlterTableStmt *stmt) { ListCell *lc; if (stmt->objtype != OBJECT_INDEX) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); if (cmd->subtype == AT_SetTableSpace) return true; } return false; } static bool tp_alter_table_may_rewrite(AlterTableStmt *stmt) { ListCell *lc; if (stmt->objtype != OBJECT_TABLE && stmt->objtype != OBJECT_MATVIEW) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); switch (cmd->subtype) { case AT_AddColumn: case AT_AlterColumnType: case AT_SetExpression: case AT_SetLogged: case AT_SetUnLogged: case AT_SetAccessMethod: return true; default: break; } } return false; } static bool tp_alter_table_rewrite_recurses(AlterTableStmt *stmt) { ListCell *lc; if (!stmt->relation->inh) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); switch (cmd->subtype) { case AT_AddColumn: case AT_AlterColumnType: case AT_SetExpression: return true; default: break; } } return false; } static List * tp_alter_table_rewrite_indexes( AlterTableStmt *stmt, Oid relation_oid, LOCKMODE lockmode) { List *relation_oids; List *indexoids = NIL; ListCell *lc; if (!tp_alter_table_rewrite_recurses(stmt)) return tp_relation_indexes_locked(relation_oid, NoLock); relation_oids = find_all_inheritors(relation_oid, lockmode, NULL); foreach (lc, relation_oids) { Oid child_oid = lfirst_oid(lc); List *child_indexes; child_indexes = tp_relation_indexes_locked(child_oid, NoLock); indexoids = list_concat_unique_oid(indexoids, child_indexes); } list_free(relation_oids); return indexoids; } static bool tp_alter_table_attaches_partition(AlterTableStmt *stmt) { ListCell *lc; if (stmt->objtype != OBJECT_TABLE && stmt->objtype != OBJECT_INDEX) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); if (cmd->subtype == AT_AttachPartition) return true; } return false; } static bool tp_alter_table_detaches_partition(AlterTableStmt *stmt) { ListCell *lc; if (stmt->objtype != OBJECT_TABLE) return false; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); if (cmd->subtype == AT_DetachPartition || cmd->subtype == AT_DetachPartitionFinalize) return true; } return false; } static List * tp_detached_partition_oids(AlterTableStmt *stmt) { List *partition_oids = NIL; ListCell *lc; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); PartitionCmd *partition_cmd; Oid partition_oid; if (cmd->subtype != AT_DetachPartition && cmd->subtype != AT_DetachPartitionFinalize) continue; partition_cmd = castNode(PartitionCmd, cmd->def); partition_oid = RangeVarGetRelidExtended( partition_cmd->name, NoLock, RVR_MISSING_OK, NULL, NULL); if (OidIsValid(partition_oid)) partition_oids = list_append_unique_oid(partition_oids, partition_oid); } return partition_oids; } static void tp_reconcile_completed_partition_detaches(List *partition_oids) { ListCell *lc; foreach (lc, partition_oids) { Oid partition_oid = lfirst_oid(lc); if (!SearchSysCacheExists1(RELOID, ObjectIdGetDatum(partition_oid)) || get_rel_relispartition(partition_oid)) continue; tp_reconcile_detached_partition_lineages(partition_oid); } } static void tp_truncate_check_relation(Oid relation_oid, const char *relation_name) { HeapTuple tuple; Form_pg_class relation_form; AclResult aclresult; tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relation_oid)); if (!HeapTupleIsValid(tuple)) elog(ERROR, "cache lookup failed for relation %u", relation_oid); relation_form = (Form_pg_class)GETSTRUCT(tuple); if (relation_form->relkind != RELKIND_RELATION && relation_form->relkind != RELKIND_PARTITIONED_TABLE && relation_form->relkind != RELKIND_FOREIGN_TABLE) { ReleaseSysCache(tuple); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("\"%s\" is not a table", relation_name))); } aclresult = pg_class_aclcheck(relation_oid, GetUserId(), ACL_TRUNCATE); if (aclresult != ACLCHECK_OK) { ObjectType object_type = get_relkind_objtype(relation_form->relkind); ReleaseSysCache(tuple); aclcheck_error(aclresult, object_type, relation_name); } ReleaseSysCache(tuple); } static void tp_truncate_lookup_callback( const RangeVar *relation, Oid relid, Oid old_relid pg_attribute_unused(), void *arg pg_attribute_unused()) { if (OidIsValid(relid)) tp_truncate_check_relation(relid, relation->relname); } static List * tp_truncate_rewrite_indexes(TruncateStmt *stmt) { List *indexoids = NIL; List *relation_oids = NIL; ListCell *lc; foreach (lc, stmt->relations) { RangeVar *relation = lfirst_node(RangeVar, lc); Oid relation_oid; relation_oid = RangeVarGetRelidExtended( relation, AccessExclusiveLock, 0, tp_truncate_lookup_callback, NULL); relation_oids = list_append_unique_oid(relation_oids, relation_oid); if (relation->inh) { List *children = find_all_inheritors( relation_oid, AccessExclusiveLock, NULL); relation_oids = list_concat_unique_oid(relation_oids, children); } } if (stmt->behavior == DROP_CASCADE) { for (;;) { List *new_relation_oids = heap_truncate_find_FKs(relation_oids); if (new_relation_oids == NIL) break; foreach (lc, new_relation_oids) { Oid relation_oid = lfirst_oid(lc); char *relation_name; LockRelationOid(relation_oid, AccessExclusiveLock); relation_name = get_rel_name(relation_oid); if (relation_name == NULL) continue; tp_truncate_check_relation(relation_oid, relation_name); pfree(relation_name); relation_oids = list_append_unique_oid(relation_oids, relation_oid); } list_free(new_relation_oids); } } foreach (lc, relation_oids) { List *relation_indexes = tp_relation_indexes_locked(lfirst_oid(lc), NoLock); indexoids = list_concat_unique_oid(indexoids, relation_indexes); } list_free(relation_oids); return indexoids; } static List * tp_refresh_matview_rewrite_indexes(RefreshMatViewStmt *stmt) { LOCKMODE lockmode = stmt->concurrent ? ExclusiveLock : AccessExclusiveLock; Oid relation_oid; relation_oid = RangeVarGetRelidExtended( stmt->relation, lockmode, 0, RangeVarCallbackMaintainsTable, NULL); return tp_relation_indexes_locked(relation_oid, NoLock); } static void tp_process_tracked_rewrite( PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc, List *indexoids, bool nowait) { TpReindexState *rewrite_state = NULL; List *candidates; candidates = tp_physical_bm25_indexes_for_rewrite(indexoids, nowait); rewrite_state = tp_reindex_tracking_begin( candidates, REINDEX_OBJECT_INDEX, InvalidOid, false, false, nowait); list_free(candidates); list_free(indexoids); PG_TRY(); { if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); if (rewrite_state != NULL) tp_collect_reindex_state_intents(rewrite_state, true, true); } PG_FINALLY(); { tp_reindex_tracking_end(rewrite_state); } PG_END_TRY(); } static RoleSpec * tp_alter_new_owner(AlterTableStmt *stmt) { ListCell *lc; foreach (lc, stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); if (cmd->subtype != AT_ChangeOwner) continue; return cmd->newowner; } return NULL; } static void tp_alter_owner_preflight(Oid relation_oid, RoleSpec *new_owner) { HeapTuple relation_tuple; Form_pg_class relation_form; Oid new_owner_oid = get_rolespec_oid(new_owner, false); relation_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(relation_oid)); if (!HeapTupleIsValid(relation_tuple)) elog(ERROR, "cache lookup failed for relation %u", relation_oid); relation_form = (Form_pg_class)GETSTRUCT(relation_tuple); if (relation_form->relowner != new_owner_oid && !superuser()) { AclResult aclresult; check_can_set_role(GetUserId(), new_owner_oid); aclresult = object_aclcheck( NamespaceRelationId, relation_form->relnamespace, new_owner_oid, ACL_CREATE); if (aclresult != ACLCHECK_OK) aclcheck_error( aclresult, OBJECT_SCHEMA, get_namespace_name(relation_form->relnamespace)); } ReleaseSysCache(relation_tuple); } /* * ProcessUtility hook - detect CREATE INDEX USING bm25 and wrap * with build progress tracking. This collapses per-partition * NOTICEs into a single summary for partitioned tables. */ static void initialize_utility_context( TpProcessUtilityContext *utility_context, Node *stmt) { memset(utility_context, 0, sizeof(*utility_context)); utility_context->previous = current_utility_context; utility_context->allow_rls = tp_allow_rls; if (IsA(stmt, CreateExtensionStmt)) { CreateExtensionStmt *create_stmt = castNode(CreateExtensionStmt, stmt); utility_context->extension_lifecycle = strcmp(create_stmt->extname, "pg_textsearch") == 0; } else if (IsA(stmt, AlterExtensionStmt)) { AlterExtensionStmt *alter_stmt = castNode(AlterExtensionStmt, stmt); utility_context->extension_lifecycle = strcmp(alter_stmt->extname, "pg_textsearch") == 0; } else if (IsA(stmt, AlterExtensionContentsStmt)) { AlterExtensionContentsStmt *alter_stmt = castNode(AlterExtensionContentsStmt, stmt); utility_context->extension_lifecycle = strcmp(alter_stmt->extname, "pg_textsearch") == 0; } else if (IsA(stmt, DropStmt)) { DropStmt *drop_stmt = castNode(DropStmt, stmt); ListCell *lc; if (drop_stmt->removeType == OBJECT_EXTENSION) { foreach (lc, drop_stmt->objects) { List *name = lfirst(lc); if (list_length(name) == 1 && strcmp(strVal(linitial(name)), "pg_textsearch") == 0) { utility_context->extension_lifecycle = true; break; } } } } if (utility_context->extension_lifecycle) { utility_context->serialize_rls_ddl = true; utility_context->retain_rls_ddl_lock = true; } else if (IsA(stmt, IndexStmt)) { IndexStmt *index_stmt = castNode(IndexStmt, stmt); utility_context->track_index_build = true; utility_context->serialize_rls_ddl = index_stmt->accessMethod != NULL && strcmp(index_stmt->accessMethod, "bm25") == 0; } else if (IsA(stmt, ReindexStmt)) utility_context->serialize_rls_ddl = true; else if (IsA(stmt, CreateStmt)) { CreateStmt *create_stmt = castNode(CreateStmt, stmt); if (create_stmt->inhRelations != NIL) { utility_context->serialize_rls_ddl = true; utility_context->track_relation_create = true; } } else if (IsA(stmt, AlterTableStmt)) { AlterTableStmt *alter_stmt = castNode(AlterTableStmt, stmt); ListCell *lc; foreach (lc, alter_stmt->cmds) { AlterTableCmd *cmd = lfirst_node(AlterTableCmd, lc); if (cmd->subtype == AT_EnableRowSecurity) { utility_context->check_rls_enable = true; utility_context->serialize_rls_ddl = true; } else if ( cmd->subtype == AT_AddInherit || (cmd->subtype == AT_AttachPartition && alter_stmt->objtype == OBJECT_TABLE)) { utility_context->check_hierarchy_change = true; utility_context->serialize_rls_ddl = true; } } } } static bool relation_exists(Oid relid) { Relation class_rel; ScanKeyData key; SysScanDesc scan; bool exists; /* End triggers can delete the tuple in the current command. */ class_rel = table_open(RelationRelationId, AccessShareLock); ScanKeyInit( &key, Anum_pg_class_oid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan( class_rel, ClassOidIndexId, true, SnapshotSelf, 1, &key); exists = HeapTupleIsValid(systable_getnext(scan)); systable_endscan(scan); table_close(class_rel, AccessShareLock); return exists; } static void validate_utility_context(TpProcessUtilityContext *utility_context) { ListCell *lc; foreach (lc, utility_context->altered_relids) { Oid relid = lfirst_oid(lc); if (relation_exists(relid)) tp_check_rls_enable_allowed(relid); } foreach (lc, utility_context->hierarchy_relids) { Oid relid = lfirst_oid(lc); if (relation_exists(relid)) tp_check_bm25_hierarchy_allowed(relid); } } static void call_next_process_utility( PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc) { tp_process_utility_depth++; PG_TRY(); { tp_process_utility_impl( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); } PG_FINALLY(); { tp_process_utility_depth--; } PG_END_TRY(); } static void tp_process_utility_impl( PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc) { Node *parsetree = pstmt->utilityStmt; if (IsA(parsetree, DropdbStmt)) { TpDropDatabaseContext *drop_context; Oid database_oid; drop_context = palloc(sizeof(*drop_context)); drop_context->previous = active_drop_database_context; drop_context->database_oid = InvalidOid; active_drop_database_context = drop_context; PG_TRY(); { if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); } PG_CATCH(); { active_drop_database_context = drop_context->previous; pfree(drop_context); PG_RE_THROW(); } PG_END_TRY(); active_drop_database_context = drop_context->previous; database_oid = drop_context->database_oid; pfree(drop_context); if (OidIsValid(database_oid)) tp_cleanup_database_shared_memory(database_oid); return; } if (IsA(parsetree, VacuumStmt) && tp_vacuum_rewrites_storage(castNode(VacuumStmt, parsetree))) { PreventInTransactionBlock( context == PROCESS_UTILITY_TOPLEVEL, "VACUUM"); tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, tp_vacuum_rewrite_indexes(castNode(VacuumStmt, parsetree)), false); return; } #if PG_VERSION_NUM >= 190000 if (IsA(parsetree, RepackStmt) && castNode(RepackStmt, parsetree)->command == REPACK_COMMAND_CLUSTER) #else if (IsA(parsetree, ClusterStmt)) #endif { bool multi_transaction; #if PG_VERSION_NUM >= 190000 List *indexoids = tp_cluster_rewrite_indexes( castNode(RepackStmt, parsetree), &multi_transaction); #else List *indexoids = tp_cluster_rewrite_indexes( castNode(ClusterStmt, parsetree), &multi_transaction); #endif if (multi_transaction) PreventInTransactionBlock( context == PROCESS_UTILITY_TOPLEVEL, "CLUSTER"); tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, indexoids, false); return; } if (IsA(parsetree, TruncateStmt)) { tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, tp_truncate_rewrite_indexes(castNode(TruncateStmt, parsetree)), false); return; } if (IsA(parsetree, RefreshMatViewStmt)) { tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, tp_refresh_matview_rewrite_indexes( castNode(RefreshMatViewStmt, parsetree)), false); return; } if (IsA(parsetree, AlterTableMoveAllStmt)) { AlterTableMoveAllStmt *stmt = castNode(AlterTableMoveAllStmt, parsetree); if (stmt->objtype == OBJECT_INDEX) { Oid tablespace_oid; List *owner_oids = tp_role_oids(stmt->roles); if (tp_bulk_move_preflight(stmt, &tablespace_oid)) { tablespace_oid = tp_catalog_tablespace_oid(tablespace_oid); if (superuser() || !tp_bulk_move_has_unauthorized_index( tablespace_oid, owner_oids)) { tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, tp_all_bm25_indexes( tablespace_oid, true, owner_oids, !superuser()), stmt->nowait); list_free(owner_oids); return; } } list_free(owner_oids); } } if (IsA(parsetree, ReassignOwnedStmt)) { List *indexoids = tp_reassign_owned_indexes( castNode(ReassignOwnedStmt, parsetree)); List *candidates = tp_physical_bm25_indexes(indexoids, true); List *owner_targets; tp_prelock_owner_change_heaps(candidates); owner_targets = tp_capture_owner_change_targets(candidates); list_free(candidates); list_free(indexoids); PG_TRY(); { if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); tp_collect_owner_change_targets(owner_targets); } PG_FINALLY(); { tp_free_owner_change_targets(owner_targets); } PG_END_TRY(); return; } if (IsA(parsetree, AlterTableStmt)) { AlterTableStmt *stmt = (AlterTableStmt *)parsetree; RoleSpec *new_owner; tp_reject_user_lineage_alter(stmt); if (tp_alter_index_sets_tablespace(stmt)) { Oid indexoid = RangeVarGetRelidExtended( stmt->relation, AlterTableGetLockLevel(stmt->cmds), stmt->missing_ok ? RVR_MISSING_OK : 0, RangeVarCallbackOwnsRelation, NULL); if (OidIsValid(indexoid)) tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, tp_index_tree_locked(indexoid, AccessShareLock), false); else if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); return; } if (tp_alter_table_attaches_partition(stmt)) { TpCreateIndexState *create_state = NULL; LOCKMODE lockmode = AlterTableGetLockLevel(stmt->cmds); Oid relation_oid; relation_oid = RangeVarGetRelidExtended( stmt->relation, lockmode, stmt->missing_ok ? RVR_MISSING_OK : 0, RangeVarCallbackOwnsRelation, NULL); if (!OidIsValid(relation_oid)) { if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); return; } PG_TRY(); { if (stmt->objtype == OBJECT_TABLE) create_state = tp_create_index_tracking_begin( relation_oid); if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); if (stmt->objtype == OBJECT_TABLE) tp_reconcile_partition_index_options(relation_oid); else tp_reconcile_attached_index_options(relation_oid); } PG_FINALLY(); { tp_create_index_tracking_end(create_state); } PG_END_TRY(); return; } if (tp_alter_table_detaches_partition(stmt)) { List *partition_oids = tp_detached_partition_oids(stmt); if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); tp_reconcile_completed_partition_detaches(partition_oids); list_free(partition_oids); return; } if (tp_alter_table_may_rewrite(stmt)) { LOCKMODE lockmode = AlterTableGetLockLevel(stmt->cmds); Oid relation_oid; relation_oid = RangeVarGetRelidExtended( stmt->relation, lockmode, stmt->missing_ok ? RVR_MISSING_OK : 0, RangeVarCallbackOwnsRelation, NULL); if (OidIsValid(relation_oid)) tp_process_tracked_rewrite( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc, tp_alter_table_rewrite_indexes( stmt, relation_oid, lockmode), false); else if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); return; } new_owner = tp_alter_new_owner(stmt); if (new_owner != NULL && (stmt->objtype == OBJECT_TABLE || stmt->objtype == OBJECT_MATVIEW)) { Oid relation_oid; List *owner_targets = NIL; relation_oid = AlterTableLookupRelation( stmt, AlterTableGetLockLevel(stmt->cmds)); if (OidIsValid(relation_oid) && object_ownercheck( RelationRelationId, relation_oid, GetUserId())) { List *indexoids; List *candidates; tp_alter_owner_preflight(relation_oid, new_owner); indexoids = tp_relation_indexes_locked(relation_oid, NoLock); candidates = tp_physical_bm25_indexes(indexoids, true); owner_targets = tp_capture_owner_change_targets(candidates); list_free(candidates); list_free(indexoids); } PG_TRY(); { if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); tp_collect_owner_change_targets(owner_targets); } PG_FINALLY(); { tp_free_owner_change_targets(owner_targets); } PG_END_TRY(); return; } if (tp_alter_index_refreshes_background(stmt)) { Oid indexoid; if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); indexoid = RangeVarGetRelid( stmt->relation, AccessShareLock, stmt->missing_ok); if (!OidIsValid(indexoid)) return; if (tp_is_background_physical_index(indexoid)) tp_collect_managed_intent( indexoid, NULL, NULL, NULL, TP_MANAGED_INTENT_REFRESH_DEFAULT); else tp_collect_managed_intent( indexoid, NULL, NULL, NULL, TP_MANAGED_INTENT_DISABLE); return; } } if (IsA(parsetree, ReindexStmt)) { ReindexStmt *stmt = (ReindexStmt *)parsetree; List *indexoids; bool tracks_commits; TpReindexState *reindex_state = NULL; bool is_top_level = context == PROCESS_UTILITY_TOPLEVEL; Oid scope_oid = InvalidOid; tp_reindex_preflight(stmt, is_top_level); if (stmt->kind == REINDEX_OBJECT_SCHEMA || stmt->kind == REINDEX_OBJECT_DATABASE) PreventInTransactionBlock(is_top_level, "REINDEX"); indexoids = tp_reindex_initial_indexes( stmt, &tracks_commits, is_top_level, &scope_oid); PG_TRY(); { List *candidates = tp_physical_bm25_indexes(indexoids, false); reindex_state = tp_reindex_tracking_begin( candidates, stmt->kind, scope_oid, OidIsValid(scope_oid), tracks_commits, false); list_free(candidates); list_free(indexoids); if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); if (reindex_state != NULL) tp_collect_reindex_state_intents(reindex_state, true, true); else if ( !tracks_commits && (stmt->kind == REINDEX_OBJECT_INDEX || stmt->kind == REINDEX_OBJECT_TABLE)) { indexoids = tp_reindex_current_indexes(stmt); tp_collect_background_indexes(indexoids, true); list_free(indexoids); } } PG_FINALLY(); { tp_reindex_tracking_end(reindex_state); } PG_END_TRY(); return; } if (IsA(parsetree, CreateStmt)) { TpCreateIndexState *create_state = NULL; PG_TRY(); { create_state = tp_create_index_tracking_begin(InvalidOid); if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); tp_collect_background_indexes(create_state->created_indexes, true); } PG_FINALLY(); { tp_create_index_tracking_end(create_state); } PG_END_TRY(); return; } if (IsA(parsetree, IndexStmt)) { IndexStmt *stmt = (IndexStmt *)parsetree; if (stmt->accessMethod && strcmp(stmt->accessMethod, "bm25") == 0) { Oid heapoid; Relation heap_rel; TpCreateIndexState *create_state = NULL; const char *compaction; bool lineage_supplied; if (readOnlyTree) { pstmt = copyObject(pstmt); parsetree = pstmt->utilityStmt; stmt = castNode(IndexStmt, parsetree); readOnlyTree = false; } heapoid = RangeVarGetRelidExtended( stmt->relation, stmt->concurrent ? ShareUpdateExclusiveLock : ShareLock, 0, RangeVarCallbackOwnsRelation, NULL); heap_rel = relation_open(heapoid, NoLock); compaction = tp_index_stmt_option(stmt, "compaction"); tp_index_stmt_validate_supplied_lineage( stmt, heapoid, heap_rel->rd_rel->relowner); lineage_supplied = tp_index_stmt_option(stmt, "compaction_lineage") != NULL; if (compaction != NULL && pg_strcasecmp(compaction, "background") == 0 && RelationUsesLocalBuffers(heap_rel)) { relation_close(heap_rel, NoLock); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("background compaction is not supported for " "temporary indexes"))); } if (stmt->concurrent && lineage_supplied && compaction != NULL && pg_strcasecmp(compaction, "background") == 0) { relation_close(heap_rel, NoLock); ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("explicit background compaction lineage is " "not supported with CREATE INDEX " "CONCURRENTLY"), errhint("Create the index without CONCURRENTLY, or " "omit compaction_lineage."))); } if (compaction != NULL && pg_strcasecmp(compaction, "background") == 0) tp_index_stmt_ensure_lineage( stmt, heapoid, heap_rel->rd_rel->relowner); /* * CIC commits its catalog shell before returning to this hook. * Once PostgreSQL's ownership check is known to succeed, validate * deterministic owner admission and workflow construction before * those irreversible phases. */ if (tp_background_cic_needs_preflight(stmt, heap_rel) && object_ownercheck(RelationRelationId, heapoid, GetUserId())) { const char *schedule = tp_index_stmt_option(stmt, "compaction_schedule"); if (schedule == NULL) schedule = tp_background_compaction_schedule; tp_compaction_job_preflight( heap_rel->rd_rel->relowner, schedule); } relation_close(heap_rel, NoLock); PG_TRY(); { create_state = tp_create_index_tracking_begin(heapoid); if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); if (compaction != NULL && pg_strcasecmp(compaction, "background") == 0) tp_collect_created_background_indexes( create_state->created_indexes, create_state->heap_oid, tp_index_stmt_option(stmt, "compaction_schedule"), tp_index_stmt_option(stmt, "compaction_lineage"), lineage_supplied); else tp_collect_background_indexes( create_state->created_indexes, true); } PG_FINALLY(); { tp_create_index_tracking_end(create_state); } PG_END_TRY(); return; } } /* Not a bm25 CREATE INDEX - pass through */ if (prev_process_utility_hook) prev_process_utility_hook( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); else standard_ProcessUtility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); } /* * ProcessUtility hook - isolate each utility command's object-access events, * post-validate ALTER TABLE catalog state, and preserve partitioned build * progress tracking. */ static void tp_process_utility( PlannedStmt *pstmt, const char *queryString, bool readOnlyTree, ProcessUtilityContext context, ParamListInfo params, QueryEnvironment *queryEnv, DestReceiver *dest, QueryCompletion *qc) { TpProcessUtilityContext *utility_context; ListCell *lc; if (tp_managed_reconciling && !IsA(pstmt->utilityStmt, GrantStmt)) ereport(ERROR, (errmsg("cannot execute DDL during background compaction " "reconciliation"))); utility_context = MemoryContextAllocZero(TopMemoryContext, sizeof(*utility_context)); initialize_utility_context(utility_context, pstmt->utilityStmt); current_utility_context = utility_context; tp_build_progress_set_owner(utility_context); PG_TRY(); { if (utility_context->serialize_rls_ddl) { utility_context->rls_ddl_lock_mode = utility_context->extension_lifecycle || !utility_context->allow_rls ? ExclusiveLock : ShareLock; utility_context->rls_ddl_lock_acquired = acquire_rls_ddl_lock( utility_context->rls_ddl_lock_mode, utility_context->previous != NULL, utility_context->extension_lifecycle, &utility_context->rls_ddl_lock_object); } call_next_process_utility( pstmt, queryString, readOnlyTree, context, params, queryEnv, dest, qc); foreach (lc, utility_context->concurrent_index_drops) tp_schedule_index_drop(lfirst_oid(lc)); list_free(utility_context->concurrent_index_drops); utility_context->concurrent_index_drops = NIL; validate_utility_context(utility_context); if (utility_context->build_progress_started) { utility_context->build_progress_started = false; tp_build_progress_end(); } if (utility_context->rls_ddl_lock_acquired) { release_rls_ddl_lock( utility_context->rls_ddl_lock_object, utility_context->rls_ddl_lock_mode, utility_context->retain_rls_ddl_lock); utility_context->rls_ddl_lock_acquired = false; } current_utility_context = utility_context->previous; tp_build_progress_set_owner(utility_context->previous); list_free(utility_context->altered_relids); list_free(utility_context->hierarchy_relids); pfree(utility_context); } PG_CATCH(); { current_utility_context = utility_context->previous; list_free(utility_context->concurrent_index_drops); if (utility_context->build_progress_started) { utility_context->build_progress_started = false; tp_build_progress_abort(); } tp_build_progress_set_owner(utility_context->previous); if (utility_context->rls_ddl_lock_acquired) { release_rls_ddl_lock( utility_context->rls_ddl_lock_object, utility_context->rls_ddl_lock_mode, false); utility_context->rls_ddl_lock_acquired = false; } pfree(utility_context); PG_RE_THROW(); } PG_END_TRY(); } /* * Deprecated stub for legacy upgrade compatibility. * * pg_textsearch--1.0.0--1.1.0.sql ships with a CREATE FUNCTION * bm25_memory_usage() that binds to this C symbol. The SRF and * its underlying soft-limit infrastructure were removed in * 1.3.0 (issue #374), and the matching SQL function is * DROPped by pg_textsearch--1.2.0--1.3.0.sql, but during an * ALTER EXTENSION UPDATE chain that walks 1.0.0 -> 1.3.0, * the CREATE in 1.0.0--1.1.0 has to find this symbol before the * DROP can run. The stub returns NULL. */ PG_FUNCTION_INFO_V1(tp_memory_usage); Datum tp_memory_usage(PG_FUNCTION_ARGS) { PG_RETURN_NULL(); } timescale-pg_textsearch-b63fe58/src/planner/000077500000000000000000000000001525762047700212075ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/planner/cost.c000066400000000000000000000131751525762047700223320ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * cost.c - Cost estimation for BM25 index scans */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/boolean.h" #include "constants.h" #include "index/metapage.h" #include "planner/cost.h" static bool tp_boolean_get_constant_query(IndexPath *path, TSQuery *query) { IndexClause *index_clause; OpExpr *clause; Node *query_node; Const *query_const; index_clause = linitial_node(IndexClause, path->indexclauses); if (index_clause->rinfo == NULL || !IsA(index_clause->rinfo->clause, OpExpr)) return false; clause = castNode(OpExpr, index_clause->rinfo->clause); if (list_length(clause->args) != 2) return false; query_node = lsecond(clause->args); while (query_node != NULL && IsA(query_node, RelabelType)) query_node = (Node *)castNode(RelabelType, query_node)->arg; if (query_node == NULL || !IsA(query_node, Const)) return false; query_const = castNode(Const, query_node); if (query_const->constisnull || query_const->consttype != TSQUERYOID) return false; *query = DatumGetTSQuery(query_const->constvalue); return true; } static bool tp_boolean_query_requires_full_scan(TSQuery query) { QueryItem *items; if (query->size == 0) return false; if (!tsquery_requires_match(GETQUERY(query))) return true; items = GETQUERY(query); for (int i = 0; i < query->size; i++) { if (items[i].type == QI_VAL && (items[i].qoperand.prefix || items[i].qoperand.weight != 0)) return true; } return false; } static void tp_disable_index_path( IndexPath *path, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, double *indexCorrelation, double *indexPages) { #if PG_VERSION_NUM >= 180000 /* A fallback can add a disabled sequential scan and sort. */ path->path.disabled_nodes += 3; #else (void)path; #endif *indexStartupCost = 0.0; *indexTotalCost = get_float8_infinity(); *indexSelectivity = 1.0; *indexCorrelation = 0.0; *indexPages = 0.0; } /* * Estimate cost of BM25 index scan */ void tp_costestimate( PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, double *indexCorrelation, double *indexPages) { GenericCosts costs; TpIndexMetaPage metap; double num_tuples = TP_DEFAULT_TUPLE_ESTIMATE; bool has_orderby = path->indexorderbys != NIL; bool has_boolean = path->indexclauses != NIL; bool boolean_full_scan = false; TSQuery boolean_query = NULL; /* * Boolean filtering and ranked scans are separate execution modes. * Multiple Boolean keys and combined filtering/ranking are follow-ups. */ if ((!has_orderby && !has_boolean) || (has_orderby && has_boolean) || (has_boolean && list_length(path->indexclauses) != 1)) { tp_disable_index_path( path, indexStartupCost, indexTotalCost, indexSelectivity, indexCorrelation, indexPages); return; } if (has_boolean) { if (!tp_boolean_get_constant_query(path, &boolean_query)) boolean_full_scan = true; else if ( tp_boolean_query_exact_operand_count(boolean_query) > TP_BOOLEAN_MAX_EXACT_OPERANDS) { tp_disable_index_path( path, indexStartupCost, indexTotalCost, indexSelectivity, indexCorrelation, indexPages); return; } else boolean_full_scan = tp_boolean_query_requires_full_scan( boolean_query); } /* Try to get actual statistics from the index */ if (path->indexinfo && path->indexinfo->indexoid != InvalidOid) { Relation index_rel = index_open(path->indexinfo->indexoid, AccessShareLock); if (index_rel) { metap = tp_get_metapage(index_rel); if (has_boolean && metap->text_config_oid != getTSCurrentConfig(true)) { pfree(metap); index_close(index_rel, AccessShareLock); tp_disable_index_path( path, indexStartupCost, indexTotalCost, indexSelectivity, indexCorrelation, indexPages); return; } if (metap && metap->total_docs > 0) num_tuples = (double)metap->total_docs; if (metap) pfree(metap); index_close(index_rel, AccessShareLock); } } /* Initialize generic costs */ MemSet (&costs, 0, sizeof(costs)) ; genericcostestimate(root, path, loop_count, &costs); /* Override with BM25-specific estimates */ *indexTotalCost = boolean_full_scan ? costs.indexTotalCost + cpu_operator_cost * num_tuples : costs.indexTotalCost * TP_INDEX_SCAN_COST_FACTOR; *indexStartupCost = has_boolean ? *indexTotalCost : costs.indexStartupCost + 0.01; /* * Calculate selectivity based on LIMIT if available, otherwise default */ if (boolean_full_scan) { *indexSelectivity = 1.0; } else if (has_boolean) { *indexSelectivity = TP_DEFAULT_INDEX_SELECTIVITY; } else if ( root && root->limit_tuples > 0 && root->limit_tuples < INT_MAX && num_tuples > 0) { /* Use LIMIT as upper bound for selectivity calculation */ double limit_selectivity = Min(1.0, root->limit_tuples / num_tuples); *indexSelectivity = Max(limit_selectivity, TP_DEFAULT_INDEX_SELECTIVITY); } else { *indexSelectivity = TP_DEFAULT_INDEX_SELECTIVITY; } *indexCorrelation = 0.0; /* No correlation assumptions */ *indexPages = Max(1.0, num_tuples / 100.0); /* Rough page estimate */ } timescale-pg_textsearch-b63fe58/src/planner/cost.h000066400000000000000000000010301525762047700223220ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * cost.h - Cost estimation for BM25 index scans */ #pragma once #include #include #include /* * Estimate cost of BM25 index scan */ void tp_costestimate( PlannerInfo *root, IndexPath *path, double loop_count, Cost *indexStartupCost, Cost *indexTotalCost, Selectivity *indexSelectivity, double *indexCorrelation, double *indexPages); timescale-pg_textsearch-b63fe58/src/planner/hooks.c000066400000000000000000001524451525762047700225110ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * hooks.c - Parse analysis and planner hooks * * When queries use the <@> operator without an explicit index, this hook * identifies the column being scored and finds a suitable BM25 index. * It also optimizes BM25 score expressions in the planner. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "compat.h" #include "index/limit.h" #include "planner/hooks.h" #include "scoring/bm25.h" #include "types/query.h" /* Previous hooks in chain */ static post_parse_analyze_hook_type prev_post_parse_analyze_hook = NULL; static planner_hook_type prev_planner_hook = NULL; static set_rel_pathlist_hook_type prev_set_rel_pathlist_hook = NULL; /* * Structure to hold looked-up OIDs for a query */ typedef struct BM25OidCache { Oid bm25_am_oid; Oid tpquery_type_oid; Oid text_tpquery_operator_oid; Oid text_text_operator_oid; Oid textarray_tpquery_operator_oid; Oid textarray_text_operator_oid; } BM25OidCache; /* * Backend-local cache to avoid repeated syscache lookups. * Cache is invalidated when pg_am changes (extension install/drop). */ typedef enum { CACHE_UNKNOWN, /* Not yet checked */ CACHE_NOT_FOUND, /* Extension not installed */ CACHE_VALID /* Extension installed, OIDs cached */ } CacheState; static CacheState oid_cache_state = CACHE_UNKNOWN; static BM25OidCache cached_oids; static bool invalidation_registered = false; /* * Structure to track explicit index requirements for a relation/column. * Used during planning to force the planner to use the specified index. */ typedef struct ExplicitIndexRequirement { Oid relid; /* Table OID */ AttrNumber attnum; /* Column attr (0 for expr) */ Node *expr; /* Expression (NULL for cols) */ Oid required_index_oid; /* The index that must be used */ } ExplicitIndexRequirement; /* * Planning context - stores explicit index requirements during planning. * This is set before standard_planner and cleared after. */ typedef struct PlanningContext { List *explicit_indexes; /* List of ExplicitIndexRequirement */ Oid bm25_am_oid; /* Cached BM25 access method OID */ } PlanningContext; static PlanningContext *current_planning_context = NULL; /* * Context for query tree mutation */ typedef struct ResolveIndexContext { Query *query; BM25OidCache *oid_cache; } ResolveIndexContext; /* * Syscache invalidation callback - reset cache when pg_am or pg_type changes. */ static void bm25_cache_invalidation_callback( Datum arg pg_attribute_unused(), int cacheid pg_attribute_unused(), uint32 hashvalue pg_attribute_unused()) { oid_cache_state = CACHE_UNKNOWN; } /* * Look up OIDs for BM25 extension objects (uncached). * Returns false if extension is not installed. */ static bool lookup_bm25_oids_internal(BM25OidCache *cache) { HeapTuple tuple; List *opname; memset(cache, 0, sizeof(BM25OidCache)); /* Look up bm25 access method */ tuple = SearchSysCache1(AMNAME, CStringGetDatum("bm25")); if (HeapTupleIsValid(tuple)) { cache->bm25_am_oid = ((Form_pg_am)GETSTRUCT(tuple))->oid; ReleaseSysCache(tuple); } if (!OidIsValid(cache->bm25_am_oid)) return false; /* Look up bm25query type using the search path */ cache->tpquery_type_oid = TypenameGetTypid("bm25query"); if (!OidIsValid(cache->tpquery_type_oid)) return false; /* Look up the <@> operator for (text, bm25query) */ opname = list_make1(makeString("<@>")); cache->text_tpquery_operator_oid = OpernameGetOprid(opname, TEXTOID, cache->tpquery_type_oid); list_free(opname); /* Look up the <@> operator for (text, text) */ opname = list_make1(makeString("<@>")); cache->text_text_operator_oid = OpernameGetOprid(opname, TEXTOID, TEXTOID); list_free(opname); /* Look up the <@> operator for (text[], bm25query) */ opname = list_make1(makeString("<@>")); cache->textarray_tpquery_operator_oid = OpernameGetOprid(opname, TEXTARRAYOID, cache->tpquery_type_oid); list_free(opname); /* Look up the <@> operator for (text[], text) */ opname = list_make1(makeString("<@>")); cache->textarray_text_operator_oid = OpernameGetOprid(opname, TEXTARRAYOID, TEXTOID); list_free(opname); return true; } /* * Get BM25 OIDs with caching. * Returns false if extension is not installed. */ static bool get_bm25_oids(BM25OidCache *cache) { /* Register invalidation callbacks on first call */ if (!invalidation_registered) { CacheRegisterSyscacheCallback( AMOID, bm25_cache_invalidation_callback, (Datum)0); CacheRegisterSyscacheCallback( TYPEOID, bm25_cache_invalidation_callback, (Datum)0); invalidation_registered = true; } /* Fast path: use cached result */ switch (oid_cache_state) { case CACHE_NOT_FOUND: return false; case CACHE_VALID: *cache = cached_oids; return true; case CACHE_UNKNOWN: break; } /* Do the actual lookup */ if (!lookup_bm25_oids_internal(cache)) { oid_cache_state = CACHE_NOT_FOUND; return false; } cached_oids = *cache; oid_cache_state = CACHE_VALID; return true; } /* * Check if an index is on a specific column of a table. * Returns true if the index is on the given table and column. */ static bool index_is_on_column(Oid index_oid, Oid relid, AttrNumber attnum) { HeapTuple idx_tuple; Form_pg_index idx_form; bool result = false; idx_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (!HeapTupleIsValid(idx_tuple)) return false; idx_form = (Form_pg_index)GETSTRUCT(idx_tuple); /* Check if index is on the correct table */ if (idx_form->indrelid == relid) { /* Check if index is on the correct column */ int nkeys = idx_form->indnatts; for (int i = 0; i < nkeys; i++) { if (idx_form->indkey.values[i] == attnum) { result = true; break; } } } ReleaseSysCache(idx_tuple); return result; } /* * Find BM25 index for a column. */ static Oid find_bm25_index_for_column(Oid relid, AttrNumber attnum, Oid bm25_am_oid) { Relation indexRelation; SysScanDesc scan; HeapTuple indexTuple; Oid result = InvalidOid; int index_count = 0; int partial_skipped = 0; ScanKeyData scanKey; if (!OidIsValid(bm25_am_oid)) return InvalidOid; indexRelation = table_open(IndexRelationId, AccessShareLock); ScanKeyInit( &scanKey, Anum_pg_index_indrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan( indexRelation, IndexIndrelidIndexId, true, NULL, 1, &scanKey); while ((indexTuple = systable_getnext(scan)) != NULL) { Form_pg_index indexForm = (Form_pg_index)GETSTRUCT(indexTuple); Oid indexOid = indexForm->indexrelid; HeapTuple classTuple; Form_pg_class classForm; classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(indexOid)); if (!HeapTupleIsValid(classTuple)) continue; classForm = (Form_pg_class)GETSTRUCT(classTuple); if (classForm->relam == bm25_am_oid) { bool predNull; Datum predDatum; int nkeys = indexForm->indnatts; /* * Skip partial indexes for implicit * resolution. Users must name them * explicitly via to_bm25query. */ predDatum = SysCacheGetAttr( INDEXRELID, indexTuple, Anum_pg_index_indpred, &predNull); (void)predDatum; if (!predNull) { for (int i = 0; i < nkeys; i++) { if (indexForm->indkey.values[i] == attnum) { partial_skipped++; break; } } ReleaseSysCache(classTuple); continue; } for (int i = 0; i < nkeys; i++) { if (indexForm->indkey.values[i] == attnum) { index_count++; if (result == InvalidOid) result = indexOid; break; } } } ReleaseSysCache(classTuple); } systable_endscan(scan); table_close(indexRelation, AccessShareLock); if (index_count > 1) ereport(WARNING, (errmsg("multiple BM25 indexes exist on the same column"), errhint("Use explicit to_bm25query('query', 'index_name') " "to specify which index to use."))); if (index_count == 0 && partial_skipped > 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("no non-partial BM25 index found " "for this column " "(%d partial index(es) skipped)", partial_skipped), errhint("Use to_bm25query('query', " "'index_name') to specify which " "partial index to use."))); return result; } /* * Extract relation OID and attribute number from a Var node. */ static bool get_var_relation_and_attnum( Var *var, Query *query, Oid *relid, AttrNumber *attnum) { RangeTblEntry *rte; if (var->varno < 1 || var->varno > list_length(query->rtable)) return false; rte = rt_fetch(var->varno, query->rtable); if (rte->rtekind != RTE_RELATION) return false; *relid = rte->relid; *attnum = var->varattno; return true; } /* * Extract relation OID from an arbitrary expression. * Returns InvalidOid if vars reference multiple relations * or none. Sets *varno_out to the varno of the relation. */ static Oid get_expr_relid(Node *expr, Query *query, Index *varno_out) { List *vars; ListCell *lc; Oid relid = InvalidOid; Index varno = 0; vars = pull_var_clause(expr, 0); if (vars == NIL) return InvalidOid; foreach (lc, vars) { Var *var = (Var *)lfirst(lc); RangeTblEntry *rte; if (var->varlevelsup != 0) { list_free(vars); return InvalidOid; } if (var->varno < 1 || var->varno > list_length(query->rtable)) { list_free(vars); return InvalidOid; } rte = rt_fetch(var->varno, query->rtable); if (rte->rtekind != RTE_RELATION) { list_free(vars); return InvalidOid; } if (relid == InvalidOid) { relid = rte->relid; varno = var->varno; } else if (rte->relid != relid) { list_free(vars); return InvalidOid; } } list_free(vars); if (varno_out) *varno_out = varno; return relid; } /* * Find BM25 index OID from a Var node representing the indexed column. */ static Oid find_index_for_var(Var *var, ResolveIndexContext *context) { Oid relid; AttrNumber attnum; if (!get_var_relation_and_attnum(var, context->query, &relid, &attnum)) return InvalidOid; return find_bm25_index_for_column( relid, attnum, context->oid_cache->bm25_am_oid); } /* * Find BM25 index matching an expression on a relation. * For plain Var, delegates to find_bm25_index_for_column. * For complex expressions, compares expression trees. */ static Oid find_bm25_index_for_expr(Node *expr, Oid relid, Index varno, Oid bm25_am_oid) { Relation indexRelation; SysScanDesc scan; HeapTuple indexTuple; Oid result = InvalidOid; int index_count = 0; int partial_skipped = 0; ScanKeyData scanKey; if (!OidIsValid(bm25_am_oid)) return InvalidOid; /* Fast path: plain column reference */ if (IsA(expr, Var)) { Var *var = (Var *)expr; return find_bm25_index_for_column(relid, var->varattno, bm25_am_oid); } /* Slow path: expression comparison */ indexRelation = table_open(IndexRelationId, AccessShareLock); ScanKeyInit( &scanKey, Anum_pg_index_indrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(relid)); scan = systable_beginscan( indexRelation, IndexIndrelidIndexId, true, NULL, 1, &scanKey); while ((indexTuple = systable_getnext(scan)) != NULL) { Form_pg_index indexForm = (Form_pg_index)GETSTRUCT(indexTuple); Oid indexOid = indexForm->indexrelid; HeapTuple classTuple; Oid indexAmOid; classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(indexOid)); if (!HeapTupleIsValid(classTuple)) continue; indexAmOid = ((Form_pg_class)GETSTRUCT(classTuple))->relam; ReleaseSysCache(classTuple); if (indexAmOid != bm25_am_oid) continue; /* Must be an expression index */ if (indexForm->indkey.values[0] != 0) continue; /* * Skip partial indexes for implicit resolution. * Users must name them explicitly via * to_bm25query. */ { Datum predDatum; bool predNull; predDatum = SysCacheGetAttr( INDEXRELID, indexTuple, Anum_pg_index_indpred, &predNull); (void)predDatum; if (!predNull) { partial_skipped++; continue; } } { Datum exprDatum; bool isnull; List *idx_exprs; Node *idx_expr; exprDatum = SysCacheGetAttr( INDEXRELID, indexTuple, Anum_pg_index_indexprs, &isnull); if (isnull) continue; idx_exprs = (List *)stringToNode(TextDatumGetCString(exprDatum)); if (idx_exprs == NIL) continue; idx_expr = (Node *)linitial(idx_exprs); /* * pg_index stores expressions with varno=1. * Normalize to match the query's varno. */ ChangeVarNodes(idx_expr, 1, varno, 0); if (equal(idx_expr, expr)) { index_count++; if (result == InvalidOid) result = indexOid; } } } systable_endscan(scan); table_close(indexRelation, AccessShareLock); if (index_count > 1) ereport(WARNING, (errmsg("multiple BM25 indexes match the " "expression"), errhint("Use explicit to_bm25query('query'," " 'index_name') to specify which " "index."))); if (index_count == 0 && partial_skipped > 0) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("no non-partial BM25 index found " "for this expression " "(%d partial index(es) skipped)", partial_skipped), errhint("Use to_bm25query('query', " "'index_name') to specify which " "partial index to use."))); return result; } /* * Create a new tpquery Const with the resolved index OID. */ static Const * create_resolved_tpquery_const(Const *original, Oid index_oid) { TpQuery *old_tpquery = (TpQuery *)DatumGetPointer(original->constvalue); char *query_text = get_tpquery_text(old_tpquery); TpQuery *new_tpquery = create_tpquery(query_text, index_oid); return makeConst( original->consttype, original->consttypmod, original->constcollid, -1, /* bm25query is variable-length, including private hints */ PointerGetDatum(new_tpquery), false, false); } /* * Create a new OpExpr with the given arguments. */ static OpExpr * create_opexpr(Oid opno, Node *left, Node *right, Oid inputcollid, int location) { OpExpr *new_opexpr = makeNode(OpExpr); new_opexpr->opno = opno; new_opexpr->opfuncid = get_opcode(opno); new_opexpr->opresulttype = FLOAT8OID; new_opexpr->opretset = false; new_opexpr->opcollid = InvalidOid; new_opexpr->inputcollid = inputcollid; new_opexpr->args = list_make2(copyObject(left), right); new_opexpr->location = location; return new_opexpr; } /* * Transform text <@> bm25query with unresolved index. * Returns transformed OpExpr or NULL if no transformation needed. * Also validates that explicit indexes are on the correct column. */ static Node * transform_tpquery_opexpr(OpExpr *opexpr, ResolveIndexContext *context) { BM25OidCache *oids = context->oid_cache; Node *left; Node *right; Const *constNode; TpQuery *tpquery; Oid index_oid; Const *new_const; if (opexpr->opno != oids->text_tpquery_operator_oid && opexpr->opno != oids->textarray_tpquery_operator_oid) return NULL; if (list_length(opexpr->args) != 2) return NULL; left = linitial(opexpr->args); right = lsecond(opexpr->args); /* Try to simplify function calls (e.g., to_bm25query()) to constants */ if (IsA(right, FuncExpr)) right = eval_const_expressions(NULL, right); if (!IsA(right, Const)) return NULL; constNode = (Const *)right; if (constNode->consttype != oids->tpquery_type_oid || constNode->constisnull) return NULL; tpquery = (TpQuery *)DatumGetPointer(constNode->constvalue); /* * Reject constant left operands (no table column refs). * Expression index operands contain Var nodes referencing * table columns; a bare constant like 'hello' does not. */ if (!IsA(left, Var) && !contain_var_clause(left)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("left operand of <@> must reference " "a table column"), errhint("Use column_name <@> bm25query, not " "a constant."))); } /* * If the bm25query has an explicit index, validate that the * index matches the column or expression being queried. */ if (OidIsValid(tpquery->index_oid)) { if (IsA(left, Var)) { Var *var = (Var *)left; Oid relid; AttrNumber attnum; if (get_var_relation_and_attnum( var, context->query, &relid, &attnum)) { if (!index_is_on_column(tpquery->index_oid, relid, attnum)) { char *index_name = get_rel_name(tpquery->index_oid); char *col_name = get_attname(relid, attnum, false); char *table_name = get_rel_name(relid); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("index \"%s\" is not on " "column \"%s\"", index_name ? index_name : "(unknown)", col_name ? col_name : "(unknown)"), errdetail( "The explicitly specified " "index is not built on the " "column being searched."), errhint("Use an index that is built" " on column \"%s\" of table" " \"%s\", or omit the index" " name to use automatic " "index resolution.", col_name ? col_name : "(unknown)", table_name ? table_name : "(unknown)"))); } } } /* * For expressions, trust the user's explicit index * specification. Full validation would require * deserializing pg_index.indexprs and comparing * trees, but the user explicitly named the index. */ return NULL; /* Already resolved */ } /* * Implicit index resolution: find a BM25 index that * matches the left operand. */ if (IsA(left, Var)) { index_oid = find_index_for_var((Var *)left, context); } else { Index varno; Oid relid; relid = get_expr_relid(left, context->query, &varno); if (OidIsValid(relid)) index_oid = find_bm25_index_for_expr( left, relid, varno, context->oid_cache->bm25_am_oid); else index_oid = InvalidOid; } if (!OidIsValid(index_oid)) return NULL; new_const = create_resolved_tpquery_const(constNode, index_oid); return (Node *)create_opexpr( opexpr->opno, left, (Node *)new_const, opexpr->inputcollid, opexpr->location); } /* * Transform text <@> text to text <@> bm25query. * Returns transformed OpExpr or NULL if no transformation needed. */ static Node * transform_text_text_opexpr(OpExpr *opexpr, ResolveIndexContext *context) { BM25OidCache *oids = context->oid_cache; Node *left; Node *right; Const *text_const; Oid index_oid; char *query_text; TpQuery *tpquery; Const *tpquery_const; bool is_text_array_op = (opexpr->opno == oids->textarray_text_operator_oid); if (opexpr->opno != oids->text_text_operator_oid && !is_text_array_op) return NULL; if (list_length(opexpr->args) != 2) return NULL; left = linitial(opexpr->args); right = lsecond(opexpr->args); if (!IsA(right, Const)) return NULL; text_const = (Const *)right; if (text_const->consttype != TEXTOID || text_const->constisnull) return NULL; /* * Reject constant left operands (no table column refs). */ if (!IsA(left, Var) && !contain_var_clause(left)) { ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("left operand of <@> must reference " "a table column"), errhint("Use column_name <@> 'query text', " "not a constant."))); } /* Find index: plain column or expression */ if (IsA(left, Var)) { index_oid = find_index_for_var((Var *)left, context); } else { Index varno; Oid relid; relid = get_expr_relid(left, context->query, &varno); if (OidIsValid(relid)) index_oid = find_bm25_index_for_expr( left, relid, varno, context->oid_cache->bm25_am_oid); else index_oid = InvalidOid; } if (!OidIsValid(index_oid)) return NULL; query_text = TextDatumGetCString(text_const->constvalue); tpquery = create_tpquery(query_text, index_oid); pfree(query_text); tpquery_const = makeConst( oids->tpquery_type_oid, -1, InvalidOid, -1, /* varlena type */ PointerGetDatum(tpquery), false, false); return (Node *)create_opexpr( is_text_array_op ? oids->textarray_tpquery_operator_oid : oids->text_tpquery_operator_oid, left, (Node *)tpquery_const, opexpr->inputcollid, opexpr->location); } /* Forward declaration */ static void resolve_indexes_in_query(Query *query); /* * Mutator function to resolve unresolved tpquery constants. */ static Node * resolve_index_mutator(Node *node, ResolveIndexContext *context) { Node *result; if (node == NULL) return NULL; if (IsA(node, OpExpr)) { OpExpr *opexpr = (OpExpr *)node; /* Try text <@> bm25query transformation */ result = transform_tpquery_opexpr(opexpr, context); if (result) return result; /* Try text <@> text transformation */ result = transform_text_text_opexpr(opexpr, context); if (result) return result; } /* * expression_tree_mutator does not descend into SubLink * subselects (they are Query nodes, not expression nodes). * Process them here so implicit resolution works inside * subqueries of DML statements like DELETE and UPDATE. */ if (IsA(node, SubLink)) { SubLink *sublink = (SubLink *)node; if (sublink->subselect && IsA(sublink->subselect, Query)) resolve_indexes_in_query((Query *)sublink->subselect); } return expression_tree_mutator(node, resolve_index_mutator, context); } /* * Process a single query's target list. */ static void resolve_indexes_in_targetlist(Query *query, ResolveIndexContext *context) { ListCell *lc; if (!query->targetList) return; foreach (lc, query->targetList) { TargetEntry *tle = (TargetEntry *)lfirst(lc); tle->expr = (Expr *)resolve_index_mutator((Node *)tle->expr, context); } } /* * Process subqueries in CTEs and FROM clause. */ static void resolve_indexes_in_subqueries(Query *query) { ListCell *lc; /* Process CTEs */ foreach (lc, query->cteList) { CommonTableExpr *cte = (CommonTableExpr *)lfirst(lc); if (cte->ctequery && IsA(cte->ctequery, Query)) resolve_indexes_in_query((Query *)cte->ctequery); } /* Process FROM subqueries */ foreach (lc, query->rtable) { RangeTblEntry *rte = (RangeTblEntry *)lfirst(lc); if (rte->rtekind == RTE_SUBQUERY && rte->subquery) resolve_indexes_in_query(rte->subquery); } } static void resolve_indexes_in_query(Query *query) { ResolveIndexContext context; BM25OidCache oid_cache; if (!get_bm25_oids(&oid_cache)) return; context.query = query; context.oid_cache = &oid_cache; /* Process target list */ resolve_indexes_in_targetlist(query, &context); /* Process WHERE clause */ if (query->jointree && query->jointree->quals) query->jointree->quals = resolve_index_mutator(query->jointree->quals, &context); /* Process HAVING clause */ if (query->havingQual) query->havingQual = resolve_index_mutator(query->havingQual, &context); /* Process subqueries */ resolve_indexes_in_subqueries(query); } /* * Post parse analysis hook function. * * Performance note: For non-BM25 queries, this does minimal work - just a * cached OID lookup and a quick walk of expressions to check operator OIDs. * Expensive operations (syscache lookups, index resolution) only happen * for actual BM25 operator expressions. */ static void tp_post_parse_analyze_hook( ParseState *pstate pg_attribute_unused(), Query *query, TP_JUMBLE_STATE *jstate pg_attribute_unused()) { /* * Skip index resolution if we're not in a valid transaction state. * This happens when a statement is parsed after an error in a * transaction block (e.g., ROLLBACK after a failed query). In that * state, catalog lookups are forbidden. */ if (IsTransactionState()) resolve_indexes_in_query(query); if (prev_post_parse_analyze_hook) prev_post_parse_analyze_hook(pstate, query, jstate); } /* ============================================================================ * Planner hook: Replace BM25 score expressions with stub function * ============================================================================ */ /* * Check if an expression is our BM25 scoring operator (text <@> bm25query). */ static bool is_bm25_score_opexpr(Node *node, BM25OidCache *oids) { OpExpr *opexpr; if (!IsA(node, OpExpr)) return false; opexpr = (OpExpr *)node; return opexpr->opno == oids->text_tpquery_operator_oid || opexpr->opno == oids->textarray_tpquery_operator_oid; } /* * Create a FuncExpr calling bm25_get_current_score(). */ static FuncExpr * make_stub_funcexpr(void) { FuncExpr *funcexpr; Oid funcoid; /* Look up bm25_get_current_score function */ funcoid = LookupFuncName( list_make1(makeString("bm25_get_current_score")), 0, NULL, false); if (!OidIsValid(funcoid)) return NULL; funcexpr = makeNode(FuncExpr); funcexpr->funcid = funcoid; funcexpr->funcresulttype = FLOAT8OID; funcexpr->funcretset = false; funcexpr->funcvariadic = false; funcexpr->funcformat = COERCE_EXPLICIT_CALL; funcexpr->funccollid = InvalidOid; funcexpr->inputcollid = InvalidOid; funcexpr->args = NIL; funcexpr->location = -1; return funcexpr; } /* * Replace BM25 score expressions in a plan node's targetlist. */ static void replace_scores_in_targetlist(List *targetlist, BM25OidCache *oids) { ListCell *lc; Expr *orderby_expr = NULL; /* First pass: find the resjunk ORDER BY expression */ foreach (lc, targetlist) { TargetEntry *tle = (TargetEntry *)lfirst(lc); if (tle->resjunk && is_bm25_score_opexpr((Node *)tle->expr, oids)) { orderby_expr = tle->expr; break; } } if (orderby_expr == NULL) return; /* Second pass: replace the ORDER BY expr and any matching SELECT exprs */ foreach (lc, targetlist) { TargetEntry *tle = (TargetEntry *)lfirst(lc); if (is_bm25_score_opexpr((Node *)tle->expr, oids) && equal(tle->expr, orderby_expr)) { FuncExpr *stub = make_stub_funcexpr(); if (stub != NULL) tle->expr = (Expr *)stub; } } } /* * Check if the plan contains a BM25 IndexScan that will compute scores. */ static bool plan_has_bm25_indexscan(Plan *plan, BM25OidCache *oids) { ListCell *lc; if (plan == NULL) return false; /* Check if this is a BM25 IndexScan */ if (IsA(plan, IndexScan)) { IndexScan *indexscan = (IndexScan *)plan; Oid indexamid; HeapTuple classTuple; /* Look up the index's access method */ classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(indexscan->indexid)); if (HeapTupleIsValid(classTuple)) { Form_pg_class classForm = (Form_pg_class)GETSTRUCT(classTuple); indexamid = classForm->relam; ReleaseSysCache(classTuple); if (indexamid == oids->bm25_am_oid) return true; } } /* Recurse into child plans */ if (plan_has_bm25_indexscan(plan->lefttree, oids)) return true; if (plan_has_bm25_indexscan(plan->righttree, oids)) return true; /* Handle plan-specific children */ switch (nodeTag(plan)) { case T_Append: { Append *append = (Append *)plan; foreach (lc, append->appendplans) if (plan_has_bm25_indexscan((Plan *)lfirst(lc), oids)) return true; } break; case T_MergeAppend: { MergeAppend *merge = (MergeAppend *)plan; foreach (lc, merge->mergeplans) if (plan_has_bm25_indexscan((Plan *)lfirst(lc), oids)) return true; } break; case T_SubqueryScan: { SubqueryScan *subquery = (SubqueryScan *)plan; if (plan_has_bm25_indexscan(subquery->subplan, oids)) return true; } break; case T_CustomScan: { CustomScan *cscan = (CustomScan *)plan; foreach (lc, cscan->custom_plans) if (plan_has_bm25_indexscan((Plan *)lfirst(lc), oids)) return true; } break; default: break; } return false; } /* * Recursively walk the plan tree and replace BM25 score expressions. */ static void replace_scores_in_plan(Plan *plan, BM25OidCache *oids) { if (plan == NULL) return; /* Process this node's targetlist */ replace_scores_in_targetlist(plan->targetlist, oids); /* Recurse into child plans */ replace_scores_in_plan(plan->lefttree, oids); replace_scores_in_plan(plan->righttree, oids); /* Handle plan-specific children */ switch (nodeTag(plan)) { case T_Append: { Append *append = (Append *)plan; ListCell *lc; foreach (lc, append->appendplans) replace_scores_in_plan((Plan *)lfirst(lc), oids); } break; case T_MergeAppend: { MergeAppend *merge = (MergeAppend *)plan; ListCell *lc; foreach (lc, merge->mergeplans) replace_scores_in_plan((Plan *)lfirst(lc), oids); } break; case T_SubqueryScan: { SubqueryScan *subquery = (SubqueryScan *)plan; replace_scores_in_plan(subquery->subplan, oids); } break; case T_CustomScan: { CustomScan *cscan = (CustomScan *)plan; ListCell *lc2; foreach (lc2, cscan->custom_plans) replace_scores_in_plan((Plan *)lfirst(lc2), oids); } break; default: break; } } /* * Extract a TpQuery from an expression, if present. * Returns NULL if the expression doesn't contain a bm25query constant. */ static TpQuery * extract_tpquery_from_expr(Node *node, BM25OidCache *oids) { if (node == NULL) return NULL; if (IsA(node, OpExpr)) { OpExpr *opexpr = (OpExpr *)node; if ((opexpr->opno == oids->text_tpquery_operator_oid || opexpr->opno == oids->textarray_tpquery_operator_oid) && list_length(opexpr->args) == 2) { Node *right = lsecond(opexpr->args); if (IsA(right, Const)) { Const *constNode = (Const *)right; if (constNode->consttype == oids->tpquery_type_oid && !constNode->constisnull) { return (TpQuery *)DatumGetPointer(constNode->constvalue); } } } } return NULL; } /* * Find a TpQuery in a list of expressions. * Returns NULL if no bm25query constant found. */ static TpQuery * find_tpquery_in_list(List *exprlist, BM25OidCache *oids) { ListCell *lc; foreach (lc, exprlist) { Node *node = (Node *)lfirst(lc); TpQuery *tpquery = extract_tpquery_from_expr(node, oids); if (tpquery != NULL) return tpquery; } return NULL; } /* Forward declaration */ static Oid get_index_table_oid(Oid index_oid); /* * Check if scan_index is a descendant of specified_index via partitioned * index inheritance or regular table inheritance. * * For partitioned indexes, walks up the index inheritance chain. * For regular indexes, checks if the scan index's table inherits * from the specified index's table (PG17 planner may choose child * table indexes). */ static bool is_child_partition_index(Oid specified_index_oid, Oid scan_index_oid) { char relkind; Oid current_oid; int depth; /* * Check if specified index is a partitioned index or a regular * index on an inheritance parent table. For regular inheritance, * PG17's planner may choose the child table's index instead of * the parent's. */ relkind = get_rel_relkind(specified_index_oid); if (relkind != RELKIND_PARTITIONED_INDEX && relkind != RELKIND_INDEX) return false; /* * For regular (non-partitioned) indexes, check table-level * inheritance: does the scan index's table inherit from the * specified index's table? */ if (relkind == RELKIND_INDEX) { Oid specified_table = get_index_table_oid(specified_index_oid); Oid scan_table = get_index_table_oid(scan_index_oid); if (!OidIsValid(specified_table) || !OidIsValid(scan_table)) return false; /* Must be a different table (child), not the same table */ if (specified_table == scan_table) return false; return list_member_oid( find_all_inheritors(specified_table, NoLock, NULL), scan_table); } /* * Walk up the inheritance chain from scan_index_oid. At each step, look * up the parent in pg_inherits. If we reach specified_index_oid, return * true. Limit depth to prevent infinite loops. */ current_oid = scan_index_oid; for (depth = 0; depth < 100; depth++) { Relation inhrel; SysScanDesc scan; HeapTuple tuple; ScanKeyData skey; Oid parent_oid = InvalidOid; inhrel = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit( &skey, Anum_pg_inherits_inhrelid, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(current_oid)); scan = systable_beginscan( inhrel, InheritsRelidSeqnoIndexId, true, NULL, 1, &skey); tuple = systable_getnext(scan); if (tuple != NULL) { Form_pg_inherits inhform = (Form_pg_inherits)GETSTRUCT(tuple); parent_oid = inhform->inhparent; } systable_endscan(scan); table_close(inhrel, AccessShareLock); if (!OidIsValid(parent_oid)) return false; /* No parent, reached top without finding match */ if (parent_oid == specified_index_oid) return true; /* Found the specified index in the chain */ current_oid = parent_oid; } return false; /* Depth limit exceeded */ } /* * Get the table OID that an index is built on. */ static Oid get_index_table_oid(Oid index_oid) { HeapTuple idx_tuple; Form_pg_index idx_form; Oid result = InvalidOid; idx_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(index_oid)); if (HeapTupleIsValid(idx_tuple)) { idx_form = (Form_pg_index)GETSTRUCT(idx_tuple); result = idx_form->indrelid; ReleaseSysCache(idx_tuple); } return result; } /* * Check if an index is a BM25 index. */ static bool is_bm25_index(Oid index_oid, Oid bm25_am_oid) { HeapTuple classTuple; bool result = false; classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(index_oid)); if (HeapTupleIsValid(classTuple)) { Form_pg_class classForm = (Form_pg_class)GETSTRUCT(classTuple); result = (classForm->relam == bm25_am_oid); ReleaseSysCache(classTuple); } return result; } /* * Walker context for collecting explicit index requirements from a query. */ typedef struct CollectExplicitIndexContext { BM25OidCache *oid_cache; List *requirements; /* List of ExplicitIndexRequirement */ } CollectExplicitIndexContext; /* Detect BM25 operators, including those in nested queries. */ static bool has_bm25_operator_walker(Node *node, BM25OidCache *oids) { if (node == NULL) return false; if (IsA(node, OpExpr)) { OpExpr *opexpr = (OpExpr *)node; if (opexpr->opno == oids->text_tpquery_operator_oid || opexpr->opno == oids->textarray_tpquery_operator_oid || opexpr->opno == oids->text_text_operator_oid || opexpr->opno == oids->textarray_text_operator_oid) return true; } if (IsA(node, Query)) return query_tree_walker( (Query *)node, has_bm25_operator_walker, oids, 0); return expression_tree_walker(node, has_bm25_operator_walker, oids); } /* * Walker to find explicit index requirements in query expressions. */ static bool collect_explicit_indexes_walker( Node *node, CollectExplicitIndexContext *context) { if (node == NULL) return false; /* Nested requirements must not constrain another query's index paths. */ if (IsA(node, Query)) return false; if (IsA(node, OpExpr)) { OpExpr *opexpr = (OpExpr *)node; if ((opexpr->opno == context->oid_cache->text_tpquery_operator_oid || opexpr->opno == context->oid_cache->textarray_tpquery_operator_oid) && list_length(opexpr->args) == 2) { Node *left = linitial(opexpr->args); Node *right = lsecond(opexpr->args); /* * Try to fold FuncExpr (e.g., to_bm25query()) * to a Const. */ if (IsA(right, FuncExpr)) right = eval_const_expressions(NULL, right); if (IsA(right, Const)) { Const *constNode = (Const *)right; if (constNode->consttype == context->oid_cache->tpquery_type_oid && !constNode->constisnull) { TpQuery *tpquery = (TpQuery *)DatumGetPointer( constNode->constvalue); /* * Only collect explicitly specified indexes * (from to_bm25query with index name). */ if (tpquery_is_explicit_index(tpquery)) { Oid index_oid = get_tpquery_index_oid(tpquery); if (OidIsValid(index_oid)) { Oid table_oid = get_index_table_oid(index_oid); if (OidIsValid(table_oid)) { ExplicitIndexRequirement *req; req = palloc(sizeof(ExplicitIndexRequirement)); req->relid = table_oid; req->required_index_oid = index_oid; if (IsA(left, Var)) { Var *var = (Var *)left; req->attnum = var->varattno; req->expr = NULL; } else { req->attnum = 0; req->expr = copyObject(left); } context->requirements = lappend(context->requirements, req); } } } } } } } return expression_tree_walker( node, collect_explicit_indexes_walker, context); } /* * Collect all explicit index requirements from a query. * * Performance note: This walks the query tree, which is O(n) where n is the * number of nodes. However, the per-node work is minimal - just a type check * and integer comparison for non-BM25 queries. The expensive work (folding * FuncExpr, syscache lookups) only happens for actual BM25 operator * expressions with explicit index names, which is rare. */ static List * collect_explicit_index_requirements(Query *parse, BM25OidCache *oid_cache) { CollectExplicitIndexContext context; context.oid_cache = oid_cache; context.requirements = NIL; /* Collect requirements at this query level only. */ query_tree_walker(parse, collect_explicit_indexes_walker, &context, 0); return context.requirements; } /* * Find an explicit index requirement for a given relation and * column or expression. For plain columns (attnum > 0), matches * by relid and attnum. For expression indexes (attnum == 0), * matches by relid and expression tree equality. * Returns NULL if no explicit requirement exists. */ static ExplicitIndexRequirement * find_explicit_requirement(Oid relid, AttrNumber attnum, Node *expr) { ListCell *lc; if (current_planning_context == NULL) return NULL; foreach (lc, current_planning_context->explicit_indexes) { ExplicitIndexRequirement *req = (ExplicitIndexRequirement *)lfirst(lc); if (req->relid != relid) continue; if (attnum != 0 && req->attnum == attnum) return req; if (attnum == 0 && req->attnum == 0 && expr != NULL && req->expr != NULL && equal(expr, req->expr)) return req; } return NULL; } /* * Check if a path uses a specific BM25 index. * Returns the index OID if it's a BM25 IndexPath, InvalidOid otherwise. */ static Oid get_path_bm25_index_oid(Path *path, Oid bm25_am_oid) { if (IsA(path, IndexPath)) { IndexPath *indexpath = (IndexPath *)path; if (is_bm25_index(indexpath->indexinfo->indexoid, bm25_am_oid)) return indexpath->indexinfo->indexoid; } return InvalidOid; } /* * Find the IndexOptInfo for a specific index in the relation's indexlist. */ static IndexOptInfo * find_index_opt_info(RelOptInfo *rel, Oid index_oid) { ListCell *lc; foreach (lc, rel->indexlist) { IndexOptInfo *indexinfo = (IndexOptInfo *)lfirst(lc); if (indexinfo->indexoid == index_oid) return indexinfo; } return NULL; } /* * Fix BM25 IndexPaths in a path list to use the explicitly specified index. * * For each BM25 IndexPath, finds the matching explicit requirement by * column and swaps the indexinfo if needed. */ static void fix_bm25_indexpaths( List *pathlist, RelOptInfo *rel, Oid relid, Oid bm25_am_oid) { ListCell *lc; foreach (lc, pathlist) { Path *path = (Path *)lfirst(lc); IndexPath *indexpath; Oid index_oid; AttrNumber attnum; ExplicitIndexRequirement *req; IndexOptInfo *correct_indexinfo; if (!IsA(path, IndexPath)) continue; index_oid = get_path_bm25_index_oid(path, bm25_am_oid); if (!OidIsValid(index_oid)) continue; indexpath = (IndexPath *)path; if (indexpath->indexinfo->nkeycolumns < 1) continue; attnum = indexpath->indexinfo->indexkeys[0]; { Node *idx_expr = NULL; if (attnum == 0 && indexpath->indexinfo->indexprs) idx_expr = (Node *)linitial(indexpath->indexinfo->indexprs); req = find_explicit_requirement(relid, attnum, idx_expr); } if (req == NULL) continue; if (index_oid == req->required_index_oid) continue; if (is_child_partition_index(req->required_index_oid, index_oid)) continue; correct_indexinfo = find_index_opt_info(rel, req->required_index_oid); if (correct_indexinfo == NULL) continue; indexpath->indexinfo = correct_indexinfo; } } /* * set_rel_pathlist_hook: Replace BM25 IndexPaths that use the wrong index. * * When a query specifies an explicit BM25 index via to_bm25query(), we need * to ensure the planner uses that index. This hook finds BM25 IndexPaths * using the wrong index and updates them to use the correct one. * * For partitioned tables, we allow child partition indexes of the specified * parent index. * * Performance note: This hook returns immediately for non-BM25 queries since * current_planning_context will be NULL (we only set it when there are * explicit index requirements). The overhead for non-BM25 queries is just * a single pointer comparison. */ static void tp_set_rel_pathlist_hook( PlannerInfo *root, RelOptInfo *rel, Index rti, RangeTblEntry *rte) { if (prev_set_rel_pathlist_hook) prev_set_rel_pathlist_hook(root, rel, rti, rte); /* Only process base relations */ if (rel->reloptkind != RELOPT_BASEREL) return; /* Check if we have planning context */ if (current_planning_context == NULL) return; fix_bm25_indexpaths( rel->pathlist, rel, rte->relid, current_planning_context->bm25_am_oid); fix_bm25_indexpaths( rel->partial_pathlist, rel, rte->relid, current_planning_context->bm25_am_oid); } /* * Check an IndexScan node for explicit index mismatch. * If the IndexScan uses a different index than what's specified in the * bm25query, raise an error (if explicit) or warning (if implicit). * * Exception: partitioned indexes - if the specified index is a partitioned * index and the scan uses a child partition index, that's allowed. */ static void validate_indexscan_explicit_index(IndexScan *indexscan, BM25OidCache *oids) { TpQuery *tpquery; Oid specified_index_oid; HeapTuple classTuple; Oid indexamid; /* First check this is a BM25 index scan */ classTuple = SearchSysCache1(RELOID, ObjectIdGetDatum(indexscan->indexid)); if (!HeapTupleIsValid(classTuple)) return; indexamid = ((Form_pg_class)GETSTRUCT(classTuple))->relam; ReleaseSysCache(classTuple); if (indexamid != oids->bm25_am_oid) return; /* Not a BM25 index scan */ /* Look for bm25query in the indexorderby expressions */ tpquery = find_tpquery_in_list(indexscan->indexorderby, oids); if (tpquery == NULL) { /* Also check the indexqual */ tpquery = find_tpquery_in_list(indexscan->indexqual, oids); } if (tpquery == NULL) return; /* No bm25query found */ specified_index_oid = get_tpquery_index_oid(tpquery); if (!OidIsValid(specified_index_oid)) return; /* No index specified in query */ /* Validate they match */ if (specified_index_oid != indexscan->indexid) { /* * Allow partitioned indexes: if specified is parent and scan is child */ if (is_child_partition_index(specified_index_oid, indexscan->indexid)) return; /* Child partition index - allowed */ /* * When the user names an index explicitly via * to_bm25query(text, index_name), that index must be used for the * scan to ensure consistent tokenization; error if the planner chose * a different one. Implicitly resolved indexes (e.g. text <@> text) * impose no such constraint, so no diagnostic is emitted. */ if (tpquery_is_explicit_index(tpquery)) { char *specified_name = get_rel_name(specified_index_oid); char *scan_name = get_rel_name(indexscan->indexid); ereport(ERROR, (errcode(ERRCODE_WRONG_OBJECT_TYPE), errmsg("query specifies index \"%s\" but planner chose " "index \"%s\"", specified_name ? specified_name : "(unknown)", scan_name ? scan_name : "(unknown)"), errdetail( "When an explicit index is specified in " "to_bm25query(), that index must be used for " "the scan to ensure consistent tokenization."), errhint("Use a planner hint to force the specified " "index, or remove the explicit index name to let " "the planner choose automatically."))); } } } /* * Validate that when an explicit index is specified in bm25query, the planner * uses that same index for the scan. * * Walks the plan tree looking for BM25 IndexScan nodes and validates their * ORDER BY expressions. */ static void validate_explicit_index_usage(Plan *plan, BM25OidCache *oids) { ListCell *lc; if (plan == NULL) return; /* Check this node if it's an IndexScan */ if (IsA(plan, IndexScan)) { validate_indexscan_explicit_index((IndexScan *)plan, oids); } /* Recurse into child plans */ validate_explicit_index_usage(plan->lefttree, oids); validate_explicit_index_usage(plan->righttree, oids); /* Handle plan-specific children */ switch (nodeTag(plan)) { case T_Append: { Append *append = (Append *)plan; foreach (lc, append->appendplans) validate_explicit_index_usage((Plan *)lfirst(lc), oids); } break; case T_MergeAppend: { MergeAppend *merge = (MergeAppend *)plan; foreach (lc, merge->mergeplans) validate_explicit_index_usage((Plan *)lfirst(lc), oids); } break; case T_SubqueryScan: { SubqueryScan *subquery = (SubqueryScan *)plan; validate_explicit_index_usage(subquery->subplan, oids); } break; case T_CustomScan: { CustomScan *cscan = (CustomScan *)plan; foreach (lc, cscan->custom_plans) validate_explicit_index_usage((Plan *)lfirst(lc), oids); } break; default: break; } } /* * Planner hook: sets up planning context and post-processes the plan. * * We set up the planning context BEFORE calling standard_planner so that * set_rel_pathlist_hook can filter out unwanted BM25 index paths. * * PG19 added a trailing ExplainState *es parameter to planner_hook, * standard_planner() and prev planner hooks. These macros splice that * parameter into the signature and the pass-through calls only when * building against PG19+. */ #if PG_VERSION_NUM >= 190000 #define TP_PLANNER_HOOK_DECL_EXTRA , ExplainState *es #define TP_PLANNER_HOOK_PASS_EXTRA , es #else #define TP_PLANNER_HOOK_DECL_EXTRA #define TP_PLANNER_HOOK_PASS_EXTRA #endif static bool tp_limit_const_value(Node *node, int64 *value) { Const *c; if (node == NULL || !IsA(node, Const)) return false; c = (Const *)node; if (c->constisnull || c->consttype != INT8OID) return false; *value = DatumGetInt64(c->constvalue); return true; } static bool tp_limit_k(Limit *limit, int64 *k) { int64 count; int64 offset = 0; if (!tp_limit_const_value(limit->limitCount, &count) || count <= 0) return false; if (limit->limitOffset != NULL && !tp_limit_const_value(limit->limitOffset, &offset)) return false; if (offset < 0) return false; if (count >= INT_MAX - offset) return false; *k = count + offset; return true; } static void tp_attach_seed_hint( IndexScan *scan, Limit *limit, List *rtable, BM25OidCache *oids) { ListCell *lc; int64 k; RangeTblEntry *rte; Relation heap; double selectivity = 0.0; if (list_length(scan->indexorderby) != 1 || scan->indexqual != NIL || !tp_limit_k(limit, &k) || scan->scan.scanrelid <= 0) return; if (get_rel_relam(scan->indexid) != oids->bm25_am_oid) return; rte = rt_fetch(scan->scan.scanrelid, rtable); if (rte == NULL || !OidIsValid(rte->relid)) return; heap = table_open(rte->relid, AccessShareLock); if (scan->scan.plan.qual != NIL && heap->rd_rel->reltuples > 0) selectivity = scan->scan.plan.plan_rows / heap->rd_rel->reltuples; table_close(heap, AccessShareLock); foreach (lc, scan->indexorderby) { Node *expr = (Node *)lfirst(lc); if (IsA(expr, OpExpr) && list_length(((OpExpr *)expr)->args) == 2) { Node *right = lsecond(((OpExpr *)expr)->args); if (IsA(right, Const) && ((Const *)right)->consttype == oids->tpquery_type_oid && !((Const *)right)->constisnull) { Const *original = (Const *)right; TpQuery *query = (TpQuery *)DatumGetPointer( original->constvalue); TpQuery *hinted = tpquery_copy_with_seed_hint(query, k, selectivity); Const *replacement = makeConst( original->consttype, original->consttypmod, original->constcollid, -1, PointerGetDatum(hinted), false, false); replacement->location = original->location; /* Original nodes may be shared; their context owns them. */ lsecond(((OpExpr *)expr)->args) = replacement; } } } } static void tp_attach_seed_hints(Plan *plan, List *rtable, BM25OidCache *oids) { ListCell *lc; if (plan == NULL) return; if (IsA(plan, Limit) && plan->lefttree != NULL && IsA(plan->lefttree, IndexScan)) tp_attach_seed_hint( (IndexScan *)plan->lefttree, (Limit *)plan, rtable, oids); tp_attach_seed_hints(plan->lefttree, rtable, oids); tp_attach_seed_hints(plan->righttree, rtable, oids); switch (nodeTag(plan)) { case T_Append: foreach (lc, ((Append *)plan)->appendplans) tp_attach_seed_hints(lfirst(lc), rtable, oids); break; case T_MergeAppend: foreach (lc, ((MergeAppend *)plan)->mergeplans) tp_attach_seed_hints(lfirst(lc), rtable, oids); break; case T_SubqueryScan: tp_attach_seed_hints(((SubqueryScan *)plan)->subplan, rtable, oids); break; case T_CustomScan: foreach (lc, ((CustomScan *)plan)->custom_plans) tp_attach_seed_hints(lfirst(lc), rtable, oids); break; default: break; } } static PlannedStmt * tp_planner_hook( Query *parse, const char *query_string, int cursorOptions, ParamListInfo boundParams TP_PLANNER_HOOK_DECL_EXTRA) { PlannedStmt *result; BM25OidCache oid_cache; PlanningContext planning_context; PlanningContext *saved_context; List *explicit_indexes; bool query_has_bm25_operators; /* Get BM25 OIDs - if extension not installed, just pass through */ if (!get_bm25_oids(&oid_cache)) { if (prev_planner_hook) return prev_planner_hook( parse, query_string, cursorOptions, boundParams TP_PLANNER_HOOK_PASS_EXTRA); else return standard_planner( parse, query_string, cursorOptions, boundParams TP_PLANNER_HOOK_PASS_EXTRA); } /* * Collect explicit index requirements from the query. This walks the * query tree looking for to_bm25query() calls with explicit index names. * * Performance optimization: Only set up the planning context if we * actually find explicit requirements. For non-BM25 queries or queries * without explicit index names, this avoids any overhead in the * set_rel_pathlist_hook. */ query_has_bm25_operators = has_bm25_operator_walker((Node *)parse, &oid_cache); explicit_indexes = query_has_bm25_operators ? collect_explicit_index_requirements(parse, &oid_cache) : NIL; if (explicit_indexes != NIL) { planning_context.explicit_indexes = explicit_indexes; planning_context.bm25_am_oid = oid_cache.bm25_am_oid; /* Save and set current planning context */ saved_context = current_planning_context; current_planning_context = &planning_context; } else { saved_context = NULL; } /* Call previous hook or standard planner */ PG_TRY(); { if (prev_planner_hook) result = prev_planner_hook( parse, query_string, cursorOptions, boundParams TP_PLANNER_HOOK_PASS_EXTRA); else result = standard_planner( parse, query_string, cursorOptions, boundParams TP_PLANNER_HOOK_PASS_EXTRA); } PG_FINALLY(); { /* Restore previous context and clean up if we set one up */ if (explicit_indexes != NIL) { current_planning_context = saved_context; list_free_deep(explicit_indexes); } } PG_END_TRY(); /* * Post-process the plan if it may have BM25 index scans. * * Performance optimization: Only check for BM25 IndexScans if the * current query contains BM25 operators. This avoids expensive * syscache lookups in plan_has_bm25_indexscan() for non-BM25 queries. */ if (query_has_bm25_operators && result->planTree != NULL && plan_has_bm25_indexscan(result->planTree, &oid_cache)) { /* * Validate implicit index resolution - for explicit indexes, the * set_rel_pathlist_hook should have already forced the correct index. * This validation errors when an explicitly named index differs from * the one the planner scanned; implicit resolution mismatches are * allowed and emit no diagnostic. */ validate_explicit_index_usage(result->planTree, &oid_cache); replace_scores_in_plan(result->planTree, &oid_cache); } /* * Inspect the actual plan, including separately stored SubPlans. * A saved query can be planned after another statement has reset the * post-parse flag, so that flag cannot gate hint attachment. */ if (result->planTree != NULL) { ListCell *lc; tp_attach_seed_hints(result->planTree, result->rtable, &oid_cache); foreach (lc, result->subplans) tp_attach_seed_hints( (Plan *)lfirst(lc), result->rtable, &oid_cache); } return result; } /* * Initialize planner hooks (called from _PG_init). */ void tp_planner_hook_init(void) { prev_post_parse_analyze_hook = post_parse_analyze_hook; post_parse_analyze_hook = tp_post_parse_analyze_hook; prev_set_rel_pathlist_hook = set_rel_pathlist_hook; set_rel_pathlist_hook = tp_set_rel_pathlist_hook; prev_planner_hook = planner_hook; planner_hook = tp_planner_hook; } timescale-pg_textsearch-b63fe58/src/planner/hooks.h000066400000000000000000000005601525762047700225040ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * planner/hooks.h - Planner hooks for BM25 query optimization */ #pragma once #include /* * Initialize planner hooks (called from _PG_init). * Registers post_parse_analyze_hook and planner_hook. */ void tp_planner_hook_init(void); timescale-pg_textsearch-b63fe58/src/scoring/000077500000000000000000000000001525762047700212145ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/scoring/bm25.c000066400000000000000000000176351525762047700221410ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * score.c - BM25 scoring operators and document ranking */ #include #include #include #include #include #include "debug/injection.h" #include "index/metapage.h" #include "index/source.h" #include "index/state.h" #include "memtable/cache_source.h" #include "memtable/chain_source.h" #include "scoring/bm25.h" #include "scoring/bmw.h" #include "scoring/snapshot.h" #include "segment/graph_snapshot.h" #include "segment/segment.h" /* * Centralized IDF calculation (basic version) * Calculates IDF using BM25 formula: log(1 + (N - df + 0.5) / (df + 0.5)) * This formula ensures IDF is always non-negative since log(1 + x) >= 0 * for all x >= 0. */ float4 tp_calculate_idf(int32 doc_freq, int32 total_docs) { double idf_numerator = (double)(total_docs - doc_freq + 0.5); double idf_denominator = (double)(doc_freq + 0.5); double idf_ratio = idf_numerator / idf_denominator; return (float4)log(1.0 + idf_ratio); } /* * Get unified doc_freq for a term (memtable + all segments). * Returns 0 if term not found in any source. * * Per issue #374: `memtable_src` is a (possibly NULL) chain * source; we read the per-term doc_freq via the source op without * materializing the full posting list (which would be O(count) and * is wasteful for IDF lookup). */ static uint32 tp_get_unified_doc_freq( TpDataSource *memtable_src, Relation index, const char *term, const TpSegmentGraphSnapshot *snapshot) { uint32 doc_freq = 0; /* Get doc_freq from memtable chain source */ if (memtable_src != NULL) doc_freq = tp_source_get_doc_freq(memtable_src, term); doc_freq += tp_segment_roots_get_doc_freq( index, snapshot->roots, snapshot->root_count, term); return doc_freq; } /* * Batch get unified doc_freq for multiple terms (memtable + all segments). * Much faster than calling tp_get_unified_doc_freq in a loop because * it opens each segment only once instead of once per term. * * Per issue #374: `memtable_src` is a (possibly NULL) chain * source; we read each term's doc_freq via the source op without * materializing posting lists. */ static void tp_batch_get_unified_doc_freq( TpDataSource *memtable_src, Relation index, char **terms, int term_count, const TpSegmentGraphSnapshot *snapshot, uint32 *doc_freqs) { int i; /* Initialize doc_freqs with memtable counts */ for (i = 0; i < term_count; i++) { doc_freqs[i] = 0; if (memtable_src != NULL) doc_freqs[i] = tp_source_get_doc_freq(memtable_src, terms[i]); } tp_batch_get_segment_roots_doc_freq( index, snapshot->roots, snapshot->root_count, terms, term_count, doc_freqs); } /* * Score documents using BM25 algorithm * Returns number of documents scored */ int tp_score_documents( TpLocalIndexState *local_state, Relation index_relation, char **query_terms, int32 *query_frequencies, int query_term_count, int max_results, ItemPointer result_ctids, float4 **result_scores) { float4 avg_doc_len; int32 total_docs; float4 k1; float4 b; TpSegmentGraphSnapshot *snapshot; TpDataSource *memtable_src = NULL; int i; int result_count = 0; TpScoringSnapshot *scoring; /* Basic sanity checks */ Assert(local_state != NULL); Assert(query_terms != NULL); Assert(result_ctids != NULL); Assert(result_scores != NULL); if (query_term_count <= 0 || max_results <= 0) return 0; if (!local_state->shared) { elog(ERROR, "tp_score_documents: shared state is NULL!"); return 0; } scoring = tp_scoring_snapshot_get( index_relation, (const char *const *)query_terms, query_term_count, &memtable_src); snapshot = scoring->graph; if (memtable_src == NULL) memtable_src = tp_memtable_source_create_for_snapshot( local_state, index_relation, &snapshot->memtable, scoring->recovery, (const char *const *)query_terms, query_term_count); TP_INJECTION_POINT(TP_INJECTION_SCORING_SOURCE); k1 = snapshot->metapage.k1; b = snapshot->metapage.b; total_docs = scoring->total_docs; avg_doc_len = scoring->avg_doc_len; if (total_docs <= 0 || avg_doc_len <= 0.0f) { if (memtable_src != NULL) tp_source_close(memtable_src); return 0; } /* * BMW fast path for single-term queries. * Uses Block-Max WAND to skip blocks that can't contribute to top-k. */ if (query_term_count == 1) { const char *term = query_terms[0]; uint32 doc_freq; float4 idf; float4 *scores; TpBMWStats stats; /* Get unified doc_freq across memtable and segments */ doc_freq = tp_get_unified_doc_freq( memtable_src, index_relation, term, snapshot); if (doc_freq == 0) { if (memtable_src != NULL) tp_source_close(memtable_src); return 0; } /* Calculate IDF */ idf = tp_calculate_idf(doc_freq, total_docs); /* Allocate scores array */ scores = (float4 *)palloc(max_results * sizeof(float4)); /* Run BMW scoring */ result_count = tp_score_single_term_bmw( local_state, index_relation, snapshot, memtable_src, term, idf, k1, b, avg_doc_len, max_results, result_ctids, scores, &stats); /* Log BMW stats if enabled */ if (tp_log_bmw_stats) { elog(LOG, "BMW stats: memtable=%lu docs, segments=%lu docs " "(blocks: %lu scanned, %lu skipped, %.1f%% skip), " "seeks=%lu, results=%lu", (unsigned long)stats.memtable_docs, (unsigned long)stats.segment_docs_scored, (unsigned long)stats.blocks_scanned, (unsigned long)stats.blocks_skipped, (stats.blocks_scanned + stats.blocks_skipped) > 0 ? 100.0 * stats.blocks_skipped / (stats.blocks_scanned + stats.blocks_skipped) : 0.0, (unsigned long)stats.seeks_performed, (unsigned long)stats.docs_in_results); } *result_scores = scores; if (memtable_src != NULL) tp_source_close(memtable_src); return result_count; } /* * BMW fast path for multi-term queries (query_term_count >= 2). * Uses block-level upper bounds to skip non-contributing blocks. */ { uint32 *doc_freqs; float4 *idfs; float4 *scores; TpBMWStats stats; /* Batch lookup doc_freqs for all terms (opens each segment once) */ doc_freqs = palloc(query_term_count * sizeof(uint32)); tp_batch_get_unified_doc_freq( memtable_src, index_relation, query_terms, query_term_count, snapshot, doc_freqs); /* Convert doc_freqs to IDFs */ idfs = palloc(query_term_count * sizeof(float4)); for (i = 0; i < query_term_count; i++) { idfs[i] = (doc_freqs[i] > 0) ? tp_calculate_idf(doc_freqs[i], total_docs) : 0.0f; } pfree(doc_freqs); /* Allocate scores array */ scores = (float4 *)palloc(max_results * sizeof(float4)); /* Run multi-term BMW scoring */ result_count = tp_score_multi_term_bmw( local_state, index_relation, snapshot, memtable_src, query_terms, query_term_count, query_frequencies, idfs, k1, b, avg_doc_len, max_results, result_ctids, scores, &stats); pfree(idfs); /* Log BMW stats if enabled */ if (tp_log_bmw_stats) { elog(LOG, "BMW stats: memtable=%lu docs, segments=%lu docs " "(blocks: %lu scanned, %lu skipped, %.1f%% skip), " "seeks=%lu, results=%lu", (unsigned long)stats.memtable_docs, (unsigned long)stats.segment_docs_scored, (unsigned long)stats.blocks_scanned, (unsigned long)stats.blocks_skipped, (stats.blocks_scanned + stats.blocks_skipped) > 0 ? 100.0 * stats.blocks_skipped / (stats.blocks_scanned + stats.blocks_skipped) : 0.0, (unsigned long)stats.seeks_performed, (unsigned long)stats.docs_in_results); } *result_scores = scores; if (memtable_src != NULL) tp_source_close(memtable_src); return result_count; } } timescale-pg_textsearch-b63fe58/src/scoring/bm25.h000066400000000000000000000016721525762047700221400ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * score.h - BM25 scoring operator interface */ #pragma once #include #include typedef struct TpLocalIndexState TpLocalIndexState; /* * Document score entry for query result accumulation. * Used by bmw.c for BM25 scoring. */ typedef struct DocumentScoreEntry { ItemPointerData ctid; float4 score; float4 doc_length; } DocumentScoreEntry; extern int tp_score_documents( TpLocalIndexState *local_state, Relation index_relation, char **query_terms, int32 *query_frequencies, int query_term_count, int max_results, ItemPointer result_ctids, float4 **result_scores); /* IDF calculation */ extern float4 tp_calculate_idf(int32 doc_freq, int32 total_docs); /* GUC variable for BMW stats logging - defined in mod.c */ extern bool tp_log_bmw_stats; timescale-pg_textsearch-b63fe58/src/scoring/bmw.c000066400000000000000000001252011525762047700221460ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * bmw.c - Block-Max WAND query optimization implementation */ #include #include #include #include #include #include "constants.h" #include "index/metapage.h" #include "index/source.h" #include "memtable/chain_source.h" #include "scoring/bm25.h" #include "scoring/bmw.h" #include "segment/alive_bitset.h" #include "segment/compression.h" #include "segment/fieldnorm.h" #include "segment/io.h" /* * ------------------------------------------------------------ * Top-K Min-Heap Implementation * ------------------------------------------------------------ */ void tp_topk_init(TpTopKHeap *heap, int k, MemoryContext ctx) { MemoryContext old_ctx = MemoryContextSwitchTo(ctx); heap->ctids = palloc(k * sizeof(ItemPointerData)); heap->seg_blocks = palloc(k * sizeof(BlockNumber)); heap->doc_ids = palloc(k * sizeof(uint32)); heap->scores = palloc(k * sizeof(float4)); heap->capacity = k; heap->size = 0; MemoryContextSwitchTo(old_ctx); } void tp_topk_free(TpTopKHeap *heap) { if (heap->ctids) { pfree(heap->ctids); heap->ctids = NULL; } if (heap->seg_blocks) { pfree(heap->seg_blocks); heap->seg_blocks = NULL; } if (heap->doc_ids) { pfree(heap->doc_ids); heap->doc_ids = NULL; } if (heap->scores) { pfree(heap->scores); heap->scores = NULL; } heap->capacity = 0; heap->size = 0; } /* * Swap two heap entries. */ static inline void heap_swap(TpTopKHeap *heap, int i, int j) { ItemPointerData tmp_ctid = heap->ctids[i]; BlockNumber tmp_seg_block = heap->seg_blocks[i]; uint32 tmp_doc_id = heap->doc_ids[i]; float4 tmp_score = heap->scores[i]; heap->ctids[i] = heap->ctids[j]; heap->seg_blocks[i] = heap->seg_blocks[j]; heap->doc_ids[i] = heap->doc_ids[j]; heap->scores[i] = heap->scores[j]; heap->ctids[j] = tmp_ctid; heap->seg_blocks[j] = tmp_seg_block; heap->doc_ids[j] = tmp_doc_id; heap->scores[j] = tmp_score; } /* * Compare two heap entries for min-heap ordering. * Returns true if entry at position a should be closer to the root * (i.e., is a weaker result that should be evicted first). * * Primary: lower score = weaker = closer to root. * Secondary: among equal scores, higher CTID = weaker (lower CTID wins * ties in the final output). For segment entries whose CTIDs are not * yet resolved, doc_id within the segment is a reliable proxy. */ static inline bool heap_less(TpTopKHeap *heap, int a, int b) { if (heap->scores[a] < heap->scores[b]) return true; if (heap->scores[a] > heap->scores[b]) return false; /* Equal scores: higher CTID is weaker (closer to root) */ if (ItemPointerIsValid(&heap->ctids[a]) && ItemPointerIsValid(&heap->ctids[b])) return ItemPointerCompare(&heap->ctids[a], &heap->ctids[b]) > 0; /* Segment entries: doc_id proxy (higher doc_id = weaker) */ if (heap->doc_ids[a] != heap->doc_ids[b]) return heap->doc_ids[a] > heap->doc_ids[b]; return heap->seg_blocks[a] > heap->seg_blocks[b]; } /* * Sift up: restore heap property after insertion at position i. * For min-heap: parent <= children (using heap_less for comparison). */ static void heap_sift_up(TpTopKHeap *heap, int i) { while (i > 0) { int parent = (i - 1) / 2; if (!heap_less(heap, i, parent)) break; heap_swap(heap, i, parent); i = parent; } } /* * Sift down: restore heap property after replacement at position i. * For min-heap: parent <= children (using heap_less for comparison). */ static void heap_sift_down(TpTopKHeap *heap, int i) { while (true) { int left = 2 * i + 1; int right = 2 * i + 2; int smallest = i; if (left < heap->size && heap_less(heap, left, smallest)) smallest = left; if (right < heap->size && heap_less(heap, right, smallest)) smallest = right; if (smallest == i) break; heap_swap(heap, i, smallest); i = smallest; } } /* * Check if a candidate entry should replace the heap root. * * Higher score always wins. For equal scores, lower CTID wins * (matching the extraction sort order). When CTIDs are not both * available (memtable vs segment), doc_id serves as proxy. */ static inline bool heap_beats_root( TpTopKHeap *heap, float4 score, ItemPointerData *ctid, uint32 doc_id) { if (score > heap->scores[0]) return true; if (score < heap->scores[0]) return false; /* Equal score: lower CTID beats higher CTID */ if (ctid && ItemPointerIsValid(ctid) && ItemPointerIsValid(&heap->ctids[0])) return ItemPointerCompare(ctid, &heap->ctids[0]) < 0; /* Both segment entries: lower doc_id proxy */ if (ctid == NULL && !ItemPointerIsValid(&heap->ctids[0])) return doc_id < heap->doc_ids[0]; /* Mixed memtable/segment: can't compare, don't evict */ return false; } /* * Add a memtable result to the top-k heap. * CTID is known immediately for memtable entries. */ void tp_topk_add_memtable(TpTopKHeap *heap, ItemPointerData ctid, float4 score) { if (heap->size < heap->capacity) { /* Heap not full - just add */ int i = heap->size++; heap->ctids[i] = ctid; heap->seg_blocks[i] = InvalidBlockNumber; /* Marks as memtable entry */ heap->doc_ids[i] = 0; heap->scores[i] = score; heap_sift_up(heap, i); } else if (heap_beats_root(heap, score, &ctid, 0)) { /* Heap full but new entry beats root */ heap->ctids[0] = ctid; heap->seg_blocks[0] = InvalidBlockNumber; heap->doc_ids[0] = 0; heap->scores[0] = score; heap_sift_down(heap, 0); } /* else: doesn't qualify for top-k, ignore */ } /* * Add a segment result to the top-k heap. * CTID resolution is deferred until extraction. */ void tp_topk_add_segment( TpTopKHeap *heap, BlockNumber seg_block, uint32 doc_id, float4 score) { if (heap->size < heap->capacity) { /* Heap not full - just add */ int i = heap->size++; ItemPointerSetInvalid(&heap->ctids[i]); /* Will be resolved later */ heap->seg_blocks[i] = seg_block; heap->doc_ids[i] = doc_id; heap->scores[i] = score; heap_sift_up(heap, i); } else if (heap_beats_root(heap, score, NULL, doc_id)) { /* Heap full but new entry beats root */ ItemPointerSetInvalid(&heap->ctids[0]); heap->seg_blocks[0] = seg_block; heap->doc_ids[0] = doc_id; heap->scores[0] = score; heap_sift_down(heap, 0); } /* else: doesn't qualify for top-k, ignore */ } /* * Resolve CTIDs for segment results in the heap. * Batches lookups by segment - opens each unique segment once, resolves all * CTIDs from that segment, then closes it. This avoids O(k) segment opens. */ void tp_topk_resolve_ctids(TpTopKHeap *heap, Relation index) { int i, j; for (i = 0; i < heap->size; i++) { TpSegmentReader *reader; BlockNumber seg_block; /* Skip memtable entries and already-resolved entries */ if (heap->seg_blocks[i] == InvalidBlockNumber) continue; seg_block = heap->seg_blocks[i]; /* Open segment once for all entries from this segment */ reader = tp_segment_open_ex(index, seg_block, false); if (reader == NULL) continue; /* Resolve all CTIDs from this segment */ for (j = i; j < heap->size; j++) { if (heap->seg_blocks[j] == seg_block) { tp_segment_lookup_ctid( reader, heap->doc_ids[j], &heap->ctids[j]); /* Mark as resolved by setting seg_block to invalid */ heap->seg_blocks[j] = InvalidBlockNumber; } } tp_segment_close(reader); } } /* * Compare function for qsort: sort by (score DESC, CTID ASC). * This matches the exhaustive path's tie-breaking for deterministic results. */ static int compare_heap_entries(const void *a, const void *b, void *arg) { int i = *(const int *)a; int j = *(const int *)b; TpTopKHeap *heap = (TpTopKHeap *)arg; int cmp; /* Primary: higher score first (descending) */ if (heap->scores[i] > heap->scores[j]) return -1; if (heap->scores[i] < heap->scores[j]) return 1; /* Secondary: lower CTID first (ascending) for deterministic tie-breaking */ cmp = ItemPointerCompare(&heap->ctids[i], &heap->ctids[j]); return cmp; } /* * Extract sorted results (descending by score, CTID tie-breaking). * * Note: Call tp_topk_resolve_ctids first if heap contains segment results. * This ensures CTIDs are available for tie-breaking to match exhaustive path. */ int tp_topk_extract(TpTopKHeap *heap, ItemPointerData *ctids, float4 *scores) { int count = heap->size; int *indices; int i; if (count == 0) return 0; /* Create index array for sorting */ indices = palloc(count * sizeof(int)); for (i = 0; i < count; i++) indices[i] = i; /* Sort indices by (score DESC, CTID ASC) */ qsort_arg(indices, count, sizeof(int), compare_heap_entries, heap); /* Copy to output in sorted order */ for (i = 0; i < count; i++) { int idx = indices[i]; ctids[i] = heap->ctids[idx]; scores[i] = heap->scores[idx]; } pfree(indices); heap->size = 0; return count; } /* * ------------------------------------------------------------ * Block Max Score Computation * ------------------------------------------------------------ */ float4 tp_compute_block_max_score( TpSkipEntry *skip, float4 idf, float4 k1, float4 b, float4 avg_doc_len) { float4 tf = (float4)skip->block_max_tf; float4 dl = (float4)decode_fieldnorm(skip->block_max_norm); /* BM25 formula with max TF and min doc length in block */ float4 len_norm = 1.0f - b + b * (dl / avg_doc_len); float4 tf_component = (tf * (k1 + 1.0f)) / (tf + k1 * len_norm); return idf * tf_component; } /* * Compute BM25 score for a single posting. */ static inline float4 compute_bm25_score( float4 idf, int32 tf, int32 doc_len, float4 k1, float4 b, float4 avg_doc_len) { float4 len_norm = 1.0f - b + b * ((float4)doc_len / avg_doc_len); float4 tf_component = ((float4)tf * (k1 + 1.0f)) / ((float4)tf + k1 * len_norm); return idf * tf_component; } /* * ------------------------------------------------------------ * Single-Term BMW Scoring * ------------------------------------------------------------ */ /* * Score memtable postings for a single term. * Memtable has no skip index, so we score all postings exhaustively. * * Caller owns `source` (a TpMemtableChainSource over the index's * on-disk chain) and is responsible for closing it. If `source` * is NULL we skip memtable scoring entirely. */ static void score_memtable_single_term( TpTopKHeap *heap, TpDataSource *source, const char *term, float4 idf, float4 k1, float4 b, float4 avg_doc_len, TpBMWStats *stats) { TpPostingData *postings; int i; if (!source) return; /* Get postings for this term */ postings = tp_source_get_postings(source, term); if (!postings || postings->count == 0) { if (postings) tp_source_free_postings(source, postings); return; } /* Score each posting */ for (i = 0; i < postings->count; i++) { ItemPointerData *ctid = &postings->ctids[i]; int32 tf = postings->frequencies[i]; int32 doc_len; float4 score; if ((i & 0xFFF) == 0) CHECK_FOR_INTERRUPTS(); /* Get document length */ doc_len = tp_source_get_doc_length(source, ctid); if (doc_len <= 0) doc_len = 1; /* Fallback for missing entries */ score = compute_bm25_score(idf, tf, doc_len, k1, b, avg_doc_len); if (!tp_topk_dominated(heap, score)) tp_topk_add_memtable(heap, *ctid, score); if (stats) stats->memtable_docs++; } tp_source_free_postings(source, postings); } /* * Score segment postings for a single term using BMW. */ static void score_segment_single_term_bmw( TpTopKHeap *heap, TpSegmentReader *reader, const char *term, float4 idf, float4 k1, float4 b, float4 avg_doc_len, TpBMWStats *stats) { TpSegmentPostingIterator iter; TpSegmentPosting *posting; TpDictEntry *dict_entry; uint32 block_count; float4 *block_max_scores; uint32 i; /* Initialize iterator for this term */ if (!tp_segment_posting_iterator_init(&iter, reader, term)) return; /* Term not found in segment */ /* Get dictionary entry for block count and skip index */ dict_entry = &iter.dict_entry; block_count = dict_entry->block_count; /* Pre-compute block max scores */ block_max_scores = palloc(block_count * sizeof(float4)); for (i = 0; i < block_count; i++) { TpSkipEntry skip; tp_segment_read_skip_entry( reader, dict_entry->skip_index_offset, i, &skip); block_max_scores[i] = tp_compute_block_max_score(&skip, idf, k1, b, avg_doc_len); } /* Process blocks with BMW */ for (i = 0; i < block_count; i++) { float4 threshold = tp_topk_threshold(heap); float4 block_max = block_max_scores[i]; CHECK_FOR_INTERRUPTS(); /* Skip block if it can't beat threshold */ if (block_max < threshold) { if (stats) stats->blocks_skipped++; continue; } if (stats) stats->blocks_scanned++; /* Load and score this block */ iter.current_block = i; iter.finished = false; /* Reset so we can process this block */ tp_segment_posting_iterator_load_block(&iter); while (tp_segment_posting_iterator_next(&iter, &posting)) { float4 score; /* * Break if iterator auto-advanced to next block. * This ensures we only process block i, allowing the outer * for loop to apply threshold checks to subsequent blocks. */ if (iter.current_block != i) break; /* Skip dead docs */ if (!tp_segment_is_alive(reader, posting->doc_id)) { if (stats) stats->dead_docs_skipped++; continue; } score = compute_bm25_score( idf, posting->frequency, posting->doc_length, k1, b, avg_doc_len); if (!tp_topk_dominated(heap, score)) { tp_topk_add_segment( heap, reader->root_block, posting->doc_id, score); } if (stats) stats->segment_docs_scored++; } } pfree(block_max_scores); tp_segment_posting_iterator_free(&iter); } int tp_score_single_term_bmw( TpLocalIndexState *local_state, Relation index, const TpSegmentGraphSnapshot *snapshot, TpDataSource *memtable_src, const char *term, float4 idf, float4 k1, float4 b, float4 avg_doc_len, int max_results, ItemPointerData *result_ctids, float4 *result_scores, TpBMWStats *stats) { TpTopKHeap heap; int result_count; (void)local_state; /* reserved for future use */ /* Initialize stats */ if (stats) memset(stats, 0, sizeof(TpBMWStats)); /* Initialize top-k heap */ tp_topk_init(&heap, max_results, CurrentMemoryContext); /* Score memtable (exhaustive - no skip index) */ score_memtable_single_term( &heap, memtable_src, term, idf, k1, b, avg_doc_len, stats); /* Score each segment level with BMW */ for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { uint32 root_count; const BlockNumber *roots = tp_segment_graph_snapshot_level(snapshot, level, &root_count); for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentReader *reader = tp_segment_open(index, roots[root_idx]); CHECK_FOR_INTERRUPTS(); score_segment_single_term_bmw( &heap, reader, term, idf, k1, b, avg_doc_len, stats); tp_segment_close(reader); } } /* Resolve CTIDs for segment results before extraction */ tp_topk_resolve_ctids(&heap, index); /* Extract sorted results */ result_count = tp_topk_extract(&heap, result_ctids, result_scores); if (stats) stats->docs_in_results = result_count; return result_count; } /* * ------------------------------------------------------------ * Multi-Term BMW Scoring * ------------------------------------------------------------ */ /* * Per-term state for multi-term BMW scoring. */ typedef struct TpTermState { const char *term; float4 idf; int32 query_freq; /* Query term frequency (for boosting) */ /* Global maximum score across all blocks (for WAND pivot) */ float4 max_score; /* Segment-specific state (reset per segment) */ bool found; /* Term found in current segment */ TpSegmentPostingIterator iter; /* Iterator (contains dict_entry) */ float4 *block_max_scores; /* Pre-computed block max scores */ uint32 *block_last_doc_ids; /* Cached last_doc_id per block */ uint32 cur_doc_id; /* Cached current doc ID */ } TpTermState; /* * Refresh cached doc ID from iterator state. */ static inline void refresh_cur_doc_id(TpTermState *ts) { if (!ts->found || ts->iter.finished) ts->cur_doc_id = UINT32_MAX; else ts->cur_doc_id = tp_segment_posting_iterator_current_doc_id(&ts->iter); } /* * Get current doc ID for a term, or UINT32_MAX if exhausted. * Uses cached value maintained by advance/seek/init functions. */ static inline uint32 term_current_doc_id(TpTermState *ts) { return ts->cur_doc_id; } /* * Score memtable postings for multiple terms. * Memtable has no skip index, so we score all postings exhaustively. */ static void score_memtable_multi_term( TpTopKHeap *heap, TpDataSource *source, TpTermState **terms, int term_count, float4 k1, float4 b, float4 avg_doc_len, TpBMWStats *stats) { HTAB *doc_accum; HASHCTL hash_ctl; int term_idx; if (!source) return; /* Create hash table for document score accumulation */ memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(ItemPointerData); hash_ctl.entrysize = sizeof(DocumentScoreEntry); hash_ctl.hcxt = CurrentMemoryContext; doc_accum = hash_create( "Memtable Doc Accum", 1024, &hash_ctl, HASH_ELEM | HASH_BLOBS); /* Process each term */ for (term_idx = 0; term_idx < term_count; term_idx++) { TpTermState *ts = terms[term_idx]; TpPostingData *postings; int i; postings = tp_source_get_postings(source, ts->term); if (!postings || postings->count == 0) { if (postings) tp_source_free_postings(source, postings); continue; } /* Score each posting for this term */ for (i = 0; i < postings->count; i++) { ItemPointerData *ctid = &postings->ctids[i]; int32 tf = postings->frequencies[i]; int32 doc_len; float4 term_score; DocumentScoreEntry *entry; bool found; /* CHECK every 4096 postings; this loop can be very long * when the memtable holds millions of postings -- gating * amortizes the overhead. */ if ((i & 0xFFF) == 0) CHECK_FOR_INTERRUPTS(); /* Get document length */ doc_len = tp_source_get_doc_length(source, ctid); if (doc_len <= 0) doc_len = 1; /* Compute BM25 term contribution */ term_score = compute_bm25_score( ts->idf, tf, doc_len, k1, b, avg_doc_len) * ts->query_freq; /* Accumulate in hash table */ entry = (DocumentScoreEntry *) hash_search(doc_accum, ctid, HASH_ENTER, &found); if (!found) { entry->ctid = *ctid; entry->score = term_score; entry->doc_length = (float4)doc_len; } else { entry->score += term_score; } } tp_source_free_postings(source, postings); } /* Add accumulated documents to heap */ { HASH_SEQ_STATUS seq; DocumentScoreEntry *entry; hash_seq_init(&seq, doc_accum); while ((entry = hash_seq_search(&seq)) != NULL) { if (!tp_topk_dominated(heap, entry->score)) tp_topk_add_memtable(heap, entry->ctid, entry->score); if (stats) stats->memtable_docs++; } } hash_destroy(doc_accum); } /* * Advance a term iterator to the next document. * Returns true if iterator is still active, false if exhausted. */ static bool advance_term_iterator(TpTermState *ts) { ts->iter.current_in_block++; if (ts->iter.current_in_block >= ts->iter.skip_entry.doc_count) { ts->iter.current_block++; if (ts->iter.current_block >= ts->iter.dict_entry.block_count) { ts->iter.finished = true; ts->cur_doc_id = UINT32_MAX; return false; } ts->iter.current_in_block = 0; if (!tp_segment_posting_iterator_load_block(&ts->iter)) { ts->iter.finished = true; ts->cur_doc_id = UINT32_MAX; return false; } } ts->cur_doc_id = ts->iter.block_postings[ts->iter.current_in_block].doc_id; return true; } /* * Seek a term iterator to target doc ID using binary search on cached skip * data. Returns true if iterator is still active (positioned at doc >= * target), false if exhausted. * * Uses pre-loaded block_last_doc_ids for O(log blocks) in-memory binary * search, avoiding I/O during the search. Only loads the target block from * disk. * * Post-condition on `return true`: ts->cur_doc_id >= target_doc_id. * * Both the in-block linear scan and the binary-search-selected block can * exhaust without finding doc >= target if the cached skip data * (skip_entry.last_doc_id, block_last_doc_ids[]) is inconsistent with * on-disk block_postings -- a condition observed on MS MARCO under * concurrent-insert segment topology. In that case we keep loading * subsequent blocks until we find one whose linear scan succeeds, or we * exhaust the iterator. Without this guarantee, callers like seek_to_pivot * spin forever (gdb stack on the production hang showed * seek_to_pivot:1329 with cur_doc_id < pivot_doc_id after a `return true` * from this function). */ static bool seek_term_to_doc(TpTermState *ts, uint32 target_doc_id) { uint32 block_count; int left, right, mid; uint32 target_block; if (!ts->found || ts->iter.finished) return false; /* * Fast path: target is in (or before) the current block, per the * cached last_doc_id. Linear-scan from current_in_block. * * If the cached last_doc_id is accurate, the loop below will find a * doc >= target before exhausting the block. If it is *not* accurate * (i.e., last_doc_id >= target but no posting in the block is >= * target), the loop falls through and the block-advancing loop below * picks up where we left off. */ if (target_doc_id <= ts->iter.skip_entry.last_doc_id) { while (ts->iter.current_in_block < ts->iter.skip_entry.doc_count) { uint32 doc_id = ts->iter.block_postings[ts->iter.current_in_block].doc_id; if (doc_id >= target_doc_id) { ts->cur_doc_id = doc_id; return true; } ts->iter.current_in_block++; } /* Fall through: advance past current block. */ } else { /* * Target is past current block's cached last_doc_id. Binary * search the per-term skip cache to find the first block whose * last_doc_id >= target. */ block_count = ts->iter.dict_entry.block_count; left = ts->iter.current_block + 1; right = (int)block_count - 1; while (left < right) { mid = left + (right - left) / 2; if (ts->block_last_doc_ids[mid] < target_doc_id) left = mid + 1; else right = mid; } target_block = left; if (target_block >= block_count) { ts->iter.finished = true; ts->cur_doc_id = UINT32_MAX; return false; } /* Position at the start of the binary-search-selected block. */ ts->iter.current_block = target_block; ts->iter.current_in_block = 0; if (!tp_segment_posting_iterator_load_block(&ts->iter)) { ts->iter.finished = true; ts->cur_doc_id = UINT32_MAX; return false; } } /* * Block-advancing scan loop. Keep loading subsequent blocks until * we find a posting >= target_doc_id or exhaust the iterator. * * This loop is the durable post-condition guarantor: it terminates * only by finding doc >= target (and returning true) or by * exhausting all blocks (and returning false). It does not rely on * cached skip data being consistent with block_postings -- if a * block's actual postings are all < target despite a cached * last_doc_id >= target, we simply move on to the next block. * * Termination: each iteration of the outer loop strictly increments * current_block, bounded by dict_entry.block_count. The inner * while-loop scans a single block, bounded by skip_entry.doc_count. */ for (;;) { /* * Cancelability: each iteration of the outer loop calls * tp_segment_posting_iterator_load_block, which performs disk * I/O and possibly block decompression. Under the cache- * inconsistency scenario this function defends against, the * loop can iterate across many blocks. Without this CHECK, * statement_timeout / pg_cancel_backend cannot abort the * scan, and seek_to_pivot's CHECK_FOR_INTERRUPTS one level up * never fires because we never return from this function. */ CHECK_FOR_INTERRUPTS(); while (ts->iter.current_in_block < ts->iter.skip_entry.doc_count) { uint32 doc_id = ts->iter.block_postings[ts->iter.current_in_block].doc_id; if (doc_id >= target_doc_id) { ts->cur_doc_id = doc_id; return true; } ts->iter.current_in_block++; } ts->iter.current_block++; if (ts->iter.current_block >= ts->iter.dict_entry.block_count) { ts->iter.finished = true; ts->cur_doc_id = UINT32_MAX; return false; } ts->iter.current_in_block = 0; if (!tp_segment_posting_iterator_load_block(&ts->iter)) { ts->iter.finished = true; ts->cur_doc_id = UINT32_MAX; return false; } } } /* * Initialize term states for a segment. * Returns count of active iterators (terms found in segment). * Terms array is sorted by doc ID after initialization. */ static int init_segment_term_states( TpTermState **terms, int term_count, TpSegmentReader *reader, float4 k1, float4 b, float4 avg_doc_len) { int active_count = 0; int term_idx; for (term_idx = 0; term_idx < term_count; term_idx++) { TpTermState *ts = terms[term_idx]; ts->found = false; ts->max_score = 0.0f; ts->cur_doc_id = UINT32_MAX; ts->block_max_scores = NULL; ts->block_last_doc_ids = NULL; if (!tp_segment_posting_iterator_init(&ts->iter, reader, ts->term)) continue; ts->found = true; /* Pre-load skip entries for BMW threshold checks and fast seeking */ if (ts->iter.dict_entry.block_count > 0) { uint32 block_idx; uint32 block_count = ts->iter.dict_entry.block_count; TpSkipEntry *skip_cache; ts->block_max_scores = palloc(block_count * sizeof(float4)); ts->block_last_doc_ids = palloc(block_count * sizeof(uint32)); /* Allocate and populate skip entry cache for the iterator */ skip_cache = palloc(block_count * sizeof(TpSkipEntry)); for (block_idx = 0; block_idx < block_count; block_idx++) { tp_segment_read_skip_entry( reader, ts->iter.dict_entry.skip_index_offset, block_idx, &skip_cache[block_idx]); ts->block_max_scores[block_idx] = tp_compute_block_max_score( &skip_cache[block_idx], ts->idf, k1, b, avg_doc_len); ts->block_last_doc_ids[block_idx] = skip_cache[block_idx].last_doc_id; if (ts->block_max_scores[block_idx] > ts->max_score) ts->max_score = ts->block_max_scores[block_idx]; } ts->max_score *= ts->query_freq; /* Set caches on iterator for load_block to use */ ts->iter.cached_skip_entries = skip_cache; ts->iter.compressed_buf_cache = palloc( TP_MAX_COMPRESSED_BLOCK_SIZE); } if (tp_segment_posting_iterator_load_block(&ts->iter)) { refresh_cur_doc_id(ts); active_count++; } } return active_count; } /* * Free term state resources for a segment. */ static void cleanup_segment_term_states(TpTermState **terms, int term_count) { int term_idx; for (term_idx = 0; term_idx < term_count; term_idx++) { TpTermState *ts = terms[term_idx]; /* * Free BMW-owned caches before iterator_free (which NULLs * the borrowed pointers but doesn't free them). */ if (ts->iter.cached_skip_entries) pfree(ts->iter.cached_skip_entries); if (ts->iter.compressed_buf_cache) pfree(ts->iter.compressed_buf_cache); if (ts->found) tp_segment_posting_iterator_free(&ts->iter); if (ts->block_max_scores) pfree(ts->block_max_scores); if (ts->block_last_doc_ids) pfree(ts->block_last_doc_ids); } } /* * Compare terms by current doc_id for initial sort. * Exhausted terms (UINT32_MAX) sort to the end. */ static int compare_term_doc_id(const void *a, const void *b) { TpTermState *const *pa = (TpTermState *const *)a; TpTermState *const *pb = (TpTermState *const *)b; uint32 da = term_current_doc_id(*pa); uint32 db = term_current_doc_id(*pb); if (da < db) return -1; if (da > db) return 1; return 0; } /* * Sort term states by current doc_id. * Used once after init_segment_term_states. */ static void sort_terms_by_doc_id(TpTermState **terms, int term_count) { qsort(terms, term_count, sizeof(TpTermState *), compare_term_doc_id); } /* * Restore sorted order after term at position 'ord' advanced. * The term's doc_id increased, so it may need to move right. * Uses linear insertion -- O(1) typical, O(T) worst case. */ static void restore_ordering(TpTermState **terms, int term_count, int ord) { TpTermState *tmp; uint32 doc_id = term_current_doc_id(terms[ord]); int i; for (i = ord + 1; i < term_count; i++) { if (term_current_doc_id(terms[i]) >= doc_id) break; } /* Term needs to move from ord to i-1 */ if (i > ord + 1) { tmp = terms[ord]; memmove(&terms[ord], &terms[ord + 1], (i - ord - 1) * sizeof(TpTermState *)); terms[i - 1] = tmp; } } /* * WAND pivot selection. * * Given terms sorted by current doc_id, walk from lowest doc_id * accumulating each term's max_score. When the sum exceeds * threshold, we've found the pivot. * * Returns true if a pivot was found, false if no term combination * can beat threshold. Sets *pivot_len_out to the number of terms * participating (terms[0..pivot_len-1]) and *pivot_doc_id_out to * the pivot document. */ static bool find_wand_pivot( TpTermState **terms, int term_count, float4 threshold, int *pivot_len_out, uint32 *pivot_doc_id_out) { float4 accumulated = 0.0f; int i; for (i = 0; i < term_count; i++) { uint32 doc_id = term_current_doc_id(terms[i]); if (doc_id == UINT32_MAX) break; /* No more active terms */ accumulated += terms[i]->max_score; if (accumulated > threshold) { /* * Found pivot: terms[i] is the pivot term. * Include any subsequent terms at the same doc_id. */ uint32 pivot_doc = doc_id; int pivot_len = i + 1; while (pivot_len < term_count && term_current_doc_id(terms[pivot_len]) == pivot_doc) pivot_len++; *pivot_len_out = pivot_len; *pivot_doc_id_out = pivot_doc; return true; } } return false; /* Can't beat threshold */ } /* * Compute block-max score upper bound at pivot. * * After WAND pivot selection using global max_scores, refine * with actual block-level upper bounds. Only considers terms * [0..pivot_len-1] since they're the only ones at or before * the pivot. */ static float4 compute_block_max_at_pivot(TpTermState **terms, int pivot_len) { float4 upper_bound = 0.0f; int i; for (i = 0; i < pivot_len; i++) { TpTermState *ts = terms[i]; uint32 block; if (!ts->found || ts->iter.finished) continue; if (ts->block_max_scores == NULL) continue; block = ts->iter.current_block; if (block < ts->iter.dict_entry.block_count) upper_bound += ts->block_max_scores[block] * ts->query_freq; } return upper_bound; } /* * When block-max upper bound < threshold, advance one scorer. * * We pick the pivot term whose current block ends *soonest* * (min_block_end), because seeking that term to min_block_end+1 is * guaranteed to advance it past its current block boundary, which * guarantees forward progress of the outer WAND loop. * * A previous version picked the pivot term with the highest max_score * and seeked it to min_block_end+1. That term's cur_doc_id could be * greater than min_block_end+1, in which case seek_term_to_doc was a * no-op and the outer WAND loop would spin forever (issue #355). * High-max-score selection is a performance heuristic; forward progress * is the correctness requirement. */ static void block_max_skip_advance( TpTermState **terms, int term_count, int pivot_len, int *active_count, TpBMWStats *stats) { int seek_term_idx = -1; uint32 min_block_end = UINT32_MAX; uint32 seek_target; int i; /* Find pivot term whose current block ends soonest. */ for (i = 0; i < pivot_len; i++) { TpTermState *ts = terms[i]; uint32 doc_id; uint32 block; uint32 block_last; doc_id = term_current_doc_id(ts); if (doc_id == UINT32_MAX) continue; block = ts->iter.current_block; if (ts->block_last_doc_ids != NULL && block < ts->iter.dict_entry.block_count) { block_last = ts->block_last_doc_ids[block]; if (block_last < min_block_end) { min_block_end = block_last; seek_term_idx = i; } } else if (seek_term_idx < 0) { /* * Fallback: term has no cached block_last_doc_ids. Pick * the first such term so we still make progress. */ seek_term_idx = i; } } if (seek_term_idx < 0) return; /* Seek target: past the chosen term's current block. */ if (min_block_end == UINT32_MAX) seek_target = term_current_doc_id(terms[seek_term_idx]) + 1; else seek_target = min_block_end + 1; /* Don't skip past the first non-pivot term */ if (pivot_len < term_count) { uint32 next_doc = term_current_doc_id(terms[pivot_len]); if (next_doc < seek_target) seek_target = next_doc; } /* * Defense-in-depth: seek_term_to_doc(target <= cur_doc_id) is a * no-op and would leave us in an infinite loop. The selection * above guarantees forward progress in the normal case (the * min-block-end term is by construction at cur_doc_id <= * min_block_end < seek_target), but the non-pivot cap above can * pull seek_target back to next_doc, which can equal the chosen * term's cur_doc_id when pivot_doc_id and next_doc coincide on a * tie boundary. In that edge case force a single posting * advance to guarantee progress. */ if (seek_target <= term_current_doc_id(terms[seek_term_idx])) { if (!advance_term_iterator(terms[seek_term_idx])) (*active_count)--; if (stats) stats->blocks_skipped++; } else { if (!seek_term_to_doc(terms[seek_term_idx], seek_target)) (*active_count)--; if (stats) { stats->blocks_skipped++; stats->seeks_performed++; } } restore_ordering(terms, term_count, seek_term_idx); } /* * Seek pre-pivot terms to pivot_doc_id. * Returns true if all terms are still active, false if a term * was exhausted (caller should re-pivot). */ static bool seek_to_pivot( TpTermState **terms, int term_count, int pivot_len, uint32 pivot_doc_id, int *active_count) { int i; for (i = 0; i < pivot_len; i++) { uint32 doc_id; /* * Cancelability: the for loop's `i--; continue;` re-entry * pattern, combined with restore_ordering rotating terms * through slot `i`, was the gdb-observed spin site for the * MS MARCO bucket-8 hang. The underlying cause (seek_term_to_doc * returning true without advancing past target) is now fixed at * the source, but keep this CHECK so any future regression in * BMW invariants is interruptible from SQL rather than * requiring SIGKILL. (The outer WAND loop's CFI is too coarse: * if seek_to_pivot fails to terminate it never returns there.) */ CHECK_FOR_INTERRUPTS(); doc_id = term_current_doc_id(terms[i]); if (doc_id == UINT32_MAX) { (*active_count)--; return false; } if (doc_id < pivot_doc_id) { if (!seek_term_to_doc(terms[i], pivot_doc_id)) { (*active_count)--; restore_ordering(terms, term_count, i); return false; } restore_ordering(terms, term_count, i); i--; /* Re-check: a new term slid into position i */ continue; } } return true; } /* * Verify all pivot terms are aligned at pivot_doc_id. * Seeks may have overshot; if any term is past pivot, return false * so the caller can re-pivot. */ static bool verify_pivot_alignment(TpTermState **terms, int pivot_len, uint32 pivot_doc_id) { int i; for (i = 0; i < pivot_len; i++) { if (term_current_doc_id(terms[i]) != pivot_doc_id) return false; } return true; } /* * Score pivot document by accumulating BM25 contributions from * all confirmed pivot terms. * * Only called after verify_pivot_alignment confirms all pivot * terms are positioned at pivot_doc_id, so each term's current * posting is guaranteed to be the pivot document. */ static float4 score_pivot_document( TpTermState **terms, int pivot_len, float4 k1, float4 b, float4 avg_doc_len) { float4 doc_score = 0.0f; int i; for (i = 0; i < pivot_len; i++) { TpTermState *ts = terms[i]; TpBlockPosting *bp; float4 term_score; if (!ts->found || ts->iter.finished) continue; bp = &ts->iter.block_postings[ts->iter.current_in_block]; term_score = compute_bm25_score( ts->idf, bp->frequency, (int32)decode_fieldnorm(bp->fieldnorm), k1, b, avg_doc_len) * ts->query_freq; doc_score += term_score; } return doc_score; } /* * Score segment postings for multiple terms using WAND traversal. * * Classic WAND: terms are sorted by current doc_id. We walk from * lowest doc_id, accumulating max_scores until the threshold is * exceeded to find the "pivot" document. This skips large doc_id * ranges that can't contribute to top-k results. * * Block-max refinement then checks if block-level upper bounds * at the pivot still beat the threshold, and uses Tantivy-style * skip advancement when they don't. */ static void score_segment_multi_term_bmw( TpTopKHeap *heap, TpSegmentReader *reader, TpTermState **terms, int term_count, float4 k1, float4 b, float4 avg_doc_len, TpBMWStats *stats) { int active_count; active_count = init_segment_term_states( terms, term_count, reader, k1, b, avg_doc_len); if (active_count == 0) { cleanup_segment_term_states(terms, term_count); return; } /* Sort terms by current doc_id for WAND traversal */ sort_terms_by_doc_id(terms, term_count); /* WAND main loop */ while (active_count > 0) { int pivot_len; uint32 pivot_doc_id; float4 threshold; float4 block_upper; float4 doc_score; int i; CHECK_FOR_INTERRUPTS(); threshold = tp_topk_threshold(heap); /* Step 1: Find WAND pivot */ if (!find_wand_pivot( terms, term_count, threshold, &pivot_len, &pivot_doc_id)) break; /* No term combination can beat threshold */ /* Step 2: Seek pre-pivot terms to pivot_doc_id */ if (!seek_to_pivot( terms, term_count, pivot_len, pivot_doc_id, &active_count)) continue; /* Re-pivot with updated positions */ /* * Step 3: Block-max refinement. * Check if block-level upper bound still beats threshold. * * Correctness note (#365): block_max_skip_advance() advances * a pivot term past its current block via seek_term_to_doc. * Docs in the skipped block that exist in *non-pivot* terms' * posting lists too can still be top-K candidates -- their * total true score includes non-pivot term contributions * that the pivot's block_upper alone doesn't capture. * * The safe-skip condition is therefore: * block_upper(pivot) + max_score_sum(non-pivot) <= threshold * (an upper bound on a hypothetical doc's score using * pivot terms' actual block-max plus non-pivot terms' global * max). Only then can we skip without losing a top-K * candidate -- or under-scoring one we later see via another * pivot, after a relevant term already jumped over it. * * Previous version used block_upper alone, which let * block_max_skip_advance jump a term past a doc that other * pivot iterations would later pick up with one term missing * from its score sum. Manifests at large K (low heap * threshold) where the looser check fires for blocks whose * docs are still top-K when non-pivot contributions count. */ block_upper = compute_block_max_at_pivot(terms, pivot_len); { float4 non_pivot_max = 0.0f; int np; for (np = pivot_len; np < term_count; np++) { TpTermState *ts = terms[np]; if (!ts->found || ts->iter.finished) continue; non_pivot_max += ts->max_score; } if ((block_upper + non_pivot_max) <= threshold) { block_max_skip_advance( terms, term_count, pivot_len, &active_count, stats); continue; } } if (stats) stats->blocks_scanned++; /* Step 4: Verify all pivot terms are at pivot_doc_id */ if (!verify_pivot_alignment(terms, pivot_len, pivot_doc_id)) continue; /* Re-pivot with new positions */ /* Skip dead docs */ if (!tp_segment_is_alive(reader, pivot_doc_id)) { if (stats) stats->dead_docs_skipped++; } else { /* Step 5: Score the pivot document */ doc_score = score_pivot_document(terms, pivot_len, k1, b, avg_doc_len); if (doc_score > 0.0f && !tp_topk_dominated(heap, doc_score)) tp_topk_add_segment( heap, reader->root_block, pivot_doc_id, doc_score); if (stats) stats->segment_docs_scored++; } /* * Step 6: Advance all pivot terms past pivot_doc_id. * Iterate backward so restore_ordering shifts don't * skip any terms. */ for (i = pivot_len - 1; i >= 0; i--) { if (term_current_doc_id(terms[i]) == pivot_doc_id) { if (!advance_term_iterator(terms[i])) active_count--; restore_ordering(terms, term_count, i); } } } cleanup_segment_term_states(terms, term_count); } /* * Score documents using multi-term Block-Max WAND. */ int tp_score_multi_term_bmw( TpLocalIndexState *local_state, Relation index, const TpSegmentGraphSnapshot *snapshot, TpDataSource *memtable_src, char **query_terms, int term_count, int32 *query_freqs, float4 *idfs, float4 k1, float4 b, float4 avg_doc_len, int max_results, ItemPointerData *result_ctids, float4 *result_scores, TpBMWStats *stats) { TpTopKHeap heap; TpTermState **terms; int result_count; int i; (void)local_state; /* reserved for future use */ /* Initialize stats */ if (stats) memset(stats, 0, sizeof(TpBMWStats)); /* Initialize top-k heap */ tp_topk_init(&heap, max_results, CurrentMemoryContext); /* Initialize term states */ terms = palloc(term_count * sizeof(TpTermState *)); for (i = 0; i < term_count; i++) { terms[i] = palloc(sizeof(TpTermState)); terms[i]->term = query_terms[i]; terms[i]->idf = idfs[i]; terms[i]->query_freq = query_freqs[i]; } /* Score memtable (exhaustive - no skip index) */ score_memtable_multi_term( &heap, memtable_src, terms, term_count, k1, b, avg_doc_len, stats); /* Score each segment with block-based BMW */ for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { uint32 root_count; const BlockNumber *roots = tp_segment_graph_snapshot_level(snapshot, level, &root_count); for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentReader *reader = tp_segment_open(index, roots[root_idx]); CHECK_FOR_INTERRUPTS(); score_segment_multi_term_bmw( &heap, reader, terms, term_count, k1, b, avg_doc_len, stats); tp_segment_close(reader); } } for (i = 0; i < term_count; i++) pfree(terms[i]); pfree(terms); /* Resolve CTIDs for segment results before extraction */ tp_topk_resolve_ctids(&heap, index); /* Extract sorted results */ result_count = tp_topk_extract(&heap, result_ctids, result_scores); if (stats) stats->docs_in_results = result_count; return result_count; } timescale-pg_textsearch-b63fe58/src/scoring/bmw.h000066400000000000000000000133151525762047700221550ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * bmw.h - Block-Max WAND query optimization * * Implements early termination for top-k queries by computing upper bounds * on block scores and skipping blocks that cannot contribute to results. */ #pragma once #include #include #include #include "index/source.h" #include "index/state.h" #include "segment/graph_snapshot.h" #include "segment/segment.h" /* * Top-K min-heap for maintaining threshold during scoring. * * Heap property: parent score <= child scores (minimum at root). * This allows O(1) threshold access and O(log k) updates. * When heap is full, threshold = root score (minimum in top-k). * * Supports deferred CTID resolution: segment results store (seg_block, doc_id) * and resolve CTIDs only at extraction time. Memtable results have CTIDs * immediately (seg_block = InvalidBlockNumber). */ typedef struct TpTopKHeap { ItemPointerData *ctids; /* Array of k CTIDs (resolved at extraction) */ BlockNumber *seg_blocks; /* Segment root block (InvalidBlockNumber = memtable) */ uint32 *doc_ids; /* Segment-local doc IDs */ float4 *scores; /* Parallel array of scores */ int capacity; /* k - maximum results */ int size; /* Current entries (0 to k) */ } TpTopKHeap; /* * Initialize a top-k heap. * Allocates arrays in the given memory context. */ extern void tp_topk_init(TpTopKHeap *heap, int k, MemoryContext ctx); /* * Free a top-k heap's allocated memory. */ extern void tp_topk_free(TpTopKHeap *heap); /* * Get current threshold (minimum score to enter top-k). * Returns 0 if heap not yet full. */ static inline float4 tp_topk_threshold(TpTopKHeap *heap) { return (heap->size >= heap->capacity) ? heap->scores[0] : 0.0f; } /* * Check if a score is definitely dominated (cannot enter top-k). * Quick check to avoid heap operations for non-competitive docs. * Returns false for equal scores since they may qualify via CTID tie-breaking. */ static inline bool tp_topk_dominated(TpTopKHeap *heap, float4 score) { return heap->size >= heap->capacity && score < heap->scores[0]; } /* * Add a memtable result to the top-k heap. * CTID is known immediately for memtable entries. */ extern void tp_topk_add_memtable(TpTopKHeap *heap, ItemPointerData ctid, float4 score); /* * Add a segment result to the top-k heap. * CTID resolution is deferred until extraction. */ extern void tp_topk_add_segment( TpTopKHeap *heap, BlockNumber seg_block, uint32 doc_id, float4 score); /* * Resolve CTIDs for segment results in the heap. * Must be called before tp_topk_extract. * Opens segments as needed to look up CTIDs. */ extern void tp_topk_resolve_ctids(TpTopKHeap *heap, Relation index); /* * Extract sorted results from heap (descending by score). * Returns number of results extracted. * After extraction, heap is empty. * Note: Call tp_topk_resolve_ctids first if heap contains segment results. */ extern int tp_topk_extract(TpTopKHeap *heap, ItemPointerData *ctids, float4 *scores); /* * BMW statistics for debugging/EXPLAIN ANALYZE */ typedef struct TpBMWStats { uint64 blocks_scanned; /* Segment blocks actually scored */ uint64 blocks_skipped; /* Segment blocks skipped by BMW */ uint64 memtable_docs; /* Documents scored from memtable (exhaustive) */ uint64 segment_docs_scored; /* Documents scored from segments */ uint64 docs_in_results; /* Documents in final results */ uint64 seeks_performed; /* Binary search seeks executed */ uint64 dead_docs_skipped; /* Dead docs filtered by alive bitset */ } TpBMWStats; /* * Score documents using single-term Block-Max WAND. * * For single-term queries, uses block-level upper bounds to skip * blocks that cannot contribute to top-k results. * * `memtable_src` is the already-built chain source over the on-disk * memtable chain (may be NULL when the chain is empty). The caller * retains ownership and is responsible for closing it after BMW * returns. Threading the source in (instead of re-creating it * here) avoids a second full chain walk on every query. * * Returns number of results (up to max_results). */ extern int tp_score_single_term_bmw( TpLocalIndexState *local_state, Relation index, const TpSegmentGraphSnapshot *snapshot, TpDataSource *memtable_src, const char *term, float4 idf, float4 k1, float4 b, float4 avg_doc_len, int max_results, ItemPointerData *result_ctids, float4 *result_scores, TpBMWStats *stats); /* * Score documents using multi-term Block-Max WAND. * * For multi-term queries, uses the WAND algorithm with block-level * upper bounds to find top-k documents efficiently. * * `memtable_src` is the already-built chain source over the on-disk * memtable chain (may be NULL when the chain is empty). The caller * retains ownership. * * Returns number of results (up to max_results). */ extern int tp_score_multi_term_bmw( TpLocalIndexState *local_state, Relation index, const TpSegmentGraphSnapshot *snapshot, TpDataSource *memtable_src, char **terms, int term_count, int32 *query_freqs, float4 *idfs, float4 k1, float4 b, float4 avg_doc_len, int max_results, ItemPointerData *result_ctids, float4 *result_scores, TpBMWStats *stats); /* * Compute block maximum BM25 score from skip entry metadata. * * Uses block_max_tf and block_max_norm to compute upper bound on * any document's score in the block. */ extern float4 tp_compute_block_max_score( TpSkipEntry *skip, float4 idf, float4 k1, float4 b, float4 avg_doc_len); timescale-pg_textsearch-b63fe58/src/scoring/snapshot.c000066400000000000000000000122401525762047700232160ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * One published index generation per executor, shared by ranked scans and * score expressions. No buffer or LWLock survives a scoring call. */ #include #include #include #include #include #include #include "index/state.h" #include "memtable/cache_source.h" #include "scoring/snapshot.h" typedef struct TpScoringIndex { Oid index_oid; RelFileLocator locator; TpScoringSnapshot snapshot; struct TpScoringIndex *next; } TpScoringIndex; typedef struct TpScoringStatement { MemoryContext context; MemoryContextCallback cleanup; TpScoringIndex *indexes; struct TpScoringStatement *next; } TpScoringStatement; static ExecutorRun_hook_type previous_run; static ExecutorFinish_hook_type previous_finish; static MemoryContext current_query_context; static TpScoringStatement *statements; static uint64 next_serial; static void tp_scoring_statement_cleanup(void *arg) { TpScoringStatement *statement = arg; TpScoringStatement **link = &statements; while (*link != statement) { Assert(*link != NULL); link = &(*link)->next; } *link = statement->next; } static void tp_scoring_executor_run( QueryDesc *query, ScanDirection direction, uint64 count #if PG_VERSION_NUM < 180000 , bool execute_once #endif ) { MemoryContext previous_context = current_query_context; current_query_context = query->estate->es_query_cxt; PG_TRY(); { if (previous_run) previous_run( query, direction, count #if PG_VERSION_NUM < 180000 , execute_once #endif ); else standard_ExecutorRun( query, direction, count #if PG_VERSION_NUM < 180000 , execute_once #endif ); } PG_FINALLY(); { current_query_context = previous_context; } PG_END_TRY(); } static void tp_scoring_executor_finish(QueryDesc *query) { MemoryContext previous_context = current_query_context; current_query_context = query->estate->es_query_cxt; PG_TRY(); { if (previous_finish) previous_finish(query); else standard_ExecutorFinish(query); } PG_FINALLY(); { current_query_context = previous_context; } PG_END_TRY(); } void tp_scoring_snapshot_init(void) { previous_run = ExecutorRun_hook; ExecutorRun_hook = tp_scoring_executor_run; previous_finish = ExecutorFinish_hook; ExecutorFinish_hook = tp_scoring_executor_finish; } TpScoringSnapshot * tp_scoring_snapshot_get( Relation index, const char *const *query_terms, int query_term_count, TpDataSource **initial_source) { MemoryContext context = current_query_context != NULL ? current_query_context : CurrentMemoryContext; MemoryContext old; TpScoringStatement *statement; TpScoringIndex *entry; TpDataSource *source; TpLocalIndexState *state; const char *no_terms = NULL; int64 total_docs; int64 total_len; if (initial_source != NULL) *initial_source = NULL; for (statement = statements; statement; statement = statement->next) if (statement->context == context) break; if (statement == NULL) { statement = MemoryContextAllocZero(context, sizeof(*statement)); statement->context = context; statement->cleanup.func = tp_scoring_statement_cleanup; statement->cleanup.arg = statement; MemoryContextRegisterResetCallback(context, &statement->cleanup); statement->next = statements; statements = statement; } for (entry = statement->indexes; entry; entry = entry->next) if (entry->index_oid == RelationGetRelid(index) && RelFileLocatorEquals(entry->locator, index->rd_locator)) return &entry->snapshot; /* Exclude concurrent physical rewrites between standalone calls. */ LockRelationOid(RelationGetRelid(index), AccessShareLock); state = tp_get_local_index_state(RelationGetRelid(index)); if (state == NULL) elog(ERROR, "could not get BM25 index state"); old = MemoryContextSwitchTo(context); entry = palloc0(sizeof(*entry)); entry->index_oid = RelationGetRelid(index); entry->locator = index->rd_locator; entry->snapshot.serial = ++next_serial; entry->snapshot.recovery = RecoveryInProgress(); entry->snapshot.graph = tp_segment_graph_snapshot_create(index); MemoryContextSwitchTo(old); source = tp_memtable_source_create_for_snapshot( state, index, &entry->snapshot.graph->memtable, entry->snapshot.recovery, initial_source != NULL ? query_terms : &no_terms, initial_source != NULL ? query_term_count : 0); total_docs = entry->snapshot.graph->metapage.total_docs; total_len = entry->snapshot.graph->metapage.total_len; if (source != NULL) { total_docs += source->total_docs; total_len += source->total_len; if (initial_source == NULL) tp_source_close(source); } entry->snapshot.total_docs = Min(total_docs, PG_INT32_MAX); entry->snapshot.avg_doc_len = total_docs > 0 ? (float4)((double)total_len / entry->snapshot.total_docs) : 0; entry->next = statement->indexes; statement->indexes = entry; if (initial_source != NULL) *initial_source = source; return &entry->snapshot; } timescale-pg_textsearch-b63fe58/src/scoring/snapshot.h000066400000000000000000000014041525762047700232230ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #pragma once #include #include "index/source.h" #include "segment/graph_snapshot.h" typedef struct TpScoringSnapshot { uint64 serial; TpSegmentGraphSnapshot *graph; int32 total_docs; float4 avg_doc_len; bool recovery; } TpScoringSnapshot; extern void tp_scoring_snapshot_init(void); /* * Optionally return a new snapshot's totals source for the caller to close. * Standalone callers tokenize without source locks and request only totals. */ extern TpScoringSnapshot *tp_scoring_snapshot_get( Relation index, const char *const *query_terms, int query_term_count, TpDataSource **initial_source); timescale-pg_textsearch-b63fe58/src/segment/000077500000000000000000000000001525762047700212125ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/segment/alive_bitset.c000066400000000000000000000136121525762047700240330ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * alive_bitset.c - Per-segment alive bitset implementation */ #include #include #include #include #include #include #include "segment/alive_bitset.h" #include "segment/io.h" #include "segment/pagemapper.h" /* * Initialize a raw bitset buffer to all-alive for num_docs. * Sets all bits to 1, then clears trailing bits in the last byte. */ void tp_alive_bitset_init_data(uint8 *buf, uint32 num_docs) { uint32 nbytes = tp_alive_bitset_size(num_docs); memset(buf, 0xFF, nbytes); if (num_docs % 8 != 0) { uint8 mask = (1 << (num_docs % 8)) - 1; buf[nbytes - 1] &= mask; } } /* * Create an in-memory bitset with all docs alive. * * Allocates a TpAliveBitset with all bits set to 1, then clears * any trailing bits beyond num_docs in the final byte. */ TpAliveBitset * tp_alive_bitset_create(uint32 num_docs) { TpAliveBitset *bitset; uint32 nbytes; bitset = palloc(sizeof(TpAliveBitset)); bitset->num_docs = num_docs; bitset->alive_count = num_docs; nbytes = tp_alive_bitset_size(num_docs); bitset->bits = palloc(nbytes); tp_alive_bitset_init_data(bitset->bits, num_docs); return bitset; } /* * Load the alive bitset from a segment into memory. * * Returns NULL if the segment has no alive bitset (pre-V5 segments * have alive_bitset_offset == 0). */ TpAliveBitset * tp_alive_bitset_load(TpSegmentReader *reader) { TpAliveBitset *bitset; uint32 nbytes; if (reader->header->alive_bitset_offset == 0) return NULL; bitset = palloc(sizeof(TpAliveBitset)); bitset->num_docs = reader->header->num_docs; bitset->alive_count = reader->header->alive_count; nbytes = tp_alive_bitset_size(bitset->num_docs); bitset->bits = palloc(nbytes); tp_segment_read( reader, reader->header->alive_bitset_offset, bitset->bits, nbytes); return bitset; } /* * Mark a document as dead in the in-memory bitset. * * Returns true if the document was previously alive (and alive_count * was decremented), false if it was already dead or out of range. */ bool tp_alive_bitset_mark_dead(TpAliveBitset *bitset, uint32 doc_id) { uint32 byte_idx; uint8 bit_mask; Assert(doc_id < bitset->num_docs); /* * Defense in depth for non-assert builds: a doc_id past num_docs is * stale -- e.g. a segment shrunk by a concurrent merge between VACUUM's * identify and mark phases. Callers serialize source replacement with * the maintenance object lock, but the per-index lock is not held across * bitmap updates. */ if (doc_id >= bitset->num_docs) return false; byte_idx = doc_id >> 3; bit_mask = (uint8)(1 << (doc_id & 7)); if (!(bitset->bits[byte_idx] & bit_mask)) return false; /* already dead */ bitset->bits[byte_idx] &= ~bit_mask; bitset->alive_count--; return true; } /* * Write the in-memory bitset back to segment pages using * GenericXLog, then update alive_count in the segment header. * * The header update is batched into the final bitset page's * GenericXLog transaction so that the bitset data and * alive_count are crash-atomic. If the header page happens * to be the same physical page as the last bitset page, both * updates go into a single GenericXLog with one buffer. */ void tp_alive_bitset_write( TpAliveBitset *bitset, TpSegmentReader *reader, Relation index) { uint64 bitset_offset; uint32 nbytes; uint32 bytes_written; bitset_offset = reader->header->alive_bitset_offset; nbytes = tp_alive_bitset_size(bitset->num_docs); bytes_written = 0; /* * Write bitset data page by page. On the last page, * batch the header alive_count update into the same * GenericXLog transaction for crash atomicity. */ while (bytes_written < nbytes) { uint64 logical_offset; uint32 logical_page; uint32 page_off; uint32 bytes_on_page; BlockNumber physical_block; Buffer buf; Page page; GenericXLogState *xstate; bool is_last_page; Buffer header_buf = InvalidBuffer; logical_offset = bitset_offset + bytes_written; logical_page = tp_logical_page(logical_offset); page_off = tp_page_offset(logical_offset); Assert(logical_page < reader->num_pages); physical_block = reader->page_map[logical_page]; bytes_on_page = SEGMENT_DATA_PER_PAGE - page_off; if (bytes_on_page > nbytes - bytes_written) bytes_on_page = nbytes - bytes_written; is_last_page = (bytes_written + bytes_on_page >= nbytes); buf = ReadBuffer(index, physical_block); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); xstate = GenericXLogStart(index); page = GenericXLogRegisterBuffer(xstate, buf, 0); memcpy((char *)page + SizeOfPageHeaderData + page_off, bitset->bits + bytes_written, bytes_on_page); /* * On the last bitset page, also update alive_count * in the segment header within the same GenericXLog * transaction for crash atomicity. */ if (is_last_page) { if (physical_block == reader->root_block) { /* * Header is on the same page — update the * already-registered buffer. */ TpSegmentHeader *hdr = (TpSegmentHeader *)PageGetContents( page); hdr->alive_count = bitset->alive_count; } else { Page header_page; header_buf = ReadBuffer(index, reader->root_block); LockBuffer(header_buf, BUFFER_LOCK_EXCLUSIVE); header_page = GenericXLogRegisterBuffer(xstate, header_buf, 0); ((TpSegmentHeader *)PageGetContents(header_page)) ->alive_count = bitset->alive_count; } } GenericXLogFinish(xstate); UnlockReleaseBuffer(buf); if (BufferIsValid(header_buf)) UnlockReleaseBuffer(header_buf); bytes_written += bytes_on_page; } } /* * Free an in-memory alive bitset. */ void tp_alive_bitset_free(TpAliveBitset *bitset) { if (bitset == NULL) return; if (bitset->bits) pfree(bitset->bits); pfree(bitset); } timescale-pg_textsearch-b63fe58/src/segment/alive_bitset.h000066400000000000000000000060061525762047700240370ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * alive_bitset.h - Per-segment alive bitset for tombstone tracking * * Each segment contains a flat bitset with one bit per document. * Bit 1 = alive, bit 0 = dead. Pre-allocated at segment creation * with all bits set. VACUUM clears bits for dead documents. */ #pragma once #include #include #include #include "segment/io.h" /* * In-memory alive bitset, used during VACUUM to batch-update * the on-disk bitset. Not used during scoring — scoring reads * bitset bytes directly from segment pages via paged I/O. */ typedef struct TpAliveBitset { uint8 *bits; /* Bitset data (1=alive, 0=dead) */ uint32 num_docs; /* Total doc count (bitset capacity) */ uint32 alive_count; /* Number of alive docs */ } TpAliveBitset; /* * Byte size of bitset data for a given doc count. */ static inline uint32 tp_alive_bitset_size(uint32 num_docs) { return num_docs / 8 + (num_docs % 8 != 0); } /* * Check if a doc is alive by reading from segment pages. * Used in the hot scoring path — reads one byte via paged * I/O. Returns true if the segment has no bitset (pre-V5) * or if all docs are alive, or if the doc's bit is set. */ static inline bool tp_segment_is_alive(TpSegmentReader *reader, uint32 doc_id) { uint64 byte_offset; uint8 byte; if (reader->header->alive_bitset_offset == 0 || reader->header->alive_count == reader->header->num_docs) return true; byte_offset = reader->header->alive_bitset_offset + (doc_id >> 3); tp_segment_read(reader, byte_offset, &byte, 1); return (byte >> (doc_id & 7)) & 1; } /* * Initialize a raw bitset buffer to all-alive for num_docs. * Sets all bits to 1, then clears trailing bits beyond num_docs. * Used by segment writers to avoid duplicating the mask logic. */ extern void tp_alive_bitset_init_data(uint8 *buf, uint32 num_docs); /* * Create an in-memory bitset with all docs alive. * Used during VACUUM to load, modify, and write back. */ extern TpAliveBitset *tp_alive_bitset_create(uint32 num_docs); /* * Load the alive bitset from a segment into memory. * Returns NULL if the segment has no bitset (pre-V5). */ extern TpAliveBitset *tp_alive_bitset_load(TpSegmentReader *reader); /* * Mark a doc as dead. Returns true if it was alive * (i.e., the alive_count was decremented), false if it was * already dead or out of range (doc_id >= num_docs; a no-op * that leaves alive_count unchanged). */ extern bool tp_alive_bitset_mark_dead(TpAliveBitset *bitset, uint32 doc_id); /* * Write the in-memory bitset back to segment pages and * update the segment header's alive_count. Uses * GenericXLog for crash safety. The header update is * batched into the final bitset page's GenericXLog * transaction for atomicity. */ extern void tp_alive_bitset_write( TpAliveBitset *bitset, TpSegmentReader *reader, Relation index); /* * Free an in-memory bitset. */ extern void tp_alive_bitset_free(TpAliveBitset *bitset); timescale-pg_textsearch-b63fe58/src/segment/compaction.c000066400000000000000000002415631525762047700235250ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "constants.h" #include "debug/injection.h" #include "index/metapage.h" #include "index/state.h" #include "segment/alive_bitset.h" #include "segment/compaction.h" #include "segment/format.h" #include "segment/graph_snapshot.h" #include "segment/io.h" #include "segment/merge.h" #include "segment/pagemapper.h" #include "segment/tombstone.h" #define TP_COMPACTION_MAINTENANCE_LOCK_SUBID 3 #define TP_COMPACTION_PUBLICATION_LOCK_SUBID 4 typedef struct TpSegmentEstimate { uint64 bytes; uint64 docs; uint64 terms; uint64 string_bytes; uint64 postings; uint64 skip_entries; uint64 pages; } TpSegmentEstimate; typedef struct TpCompactionSource { BlockNumber root; uint32 source_level; uint32 chain_position; bool has_dead_docs; uint64 total_tokens; uint64 live_bytes; TpSegmentEstimate estimate; } TpCompactionSource; typedef struct TpCompactionBatch { uint32 first_source; uint32 source_count; uint32 output_level; bool uncombinable; TpSegmentEstimate estimate; } TpCompactionBatch; typedef struct TpCompactionPlan { TpCompactionSource *sources; uint32 num_sources; uint32 source_capacity; TpCompactionBatch *batches; uint32 num_batches; uint32 output_capacity; uint16 prefix_counts[TP_MAX_LEVELS]; BlockNumber selected_heads[TP_MAX_LEVELS]; uint16 selected_counts[TP_MAX_LEVELS]; BlockNumber retained_heads[TP_MAX_LEVELS]; uint16 retained_counts[TP_MAX_LEVELS]; } TpCompactionPlan; typedef struct TpCompactionOutput { BlockNumber output_heads[TP_MAX_LEVELS]; uint16 output_counts[TP_MAX_LEVELS]; BlockNumber *owned_output_roots; uint32 owned_output_count; uint32 owned_output_capacity; uint64 removed_docs; uint64 removed_tokens; uint64 added_tokens; TpDetachedTombstoneBatch tombstones; bool publication_started; } TpCompactionOutput; typedef struct TpCompactionPublication { TpIndexMetaPageData metapage; BlockNumber predecessor; uint32 predecessor_level; } TpCompactionPublication; typedef enum TpStatsRebasePolicy { TP_STATS_REBASE_STRICT, TP_STATS_REBASE_CLAMP_LEGACY_VACUUM } TpStatsRebasePolicy; static bool tp_debug_compaction_build_active = false; static void tp_free_compaction_plan(TpCompactionPlan *plan); /* * Fire the injection point for an allocation step of a compaction * build. The allocation paths are shared with memtable spill, so the * points only fire while a compaction build is in progress. */ void tp_compaction_allocation_injection_point(TpCompactionAllocationPhase phase) { const char *point; if (!tp_debug_compaction_build_active) return; switch (phase) { case TP_COMPACTION_ALLOCATION_OUTPUT_DATA: point = TP_INJECTION_COMPACTION_ALLOC_OUTPUT_DATA; break; case TP_COMPACTION_ALLOCATION_PAGE_INDEX: point = TP_INJECTION_COMPACTION_ALLOC_PAGE_INDEX; break; case TP_COMPACTION_ALLOCATION_TOMBSTONE: point = TP_INJECTION_COMPACTION_ALLOC_TOMBSTONE; break; default: return; } TP_INJECTION_POINT(point); } void tp_compaction_lock(Relation index) { /* * Serializes compaction on one index. Uses an object lock in * pg_am's subid space, not a relation lock: the pre-commit * dispatcher holds ShareUpdateExclusiveLock on the index while * signaling a worker that may compact before the writer commits. * The subid keeps this distinct from managed compaction's private * lock. */ LockDatabaseObject( AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_MAINTENANCE_LOCK_SUBID, ExclusiveLock); } bool tp_try_compaction_lock(Relation index) { return ConditionalLockDatabaseObject( AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_MAINTENANCE_LOCK_SUBID, ExclusiveLock); } void tp_compaction_unlock(Relation index) { UnlockDatabaseObject( AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_MAINTENANCE_LOCK_SUBID, ExclusiveLock); } void tp_compaction_publication_lock(Relation index, LOCKMODE mode) { Assert(mode == ShareLock || mode == ExclusiveLock); LockDatabaseObject( AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_PUBLICATION_LOCK_SUBID, mode); } bool tp_try_compaction_publication_lock(Relation index, LOCKMODE mode) { Assert(mode == ShareLock || mode == ExclusiveLock); return ConditionalLockDatabaseObject( AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_PUBLICATION_LOCK_SUBID, mode); } void tp_compaction_publication_unlock(Relation index, LOCKMODE mode) { Assert(mode == ShareLock || mode == ExclusiveLock); UnlockDatabaseObject( AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_PUBLICATION_LOCK_SUBID, mode); } static bool tp_compaction_maintenance_lock_held(Relation index) { LOCKTAG tag; SET_LOCKTAG_OBJECT( tag, MyDatabaseId, AccessMethodRelationId, RelationGetRelid(index), TP_COMPACTION_MAINTENANCE_LOCK_SUBID); return LockHeldByMe(&tag, ExclusiveLock, true); } static bool tp_compaction_is_private(TpLocalIndexState *index_state, Relation index) { if (index_state == NULL || index_state->shared == NULL) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction requires a valid per-index state"))); if (index_state->lock_held) { if (index_state->lock_mode != LW_EXCLUSIVE || !LWLockHeldByMeInMode(&index_state->shared->lock, LW_EXCLUSIVE)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("private compaction requires the per-index " "exclusive lock"))); return true; } if (!tp_compaction_maintenance_lock_held(index)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("runtime compaction requires the per-index " "maintenance lock"))); return false; } uint64 tp_max_segment_size_bytes(void) { return (uint64)tp_max_segment_size_mb * 1024 * 1024; } static bool tp_u64_add(uint64 left, uint64 right, uint64 *result) { return !pg_add_u64_overflow(left, right, result); } static bool tp_u64_multiply(uint64 left, uint64 right, uint64 *result) { return !pg_mul_u64_overflow(left, right, result); } static bool tp_u64_round_up_divide(uint64 value, uint64 divisor, uint64 *result) { uint64 rounded; Assert(divisor > 0); if (value == 0) { *result = 0; return true; } if (!tp_u64_add(value, divisor - 1, &rounded)) return false; *result = rounded / divisor; return true; } /* * Conservative current-format bound. Arithmetic overflow is corrupt * metadata; an otherwise valid oversized source is merely uncombinable. */ static bool tp_estimate_physical_bytes(TpSegmentEstimate *estimate, bool *representable) { uint64 bytes = sizeof(TpSegmentHeader); uint64 contribution; uint64 data_pages; uint64 entries_per_index_page; uint64 index_pages; uint64 total_pages; *representable = true; if (!tp_u64_add(bytes, sizeof(uint32), &bytes) || !tp_u64_multiply(estimate->terms, sizeof(uint32), &contribution) || !tp_u64_add(bytes, contribution, &bytes) || !tp_u64_add(bytes, estimate->string_bytes, &bytes) || !tp_u64_multiply( estimate->terms, sizeof(TpDictEntry), &contribution) || !tp_u64_add(bytes, contribution, &bytes) || !tp_u64_multiply( estimate->postings, sizeof(TpBlockPosting), &contribution) || !tp_u64_add(bytes, contribution, &bytes) || !tp_u64_multiply( estimate->skip_entries, sizeof(TpSkipEntry), &contribution) || !tp_u64_add(bytes, contribution, &bytes) || !tp_u64_multiply(estimate->docs, sizeof(uint8), &contribution) || !tp_u64_add(bytes, contribution, &bytes) || !tp_u64_multiply(estimate->docs, sizeof(BlockNumber), &contribution) || !tp_u64_add(bytes, contribution, &bytes) || !tp_u64_multiply( estimate->docs, sizeof(OffsetNumber), &contribution) || !tp_u64_add(bytes, contribution, &bytes)) return false; if (!tp_document_count_fits(estimate->docs)) *representable = false; else if (!tp_u64_add( bytes, tp_alive_bitset_size((uint32)estimate->docs), &bytes)) return false; if (!tp_u64_round_up_divide(bytes, SEGMENT_DATA_PER_PAGE, &data_pages)) return false; entries_per_index_page = (BLCKSZ - SizeOfPageHeaderData - MAXALIGN(sizeof(TpPageIndexSpecial))) / sizeof(BlockNumber); Assert(entries_per_index_page > 0); if (!tp_u64_round_up_divide( data_pages, entries_per_index_page, &index_pages) || !tp_u64_add(data_pages, index_pages, &total_pages) || !tp_u64_multiply(total_pages, BLCKSZ, &estimate->bytes)) return false; if (!tp_document_count_fits(estimate->docs) || estimate->terms > TP_MAX_DICTIONARY_TERMS || estimate->string_bytes > TP_MAX_STRING_POOL_BYTES || estimate->skip_entries > TP_MAX_GROWABLE_CAPACITY || data_pages > PG_UINT32_MAX || total_pages >= InvalidBlockNumber) *representable = false; return true; } static void tp_collect_source_estimate( TpCompactionSource *source, TpSegmentReader *reader, bool pause_source_estimate) { TpSegmentHeader *header = reader->header; bool representable; if (header->entries_offset < header->strings_offset) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment at block %u has reversed string offsets", source->root))); source->estimate.docs = header->num_docs; source->estimate.terms = header->num_terms; source->estimate.string_bytes = header->entries_offset - header->strings_offset; source->estimate.pages = header->num_pages; for (uint32 term = 0; term < header->num_terms; term++) { TpDictEntry entry; uint64 term_skip_entries; tp_segment_read_dict_entry(reader, header, term, &entry); if (!tp_u64_add( source->estimate.postings, (uint64)entry.doc_freq, &source->estimate.postings)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment posting count overflow at block %u", source->root))); if (!tp_u64_round_up_divide( (uint64)entry.doc_freq, TP_BLOCK_SIZE, &term_skip_entries) || !tp_u64_add( source->estimate.skip_entries, term_skip_entries, &source->estimate.skip_entries)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment skip entry count overflow at block %u", source->root))); } if (pause_source_estimate) TP_INJECTION_POINT(TP_INJECTION_COMPACTION_SOURCE_ESTIMATE); if (!tp_estimate_physical_bytes(&source->estimate, &representable)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment size estimate overflow at block %u", source->root))); if (!representable) source->estimate.bytes = PG_UINT64_MAX; } static BlockNumber tp_collect_source( Relation index, TpCompactionPlan *plan, BlockNumber root, uint32 level, uint32 chain_position, bool estimate_size) { TpCompactionSource *source; TpSegmentReader *reader; BlockNumber next; if (!BlockNumberIsValid(root)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment chain for level %u ended before its " "recorded count", level))); if (plan->num_sources >= plan->source_capacity) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction source plan exceeded its capacity"))); source = &plan->sources[plan->num_sources++]; source->root = root; source->source_level = level; source->chain_position = chain_position; reader = estimate_size ? tp_segment_open(index, root) : tp_segment_open_header(index, root); if (reader == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open segment at block %u", root))); if (reader->header->level != level) { uint32 recorded_level = reader->header->level; tp_segment_close(reader); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment at block %u records level %u but is " "linked from level %u", root, recorded_level, level))); } if (estimate_size) tp_collect_source_estimate( source, reader, plan->num_sources == 1 && tp_compaction_maintenance_lock_held(index)); /* Policy accounting is cheap; dictionary walks are only for size safety. */ source->live_bytes = (uint64)reader->header->num_pages * BLCKSZ; if (reader->header->alive_bitset_offset > 0 && reader->header->num_docs > 0) { uint64 alive = reader->header->alive_count; uint64 docs = reader->header->num_docs; source->live_bytes = source->live_bytes / docs * alive + source->live_bytes % docs * alive / docs; } source->live_bytes = Max(source->live_bytes, (uint64)BLCKSZ); source->has_dead_docs = reader->header->alive_bitset_offset > 0 && reader->header->alive_count < reader->header->num_docs; source->total_tokens = reader->header->total_tokens; next = reader->header->next_segment; tp_segment_close(reader); return next; } static bool tp_estimate_add( const TpSegmentEstimate *left, const TpSegmentEstimate *right, uint64 budget, TpSegmentEstimate *result) { bool representable; memset(result, 0, sizeof(*result)); if (!tp_u64_add(left->docs, right->docs, &result->docs) || !tp_u64_add(left->terms, right->terms, &result->terms) || !tp_u64_add( left->string_bytes, right->string_bytes, &result->string_bytes) || !tp_u64_add(left->postings, right->postings, &result->postings) || !tp_u64_add( left->skip_entries, right->skip_entries, &result->skip_entries) || !tp_u64_add(left->pages, right->pages, &result->pages) || !tp_estimate_physical_bytes(result, &representable)) return false; return representable && result->bytes <= budget; } static bool tp_metapage_identity_matches( const TpIndexMetaPageData *current, const TpIndexMetaPageData *snapshot); static bool tp_metapage_segment_graph_matches(Page page, const TpIndexMetaPage snapshot) { TpIndexMetaPage current = (TpIndexMetaPage)PageGetContents(page); if (!tp_metapage_identity_matches(current, snapshot)) return false; return current->total_docs == snapshot->total_docs && current->total_len == snapshot->total_len && memcmp(current->level_heads, snapshot->level_heads, sizeof(current->level_heads)) == 0 && memcmp(current->level_counts, snapshot->level_counts, sizeof(current->level_counts)) == 0 && (current->version < TP_METAPAGE_VERSION ? snapshot->capabilities == 0 : current->capabilities == snapshot->capabilities); } static void tp_collect_force_sources( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan) { uint32 total_sources = 0; memset(plan, 0, sizeof(*plan)); for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { if (pg_add_u32_overflow( total_sources, (uint32)snapshot->level_counts[level], &total_sources)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment count overflow in index \"%s\"", RelationGetRelationName(index)))); plan->selected_heads[level] = snapshot->level_heads[level]; plan->selected_counts[level] = snapshot->level_counts[level]; plan->retained_heads[level] = InvalidBlockNumber; } if (total_sources == 0) return; plan->source_capacity = total_sources; plan->sources = palloc0(sizeof(TpCompactionSource) * total_sources); for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { BlockNumber current = snapshot->level_heads[level]; for (uint32 position = 0; position < snapshot->level_counts[level]; position++) current = tp_collect_source( index, plan, current, level, position, true); if (BlockNumberIsValid(current)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment chain for level %u exceeds its " "recorded count", level))); } Assert(plan->num_sources == total_sources); plan->num_sources = total_sources; } static uint32 tp_size_class(uint64 bytes) { uint64 limit = TP_BASE_LEVEL_SIZE_BYTES; for (uint32 level = 0; level < TP_MAX_LEVELS - 1; level++) { if (bytes <= limit) return level; if (limit > PG_UINT64_MAX / (uint64)tp_segments_per_level) return TP_MAX_LEVELS - 1; limit *= (uint64)tp_segments_per_level; } return TP_MAX_LEVELS - 1; } static void tp_append_bounded_batches( TpCompactionPlan *plan, uint32 first_source, uint32 source_count) { uint64 budget = tp_max_segment_size_bytes(); uint32 source_index; uint32 source_end; if (source_count == 0) return; if (first_source > plan->num_sources || source_count > plan->num_sources - first_source) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid compaction source range"))); source_index = first_source; source_end = first_source + source_count; while (source_index < source_end) { TpCompactionBatch *batch; if (plan->num_batches >= plan->source_capacity) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction batch plan exceeded its capacity"))); batch = &plan->batches[plan->num_batches++]; batch->first_source = source_index; batch->source_count = 1; batch->estimate = plan->sources[source_index].estimate; batch->uncombinable = batch->estimate.bytes > budget; source_index++; while (!batch->uncombinable && source_index < source_end) { TpSegmentEstimate combined; if (!tp_estimate_add( &batch->estimate, &plan->sources[source_index].estimate, budget, &combined)) break; batch->estimate = combined; batch->source_count++; source_index++; } } } static void tp_build_force_batches(TpCompactionPlan *plan) { uint32 *order; uint16 assigned_counts[TP_MAX_LEVELS]; if (plan->num_sources == 0) return; plan->output_capacity = PG_UINT16_MAX; plan->batches = palloc0(sizeof(TpCompactionBatch) * plan->num_sources); tp_append_bounded_batches(plan, 0, plan->num_sources); memcpy(assigned_counts, plan->retained_counts, sizeof(assigned_counts)); order = palloc(sizeof(uint32) * plan->num_batches); for (uint32 i = 0; i < plan->num_batches; i++) order[i] = i; /* Stable insertion sort: equal-size batches retain source order. */ for (uint32 i = 1; i < plan->num_batches; i++) { uint32 batch_index = order[i]; uint32 position = i; while (position > 0 && plan->batches[order[position - 1]].estimate.bytes < plan->batches[batch_index].estimate.bytes) { order[position] = order[position - 1]; position--; } order[position] = batch_index; } for (uint32 i = 0; i < plan->num_batches; i++) { TpCompactionBatch *batch = &plan->batches[order[i]]; uint32 preferred = tp_size_class(batch->estimate.bytes); uint32 selected = TP_MAX_LEVELS; for (uint32 level = preferred; level < TP_MAX_LEVELS; level++) { if (assigned_counts[level] < PG_UINT16_MAX) { selected = level; break; } } if (selected == TP_MAX_LEVELS) { for (int level = (int)preferred - 1; level >= 0; level--) { if (assigned_counts[level] < PG_UINT16_MAX) { selected = (uint32)level; break; } } } if (selected == TP_MAX_LEVELS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("bounded compaction cannot place %u output " "segments", plan->num_batches))); batch->output_level = selected; assigned_counts[selected]++; } pfree(order); } static bool tp_plan_chains_match(Relation index, TpCompactionPlan *plan) { uint32 source_index = 0; for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { BlockNumber current = InvalidBlockNumber; bool has_selected_sources; while (source_index < plan->num_sources && plan->sources[source_index].source_level < level) source_index++; has_selected_sources = source_index < plan->num_sources && plan->sources[source_index].source_level == level; if (has_selected_sources) current = plan->sources[source_index].root; while (source_index < plan->num_sources && plan->sources[source_index].source_level == level) { TpSegmentReader *reader; if (plan->sources[source_index].root != current) return false; reader = tp_segment_open(index, current); if (reader == NULL) return false; current = reader->header->next_segment; tp_segment_close(reader); source_index++; } if (has_selected_sources && current != plan->retained_heads[level]) return false; } return source_index == plan->num_sources; } static bool tp_plan_is_noop( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan) { Buffer buf; Page page; bool matches; if (plan->num_batches != plan->num_sources) return false; for (uint32 i = 0; i < plan->num_batches; i++) { if (plan->batches[i].source_count != 1 || plan->batches[i].output_level != plan->sources[i].source_level) return false; } /* * A singleton rewrite still helps when its alive bitmap has dead * documents: merge keeps only survivors and corrects corpus * statistics. An all-zero singleton disappears. */ for (uint32 i = 0; i < plan->num_sources; i++) { if (plan->sources[i].has_dead_docs) return false; } buf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); matches = tp_metapage_segment_graph_matches(page, snapshot); UnlockReleaseBuffer(buf); return matches && tp_plan_chains_match(index, plan); } static void tp_initialize_compaction_output( const TpCompactionPlan *plan, TpCompactionOutput *output) { memset(output, 0, sizeof(*output)); memcpy(output->output_heads, plan->retained_heads, sizeof(output->output_heads)); if (plan->num_batches > 0) output->owned_output_roots = palloc( sizeof(BlockNumber) * plan->num_batches); output->owned_output_capacity = plan->num_batches; output->tombstones.head = InvalidBlockNumber; output->tombstones.tail = InvalidBlockNumber; } static void tp_discard_compaction_output(Relation index, TpCompactionOutput *output) { TpDetachedTombstoneBatch tombstones = output->tombstones; output->tombstones.head = InvalidBlockNumber; output->tombstones.tail = InvalidBlockNumber; output->tombstones.owned_pages = NULL; output->tombstones.owned_count = 0; for (uint32 i = 0; i < output->owned_output_count; i++) tp_discard_unpublished_segment(index, output->owned_output_roots[i]); if (output->owned_output_roots != NULL) pfree(output->owned_output_roots); output->owned_output_roots = NULL; output->owned_output_count = 0; output->owned_output_capacity = 0; for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { output->output_heads[level] = InvalidBlockNumber; output->output_counts[level] = 0; } if (tombstones.owned_pages != NULL) tp_tombstone_discard_detached(index, tombstones); } static bool tp_segment_page_link(Page page, uint32 *level, BlockNumber *next) { char *contents = PageGetContents(page); uint32 magic; uint32 version; memcpy(&magic, contents, sizeof(magic)); memcpy(&version, contents + sizeof(magic), sizeof(version)); if (magic != TP_SEGMENT_MAGIC) return false; if (version <= TP_SEGMENT_FORMAT_VERSION_3) { TpSegmentHeaderV3 *header = (TpSegmentHeaderV3 *)contents; *level = header->level; *next = header->next_segment; } else if (version <= TP_SEGMENT_FORMAT_VERSION_4) { TpSegmentHeaderV4 *header = (TpSegmentHeaderV4 *)contents; *level = header->level; *next = header->next_segment; } else if (version <= TP_SEGMENT_FORMAT_VERSION) { TpSegmentHeader *header = (TpSegmentHeader *)contents; *level = header->level; *next = header->next_segment; } else return false; return true; } static bool tp_read_segment_link( Relation index, BlockNumber root, uint32 expected_level, BlockNumber *next) { Buffer buf; Page page; uint32 level; BlockNumber next_segment; bool decoded; if (!BlockNumberIsValid(root)) return false; buf = ReadBuffer(index, root); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); decoded = tp_segment_page_link(page, &level, &next_segment); UnlockReleaseBuffer(buf); if (!decoded || level != expected_level) return false; *next = next_segment; return true; } static void tp_segment_page_set_link(Page page, BlockNumber next) { char *contents = PageGetContents(page); uint32 version = ((TpSegmentHeader *)contents)->version; if (version <= TP_SEGMENT_FORMAT_VERSION_3) ((TpSegmentHeaderV3 *)contents)->next_segment = next; else ((TpSegmentHeader *)contents)->next_segment = next; } /* Report whether any batch in the plan writes its output to level. */ static bool tp_plan_outputs_at_level(const TpCompactionPlan *plan, uint32 level) { for (uint32 i = 0; i < plan->num_batches; i++) { if (plan->batches[i].output_level == level) return true; } return false; } static bool tp_metapage_identity_matches( const TpIndexMetaPageData *current, const TpIndexMetaPageData *snapshot) { if (current->version != TP_METAPAGE_VERSION && current->version != TP_METAPAGE_VERSION_V8 && current->version != TP_METAPAGE_VERSION_V7 && current->version != TP_METAPAGE_VERSION_V6) return false; return current->magic == snapshot->magic && current->text_config_oid == snapshot->text_config_oid && current->_unused_total_terms == snapshot->_unused_total_terms && current->k1 == snapshot->k1 && current->b == snapshot->b && current->root_blkno == snapshot->root_blkno && current->term_stats_root == snapshot->term_stats_root && current->_unused_docid_page == snapshot->_unused_docid_page; } static BlockNumber tp_metapage_pending_free_head(const TpIndexMetaPage metap) { return metap->version < TP_METAPAGE_VERSION_V8 ? InvalidBlockNumber : metap->pending_free_head; } /* * Walk the level 0 prefix that a concurrent spill prepended ahead of the * snapshot's head, recording the last prefix segment as a splice point. * Fails if the chain does not reach the snapshot head. */ static bool tp_validate_l0_prefix( Relation index, const TpIndexMetaPage snapshot, TpIndexMetaPage current_meta, HTAB *visited, BlockNumber *current_out, BlockNumber *predecessor_out) { BlockNumber current; uint32 prefix_count; if (current_meta->level_counts[0] < snapshot->level_counts[0]) return false; prefix_count = (uint32)current_meta->level_counts[0] - (uint32)snapshot->level_counts[0]; current = current_meta->level_heads[0]; for (uint32 i = 0; i < prefix_count; i++) { BlockNumber next; bool found; if (current == snapshot->level_heads[0]) return false; (void)hash_search(visited, ¤t, HASH_ENTER, &found); if (found) return false; if (!tp_read_segment_link(index, current, 0, &next)) return false; *predecessor_out = current; current = next; } if (current != snapshot->level_heads[0]) return false; *current_out = current; return true; } /* * Validate one level's segment chain against the plan, advancing * source_index over the sources the plan expects there and recording the * splice point when this level supplies one. */ static bool tp_validate_level_run( Relation index, const TpIndexMetaPage snapshot, TpIndexMetaPage current_meta, TpCompactionPlan *plan, uint32 level, HTAB *visited, uint32 *source_index, BlockNumber *predecessor, uint32 *predecessor_level) { BlockNumber current; BlockNumber level_predecessor = InvalidBlockNumber; if ((uint32)plan->prefix_counts[level] + (uint32)plan->selected_counts[level] + (uint32)plan->retained_counts[level] != (uint32)snapshot->level_counts[level]) return false; if (level == 0) { if (!tp_validate_l0_prefix( index, snapshot, current_meta, visited, ¤t, &level_predecessor)) return false; } else { if (current_meta->level_counts[level] != snapshot->level_counts[level] || current_meta->level_heads[level] != snapshot->level_heads[level]) return false; current = current_meta->level_heads[level]; } if (plan->selected_counts[level] == 0) { if (plan->prefix_counts[level] != 0 || plan->selected_heads[level] != InvalidBlockNumber) return false; /* * A level that receives output without contributing a source * still needs a splice point: a concurrent spill may have * prepended to level 0, and replacing its head would orphan * that prefix while the metapage kept counting it. Chain the * output after the prefix. */ if (level == 0 && BlockNumberIsValid(level_predecessor) && tp_plan_outputs_at_level(plan, level)) { if (BlockNumberIsValid(*predecessor)) return false; *predecessor = level_predecessor; *predecessor_level = level; } return true; } for (uint16 i = 0; i < plan->prefix_counts[level]; i++) { BlockNumber next; bool found; if (!BlockNumberIsValid(current)) return false; (void)hash_search(visited, ¤t, HASH_ENTER, &found); if (found || !tp_read_segment_link(index, current, level, &next)) return false; level_predecessor = current; current = next; } if (plan->selected_heads[level] != current) return false; if (BlockNumberIsValid(level_predecessor)) { if (BlockNumberIsValid(*predecessor)) return false; *predecessor = level_predecessor; *predecessor_level = level; } for (uint16 i = 0; i < plan->selected_counts[level]; i++) { BlockNumber next; if (*source_index >= plan->num_sources || plan->sources[*source_index].source_level != level || plan->sources[*source_index].root != current || !tp_read_segment_link(index, current, level, &next)) return false; current = next; (*source_index)++; } if (current != plan->retained_heads[level]) return false; if (BlockNumberIsValid(current)) { bool found; (void)hash_search(visited, ¤t, HASH_FIND, &found); if (found) return false; } return true; } static bool tp_validate_selected_runs( Relation index, const TpIndexMetaPage snapshot, TpIndexMetaPage current_meta, TpCompactionPlan *plan, BlockNumber *predecessor, uint32 *predecessor_level) { HASHCTL ctl; HTAB *visited; uint32 source_index = 0; bool valid = true; *predecessor = InvalidBlockNumber; *predecessor_level = TP_MAX_LEVELS; if (!tp_metapage_identity_matches(current_meta, snapshot)) return false; memset(&ctl, 0, sizeof(ctl)); ctl.keysize = sizeof(BlockNumber); ctl.entrysize = sizeof(BlockNumber); visited = hash_create( "compaction validation roots", Max((long)plan->num_sources, 16L), &ctl, HASH_ELEM | HASH_BLOBS); for (uint32 i = 0; valid && i < plan->num_sources; i++) { bool found; (void)hash_search(visited, &plan->sources[i].root, HASH_ENTER, &found); valid = !found; } for (uint32 level = 0; valid && level < TP_MAX_LEVELS; level++) valid = tp_validate_level_run( index, snapshot, current_meta, plan, level, visited, &source_index, predecessor, predecessor_level); valid = valid && source_index == plan->num_sources; hash_destroy(visited); return valid; } static bool tp_publication_identity_matches( const TpIndexMetaPage current, const TpCompactionPublication *publication) { if (!tp_metapage_identity_matches(current, &publication->metapage)) return false; for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { if (current->level_heads[level] != publication->metapage.level_heads[level] || current->level_counts[level] != publication->metapage.level_counts[level]) return false; } return true; } static void tp_prepare_compaction_publication( TpLocalIndexState *index_state, Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan, TpCompactionPublication *publication) { volatile bool acquired_here = false; TpIndexMetaPage current_meta; BlockNumber predecessor; uint32 predecessor_level; bool valid; PG_TRY(); { if (!index_state->lock_held) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_here = true; } else if ( index_state->lock_mode != LW_EXCLUSIVE || !LWLockHeldByMeInMode( &index_state->shared->lock, LW_EXCLUSIVE)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("private compaction preparation requires the " "per-index exclusive lock"))); current_meta = tp_get_metapage(index); valid = tp_validate_selected_runs( index, snapshot, current_meta, plan, &predecessor, &predecessor_level); if (valid) { memcpy(&publication->metapage, current_meta, sizeof(publication->metapage)); publication->predecessor = predecessor; publication->predecessor_level = predecessor_level; } pfree(current_meta); if (acquired_here) { tp_release_index_lock(index_state); acquired_here = false; } } PG_FINALLY(); { if (acquired_here && index_state->lock_held) tp_release_index_lock(index_state); } PG_END_TRY(); if (!valid) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction graph changed before publication " "preparation for index \"%s\"", RelationGetRelationName(index)))); } static bool tp_validate_compaction_output( Relation index, TpCompactionPlan *plan, TpCompactionOutput *output) { for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { BlockNumber current = output->output_heads[level]; for (uint16 i = 0; i < output->output_counts[level]; i++) { BlockNumber next; if (!tp_read_segment_link(index, current, level, &next)) return false; current = next; } if (current != plan->retained_heads[level]) return false; } if (output->tombstones.owned_count == 0) return output->tombstones.head == InvalidBlockNumber && output->tombstones.tail == InvalidBlockNumber; { BlockNumber current = output->tombstones.head; for (uint32 i = 0; i < output->tombstones.owned_count; i++) { Buffer buf; Page page; TpTombstonePage tombstone; BlockNumber next; if (!BlockNumberIsValid(current)) return false; buf = ReadBuffer(index, current); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_tombstone_page_is_valid(page)) { UnlockReleaseBuffer(buf); return false; } tombstone = tp_tombstone_page(page); next = tombstone->next_page; UnlockReleaseBuffer(buf); if (i + 1 == output->tombstones.owned_count) { if (current != output->tombstones.tail || next != InvalidBlockNumber) return false; } current = next; } return current == InvalidBlockNumber; } } /* * Total the docs and tokens held by the plan's source segments. */ static void tp_sum_source_statistics( const TpCompactionPlan *plan, uint64 *docs_out, uint64 *tokens_out) { uint64 docs = 0; uint64 tokens = 0; for (uint32 i = 0; i < plan->num_sources; i++) { if (!tp_u64_add(docs, plan->sources[i].estimate.docs, &docs) || !tp_u64_add(tokens, plan->sources[i].total_tokens, &tokens)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("source segment statistics overflow"))); } *docs_out = docs; *tokens_out = tokens; } /* * Record one merged segment in the output, taking ownership of its * pages so a later error discards it. */ static void tp_record_merged_batch( Relation index, const TpCompactionPlan *plan, TpCompactionOutput *output, uint32 output_level, const TpMergedSegmentResult *result, uint64 *output_docs, uint64 *output_tokens) { if (output->owned_output_count >= output->owned_output_capacity) { tp_discard_unpublished_segment(index, result->root); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction output ownership overflow"))); } output->owned_output_roots[output->owned_output_count++] = result->root; if (output->output_counts[output_level] == PG_UINT16_MAX) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction output count overflow at level %u", output_level))); output->output_heads[output_level] = result->root; output->output_counts[output_level]++; if ((uint32)plan->retained_counts[output_level] + (uint32)output->output_counts[output_level] > plan->output_capacity) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction output exceeded level %u capacity", output_level))); if (!tp_u64_add(*output_docs, (uint64)result->num_docs, output_docs) || !tp_u64_add(*output_tokens, result->total_tokens, output_tokens)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compaction output statistics overflow"))); } /* * Merge each planned batch into an unreachable output segment. Batches * run in reverse so each level's chain is built oldest-first and the * head written last. */ static void tp_merge_planned_batches( Relation index, TpCompactionPlan *plan, TpCompactionOutput *output, uint64 *output_docs, uint64 *output_tokens) { for (uint32 reverse = plan->num_batches; reverse > 0; reverse--) { TpCompactionBatch *batch = &plan->batches[reverse - 1]; BlockNumber *roots; TpMergedSegmentResult result; uint32 output_level = batch->output_level; roots = palloc(sizeof(BlockNumber) * batch->source_count); for (uint32 i = 0; i < batch->source_count; i++) roots[i] = plan->sources[batch->first_source + i].root; if (tp_merge_segment_batch( index, roots, batch->source_count, output_level, output->output_heads[output_level], &result)) tp_record_merged_batch( index, plan, output, output_level, &result, output_docs, output_tokens); pfree(roots); } } /* * Collect every page belonging to the plan's source segments, which * publication displaces. The array grows geometrically; sizing each * step exactly makes collection quadratic in the number of sources. */ static uint32 tp_collect_displaced_pages( Relation index, const TpCompactionPlan *plan, BlockNumber **pages_out) { BlockNumber *displaced = NULL; uint32 displaced_count = 0; uint32 displaced_capacity = 0; for (uint32 i = 0; i < plan->num_sources; i++) { BlockNumber *pages; uint32 num_pages; uint32 new_count; num_pages = tp_segment_collect_pages(index, plan->sources[i].root, &pages); if (num_pages > UINT32_MAX - displaced_count) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("compaction displaced-page count overflow"))); new_count = displaced_count + num_pages; if (num_pages > 0) { if (new_count > displaced_capacity) { uint32 want = displaced_capacity == 0 ? new_count : displaced_capacity; while (want < new_count) { if (want > UINT32_MAX / 2) { want = new_count; break; } want *= 2; } displaced_capacity = want; displaced = displaced == NULL ? palloc(sizeof(BlockNumber) * displaced_capacity) : repalloc( displaced, sizeof(BlockNumber) * displaced_capacity); } memcpy(displaced + displaced_count, pages, sizeof(BlockNumber) * num_pages); } displaced_count = new_count; if (pages != NULL) pfree(pages); } *pages_out = displaced; return displaced_count; } static void tp_build_compaction_output( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan, TpCompactionOutput *output) { uint64 selected_docs = 0; uint64 selected_tokens = 0; uint64 output_docs = 0; uint64 output_tokens = 0; BlockNumber *displaced_pages = NULL; uint32 displaced_count; tp_initialize_compaction_output(plan, output); tp_debug_compaction_build_active = true; PG_TRY(); { tp_sum_source_statistics(plan, &selected_docs, &selected_tokens); tp_merge_planned_batches( index, plan, output, &output_docs, &output_tokens); if (output_docs > selected_docs || output_tokens > selected_tokens) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("compaction output statistics exceed source " "statistics"))); output->removed_docs = selected_docs - output_docs; output->removed_tokens = selected_tokens - output_tokens; if (snapshot->total_docs < output->removed_docs || snapshot->total_len < output->removed_tokens) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("compaction source statistics exceed index " "statistics"))); displaced_count = tp_collect_displaced_pages(index, plan, &displaced_pages); tp_tombstone_build_detached( index, displaced_pages, displaced_count, InvalidFullTransactionId, &output->tombstones); if (displaced_pages != NULL) { pfree(displaced_pages); displaced_pages = NULL; } /* * Output and detached tombstone pages are WAL-logged but still * unreachable. A backend crash before publication can leak an * incomplete allocation until REINDEX; handled errors discard every * complete object recorded in output. */ } PG_CATCH(); { tp_debug_compaction_build_active = false; tp_discard_compaction_output(index, output); PG_RE_THROW(); } PG_END_TRY(); tp_debug_compaction_build_active = false; } /* * Reject a publication whose level counts would exceed the segment * capacity, or whose shrinkage exceeds the statistics it rebases from. */ static void tp_check_publication_limits( const TpCompactionPlan *plan, const TpCompactionOutput *output, const uint16 current_counts[TP_MAX_LEVELS], uint64 current_docs, uint64 current_tokens, TpStatsRebasePolicy stats_policy) { for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { uint32 new_count; if (current_counts[level] < plan->selected_counts[level]) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction source count changed at level %u", level))); new_count = (uint32)current_counts[level] - (uint32)plan->selected_counts[level] + (uint32)output->output_counts[level]; if (new_count > plan->output_capacity) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("bm25 segment count limit reached at level %u", level))); } if (stats_policy == TP_STATS_REBASE_STRICT && (current_docs < output->removed_docs || current_tokens < output->removed_tokens || output->added_tokens != 0)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("compaction shrinkage exceeds current index " "statistics"))); } /* * Swap the plan's sources for the built outputs in a registered * metapage copy, and rebase the corpus statistics. */ static void tp_apply_publication_to_metapage( TpIndexMetaPage meta, const TpCompactionPlan *plan, const TpCompactionOutput *output, const uint16 current_counts[TP_MAX_LEVELS], uint64 current_docs, uint64 current_tokens, BlockNumber predecessor, uint32 predecessor_level, TpStatsRebasePolicy stats_policy) { for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { bool level_changes = plan->selected_counts[level] > 0 || output->output_counts[level] > 0; if (!level_changes) continue; if (!BlockNumberIsValid(predecessor) || level != predecessor_level) meta->level_heads[level] = output->output_heads[level]; meta->level_counts[level] = (uint16)((uint32)current_counts[level] - (uint32)plan->selected_counts[level] + (uint32)output->output_counts[level]); } if (output->tombstones.owned_count > 0) meta->pending_free_head = output->tombstones.head; if (stats_policy == TP_STATS_REBASE_CLAMP_LEGACY_VACUUM) { uint64 rebased_tokens = current_tokens >= output->removed_tokens ? current_tokens - output->removed_tokens : 0; if (pg_add_u64_overflow( rebased_tokens, output->added_tokens, &rebased_tokens)) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("legacy VACUUM token statistics overflow"))); meta->total_docs = current_docs >= output->removed_docs ? current_docs - output->removed_docs : 0; meta->total_len = rebased_tokens; } else { meta->total_docs = current_docs - output->removed_docs; meta->total_len = current_tokens - output->removed_tokens; } } /* * Drop the output's ownership of pages the metapage now references, so * a later error cannot free published pages. */ static void tp_release_published_output(TpCompactionOutput *output) { for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { output->output_heads[level] = InvalidBlockNumber; output->output_counts[level] = 0; } output->tombstones.head = InvalidBlockNumber; output->tombstones.tail = InvalidBlockNumber; if (output->tombstones.owned_pages != NULL) pfree(output->tombstones.owned_pages); output->tombstones.owned_pages = NULL; output->tombstones.owned_count = 0; if (output->owned_output_roots != NULL) pfree(output->owned_output_roots); output->owned_output_roots = NULL; output->owned_output_count = 0; output->owned_output_capacity = 0; } static bool tp_publish_compaction_output( TpLocalIndexState *index_state, Relation index, TpCompactionPlan *plan, TpCompactionOutput *output, const TpCompactionPublication *publication, TpStatsRebasePolicy stats_policy, bool stamp_with_next_fxid) { volatile Buffer metabuf = InvalidBuffer; volatile Buffer predecessor_buf = InvalidBuffer; volatile Buffer tailbuf = InvalidBuffer; GenericXLogState *volatile publication_state = NULL; volatile bool acquired_here = false; BlockNumber predecessor = publication->predecessor; uint32 predecessor_level = publication->predecessor_level; bool published = false; PG_TRY(); { Page current_page; TpIndexMetaPage current_meta; uint16 current_counts[TP_MAX_LEVELS]; uint64 current_docs; uint64 current_tokens; BlockNumber current_pending; Page meta_copy; TpIndexMetaPage meta; if (!index_state->lock_held) { tp_acquire_index_lock(index_state, LW_EXCLUSIVE); acquired_here = true; } else if ( index_state->lock_mode != LW_EXCLUSIVE || !LWLockHeldByMeInMode( &index_state->shared->lock, LW_EXCLUSIVE)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction publication requires the per-index " "exclusive lock"))); metabuf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(metabuf, BUFFER_LOCK_EXCLUSIVE); current_page = BufferGetPage(metabuf); current_meta = (TpIndexMetaPage)PageGetContents(current_page); if (!tp_publication_identity_matches(current_meta, publication)) { UnlockReleaseBuffer(metabuf); metabuf = InvalidBuffer; if (acquired_here) { tp_release_index_lock(index_state); acquired_here = false; } /* A return from inside PG_TRY is illegal; skip to its end. */ goto publication_done; } memcpy(current_counts, current_meta->level_counts, sizeof(current_counts)); current_docs = current_meta->total_docs; current_tokens = current_meta->total_len; if (stats_policy == TP_STATS_REBASE_CLAMP_LEGACY_VACUUM) current_tokens = tp_injected_vacuum_total_len(current_tokens); current_pending = tp_metapage_pending_free_head(current_meta); if (stats_policy != TP_STATS_REBASE_STRICT && stats_policy != TP_STATS_REBASE_CLAMP_LEGACY_VACUUM) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid compaction statistic rebase policy"))); tp_check_publication_limits( plan, output, current_counts, current_docs, current_tokens, stats_policy); if (BlockNumberIsValid(predecessor)) { Page predecessor_page; uint32 recorded_level; BlockNumber recorded_next; if (predecessor_level >= TP_MAX_LEVELS) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("invalid compaction predecessor level"))); predecessor_buf = ReadBuffer(index, predecessor); LockBuffer(predecessor_buf, BUFFER_LOCK_EXCLUSIVE); predecessor_page = BufferGetPage(predecessor_buf); if (!tp_segment_page_link( predecessor_page, &recorded_level, &recorded_next) || recorded_level != predecessor_level || recorded_next != plan->selected_heads[predecessor_level]) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction predecessor changed before " "publication"))); } TP_INJECTION_POINT(TP_INJECTION_BEFORE_COMPACTION_PUBLISH); /* * Every output and detached tombstone page has already been * WAL-logged. WAL insertion order places this reachability record * after those page records, without a relation-wide buffer flush. */ publication_state = GenericXLogStart(index); meta_copy = GenericXLogRegisterBuffer( (GenericXLogState *)publication_state, metabuf, 0); if (BufferIsValid(predecessor_buf)) { Page predecessor_copy; predecessor_copy = GenericXLogRegisterBuffer( (GenericXLogState *)publication_state, predecessor_buf, 0); ((PageHeader)predecessor_copy)->pd_lower = BLCKSZ; tp_segment_page_set_link( predecessor_copy, output->output_heads[predecessor_level]); } tailbuf = tp_tombstone_attach_detached( (GenericXLogState *)publication_state, index, output->tombstones, current_pending, stamp_with_next_fxid); tp_metapage_upgrade_to_current(index, meta_copy); meta = (TpIndexMetaPage)PageGetContents(meta_copy); tp_apply_publication_to_metapage( meta, plan, output, current_counts, current_docs, current_tokens, predecessor, predecessor_level, stats_policy); GenericXLogFinish((GenericXLogState *)publication_state); publication_state = NULL; output->publication_started = true; TP_INJECTION_POINT(TP_INJECTION_AFTER_COMPACTION_PUBLISH); if (BufferIsValid(tailbuf)) { UnlockReleaseBuffer(tailbuf); tailbuf = InvalidBuffer; } if (BufferIsValid(predecessor_buf)) { UnlockReleaseBuffer(predecessor_buf); predecessor_buf = InvalidBuffer; } UnlockReleaseBuffer(metabuf); metabuf = InvalidBuffer; if (acquired_here) { tp_release_index_lock(index_state); acquired_here = false; } if (stamp_with_next_fxid) tp_tombstone_restamp_batch( index, output->tombstones, ReadNextFullTransactionId()); tp_release_published_output(output); published = true; publication_done: Assert(!published || output->publication_started); } PG_CATCH(); { if (!output->publication_started) { if (publication_state != NULL) GenericXLogAbort((GenericXLogState *)publication_state); if (BufferIsValid(tailbuf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer(tailbuf); } if (BufferIsValid(predecessor_buf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer(predecessor_buf); } if (BufferIsValid(metabuf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer(metabuf); } if (acquired_here && index_state->lock_held) tp_release_index_lock(index_state); } PG_RE_THROW(); } PG_END_TRY(); return published; } static void tp_complete_compaction_publication( TpLocalIndexState *index_state, Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan, TpCompactionOutput *output, FullTransactionId reclaim_fxid, bool stamp_with_next_fxid, TpStatsRebasePolicy stats_policy) { FullTransactionId merged_fxid; TpCompactionPublication publication; volatile bool publication_locked = false; output->publication_started = false; PG_TRY(); { /* * The prepared segment and tombstone chains are unreachable and * immutable. Validate them completely before any publication lock. */ if (!tp_validate_compaction_output(index, plan, output)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid prepared compaction output for index " "\"%s\"", RelationGetRelationName(index)))); /* * Ordinary compaction assigns and restamps here; a serial prepared * replacement arrives with a VACUUM-assigned XID. Parallel VACUUM * cannot assign one, so it publishes with an invalid provisional * stamp and activates after the graph publication WAL record is * inserted. */ if (stamp_with_next_fxid) { Assert(!FullTransactionIdIsValid(reclaim_fxid)); Assert(output->tombstones.owned_count > 0); } else if (!FullTransactionIdIsValid(reclaim_fxid)) { merged_fxid = GetCurrentFullTransactionId(); tp_tombstone_restamp_batch(index, output->tombstones, merged_fxid); if (!index_state->lock_held) TP_INJECTION_POINT(TP_INJECTION_COMPACTION_AFTER_RESTAMP); } if (!index_state->lock_held) { /* Stabilize the spill-prepended prefix through publication. */ tp_compaction_publication_lock(index, ExclusiveLock); publication_locked = true; } tp_prepare_compaction_publication( index_state, index, snapshot, plan, &publication); if (!index_state->lock_held) TP_INJECTION_POINT(TP_INJECTION_COMPACTION_BEFORE_PUBLISH); if (!tp_publish_compaction_output( index_state, index, plan, output, &publication, stats_policy, stamp_with_next_fxid)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction graph changed while publication was " "serialized for index \"%s\"", RelationGetRelationName(index)))); if (publication_locked) { tp_compaction_publication_unlock(index, ExclusiveLock); publication_locked = false; } } PG_CATCH(); { if (publication_locked) tp_compaction_publication_unlock(index, ExclusiveLock); if (!output->publication_started) tp_discard_compaction_output(index, output); PG_RE_THROW(); } PG_END_TRY(); } static TpIndexMetaPage tp_prepare_single_replacement_plan( TpLocalIndexState *index_state, Relation index, uint32 level, BlockNumber source_root, TpCompactionPlan *plan) { volatile bool acquired_here = false; TpSegmentGraphSnapshot *graph; TpIndexMetaPage snapshot; const BlockNumber *roots; uint32 root_count; uint32 position; bool found = false; bool private_compaction; if (level >= TP_MAX_LEVELS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid replacement segment level %u", level))); memset(plan, 0, sizeof(*plan)); private_compaction = tp_compaction_is_private(index_state, index); PG_TRY(); { if (!private_compaction) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_here = true; } graph = tp_segment_graph_snapshot_create(index); if (acquired_here) { tp_release_index_lock(index_state); acquired_here = false; } } PG_FINALLY(); { if (acquired_here && index_state->lock_held) tp_release_index_lock(index_state); } PG_END_TRY(); roots = tp_segment_graph_snapshot_level(graph, level, &root_count); for (position = 0; position < root_count; position++) { if (roots[position] == source_root) { found = true; break; } } if (!found) { tp_segment_graph_snapshot_free(graph); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("prepared replacement source %u is no longer " "published at level %u", source_root, level))); } snapshot = palloc(sizeof(*snapshot)); *snapshot = graph->metapage; for (uint32 current_level = 0; current_level < TP_MAX_LEVELS; current_level++) { plan->selected_heads[current_level] = InvalidBlockNumber; plan->retained_heads[current_level] = snapshot->level_heads[current_level]; plan->retained_counts[current_level] = snapshot->level_counts[current_level]; } plan->source_capacity = 1; plan->output_capacity = PG_UINT16_MAX; plan->sources = palloc0(sizeof(TpCompactionSource)); plan->sources[0].root = source_root; plan->sources[0].source_level = level; plan->sources[0].chain_position = position; plan->num_sources = 1; plan->prefix_counts[level] = (uint16)position; plan->selected_heads[level] = source_root; plan->selected_counts[level] = 1; plan->retained_heads[level] = (position + 1 < root_count) ? roots[position + 1] : InvalidBlockNumber; plan->retained_counts[level] = (uint16)(root_count - position - 1); tp_segment_graph_snapshot_free(graph); return snapshot; } static void tp_prepare_single_replacement_root( Relation index, BlockNumber replacement_root, uint32 level, BlockNumber next) { volatile Buffer buf = InvalidBuffer; GenericXLogState *volatile state = NULL; if (!BlockNumberIsValid(replacement_root)) return; PG_TRY(); { Page page; Page copy; TpSegmentHeader *header; buf = ReadBuffer(index, replacement_root); LockBuffer((Buffer)buf, BUFFER_LOCK_EXCLUSIVE); page = BufferGetPage((Buffer)buf); header = (TpSegmentHeader *)PageGetContents(page); if (header->magic != TP_SEGMENT_MAGIC || header->version != TP_SEGMENT_FORMAT_VERSION) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid prepared replacement segment at " "block %u", replacement_root))); state = GenericXLogStart(index); copy = GenericXLogRegisterBuffer( (GenericXLogState *)state, (Buffer)buf, 0); ((PageHeader)copy)->pd_lower = BLCKSZ; header = (TpSegmentHeader *)PageGetContents(copy); header->level = level; header->next_segment = next; GenericXLogFinish((GenericXLogState *)state); state = NULL; UnlockReleaseBuffer((Buffer)buf); buf = InvalidBuffer; } PG_CATCH(); { if (state != NULL) GenericXLogAbort((GenericXLogState *)state); if (BufferIsValid((Buffer)buf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer((Buffer)buf); } PG_RE_THROW(); } PG_END_TRY(); } void tp_publish_prepared_segment_replacement( TpLocalIndexState *index_state, Relation index, uint32 level, BlockNumber source_root, BlockNumber replacement_root, uint64 removed_docs, uint64 removed_tokens, uint64 added_tokens, TpSegmentReplacementReclaimMode reclaim_mode) { TpCompactionPlan plan; TpCompactionOutput output; bool stamp_with_next_fxid = reclaim_mode == TP_SEGMENT_REPLACEMENT_RECLAIM_NEXT_XID; FullTransactionId reclaim_fxid = InvalidFullTransactionId; TpIndexMetaPage volatile snapshot = NULL; BlockNumber *volatile source_pages = NULL; volatile bool completion_started = false; volatile BlockNumber unclaimed_root = replacement_root; uint32 source_page_count; memset(&plan, 0, sizeof(plan)); memset(&output, 0, sizeof(output)); output.tombstones.head = InvalidBlockNumber; output.tombstones.tail = InvalidBlockNumber; PG_TRY(); { BlockNumber *pages; if (BlockNumberIsValid((BlockNumber)unclaimed_root)) { output.owned_output_roots = palloc(sizeof(BlockNumber)); output.owned_output_roots[0] = (BlockNumber)unclaimed_root; output.owned_output_count = 1; output.owned_output_capacity = 1; unclaimed_root = InvalidBlockNumber; } if (reclaim_mode != TP_SEGMENT_REPLACEMENT_RECLAIM_CURRENT_XID && reclaim_mode != TP_SEGMENT_REPLACEMENT_RECLAIM_NEXT_XID) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid prepared replacement reclaim mode"))); snapshot = tp_prepare_single_replacement_plan( index_state, index, level, source_root, &plan); memcpy(output.output_heads, plan.retained_heads, sizeof(output.output_heads)); if (BlockNumberIsValid(replacement_root)) { tp_prepare_single_replacement_root( index, replacement_root, level, plan.retained_heads[level]); output.output_heads[level] = replacement_root; output.output_counts[level] = 1; } output.removed_docs = removed_docs; output.removed_tokens = removed_tokens; output.added_tokens = added_tokens; source_page_count = tp_segment_collect_pages(index, source_root, &pages); source_pages = pages; if (source_page_count == 0) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("prepared replacement source %u has no pages", source_root))); if (!stamp_with_next_fxid) reclaim_fxid = GetCurrentFullTransactionId(); tp_tombstone_build_detached( index, (BlockNumber *)source_pages, source_page_count, reclaim_fxid, &output.tombstones); pfree((BlockNumber *)source_pages); source_pages = NULL; completion_started = true; tp_complete_compaction_publication( index_state, index, (TpIndexMetaPage)snapshot, &plan, &output, reclaim_fxid, stamp_with_next_fxid, TP_STATS_REBASE_CLAMP_LEGACY_VACUUM); } PG_CATCH(); { if (BlockNumberIsValid((BlockNumber)unclaimed_root)) tp_discard_unpublished_segment(index, (BlockNumber)unclaimed_root); else if (!completion_started && !output.publication_started) tp_discard_compaction_output(index, &output); if (source_pages != NULL) pfree((BlockNumber *)source_pages); if (snapshot != NULL) pfree((TpIndexMetaPage)snapshot); tp_free_compaction_plan(&plan); PG_RE_THROW(); } PG_END_TRY(); pfree((TpIndexMetaPage)snapshot); tp_free_compaction_plan(&plan); } static uint32 tp_compaction_candidate( const uint16 level_counts[TP_MAX_LEVELS], uint32 first_level) { /* * Every level is a candidate, including the top one. */ for (uint32 level = first_level; level < TP_MAX_LEVELS; level++) { if ((uint32)level_counts[level] >= (uint32)tp_segments_per_level) return level; } return TP_MAX_LEVELS; } static void tp_initialize_ordinary_plan( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan) { uint32 total_sources = 0; memset(plan, 0, sizeof(*plan)); for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { if (pg_add_u32_overflow( total_sources, (uint32)snapshot->level_counts[level], &total_sources)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment count overflow in index \"%s\"", RelationGetRelationName(index)))); plan->selected_heads[level] = InvalidBlockNumber; plan->retained_heads[level] = snapshot->level_heads[level]; plan->retained_counts[level] = snapshot->level_counts[level]; } Assert(total_sources > 0); plan->source_capacity = total_sources; plan->output_capacity = tp_injected_segment_count_limit(); plan->sources = palloc0( sizeof(TpCompactionSource) * plan->source_capacity); plan->batches = palloc0(sizeof(TpCompactionBatch) * plan->source_capacity); } static uint32 tp_select_level_prefix( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan, uint32 level, uint32 prefix_count) { BlockNumber current; uint32 first_source; uint32 chain_position; if (plan->retained_counts[level] < prefix_count) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("compaction level %u has no threshold-sized prefix", level))); first_source = plan->num_sources; current = plan->retained_heads[level]; chain_position = (uint32)snapshot->level_counts[level] - (uint32)plan->retained_counts[level]; if (plan->selected_counts[level] == 0) plan->selected_heads[level] = current; for (uint32 i = 0; i < prefix_count; i++) current = tp_collect_source( index, plan, current, level, chain_position + i, true); plan->selected_counts[level] += (uint16)prefix_count; plan->retained_heads[level] = current; plan->retained_counts[level] -= (uint16)prefix_count; return first_source; } /* * Drop trailing single-source batches from the selection. * * A single-source batch rewrites a segment without combining anything. * At the tail of a prefix those are the level's oldest and largest * segments, already past max_segment_size, so one combinable pair at * the head must not drag them into the pass. * * Only a trailing run can be dropped: the selection is a contiguous * head prefix, so returning it means moving the retained head back, * with no segment header rewritten. Excising a single-source batch * from the middle would mean repointing its predecessor, which the * publishing metapage record cannot express, so those are still merged. * * Callers apply this only to the level a pass chose to compact. The * capacity recourse promotes single-source batches deliberately, to * free a level slot; dropping those would turn a wasteful pass into a * failed one. */ static void tp_trim_uncombinable_tail( TpCompactionPlan *plan, uint32 first_batch, uint32 level) { while (plan->num_batches > first_batch && plan->batches[plan->num_batches - 1].source_count == 1) { TpCompactionBatch *batch = &plan->batches[plan->num_batches - 1]; plan->retained_heads[level] = plan->sources[batch->first_source].root; plan->retained_counts[level]++; Assert(plan->selected_counts[level] > 0); plan->selected_counts[level]--; plan->num_sources = batch->first_source; plan->num_batches--; } } static void tp_assign_ordinary_batches( TpCompactionPlan *plan, uint32 first_batch, uint32 planned_outputs[TP_MAX_LEVELS]) { for (uint32 i = first_batch; i < plan->num_batches; i++) { TpCompactionBatch *batch = &plan->batches[i]; TpCompactionSource *source = &plan->sources[batch->first_source]; uint64 live_bytes = 0; uint32 output_level; for (uint32 j = 0; j < batch->source_count; j++) { if (!tp_u64_add( live_bytes, plan->sources[batch->first_source + j].live_bytes, &live_bytes)) live_bytes = PG_UINT64_MAX; } output_level = Max(1U, tp_size_class(live_bytes)); /* Capacity recourse must still move an uncombinable singleton. */ if (batch->source_count == 1) output_level = Max(output_level, source->source_level + 1); else if ( output_level < source->source_level && (uint32)plan->retained_counts[output_level] + planned_outputs[output_level] >= plan->output_capacity) output_level = source->source_level; /* * Capacity-driven promotion cannot go past the terminal level. */ if (output_level > TP_MAX_LEVELS - 1) output_level = TP_MAX_LEVELS - 1; for (uint32 j = 1; j < batch->source_count; j++) { if (plan->sources[batch->first_source + j].source_level != source->source_level) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("ordinary compaction batch crosses levels"))); } batch->output_level = output_level; planned_outputs[output_level]++; } } static bool tp_build_empty_plan( Relation index, const TpIndexMetaPage snapshot, uint32 first_level, TpCompactionPlan *plan) { for (uint32 level = first_level; level < TP_MAX_LEVELS; level++) { BlockNumber current = snapshot->level_heads[level]; for (uint32 position = 0; position < (uint32)snapshot->level_counts[level]; position++) { TpSegmentReader *reader; BlockNumber next; bool empty; if (!BlockNumberIsValid(current)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment chain for level %u ended before " "its recorded count", level))); reader = tp_segment_open_header(index, current); if (reader == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open segment at block %u", current))); next = reader->header->next_segment; empty = reader->header->alive_bitset_offset > 0 && reader->header->alive_count == 0; tp_segment_close(reader); if (empty) { uint32 first_source; uint32 first_batch; uint32 prefix_count = position + 1; tp_initialize_ordinary_plan(index, snapshot, plan); first_source = tp_select_level_prefix( index, snapshot, plan, level, prefix_count); first_batch = plan->num_batches; tp_append_bounded_batches(plan, first_source, prefix_count); for (uint32 i = first_batch; i < plan->num_batches; i++) { TpCompactionBatch *batch = &plan->batches[i]; for (uint32 j = 0; j < batch->source_count; j++) { if (plan->sources[batch->first_source + j] .source_level != level) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("empty cleanup batch crosses " "levels"))); } batch->output_level = level; } return true; } current = next; } if (BlockNumberIsValid(current)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment chain for level %u exceeds its " "recorded count", level))); } return false; } static bool tp_build_ordinary_plan( Relation index, const TpIndexMetaPage snapshot, uint32 first_level, TpCompactionPlan *plan) { uint32 threshold = (uint32)tp_segments_per_level; uint32 search_level = first_level; memset(plan, 0, sizeof(*plan)); while (true) { uint32 planned_outputs[TP_MAX_LEVELS] = {0}; uint32 candidate; uint32 first_source; uint32 first_batch; candidate = tp_compaction_candidate(snapshot->level_counts, search_level); if (candidate >= TP_MAX_LEVELS) return false; tp_initialize_ordinary_plan(index, snapshot, plan); first_source = tp_select_level_prefix( index, snapshot, plan, candidate, threshold); first_batch = plan->num_batches; tp_append_bounded_batches(plan, first_source, threshold); tp_trim_uncombinable_tail(plan, first_batch, candidate); tp_assign_ordinary_batches(plan, first_batch, planned_outputs); /* * Size placement can keep outputs at the candidate or move them * down. Check every level this plan grows. */ for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { while (planned_outputs[level] > 0 && (uint64)plan->retained_counts[level] + (uint64)planned_outputs[level] > (uint64)plan->output_capacity) { if (plan->retained_counts[level] < threshold) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("bm25 segment count limit reached at " "level %u", level))); first_source = tp_select_level_prefix( index, snapshot, plan, level, threshold); first_batch = plan->num_batches; tp_append_bounded_batches(plan, first_source, threshold); tp_assign_ordinary_batches(plan, first_batch, planned_outputs); } } if (plan->num_batches < plan->num_sources) return true; /* * Promoting only singleton batches leaves the level count unchanged * and makes the driver chase the threshold up the hierarchy * indefinitely. Abandon this candidate but keep searching: a level * with no legal reduction must not starve a higher one that has it. */ tp_free_compaction_plan(plan); memset(plan, 0, sizeof(*plan)); search_level = candidate + 1; } } static void tp_free_compaction_plan(TpCompactionPlan *plan) { if (plan->sources != NULL) pfree(plan->sources); if (plan->batches != NULL) pfree(plan->batches); } /* * Rewrite one half-dead segment, including an interior chain member. * Keep its level so publication only needs that chain's splice point. */ static bool tp_build_sparse_plan( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan) { for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { BlockNumber current = snapshot->level_heads[level]; for (uint32 position = 0; position < snapshot->level_counts[level]; position++) { TpSegmentReader *reader = tp_segment_open_header(index, current); BlockNumber next; bool sparse; if (reader == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open segment at block %u", current))); next = reader->header->next_segment; sparse = reader->header->alive_bitset_offset > 0 && reader->header->num_docs > 0 && reader->header->alive_count <= reader->header->num_docs / 2; tp_segment_close(reader); if (sparse) { tp_initialize_ordinary_plan(index, snapshot, plan); (void)tp_collect_source( index, plan, current, level, position, true); plan->prefix_counts[level] = (uint16)position; plan->selected_heads[level] = current; plan->selected_counts[level] = 1; plan->retained_heads[level] = next; plan->retained_counts[level] -= (uint16)(position + 1); tp_append_bounded_batches(plan, 0, 1); plan->batches[0].output_level = level; return true; } current = next; } } return false; } /* * Merge comparable head prefixes across levels. Each source is within * a factor of two of the seed's surviving size, avoiding repeated * rewrites of a large output for each tiny spill. The existing fanout * bounds input count and conservative estimates bound output size. */ static bool tp_build_consolidation_plan( Relation index, const TpIndexMetaPage snapshot, TpCompactionPlan *plan) { uint64 budget = tp_max_segment_size_bytes(); for (uint32 seed = 0; seed < TP_MAX_LEVELS; seed++) { TpSegmentEstimate estimate = {0}; uint64 seed_bytes = 0; uint64 live_bytes = 0; uint32 output_level; if (snapshot->level_counts[seed] == 0) continue; tp_initialize_ordinary_plan(index, snapshot, plan); for (uint32 level = seed; level < TP_MAX_LEVELS; level++) { while (plan->retained_counts[level] > 0 && plan->num_sources < (uint32)tp_segments_per_level) { TpCompactionSource *source; uint32 position = plan->selected_counts[level]; BlockNumber root = plan->retained_heads[level]; BlockNumber next = tp_collect_source( index, plan, root, level, position, false); source = &plan->sources[plan->num_sources - 1]; if (plan->num_sources == 1) seed_bytes = source->live_bytes; else if ( source->live_bytes / 2 > seed_bytes || seed_bytes / 2 > source->live_bytes) { plan->num_sources--; break; } if (position == 0) plan->selected_heads[level] = root; plan->selected_counts[level]++; plan->retained_counts[level]--; plan->retained_heads[level] = next; } } if (plan->num_sources >= 2) { uint32 accepted = 0; for (uint32 i = 0; i < plan->num_sources; i++) { TpCompactionSource *source = &plan->sources[i]; TpSegmentReader *reader = tp_segment_open(index, source->root); TpSegmentEstimate combined; if (reader == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open segment at block %u", source->root))); tp_collect_source_estimate( source, reader, i == 0 && tp_compaction_maintenance_lock_held(index)); tp_segment_close(reader); if (i == 0) combined = source->estimate; else if (!tp_estimate_add( &estimate, &source->estimate, budget, &combined)) break; if (combined.bytes > budget) break; estimate = combined; live_bytes += source->live_bytes; accepted++; } while (plan->num_sources > accepted) { TpCompactionSource *source = &plan->sources[--plan->num_sources]; uint32 level = source->source_level; plan->retained_heads[level] = source->root; plan->retained_counts[level]++; if (--plan->selected_counts[level] == 0) plan->selected_heads[level] = InvalidBlockNumber; } } output_level = Max(1U, tp_size_class(live_bytes)); if (plan->num_sources >= 2 && plan->retained_counts[output_level] < plan->output_capacity) { plan->num_batches = 1; plan->batches[0].source_count = plan->num_sources; plan->batches[0].estimate = estimate; plan->batches[0].output_level = output_level; return true; } tp_free_compaction_plan(plan); memset(plan, 0, sizeof(*plan)); } return false; } static bool tp_select_compaction_plan( TpLocalIndexState *index_state, Relation index, uint32 first_level, bool empty_only, TpIndexMetaPage *snapshot_out, TpCompactionPlan *plan) { volatile bool acquired_here = false; TpIndexMetaPage snapshot; bool selected = false; bool private_compaction; Assert(snapshot_out != NULL); *snapshot_out = NULL; memset(plan, 0, sizeof(*plan)); private_compaction = tp_compaction_is_private(index_state, index); PG_TRY(); { if (!private_compaction) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_here = true; } snapshot = tp_get_metapage(index); if (acquired_here) { tp_release_index_lock(index_state); acquired_here = false; } selected = tp_build_empty_plan(index, snapshot, first_level, plan); if (!selected && !empty_only && tp_compaction_candidate(snapshot->level_counts, first_level) < TP_MAX_LEVELS) selected = tp_build_ordinary_plan(index, snapshot, first_level, plan); if (!selected && !empty_only) selected = tp_build_sparse_plan(index, snapshot, plan); if (!selected && !empty_only) selected = tp_build_consolidation_plan(index, snapshot, plan); } PG_FINALLY(); { if (acquired_here && index_state->lock_held) tp_release_index_lock(index_state); } PG_END_TRY(); if (!selected) { pfree(snapshot); tp_free_compaction_plan(plan); memset(plan, 0, sizeof(*plan)); return false; } *snapshot_out = snapshot; return true; } static bool tp_select_force_compaction_plan( TpLocalIndexState *index_state, Relation index, TpIndexMetaPage *snapshot_out, TpCompactionPlan *plan) { volatile bool acquired_here = false; TpIndexMetaPage snapshot; bool selected; bool private_compaction; Assert(snapshot_out != NULL); *snapshot_out = NULL; memset(plan, 0, sizeof(*plan)); private_compaction = tp_compaction_is_private(index_state, index); PG_TRY(); { if (!private_compaction) { tp_acquire_index_lock(index_state, LW_SHARED); acquired_here = true; } snapshot = tp_get_metapage(index); if (acquired_here) { tp_release_index_lock(index_state); acquired_here = false; } tp_collect_force_sources(index, snapshot, plan); tp_build_force_batches(plan); selected = plan->num_sources > 0 && !tp_plan_is_noop(index, snapshot, plan); } PG_FINALLY(); { if (acquired_here && index_state->lock_held) tp_release_index_lock(index_state); } PG_END_TRY(); if (!selected) { pfree(snapshot); tp_free_compaction_plan(plan); memset(plan, 0, sizeof(*plan)); return false; } *snapshot_out = snapshot; return true; } /* * Select, build, and publish one bounded compaction pass. Runtime * callers hold the per-index maintenance object lock; CREATE INDEX holds * its private per-index lock for the whole build instead, since the * index is not yet visible. * * Selection and publication preparation take LW_SHARED briefly, output * construction holds no per-index lock, and only the final GenericXLog * publication takes fair LW_EXCLUSIVE. The maintenance lock keeps * selected segment payloads immutable while a concurrent spill may * prepend an L0 prefix. */ static bool tp_compact_once( TpLocalIndexState *index_state, Relation index, uint32 first_level, bool empty_only) { TpIndexMetaPage snapshot; TpCompactionPlan plan; TpCompactionOutput output; uint32 drained; bool private_compaction; private_compaction = tp_compaction_is_private(index_state, index); if (first_level >= TP_MAX_LEVELS) return false; if (!tp_select_compaction_plan( index_state, index, first_level, empty_only, &snapshot, &plan)) return false; if (!private_compaction) TP_INJECTION_POINT(TP_INJECTION_COMPACTION_AFTER_SELECT); drained = tp_tombstone_drain( index, private_compaction ? NULL : index_state, tp_reclaim_horizon(NULL), /* own_lock */ !private_compaction); if (drained > 0) IndexFreeSpaceMapVacuum(index); tp_build_compaction_output(index, snapshot, &plan, &output); tp_complete_compaction_publication( index_state, index, snapshot, &plan, &output, InvalidFullTransactionId, false, TP_STATS_REBASE_STRICT); pfree(snapshot); tp_free_compaction_plan(&plan); return true; } bool tp_compact_step(TpLocalIndexState *index_state, Relation index) { return tp_compact_once(index_state, index, 0, false); } /* * Report whether tp_compact_step would run a pass. Runs the same * selection against the same metapage snapshot but publishes nothing, * so a level that is full of over-budget segments reports false. * * Never waits. When another session holds index maintenance it reports * true, because that session is compacting this index. */ bool tp_compaction_pass_available(TpLocalIndexState *index_state, Relation index) { TpIndexMetaPage snapshot; TpCompactionPlan plan; volatile bool selected = false; if (!tp_try_compaction_lock(index)) return true; PG_TRY(); { selected = tp_select_compaction_plan( index_state, index, 0, false, &snapshot, &plan); } PG_FINALLY(); { if (index_state->lock_held) tp_release_index_lock(index_state); tp_compaction_unlock(index); } PG_END_TRY(); if (selected) { pfree(snapshot); tp_free_compaction_plan(&plan); } return selected; } bool tp_compact_empty_step(TpLocalIndexState *index_state, Relation index) { return tp_compact_once(index_state, index, 0, true); } void tp_force_compact(TpLocalIndexState *index_state, Relation index) { TpIndexMetaPage snapshot; TpCompactionPlan plan; TpCompactionOutput output; bool private_compaction; private_compaction = tp_compaction_is_private(index_state, index); if (!tp_select_force_compaction_plan(index_state, index, &snapshot, &plan)) return; if (!private_compaction) TP_INJECTION_POINT(TP_INJECTION_COMPACTION_AFTER_SELECT); tp_build_compaction_output(index, snapshot, &plan, &output); tp_complete_compaction_publication( index_state, index, snapshot, &plan, &output, InvalidFullTransactionId, false, TP_STATS_REBASE_STRICT); pfree(snapshot); tp_free_compaction_plan(&plan); } timescale-pg_textsearch-b63fe58/src/segment/compaction.h000066400000000000000000000052351525762047700235240ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. */ #pragma once #include #include "index/metapage.h" struct TpLocalIndexState; typedef struct RelationData *Relation; typedef enum TpCompactionAllocationPhase { TP_COMPACTION_ALLOCATION_OUTPUT_DATA, TP_COMPACTION_ALLOCATION_PAGE_INDEX, TP_COMPACTION_ALLOCATION_TOMBSTONE } TpCompactionAllocationPhase; typedef enum TpSegmentReplacementReclaimMode { TP_SEGMENT_REPLACEMENT_RECLAIM_CURRENT_XID, TP_SEGMENT_REPLACEMENT_RECLAIM_NEXT_XID } TpSegmentReplacementReclaimMode; extern void tp_compaction_allocation_injection_point(TpCompactionAllocationPhase phase); extern uint64 tp_max_segment_size_bytes(void); extern void tp_compaction_lock(Relation index); extern bool tp_try_compaction_lock(Relation index); extern void tp_compaction_unlock(Relation index); extern void tp_compaction_publication_lock(Relation index, LOCKMODE mode); extern bool tp_try_compaction_publication_lock(Relation index, LOCKMODE mode); extern void tp_compaction_publication_unlock(Relation index, LOCKMODE mode); /* * Report whether tp_compact_step would run a pass, accounting for * levels whose segments are all over max_segment_size. Safe as a loop * condition. */ extern bool tp_compaction_pass_available( struct TpLocalIndexState *index_state, Relation index); /* * Run at most one bounded compaction pass and report whether one ran. * Runtime callers hold the per-index maintenance object lock. CREATE INDEX may * instead call while holding its private per-index exclusive lock. */ extern bool tp_compact_step(struct TpLocalIndexState *index_state, Relation index); /* * Run at most one below-threshold pass whose selected prefix contains an * already-empty V5 segment. The caller already holds maintenance. */ extern bool tp_compact_empty_step(struct TpLocalIndexState *index_state, Relation index); /* * Publish one already-built replacement for one published source segment. * The caller holds maintenance and passes ownership of replacement_root. * Serial VACUUM uses its current transaction ID; parallel VACUUM attaches a * provisional tombstone batch, then samples and WAL-restamps it after the * publication record is inserted. */ extern void tp_publish_prepared_segment_replacement( struct TpLocalIndexState *index_state, Relation index, uint32 level, BlockNumber source_root, BlockNumber replacement_root, uint64 removed_docs, uint64 removed_tokens, uint64 added_tokens, TpSegmentReplacementReclaimMode reclaim_mode); extern void tp_force_compact(struct TpLocalIndexState *index_state, Relation index); timescale-pg_textsearch-b63fe58/src/segment/compression.c000066400000000000000000000256561525762047700237350ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compression.c - Block compression for posting lists * * Implements delta encoding + bitpacking for posting list compression. * Decoding fuses bit extraction, delta reconstruction, and posting writes. */ #include #include #include "segment/compression.h" #if defined(__x86_64__) && !defined(_MSC_VER) && defined(__has_builtin) #if __has_attribute(target) && __has_builtin(__builtin_cpu_supports) && \ __has_builtin(__builtin_cpu_init) #define TP_HAVE_AVX2 #include #define TP_AVX2 __attribute__((target("avx2"))) #endif #endif /* * Compute minimum bits needed to represent a value. * Returns 1 for 0 (need at least 1 bit), otherwise ceil(log2(value+1)). */ uint8 tp_compute_bit_width(uint32 max_value) { uint8 bits = 1; if (max_value == 0) return 1; while (bits < 32 && (1U << bits) <= max_value) bits++; return bits; } /* * Pack an array of values into a bit stream. * Returns number of bytes written. */ static uint32 bitpack_encode(uint32 *values, uint32 count, uint8 bits, uint8 *out) { uint64 buffer = 0; /* Accumulator for bits */ int buf_bits = 0; /* Bits currently in buffer */ uint32 out_pos = 0; uint32 i; uint32 mask = (bits == 32) ? UINT32_MAX : ((1U << bits) - 1); for (i = 0; i < count; i++) { /* Add value to buffer */ buffer |= ((uint64)(values[i] & mask)) << buf_bits; buf_bits += bits; /* Flush complete bytes */ while (buf_bits >= 8) { out[out_pos++] = (uint8)(buffer & 0xFF); buffer >>= 8; buf_bits -= 8; } } /* Flush remaining bits */ if (buf_bits > 0) out[out_pos++] = (uint8)(buffer & 0xFF); return out_pos; } static inline uint32 bitpack_extract(const uint8 *in, uint32 bit_offset, uint32 mask) { uint64 raw; /* The caller's full-size buffer permits reads past a packed section. */ memcpy(&raw, in + (bit_offset >> 3), sizeof(raw)); return (uint32)(raw >> (bit_offset & 7)) & mask; } /* * Compress a block of postings. * * Steps: * 1. Delta-encode doc IDs (first doc ID stored as-is, rest as deltas) * 2. Find max delta and max frequency to determine bit widths * 3. Bitpack deltas and frequencies * 4. Copy fieldnorms as-is */ uint32 tp_compress_block(TpBlockPosting *postings, uint32 count, uint8 *out_buf) { TpCompressedBlockHeader *header; uint32 *doc_deltas; uint32 *frequencies; uint32 max_delta = 0; uint32 max_freq = 0; uint32 prev_doc = 0; uint32 out_pos; uint32 i; Assert(count <= TP_BLOCK_SIZE); if (count == 0) return 0; /* Allocate temporary arrays for deltas and frequencies */ doc_deltas = palloc(count * sizeof(uint32)); frequencies = palloc(count * sizeof(uint32)); /* Delta-encode doc IDs and extract frequencies */ for (i = 0; i < count; i++) { uint32 doc_id = postings[i].doc_id; uint32 delta = doc_id - prev_doc; doc_deltas[i] = delta; frequencies[i] = postings[i].frequency; if (delta > max_delta) max_delta = delta; if (frequencies[i] > max_freq) max_freq = frequencies[i]; prev_doc = doc_id; } /* Write header */ header = (TpCompressedBlockHeader *)out_buf; header->doc_id_bits = tp_compute_bit_width(max_delta); header->freq_bits = tp_compute_bit_width(max_freq); out_pos = sizeof(TpCompressedBlockHeader); /* Bitpack doc ID deltas */ out_pos += bitpack_encode( doc_deltas, count, header->doc_id_bits, out_buf + out_pos); /* Bitpack frequencies */ out_pos += bitpack_encode( frequencies, count, header->freq_bits, out_buf + out_pos); /* Copy fieldnorms as-is (1 byte each) */ for (i = 0; i < count; i++) out_buf[out_pos++] = postings[i].fieldnorm; pfree(doc_deltas); pfree(frequencies); return out_pos; } typedef struct TpPackedBlock { const uint8 *docs; const uint8 *freqs; const uint8 *fieldnorms; uint32 doc_mask; uint32 freq_mask; uint8 doc_bits; uint8 freq_bits; } TpPackedBlock; static inline TpPackedBlock packed_block(const uint8 *compressed, uint32 count) { const TpCompressedBlockHeader *header = (const TpCompressedBlockHeader *) compressed; TpPackedBlock block; block.doc_bits = header->doc_id_bits; block.freq_bits = header->freq_bits; block.docs = compressed + sizeof(TpCompressedBlockHeader); block.freqs = block.docs + (count * block.doc_bits + 7) / 8; block.fieldnorms = block.freqs + (count * block.freq_bits + 7) / 8; block.doc_mask = block.doc_bits == 32 ? UINT32_MAX : ((1U << block.doc_bits) - 1); block.freq_mask = (1U << block.freq_bits) - 1; return block; } static inline void decompress_scalar_range( const TpPackedBlock *block, uint32 count, uint32 start, uint32 prev_doc, TpBlockPosting *out_postings) { uint32 doc_bit_offset = start * block->doc_bits; uint32 freq_bit_offset = start * block->freq_bits; uint32 i; for (i = start; i < count; i++) { uint32 doc_id = prev_doc + bitpack_extract(block->docs, doc_bit_offset, block->doc_mask); out_postings[i].doc_id = doc_id; out_postings[i].frequency = (uint16)bitpack_extract( block->freqs, freq_bit_offset, block->freq_mask); out_postings[i].fieldnorm = block->fieldnorms[i]; out_postings[i].reserved = 0; doc_bit_offset += block->doc_bits; freq_bit_offset += block->freq_bits; prev_doc = doc_id; } } static void decompress_block_scalar( const uint8 *compressed, uint32 count, uint32 first_doc_id, TpBlockPosting *out_postings) { TpPackedBlock block = packed_block(compressed, count); decompress_scalar_range(&block, count, 0, first_doc_id, out_postings); } #ifdef TP_HAVE_AVX2 typedef struct TpUnpackPlan { __m256i shuffle; __m256i shifts; } TpUnpackPlan; StaticAssertDecl( sizeof(TpBlockPosting) == 8 && offsetof(TpBlockPosting, doc_id) == 0 && offsetof(TpBlockPosting, frequency) == 4 && offsetof(TpBlockPosting, fieldnorm) == 6 && offsetof(TpBlockPosting, reserved) == 7, "AVX2 stores require packed eight-byte postings"); static inline TP_AVX2 TpUnpackPlan unpack_plan(uint8 bits, uint32 phase) { TpUnpackPlan plan; __m256i positions = _mm256_setr_epi64x( phase, phase + bits, phase + 2 * bits, phase + 3 * bits); __m256i bytes = _mm256_srli_epi64(positions, 3); __m256i repeated = _mm256_mul_epu32(bytes, _mm256_set1_epi64x(UINT64_C(0x01010101))); repeated = _mm256_or_si256(repeated, _mm256_slli_epi64(repeated, 32)); plan.shuffle = _mm256_add_epi64( repeated, _mm256_set1_epi64x(UINT64_C(0x0706050403020100))); plan.shifts = _mm256_and_si256(positions, _mm256_set1_epi64x(7)); return plan; } static inline TP_AVX2 __m128i unpack_four( const uint8 *in, uint32 bit_offset, const TpUnpackPlan plans[2], __m128i mask) { /* Four-value groups begin at bit offset 0 or 4 within a byte. */ const TpUnpackPlan *plan = &plans[(bit_offset & 7) >> 2]; __m128i bytes = _mm_loadu_si128((const __m128i *)(in + (bit_offset >> 3))); __m256i values = _mm256_broadcastsi128_si256(bytes); /* Wrapped high shuffle bytes are discarded by the shift and mask. */ values = _mm256_shuffle_epi8(values, plan->shuffle); values = _mm256_srlv_epi64(values, plan->shifts); values = _mm256_permutevar8x32_epi32( values, _mm256_setr_epi32(0, 2, 4, 6, 0, 0, 0, 0)); return _mm_and_si128(_mm256_castsi256_si128(values), mask); } static TP_AVX2 void decompress_block_avx2( const uint8 *compressed, uint32 count, uint32 first_doc_id, TpBlockPosting *out_postings) { TpPackedBlock block = packed_block(compressed, count); uint32 doc_bit_offset = 0; uint32 freq_bit_offset = 0; uint32 prev_doc = first_doc_id; uint32 i = 0; if (count >= 4) { TpUnpackPlan doc_plans[2] = {unpack_plan(block.doc_bits, 0), unpack_plan(block.doc_bits, 4)}; TpUnpackPlan freq_plans[2] = {unpack_plan(block.freq_bits, 0), unpack_plan(block.freq_bits, 4)}; __m128i doc_vmask = _mm_set1_epi32(block.doc_mask); __m128i freq_vmask = _mm_set1_epi32(block.freq_mask); __m128i carry = _mm_set1_epi32(prev_doc); for (; i + 4 <= count; i += 4) { __m128i ids = unpack_four( block.docs, doc_bit_offset, doc_plans, doc_vmask); __m128i frequencies = unpack_four( block.freqs, freq_bit_offset, freq_plans, freq_vmask); uint32 norm_bytes; __m128i metadata; ids = _mm_add_epi32(ids, _mm_slli_si128(ids, 4)); ids = _mm_add_epi32(ids, _mm_slli_si128(ids, 8)); ids = _mm_add_epi32(ids, carry); carry = _mm_shuffle_epi32(ids, _MM_SHUFFLE(3, 3, 3, 3)); memcpy(&norm_bytes, block.fieldnorms + i, sizeof(norm_bytes)); metadata = _mm_or_si128( frequencies, _mm_slli_epi32( _mm_cvtepu8_epi32(_mm_cvtsi32_si128(norm_bytes)), 16)); _mm_storeu_si128( (__m128i *)(out_postings + i), _mm_unpacklo_epi32(ids, metadata)); _mm_storeu_si128( (__m128i *)(out_postings + i + 2), _mm_unpackhi_epi32(ids, metadata)); doc_bit_offset += 4 * block.doc_bits; freq_bit_offset += 4 * block.freq_bits; } prev_doc = (uint32)_mm_cvtsi128_si32(carry); } decompress_scalar_range(&block, count, i, prev_doc, out_postings); } #endif static void (*decompress_impl)( const uint8 *, uint32, uint32, TpBlockPosting *) = NULL; /* Validate once, then unpack directly into the final postings. */ void tp_decompress_block( const uint8 *compressed, uint32 count, uint32 first_doc_id, TpBlockPosting *out_postings) { const TpCompressedBlockHeader *header; if (count > TP_BLOCK_SIZE) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupted segment: block count %u exceeds " "maximum %u", count, (uint32)TP_BLOCK_SIZE))); if (count == 0) return; header = (const TpCompressedBlockHeader *)compressed; /* Validate header values to prevent buffer overruns */ if (header->doc_id_bits < 1 || header->doc_id_bits > 32) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupted segment: invalid doc_id bit " "width %u", header->doc_id_bits))); if (header->freq_bits < 1 || header->freq_bits > 16) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupted segment: invalid frequency bit " "width %u", header->freq_bits))); if (unlikely(decompress_impl == NULL)) { decompress_impl = decompress_block_scalar; #ifdef TP_HAVE_AVX2 /* The compiler's probe also checks OS support for saving YMM state. */ __builtin_cpu_init(); if (__builtin_cpu_supports("avx2")) decompress_impl = decompress_block_avx2; #endif } decompress_impl(compressed, count, first_doc_id, out_postings); } /* * Get the size of compressed data. */ uint32 tp_compressed_block_size(const uint8 *compressed, uint32 count) { const TpCompressedBlockHeader *header; uint32 doc_id_bytes; uint32 freq_bytes; if (count == 0) return 0; header = (const TpCompressedBlockHeader *)compressed; doc_id_bytes = (count * header->doc_id_bits + 7) / 8; freq_bytes = (count * header->freq_bits + 7) / 8; return sizeof(TpCompressedBlockHeader) + doc_id_bytes + freq_bytes + count; } timescale-pg_textsearch-b63fe58/src/segment/compression.h000066400000000000000000000056341525762047700237340ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * compression.h - Block compression for posting lists * * Implements delta encoding + bitpacking for posting list compression. * Doc IDs are delta-encoded (storing gaps instead of absolute values), * then both gaps and frequencies are bitpacked using the minimum bits needed. */ #pragma once #include #include "segment/segment.h" /* * Compressed block header - stored at start of compressed block data. * Total: 2 bytes header + variable packed data + 128 bytes fieldnorms */ typedef struct TpCompressedBlockHeader { uint8 doc_id_bits; /* Bits per doc ID delta (1-32) */ uint8 freq_bits; /* Bits per frequency (1-16) */ } TpCompressedBlockHeader; /* * Maximum compressed block size (for buffer allocation). * Header (2) + max doc_id bits (32*128/8=512) + max freq bits (16*128/8=256) * + fieldnorms (128) = 898 bytes */ #define TP_MAX_COMPRESSED_BLOCK_SIZE 898 /* Verify buffer size is sufficient for worst case */ StaticAssertDecl( TP_MAX_COMPRESSED_BLOCK_SIZE >= sizeof(TpCompressedBlockHeader) + (TP_BLOCK_SIZE * 32 + 7) / 8 + /* max doc_id bits (32 per value) */ (TP_BLOCK_SIZE * 16 + 7) / 8 + /* max freq bits (16 per value) */ TP_BLOCK_SIZE, /* fieldnorms (1 byte each) */ "TP_MAX_COMPRESSED_BLOCK_SIZE too small for worst-case compression"); /* * Compression functions */ /* * Compute minimum bits needed to represent max_value. * Returns 1-32 for doc IDs, 1-16 for frequencies. */ extern uint8 tp_compute_bit_width(uint32 max_value); /* * Compress a block of postings. * * Input: array of TpBlockPosting (uncompressed) * Output: compressed data written to out_buf * Returns: number of bytes written to out_buf * * Format: * [2 bytes: TpCompressedBlockHeader] * [ceil(count * doc_id_bits / 8) bytes: bitpacked doc ID deltas] * [ceil(count * freq_bits / 8) bytes: bitpacked frequencies] * [count bytes: fieldnorms (uncompressed)] */ extern uint32 tp_compress_block(TpBlockPosting *postings, uint32 count, uint8 *out_buf); /* * Decompress a block of postings. * * Selects AVX2 at runtime on supported x86-64 builds, otherwise scalar. * Input: at least TP_MAX_COMPRESSED_BLOCK_SIZE accessible bytes. * Unaligned loads may read beyond the encoded block's logical length. * Output: array of TpBlockPosting (caller-allocated, size count) * * first_doc_id: Base added to the delta prefix sum. The current format * stores an absolute ID as its first delta, so pass 0. */ extern void tp_decompress_block( const uint8 *compressed, uint32 count, uint32 first_doc_id, TpBlockPosting *out_postings); /* * Get the size of compressed data (for validation/debugging). * Parses header to compute actual size without decompressing. */ extern uint32 tp_compressed_block_size(const uint8 *compressed, uint32 count); timescale-pg_textsearch-b63fe58/src/segment/dictionary.c000066400000000000000000000036071525762047700235310ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * dictionary.c - Term dictionary for disk segments */ #include #include #include "constants.h" #include "segment/dictionary.h" #include "segment/io.h" /* * Free dictionary's term strings. * * Phase 4: the ctids/freqs arrays are owned by the source's memory * context (e.g. chain_source's private mcxt) and are dropped when * that context is reset. We only free the per-entry palloc'd term * string and the array itself. */ void tp_free_dictionary(TermInfo *terms, uint32 num_terms) { uint32 i; if (!terms) return; for (i = 0; i < num_terms; i++) { if (terms[i].term) pfree(terms[i].term); } pfree(terms); } /* * Read a term string from a segment's string pool at a given dictionary * index. Returns a palloc'd string that must be freed by the caller. */ char * tp_segment_read_term_at_index( TpSegmentReader *reader, TpSegmentHeader *header, uint32 *string_offsets, uint32 index) { uint32 string_offset_value; uint64 string_offset; uint32 length; char *term; if (string_offsets != NULL) string_offset_value = string_offsets[index]; else tp_segment_read( reader, header->dictionary_offset + sizeof(uint32) + (uint64)index * sizeof(uint32), &string_offset_value, sizeof(string_offset_value)); string_offset = header->strings_offset + string_offset_value; /* Read string length */ tp_segment_read(reader, string_offset, &length, sizeof(uint32)); if (length > TP_MAX_TERM_LENGTH) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupt segment: term length %u exceeds " "maximum", length))); /* Allocate and read string */ term = palloc(length + 1); tp_segment_read(reader, string_offset + sizeof(uint32), term, length); term[length] = '\0'; return term; } timescale-pg_textsearch-b63fe58/src/segment/dictionary.h000066400000000000000000000030531525762047700235310ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * dictionary.h - Term dictionary for disk segments */ #pragma once #include "postgres.h" #include "storage/itemptr.h" /* Forward declarations */ struct TpDataSource; struct TpSegmentReader; struct TpSegmentHeader; /* * Term info for building a segment dictionary. * * The ctids/freqs arrays are owned by the caller (typically the * caller's memory context, e.g. the chain source's private mcxt that * already holds the parallel ctid/freq arrays). tp_write_segment * reads them directly; it does not free or take ownership. */ typedef struct TermInfo { char *term; /* The term text */ uint32 term_len; /* Term length */ ItemPointerData *ctids; /* Array of doc CTIDs (len = count) */ int32 *freqs; /* Array of term freqs (len = count) */ uint32 count; /* Length of ctids/freqs */ uint32 doc_freq; /* Document frequency (used for IDF) */ uint32 dict_entry_idx; /* Index in dict_entries array */ } TermInfo; /* * Free a TermInfo[] previously returned by a dictionary builder. * * Only the `term` strings are freed; the ctids/freqs arrays are * presumed to live in the source's memory context and are dropped * when that context is reset. */ extern void tp_free_dictionary(TermInfo *terms, uint32 num_terms); /* Shared term-reading helper */ extern char *tp_segment_read_term_at_index( struct TpSegmentReader *reader, struct TpSegmentHeader *header, uint32 *string_offsets, uint32 index); timescale-pg_textsearch-b63fe58/src/segment/docmap.c000066400000000000000000000136471525762047700226340ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * docmap.c - Document ID mapping implementation */ #include "postgres.h" #include "segment/docmap.h" #include "segment/fieldnorm.h" #include "utils/hsearch.h" #include "utils/memutils.h" /* Initial capacity for document arrays */ #define DOCMAP_INITIAL_CAPACITY 1024 /* * Hash function for ItemPointerData keys */ static uint32 ctid_hash(const void *key, Size keysize) { const ItemPointerData *ctid = (const ItemPointerData *)key; uint32 block = ItemPointerGetBlockNumber(ctid); uint16 offset = ItemPointerGetOffsetNumber(ctid); (void)keysize; /* unused */ /* Combine block and offset into a single hash */ return block ^ ((uint32)offset << 16) ^ offset; } /* * Comparison function for ItemPointerData keys */ static int ctid_match(const void *key1, const void *key2, Size keysize) { (void)keysize; /* unused */ return ItemPointerCompare((ItemPointer)key1, (ItemPointer)key2); } TpDocMapBuilder * tp_docmap_create(void) { TpDocMapBuilder *builder; HASHCTL hash_ctl; builder = palloc0(sizeof(TpDocMapBuilder)); /* Create hash table for CTID → doc_id lookup */ memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(ItemPointerData); hash_ctl.entrysize = sizeof(TpDocMapEntry); hash_ctl.hash = ctid_hash; hash_ctl.match = ctid_match; hash_ctl.hcxt = CurrentMemoryContext; builder->ctid_to_id = hash_create( "DocMap CTID->ID", DOCMAP_INITIAL_CAPACITY, &hash_ctl, HASH_ELEM | HASH_FUNCTION | HASH_COMPARE | HASH_CONTEXT); builder->num_docs = 0; builder->capacity = 0; builder->finalized = false; builder->ctid_pages = NULL; builder->ctid_offsets = NULL; builder->fieldnorms = NULL; builder->total_tokens = 0; return builder; } uint32 tp_docmap_add(TpDocMapBuilder *builder, ItemPointer ctid, uint32 doc_length) { TpDocMapEntry *entry; bool found; Assert(!builder->finalized); /* Guard: UINT32_MAX is reserved as "not found" sentinel */ if (builder->num_docs >= UINT32_MAX - 1) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("too many documents in segment (max %u)", UINT32_MAX - 1))); /* Look up or create entry in hash table */ entry = (TpDocMapEntry *) hash_search(builder->ctid_to_id, ctid, HASH_ENTER, &found); if (found) { /* Document already exists, return existing ID */ return entry->doc_id; } /* New document - assign next sequential ID */ entry->doc_id = builder->num_docs; entry->doc_length = doc_length; builder->num_docs++; builder->total_tokens += doc_length; return entry->doc_id; } uint32 tp_docmap_lookup(TpDocMapBuilder *builder, ItemPointer ctid) { TpDocMapEntry *entry; entry = (TpDocMapEntry *) hash_search(builder->ctid_to_id, ctid, HASH_FIND, NULL); if (entry == NULL) return UINT32_MAX; return entry->doc_id; } /* * Comparison function for sorting by CTID. * This ensures doc_ids are assigned in CTID order, which means * postings sorted by CTID are also sorted by doc_id - critical * for sequential access to CTID arrays during query iteration. */ static int docmap_entry_cmp_by_ctid(const void *a, const void *b) { const TpDocMapEntry *ea = (const TpDocMapEntry *)a; const TpDocMapEntry *eb = (const TpDocMapEntry *)b; /* Cast away const - ItemPointerCompare doesn't modify arguments */ return ItemPointerCompare((ItemPointer)&ea->ctid, (ItemPointer)&eb->ctid); } void tp_docmap_finalize(TpDocMapBuilder *builder) { HASH_SEQ_STATUS scan; TpDocMapEntry *entry; TpDocMapEntry *entries; uint32 i; Assert(!builder->finalized); if (builder->num_docs == 0) { builder->finalized = true; return; } /* * Collect all entries from hash table. * Use HUGE allocation since this can exceed MaxAllocSize for * very large segments (e.g., 138M docs * 16 bytes = 2.2GB). */ entries = palloc_extended( sizeof(TpDocMapEntry) * builder->num_docs, MCXT_ALLOC_HUGE); i = 0; hash_seq_init(&scan, builder->ctid_to_id); while ((entry = (TpDocMapEntry *)hash_seq_search(&scan)) != NULL) { entries[i++] = *entry; } Assert(i == builder->num_docs); /* * Sort by CTID to assign doc_ids in CTID order. * This maintains the invariant: CTID order = doc_id order. * Postings sorted by CTID will then be sorted by doc_id, * enabling sequential access to CTID arrays during iteration. */ qsort(entries, builder->num_docs, sizeof(TpDocMapEntry), docmap_entry_cmp_by_ctid); /* Allocate output arrays (split CTID storage for better cache locality) */ builder->capacity = builder->num_docs; builder->ctid_pages = palloc_extended( sizeof(BlockNumber) * builder->num_docs, MCXT_ALLOC_HUGE); builder->ctid_offsets = palloc_extended( sizeof(OffsetNumber) * builder->num_docs, MCXT_ALLOC_HUGE); builder->fieldnorms = palloc_extended( sizeof(uint8) * builder->num_docs, MCXT_ALLOC_HUGE); /* * Fill arrays and reassign doc_ids in CTID order. * Update hash table entries so lookups return the correct new doc_id. */ for (i = 0; i < builder->num_docs; i++) { TpDocMapEntry *hash_entry; /* Array position i = doc_id i (CTID-sorted order) */ builder->ctid_pages[i] = ItemPointerGetBlockNumber(&entries[i].ctid); builder->ctid_offsets[i] = ItemPointerGetOffsetNumber( &entries[i].ctid); builder->fieldnorms[i] = encode_fieldnorm(entries[i].doc_length); /* Update hash table entry with new doc_id */ hash_entry = (TpDocMapEntry *)hash_search( builder->ctid_to_id, &entries[i].ctid, HASH_FIND, NULL); Assert(hash_entry != NULL); hash_entry->doc_id = i; } pfree(entries); builder->finalized = true; } void tp_docmap_destroy(TpDocMapBuilder *builder) { if (builder == NULL) return; if (builder->ctid_to_id) hash_destroy(builder->ctid_to_id); if (builder->ctid_pages) pfree(builder->ctid_pages); if (builder->ctid_offsets) pfree(builder->ctid_offsets); if (builder->fieldnorms) pfree(builder->fieldnorms); pfree(builder); } timescale-pg_textsearch-b63fe58/src/segment/docmap.h000066400000000000000000000070271525762047700226340ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * docmap.h - Document ID mapping for segment format * * Posting lists use compact 4-byte segment-local doc IDs * instead of 6-byte CTIDs. This module provides: * - Collection of unique documents during segment build * - Assignment of sequential doc IDs (0 to N-1) in CTID order * - CTID → doc_id lookup during posting conversion * - Arrays for CTID map and fieldnorm table * * INVARIANT: Doc IDs are assigned in CTID order after finalize. * This means CTID order = doc_id order, so postings sorted by CTID * are also sorted by doc_id, enabling sequential access to CTID arrays. */ #pragma once #include "postgres.h" #include "storage/itemptr.h" #include "utils/hsearch.h" /* * Entry in the CTID → doc_id hash table (build-time only). * Used to quickly look up a document's assigned ID when converting postings. */ typedef struct TpDocMapEntry { ItemPointerData ctid; /* Key: heap tuple location */ uint32 doc_id; /* Value: segment-local doc ID */ uint32 doc_length; /* Document length (for fieldnorm) */ } TpDocMapEntry; /* * Document map builder context. * Collects documents and assigns sequential IDs during segment write. */ typedef struct TpDocMapBuilder { HTAB *ctid_to_id; /* Hash table: CTID → doc_id */ uint32 num_docs; /* Number of documents assigned */ uint32 capacity; /* Current capacity of arrays */ bool finalized; /* True after tp_docmap_finalize called */ /* Output arrays (indexed by doc_id, valid after finalize) */ BlockNumber *ctid_pages; /* doc_id → page number (4 bytes) */ OffsetNumber *ctid_offsets; /* doc_id → tuple offset (2 bytes) */ uint8 *fieldnorms; /* doc_id → encoded length (1 byte) */ /* Sum of per-doc token counts; used for segment.total_tokens. */ uint64 total_tokens; } TpDocMapBuilder; /* * Create a new document map builder (constructor). * Call this before collecting documents. */ extern TpDocMapBuilder *tp_docmap_create(void); /* * Add a document to the map. * Returns the assigned doc_id (reuses existing ID if CTID already present). * doc_length is stored for fieldnorm encoding; if CTID already exists, the * original doc_length is kept (callers should ensure consistent lengths). */ extern uint32 tp_docmap_add(TpDocMapBuilder *builder, ItemPointer ctid, uint32 doc_length); /* * Look up doc_id for a CTID using hash table. * Returns UINT32_MAX if not found. */ extern uint32 tp_docmap_lookup(TpDocMapBuilder *builder, ItemPointer ctid); /* * Finalize the document map. * Builds the ctid arrays and fieldnorms array sorted by doc_id. * After this call, the hash table is no longer needed. */ extern void tp_docmap_finalize(TpDocMapBuilder *builder); /* * Free the document map builder and all associated memory. */ extern void tp_docmap_destroy(TpDocMapBuilder *builder); /* * Get the CTID for a doc_id. Requires finalize to have been called. * Reconstructs ItemPointerData from the split storage. */ static inline void tp_docmap_get_ctid(TpDocMapBuilder *builder, uint32 doc_id, ItemPointer result) { Assert(builder->finalized); Assert(doc_id < builder->num_docs); ItemPointerSet( result, builder->ctid_pages[doc_id], builder->ctid_offsets[doc_id]); } /* * Get the fieldnorm for a doc_id. Requires finalize to have been called. */ static inline uint8 tp_docmap_get_fieldnorm(TpDocMapBuilder *builder, uint32 doc_id) { Assert(builder->finalized); if (doc_id >= builder->num_docs) return 0; return builder->fieldnorms[doc_id]; } timescale-pg_textsearch-b63fe58/src/segment/fieldnorm.c000066400000000000000000000102101525762047700233270ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * fieldnorm.c - Document length quantization for BM25 scoring */ #include #include "segment/fieldnorm.h" /* * Precomputed decode table (Lucene SmallFloat.byte4ToInt) * * Generated by: for i in 0..255: decode_table[i] = byte4ToInt(i) * Values 0-39 map exactly, larger values use exponential encoding. */ const uint32 FIELDNORM_DECODE_TABLE[256] = { /* 0-15: exact values */ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, /* 16-31: exact values */ 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, /* 32-39: exact values */ 32, 33, 34, 35, 36, 37, 38, 39, /* 40-47: step 2 */ 40, 42, 44, 46, 48, 50, 52, 54, /* 48-55: step 4 */ 56, 60, 64, 68, 72, 76, 80, 84, /* 56-63: step 8 */ 88, 96, 104, 112, 120, 128, 136, 144, /* 64-71: step 16 */ 152, 168, 184, 200, 216, 232, 248, 264, /* 72-79: step 32 */ 280, 312, 344, 376, 408, 440, 472, 504, /* 80-87: step 64 */ 536, 600, 664, 728, 792, 856, 920, 984, /* 88-95: step 128 */ 1048, 1176, 1304, 1432, 1560, 1688, 1816, 1944, /* 96-103: step 256 */ 2072, 2328, 2584, 2840, 3096, 3352, 3608, 3864, /* 104-111: step 512 */ 4120, 4632, 5144, 5656, 6168, 6680, 7192, 7704, /* 112-119: step 1024 */ 8216, 9240, 10264, 11288, 12312, 13336, 14360, 15384, /* 120-127: step 2048 */ 16408, 18456, 20504, 22552, 24600, 26648, 28696, 30744, /* 128-135: step 4096 */ 32792, 36888, 40984, 45080, 49176, 53272, 57368, 61464, /* 136-143: step 8192 */ 65560, 73752, 81944, 90136, 98328, 106520, 114712, 122904, /* 144-151: step 16384 */ 131096, 147480, 163864, 180248, 196632, 213016, 229400, 245784, /* 152-159: step 32768 */ 262168, 294936, 327704, 360472, 393240, 426008, 458776, 491544, /* 160-167: step 65536 */ 524312, 589848, 655384, 720920, 786456, 851992, 917528, 983064, /* 168-175: step 131072 */ 1048600, 1179672, 1310744, 1441816, 1572888, 1703960, 1835032, 1966104, /* 176-183: step 262144 */ 2097176, 2359320, 2621464, 2883608, 3145752, 3407896, 3670040, 3932184, /* 184-191: step 524288 */ 4194328, 4718616, 5242904, 5767192, 6291480, 6815768, 7340056, 7864344, /* 192-199: step 1048576 */ 8388632, 9437208, 10485784, 11534360, 12582936, 13631512, 14680088, 15728664, /* 200-207: step 2097152 */ 16777240, 18874392, 20971544, 23068696, 25165848, 27263000, 29360152, 31457304, /* 208-215: step 4194304 */ 33554456, 37748760, 41943064, 46137368, 50331672, 54525976, 58720280, 62914584, /* 216-223: step 8388608 */ 67108888, 75497496, 83886104, 92274712, 100663320, 109051928, 117440536, 125829144, /* 224-231: step 16777216 */ 134217752, 150994968, 167772184, 184549400, 201326616, 218103832, 234881048, 251658264, /* 232-239: step 33554432 */ 268435480, 301989912, 335544344, 369098776, 402653208, 436207640, 469762072, 503316504, /* 240-247: step 67108864 */ 536870936, 603979800, 671088664, 738197528, 805306392, 872415256, 939524120, 1006632984, /* 248-255: step 134217728 */ 1073741848, 1207959576, 1342177304, 1476395032, 1610612760, 1744830488, 1879048216, 2013265944}; /* * Encode document length to fieldnorm byte via binary search. * * Finds the largest index i where FIELDNORM_DECODE_TABLE[i] <= length. * This matches Tantivy's fieldnorm_to_id implementation. */ uint8 encode_fieldnorm(uint32 length) { int lo = 0; int hi = 255; while (lo < hi) { int mid = (lo + hi + 1) / 2; if (FIELDNORM_DECODE_TABLE[mid] <= length) lo = mid; else hi = mid - 1; } return (uint8)lo; } /* * Decode fieldnorm byte back to approximate document length */ uint32 decode_fieldnorm(uint8 norm_id) { return FIELDNORM_DECODE_TABLE[norm_id]; } timescale-pg_textsearch-b63fe58/src/segment/fieldnorm.h000066400000000000000000000016251525762047700233460ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * fieldnorm.h - Document length quantization for BM25 scoring * * Uses Lucene's SmallFloat encoding scheme which maps document lengths * to a single byte (256 buckets). This reduces storage while maintaining * good BM25 ranking quality because: * - BM25 uses the ratio dl/avgdl, not absolute length * - Small errors become smaller in the ratio * - The b parameter (0.75) further dampens length's influence * * Key properties: * - Lengths 0-39 stored exactly (covers most short documents) * - Larger lengths use 4-bit mantissa (~6% relative error max) * - 256 buckets cover lengths from 0 to 2+ billion */ #pragma once #include "postgres.h" extern const uint32 FIELDNORM_DECODE_TABLE[256]; extern uint8 encode_fieldnorm(uint32 length); extern uint32 decode_fieldnorm(uint8 norm_id); timescale-pg_textsearch-b63fe58/src/segment/format.h000066400000000000000000000215111525762047700226530ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * format.h - On-disk format definitions for segment storage */ #pragma once #include #include #include #include "compat.h" /* * Page types for segment pages. * Magic and version constants are defined in constants.h. */ typedef enum TpPageType { TP_PAGE_SEGMENT_HEADER = 1, /* Segment header page */ TP_PAGE_FILE_INDEX, /* File index (page map) pages */ TP_PAGE_DATA /* Data pages containing actual content */ } TpPageType; /* * Special area for page index pages (goes in page special area). * Contains magic/version for validation. */ typedef struct TpPageIndexSpecial { uint32 magic; /* TP_PAGE_INDEX_MAGIC */ uint16 version; /* TP_PAGE_INDEX_VERSION */ uint16 page_type; /* TpPageType enum value */ BlockNumber next_page; /* Next page in chain, InvalidBlockNumber if last */ uint16 num_entries; /* Number of BlockNumber entries on this page */ uint16 flags; /* Reserved for future use */ } TpPageIndexSpecial; /* * Segment format versions */ #define TP_SEGMENT_FORMAT_VERSION_3 3 /* Legacy: uint32 offsets */ #define TP_SEGMENT_FORMAT_VERSION_4 4 /* Legacy: no alive bitset */ #define TP_SEGMENT_FORMAT_VERSION 5 /* Current: alive bitset */ /* * V3 legacy segment header - preserved for reading old segments. * All offsets are uint32, limiting segments to 4GB. */ typedef struct TpSegmentHeaderV3 { uint32 magic; uint32 version; TimestampTz created_at; uint32 num_pages; uint32 data_size; uint32 level; BlockNumber next_segment; uint32 dictionary_offset; uint32 strings_offset; uint32 entries_offset; uint32 postings_offset; uint32 skip_index_offset; uint32 fieldnorm_offset; uint32 ctid_pages_offset; uint32 ctid_offsets_offset; uint32 num_terms; uint32 num_docs; uint64 total_tokens; BlockNumber page_index; } TpSegmentHeaderV3; /* * V4 legacy segment header - preserved for reading old segments. * Same as V5 but without alive_bitset_offset / alive_count. */ typedef struct TpSegmentHeaderV4 { uint32 magic; uint32 version; TimestampTz created_at; uint32 num_pages; uint64 data_size; uint32 level; BlockNumber next_segment; uint64 dictionary_offset; uint64 strings_offset; uint64 entries_offset; uint64 postings_offset; uint64 skip_index_offset; uint64 fieldnorm_offset; uint64 ctid_pages_offset; uint64 ctid_offsets_offset; uint32 num_terms; uint32 num_docs; uint64 total_tokens; BlockNumber page_index; } TpSegmentHeaderV4; /* * Segment header - stored on the first page (V5: alive bitset) */ typedef struct TpSegmentHeader { /* Metadata */ uint32 magic; /* TP_SEGMENT_MAGIC */ uint32 version; /* Format version */ TimestampTz created_at; /* Creation time */ uint32 num_pages; /* Total pages in segment */ uint64 data_size; /* Total data bytes */ /* Segment management */ uint32 level; /* Storage level (0 for memtable flush) */ BlockNumber next_segment; /* Next segment in chain */ /* Section offsets in logical file */ uint64 dictionary_offset; /* Offset to TpDictionary */ uint64 strings_offset; /* Offset to string pool */ uint64 entries_offset; /* Offset to TpDictEntry array */ uint64 postings_offset; /* Offset to posting data (blocks) */ /* Block storage offsets */ uint64 skip_index_offset; /* Offset to skip index */ uint64 fieldnorm_offset; /* Offset to fieldnorm table */ uint64 ctid_pages_offset; /* Offset to BlockNumber array */ uint64 ctid_offsets_offset; /* Offset to OffsetNumber array */ /* Alive bitset for tombstone tracking (V5+) */ uint64 alive_bitset_offset; /* Offset to alive bitset data */ uint32 alive_count; /* Number of alive docs (bits set) */ /* Corpus statistics */ uint32 num_terms; /* Total unique terms */ uint32 num_docs; /* Total documents */ uint64 total_tokens; /* Sum of all document lengths */ /* Page index reference */ BlockNumber page_index; /* First page of the page index */ } TpSegmentHeader; /* * Dictionary structure for fast term lookup * * The dictionary is a sorted array of string offsets, enabling binary search. * Each string is stored as: * [length:uint32][text:char*][dict_entry_offset:uint32] This allows us to * quickly find a term and jump to its TpDictEntry. */ typedef struct TpDictionary { uint32 num_terms; /* Number of terms in dictionary */ uint32 string_offsets[FLEXIBLE_ARRAY_MEMBER]; /* Sorted array of offsets to strings */ } TpDictionary; /* * String entry in string pool * * TODO: Optimize storage for short strings. Current overhead is 8 bytes per * term (4-byte length + 4-byte dict_entry_offset). Since most stemmed terms * are short (3-10 chars), this overhead is significant. Options: * - Use 1-byte length (terms rarely > 255 chars), saves 3 bytes/term * - Eliminate dict_entry_offset by computing from term index, saves 4 * bytes/term * - Use varint encoding for length */ typedef struct TpStringEntry { uint32 length; /* String length */ char text[FLEXIBLE_ARRAY_MEMBER]; /* String text (variable length) */ /* Immediately after text: uint32 dict_entry_offset */ } TpStringEntry; /* * V3 legacy dictionary entry - 12 bytes */ typedef struct TpDictEntryV3 { uint32 skip_index_offset; uint16 block_count; uint16 reserved; uint32 doc_freq; } pg_attribute_aligned(4) TpDictEntryV3; /* * Dictionary entry - 16 bytes, block-based storage (V4: uint64 offset) * * Points to skip index instead of raw postings. The skip index * contains block_count TpSkipEntry structures, each pointing to * a block of up to TP_BLOCK_SIZE postings. * * block_count was widened from uint16 to uint32 without a format * version bump. The old layout had uint16 block_count + uint16 * reserved (always 0) at bytes 8-11. On little-endian platforms * (x86-64, ARM64) these four bytes read identically as a uint32, * so existing V4 segments are binary-compatible. */ typedef struct TpDictEntry { uint64 skip_index_offset; /* Offset to TpSkipEntry array for this term */ uint32 block_count; /* Number of blocks (and skip entries) */ uint32 doc_freq; /* Document frequency for IDF */ } pg_attribute_aligned(8) TpDictEntry; /* * Block storage constants */ #define TP_BLOCK_SIZE 128 /* Documents per block (matches Tantivy) */ #ifdef _MSC_VER #pragma pack(push, 1) /* MSVC equivalent of TP_PACKED below */ #endif /* * V3 legacy skip index entry - 16 bytes per block */ typedef struct TpSkipEntryV3 { uint32 last_doc_id; uint8 doc_count; uint16 block_max_tf; uint8 block_max_norm; uint32 posting_offset; uint8 flags; uint8 reserved[3]; } TP_PACKED TpSkipEntryV3; /* * Skip index entry - 20 bytes per block (V4: uint64 posting_offset) * * Stored separately from posting data for cache efficiency during BMW. * The skip index is a dense array of these entries, one per block. */ typedef struct TpSkipEntry { uint32 last_doc_id; /* Last segment-local doc ID in block */ uint8 doc_count; /* Number of docs in block (1-128) */ uint16 block_max_tf; /* Max term frequency in block (for BMW) */ uint8 block_max_norm; /* Min fieldnorm in block (shortest doc, for BMW) */ uint64 posting_offset; /* Byte offset from segment start to block data */ uint8 flags; /* Compression type, etc. */ uint8 reserved[3]; /* Future use */ } TP_PACKED TpSkipEntry; #ifdef _MSC_VER #pragma pack(pop) #endif /* Skip entry flags */ #define TP_BLOCK_FLAG_UNCOMPRESSED 0x00 /* Raw doc IDs and frequencies */ #define TP_BLOCK_FLAG_DELTA 0x01 /* Delta-encoded doc IDs */ #define TP_BLOCK_FLAG_FOR 0x02 /* Frame-of-reference (Phase 3) */ #define TP_BLOCK_FLAG_PFOR 0x03 /* Patched FOR (Phase 3) */ /* * Block posting entry - 8 bytes, used in uncompressed blocks * * Unlike TpSegmentPosting, uses segment-local doc ID instead of CTID. * CTID lookup happens via the doc ID -> CTID mapping table. * * Fieldnorm is stored inline (using former padding bytes) to avoid * per-posting buffer manager lookups during scoring. This is critical * for performance: each buffer manager access adds ~300-500ns even when * pages are cached, which dominates query time for large posting lists. */ typedef struct TpBlockPosting { uint32 doc_id; /* Segment-local document ID */ uint16 frequency; /* Term frequency in document */ uint8 fieldnorm; /* Quantized document length (Lucene SmallFloat) */ uint8 reserved; /* Padding for alignment */ } TpBlockPosting; /* * CTID map entry - 6 bytes per document * * Maps segment-local doc IDs to heap CTIDs. This enables using * compact 4-byte doc IDs in posting lists while still being able * to look up the actual heap tuple. */ #ifdef _MSC_VER #pragma pack(push, 1) /* MSVC equivalent of TP_PACKED below */ #endif typedef struct TpCtidMapEntry { ItemPointerData ctid; /* 6 bytes - heap tuple location */ } TP_PACKED TpCtidMapEntry; #ifdef _MSC_VER #pragma pack(pop) #endif timescale-pg_textsearch-b63fe58/src/segment/graph_snapshot.c000066400000000000000000000176671525762047700244170ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * graph_snapshot.c - Atomic segment-root and memtable read snapshots */ #include #include #include #include #include #include #include "index/metapage.h" #include "memtable/chain_walker.h" #include "segment/graph_snapshot.h" #include "segment/io.h" #include "segment/segment.h" int tp_debug_segment_graph_snapshot_pause_before_lock_ms = 0; int tp_debug_segment_graph_snapshot_pause_before_unlock_ms = 0; int tp_debug_segment_graph_snapshot_pause_ms = 0; static void tp_debug_segment_graph_snapshot_pause( Relation index, int pause_ms, const char *phase) { TimestampTz deadline; if (pause_ms <= 0) return; ereport(LOG, (errmsg("pg_textsearch segment graph snapshot pause at %s for " "index %u backend %d", phase, RelationGetRelid(index), MyProcPid))); deadline = TimestampTzPlusMilliseconds(GetCurrentTimestamp(), pause_ms); for (;;) { CHECK_FOR_INTERRUPTS(); if (GetCurrentTimestamp() >= deadline) break; pg_usleep(10000L); } ereport(LOG, (errmsg("pg_textsearch segment graph snapshot resume after %s for " "index %u backend %d", phase, RelationGetRelid(index), MyProcPid))); } static void tp_read_memtable_snapshot_candidate( Relation index, BlockNumber *head, BlockNumber *tail) { Buffer buffer; Page page; buffer = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(buffer); *head = tp_metapage_read_memtable_head(page); *tail = tp_metapage_read_memtable_tail(page); UnlockReleaseBuffer(buffer); } /* * Writers extend in tail -> new page -> metapage order. Snapshot retry paths * must therefore release metapage before tail, matching the reverse of their * tail -> metapage acquisition. */ static void tp_release_snapshot_buffers(Buffer buffer, Buffer tail_buffer) { UnlockReleaseBuffer(buffer); if (BufferIsValid(tail_buffer)) UnlockReleaseBuffer(tail_buffer); } static void tp_capture_segment_roots(Relation index, TpSegmentGraphSnapshot *snapshot) { uint32 expected_roots = 0; for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { if (pg_add_u32_overflow( expected_roots, (uint32)snapshot->metapage.level_counts[level], &expected_roots)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("BM25 segment-root count overflow"))); } if (expected_roots > MaxAllocSize / sizeof(BlockNumber)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("BM25 segment-root snapshot is too large"))); if (expected_roots > 0) snapshot->roots = palloc_array(BlockNumber, expected_roots); for (uint32 level = 0; level < TP_MAX_LEVELS; level++) { BlockNumber current = snapshot->metapage.level_heads[level]; snapshot->level_offsets[level] = snapshot->root_count; for (uint32 i = 0; i < (uint32)snapshot->metapage.level_counts[level]; i++) { BlockNumber next; if (!BlockNumberIsValid(current)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 segment chain at level %u is shorter " "than its recorded count", level))); if (!tp_segment_read_next(index, current, &next)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("could not read BM25 segment root %u at " "level %u", current, level))); snapshot->roots[snapshot->root_count++] = current; current = next; } if (BlockNumberIsValid(current)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 segment chain at level %u is longer than " "its recorded count or contains a cycle", level))); } snapshot->level_offsets[TP_MAX_LEVELS] = snapshot->root_count; Assert(snapshot->root_count == expected_roots); } static TpSegmentGraphSnapshot * tp_segment_graph_snapshot_create_internal(Relation index, bool capture_roots) { if (!RelationIsValid(index)) elog(ERROR, "invalid relation passed to segment graph snapshot"); if (capture_roots) tp_debug_segment_graph_snapshot_pause( index, tp_debug_segment_graph_snapshot_pause_before_lock_ms, "before-lock"); for (;;) { TpSegmentGraphSnapshot *snapshot; TpIndexMetaPage metap; Buffer buffer; Buffer tail_buffer = InvalidBuffer; BlockNumber candidate_head; BlockNumber candidate_tail; CHECK_FOR_INTERRUPTS(); tp_read_memtable_snapshot_candidate( index, &candidate_head, &candidate_tail); if (BlockNumberIsValid(candidate_head) != BlockNumberIsValid(candidate_tail)) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("BM25 memtable head/tail validity mismatch"))); if (BlockNumberIsValid(candidate_tail)) { /* * Never retain the candidate metapage lock while acquiring the * tail: an extending writer already holds this tail EXCLUSIVE * before it requests the metapage EXCLUSIVE. */ tail_buffer = ReadBuffer(index, candidate_tail); LockBuffer(tail_buffer, BUFFER_LOCK_SHARE); } buffer = ReadBuffer(index, TP_METAPAGE_BLKNO); if (!BufferIsValid(buffer)) elog(ERROR, "failed to read metapage buffer for BM25 index \"%s\"", RelationGetRelationName(index)); LockBuffer(buffer, BUFFER_LOCK_SHARE); metap = tp_metapage_copy_from_page(index, BufferGetPage(buffer)); if (metap->memtable_head_blkno != candidate_head || metap->memtable_tail_blkno != candidate_tail) { pfree(metap); tp_release_snapshot_buffers(buffer, tail_buffer); continue; } snapshot = palloc0(sizeof(TpSegmentGraphSnapshot)); snapshot->metapage = *metap; pfree(metap); tp_memtable_chain_snapshot_capture_locked( tail_buffer, snapshot->metapage.memtable_head_blkno, snapshot->metapage.memtable_tail_blkno, &snapshot->memtable); /* * The endpoint is captured, and the metapage SHARE lock still * blocks the chain from being extended past it. Release the tail * before the root walk so appends that fit the current page are * not stalled for its duration; the captured free offset already * hides anything written after this point. */ if (BufferIsValid(tail_buffer)) { UnlockReleaseBuffer(tail_buffer); tail_buffer = InvalidBuffer; } if (capture_roots) { tp_capture_segment_roots(index, snapshot); snapshot->roots_captured = true; } if (capture_roots) tp_debug_segment_graph_snapshot_pause( index, tp_debug_segment_graph_snapshot_pause_before_unlock_ms, "before-unlock"); UnlockReleaseBuffer(buffer); if (capture_roots) tp_debug_segment_graph_snapshot_pause( index, tp_debug_segment_graph_snapshot_pause_ms, "after-unlock"); return snapshot; } } TpSegmentGraphSnapshot * tp_segment_graph_snapshot_create(Relation index) { return tp_segment_graph_snapshot_create_internal(index, true); } TpSegmentGraphSnapshot * tp_segment_graph_snapshot_create_metadata(Relation index) { return tp_segment_graph_snapshot_create_internal(index, false); } const BlockNumber * tp_segment_graph_snapshot_level( const TpSegmentGraphSnapshot *snapshot, uint32 level, uint32 *count) { uint32 offset; Assert(snapshot != NULL); Assert(count != NULL); /* * A metadata-only snapshot reports zero roots for every level, which * is indistinguishable from an index with no segments. Fail loudly * instead of silently scoring against an empty graph. */ Assert(snapshot->roots_captured); if (level >= TP_MAX_LEVELS) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid BM25 segment level %u", level))); offset = snapshot->level_offsets[level]; *count = snapshot->level_offsets[level + 1] - offset; if (*count == 0) return NULL; return snapshot->roots + offset; } void tp_segment_graph_snapshot_free(TpSegmentGraphSnapshot *snapshot) { if (snapshot == NULL) return; if (snapshot->roots != NULL) pfree(snapshot->roots); pfree(snapshot); } timescale-pg_textsearch-b63fe58/src/segment/graph_snapshot.h000066400000000000000000000036501525762047700244070ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * graph_snapshot.h - Atomic segment-root and memtable read snapshots */ #pragma once #include #include #include #include "index/metapage.h" #include "memtable/chain_walker.h" /* * A capture of the published graph: the metapage, the memtable chain * endpoint, and the per-level segment roots. * * Lifetime contract: a snapshot owns no buffer locks and no pins once * created, only copied block numbers. Traversing them afterwards is * safe only because displaced segment pages are parked in the * deferred-free tombstone chain (issue #380) until the reclaim horizon * passes. A caller must therefore either hold the per-index lock or * run under a transaction snapshot that holds that horizon back, which * on a hot standby additionally requires hot_standby_feedback = on. * * roots_captured distinguishes a full capture from a metadata-only one; * without it an uncaptured graph is indistinguishable from an index * that genuinely has no segments. */ typedef struct TpSegmentGraphSnapshot { TpIndexMetaPageData metapage; TpMemtableChainSnapshot memtable; BlockNumber *roots; uint32 level_offsets[TP_MAX_LEVELS + 1]; uint32 root_count; bool roots_captured; } TpSegmentGraphSnapshot; extern int tp_debug_segment_graph_snapshot_pause_before_lock_ms; extern int tp_debug_segment_graph_snapshot_pause_before_unlock_ms; extern int tp_debug_segment_graph_snapshot_pause_ms; extern TpSegmentGraphSnapshot * tp_segment_graph_snapshot_create(Relation index); extern TpSegmentGraphSnapshot * tp_segment_graph_snapshot_create_metadata(Relation index); extern const BlockNumber *tp_segment_graph_snapshot_level( const TpSegmentGraphSnapshot *snapshot, uint32 level, uint32 *count); extern void tp_segment_graph_snapshot_free(TpSegmentGraphSnapshot *snapshot); timescale-pg_textsearch-b63fe58/src/segment/io.h000066400000000000000000000236321525762047700220000ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * segment_io.h - Segment reader/writer I/O interface * * Defines TpSegmentReader, TpSegmentWriter, TpSegmentDirectAccess, * and TpSegmentPostingIterator types along with all I/O function * declarations for reading, writing, and iterating segment data. * * Requires segment.h for on-disk format types (TpSegmentHeader, * TpDictEntry, TpSkipEntry, TpBlockPosting, etc.). */ #pragma once #include "segment/segment.h" /* Shared overflow guards for segment writers. */ /* uint32 length, term bytes, and uint32 dictionary offset. */ static inline uint64 tp_string_pool_entry_size(uint32 term_len) { return (uint64)sizeof(uint32) + term_len + sizeof(uint32); } #define TP_MAX_STRING_POOL_BYTES ((uint64)PG_UINT32_MAX + 1) /* uint32 offsets can address every byte in a 4 GiB string pool. */ static inline bool tp_string_pool_offset_overflows(uint64 string_pos, uint64 entry_size) { return string_pos > (uint64)PG_UINT32_MAX || entry_size > TP_MAX_STRING_POOL_BYTES - string_pos; } #define TP_MAX_DICTIONARY_TERMS \ ((uint32)((uint64)PG_UINT32_MAX / sizeof(TpDictEntry) + 1)) #define TP_MAX_GROWABLE_CAPACITY (PG_UINT32_MAX - 1) static inline bool tp_dictionary_offsets_fit(uint32 num_terms) { return num_terms <= TP_MAX_DICTIONARY_TERMS; } static inline uint64 tp_dictionary_size(uint32 num_terms) { return (uint64)num_terms * sizeof(TpDictEntry); } static inline bool tp_document_count_fits(uint64 num_docs) { return num_docs <= TP_MAX_GROWABLE_CAPACITY; } static inline uint64 tp_posting_block_count(uint64 postings) { return postings / TP_BLOCK_SIZE + (postings % TP_BLOCK_SIZE != 0); } /* * Double a uint32 capacity without wrapping, stopping at * PG_UINT32_MAX - 1. */ static inline uint32 tp_grow_capacity(uint32 capacity, uint32 initial, const char *what) { if (capacity == 0) return initial; if (capacity >= TP_MAX_GROWABLE_CAPACITY) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: too many %s in segment (max %u)", what, TP_MAX_GROWABLE_CAPACITY))); if (capacity > TP_MAX_GROWABLE_CAPACITY / 2) return TP_MAX_GROWABLE_CAPACITY; return capacity * 2; } /* * Segment reader context */ typedef struct TpSegmentReader { Relation index; BlockNumber root_block; uint32 segment_version; /* On-disk format version */ /* Cached header */ TpSegmentHeader *header; Buffer header_buffer; /* Cached page map loaded from page index */ BlockNumber *page_map; uint32 num_pages; BlockNumber nblocks; /* Relation size for validation */ /* Current data page */ Buffer current_buffer; uint32 current_logical_page; /* * CTID arrays for result lookup (loaded at segment open). * Split storage for better packing and cache locality during scoring. */ BlockNumber *cached_ctid_pages; /* Page numbers (4 bytes/doc) */ OffsetNumber *cached_ctid_offsets; /* Tuple offsets (2 bytes/doc) */ uint32 cached_num_docs; /* Number of docs cached */ /* Bounded window for repeated ordered CTID lookups. */ BlockNumber *lookup_ctid_pages; OffsetNumber *lookup_ctid_offsets; uint32 lookup_ctid_start; uint32 lookup_ctid_count; uint32 lookup_ctid_capacity; /* Independent page pins for sparse lookups in the split CTID arrays. */ Buffer ctid_pages_buffer; uint32 ctid_pages_logical_page; Buffer ctid_offsets_buffer; uint32 ctid_offsets_logical_page; /* BufFile-backed reading (for temp file segments, NULL for normal) */ BufFile *buffile; uint64 buffile_base; /* Base byte offset of segment in BufFile */ } TpSegmentReader; /* * Segment writer context */ typedef struct TpSegmentWriter { Relation index; BlockNumber *pages; /* Dynamically allocated array of page blocks */ uint32 pages_allocated; /* Number of pages allocated so far */ uint32 pages_capacity; /* Capacity of the pages array */ uint64 current_offset; /* Current write position in logical file */ char *buffer; /* Write buffer (one page) */ uint32 buffer_page; /* Which logical page is in buffer */ uint32 buffer_pos; /* Position within buffer */ /* Reusable buffer for posting list conversion */ TpSegmentPosting *posting_buffer; /* Reusable posting buffer */ uint32 posting_buffer_size; /* Current size of buffer */ } TpSegmentWriter; /* Forward declarations */ struct TermInfo; struct TpDocMapBuilder; /* * Function declarations */ /* Writer functions */ extern BlockNumber tp_write_segment( Relation index, struct TermInfo *terms, uint32 num_terms, struct TpDocMapBuilder *docmap); extern void tp_segment_writer_init(TpSegmentWriter *writer, Relation index); extern void tp_segment_writer_write(TpSegmentWriter *writer, const void *data, Size len); extern void tp_segment_writer_flush(TpSegmentWriter *writer); extern void tp_segment_writer_finish(TpSegmentWriter *writer); /* Reader functions */ /* Metadata-only reader; close with tp_segment_close, do not read data. */ extern TpSegmentReader * tp_segment_open_header(Relation index, BlockNumber root); extern TpSegmentReader * tp_segment_open_ex(Relation index, BlockNumber root, bool load_ctids); extern TpSegmentReader *tp_segment_open(Relation index, BlockNumber root); extern bool tp_segment_read_next(Relation index, BlockNumber root, BlockNumber *next); extern TpSegmentReader * tp_segment_open_from_buffile(BufFile *file, uint64 base_offset); extern void tp_segment_read( TpSegmentReader *reader, uint64 logical_offset, void *dest, uint64 len); extern void tp_segment_close(TpSegmentReader *reader); /* Lazy CTID lookup for deferred resolution */ extern void tp_segment_lookup_ctid( TpSegmentReader *reader, uint32 doc_id, ItemPointerData *ctid_out); extern void tp_segment_enable_ctid_lookup_cache(TpSegmentReader *reader); /* Zero-copy reader functions */ typedef struct TpSegmentDirectAccess { Buffer buffer; /* Pinned buffer */ Page page; /* Page pointer */ void *data; /* Pointer to data in page */ uint32 available; /* Bytes available from this position */ } TpSegmentDirectAccess; extern bool tp_segment_get_direct( TpSegmentReader *reader, uint64 logical_offset, uint32 len, TpSegmentDirectAccess *access); extern void tp_segment_release_direct(TpSegmentDirectAccess *access); /* Version-aware dictionary entry reader */ extern void tp_segment_read_dict_entry( TpSegmentReader *reader, TpSegmentHeader *header, uint32 index, TpDictEntry *entry); /* Debug functions */ struct DumpOutput; /* Forward declaration */ extern void tp_dump_segment_to_output( Relation index, BlockNumber segment_root, struct DumpOutput *out); /* Page index writing (used by segment_merge.c) */ extern BlockNumber write_page_index(Relation index, BlockNumber *pages, uint32 num_pages); extern BlockNumber write_page_index_tracked( Relation index, BlockNumber *pages, uint32 num_pages, BlockNumber **owned_pages, uint32 *owned_count); /* Page reclamation for segment compaction */ extern uint32 tp_segment_collect_pages( Relation index, BlockNumber root_block, BlockNumber **pages_out); extern void tp_segment_free_pages(Relation index, BlockNumber *pages, uint32 num_pages); /* * Segment posting iterator for block-based traversal. * Used by BMW scoring to access individual blocks and skip entries. */ typedef struct TpSegmentPostingIterator { TpSegmentReader *reader; const char *term; uint32 dict_entry_idx; TpDictEntry dict_entry; bool initialized; bool finished; /* Block iteration state */ uint32 current_block; /* Current block index (0 to block_count-1) */ uint32 current_in_block; /* Position within current block */ TpSkipEntry skip_entry; /* Current block's skip entry */ /* Zero-copy block access (preferred path) */ TpSegmentDirectAccess block_access; bool has_block_access; bool force_copy; /* Avoid retained pins across many iterators */ /* Block postings pointer - points to either direct data or fallback buf */ TpBlockPosting *block_postings; /* Fallback buffer for when block spans page boundaries */ TpBlockPosting *fallback_block; uint32 fallback_block_size; /* * BMW optimization: cached skip entries and reusable compressed buffer. * When non-NULL, load_block uses these instead of reading skip entries * from disk or palloc/pfree-ing per block. Set by BMW init code. */ TpSkipEntry *cached_skip_entries; /* Pre-loaded skip entries array */ uint8 *compressed_buf_cache; /* Reusable decompression buffer */ /* Output posting (converted for scoring compatibility) */ TpSegmentPosting output_posting; } TpSegmentPostingIterator; typedef struct TpSegmentPrefixCandidates { TpSegmentPostingIterator *iterators; uint8 *doc_bitmap; uint32 iterator_count; uint64 estimate; } TpSegmentPrefixCandidates; /* Segment posting iterator functions */ extern bool tp_segment_posting_iterator_init( TpSegmentPostingIterator *iter, TpSegmentReader *reader, const char *term); extern void tp_segment_prefix_candidates_init( TpSegmentReader *reader, const char *prefix, int prefix_length, uint32 max_iterators, TpSegmentPrefixCandidates *candidates); extern bool tp_segment_posting_iterator_load_block(TpSegmentPostingIterator *iter); extern bool tp_segment_posting_iterator_next( TpSegmentPostingIterator *iter, TpSegmentPosting **posting); extern void tp_segment_posting_iterator_free(TpSegmentPostingIterator *iter); /* Read a skip entry by block index */ extern void tp_segment_read_skip_entry( TpSegmentReader *reader, uint64 skip_index_offset, uint32 block_idx, TpSkipEntry *skip); /* Seek iterator to target doc ID (for WAND algorithm) */ extern bool tp_segment_posting_iterator_seek( TpSegmentPostingIterator *iter, uint32 target_doc_id, TpSegmentPosting **posting); /* Get current doc ID from iterator (for WAND pivot selection) */ extern uint32 tp_segment_posting_iterator_current_doc_id(TpSegmentPostingIterator *iter); timescale-pg_textsearch-b63fe58/src/segment/merge.c000066400000000000000000001332421525762047700224620ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * merge.c - Segment merge for LSM-style compaction */ #include #include #include #include #include #include #include #include #include #include #include "constants.h" #include "segment/alive_bitset.h" #include "segment/compression.h" #include "segment/dictionary.h" #include "segment/docmap.h" #include "segment/fieldnorm.h" #include "segment/io.h" #include "segment/merge.h" #include "segment/merge_internal.h" #include "segment/pagemapper.h" #include "segment/segment.h" /* Sentinel for dead docs in old_to_new mapping */ #define TP_MERGE_DOC_DEAD UINT32_MAX /* GUC variable for segment compression */ extern bool tp_compress_segments; /* * Types TpMergeSource, TpTermSegmentRef, TpMergedTerm, * TpMergePostingInfo, TpPostingMergeSource, TpMergeDocMapping, * and MergeTermBlockInfo are defined in merge_internal.h. */ /* ---------------------------------------------------------------- * Merge sink: paged I/O abstraction * ---------------------------------------------------------------- */ void merge_sink_init_pages(TpMergeSink *sink, Relation index) { memset(sink, 0, sizeof(TpMergeSink)); sink->index = index; tp_segment_writer_init(&sink->writer, index); sink->current_offset = sink->writer.current_offset; } void tp_discard_unpublished_segment(Relation index, BlockNumber root) { TpSegmentReader *reader; BlockNumber *pages; uint32 num_pages; if (!BlockNumberIsValid(root)) return; reader = tp_segment_open(index, root); if (reader == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open unpublished segment at block %u", root))); tp_segment_close(reader); num_pages = tp_segment_collect_pages(index, root, &pages); if (num_pages == 0) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("unpublished segment at block %u has no pages", root))); tp_segment_free_pages(index, pages, num_pages); pfree(pages); IndexFreeSpaceMapVacuum(index); } static void tp_discard_unpublished_pages( Relation index, BlockNumber *pages, uint32 num_pages) { if (num_pages == 0) return; Assert(pages != NULL); tp_segment_free_pages(index, pages, num_pages); IndexFreeSpaceMapVacuum(index); } /* * Sequential append to sink. */ void merge_sink_write(TpMergeSink *sink, const void *data, Size size) { tp_segment_writer_write(&sink->writer, data, size); sink->current_offset = sink->writer.current_offset; } /* * Positioned write for backpatching (dict entries, header). * * Each per-page chunk is logged as a small GenericXLog delta against the * page that the writer flush already wrote. This keeps backpatch WAL * traffic proportional to the number of bytes actually changed (a few * dict entries or the segment header struct), rather than emitting a * full-page image for every visited page. */ void merge_sink_write_at( TpMergeSink *sink, uint64 offset, const void *data, uint64 size) { const char *src = (const char *)data; uint64 remaining = size; uint64 pos = offset; while (remaining > 0) { uint32 logical_pg = tp_logical_page(pos); uint32 pg_off = tp_page_offset(pos); uint32 avail = SEGMENT_DATA_PER_PAGE - pg_off; uint32 chunk = (uint32)Min(remaining, (uint64)avail); BlockNumber physical_block; Buffer buf; Page page; GenericXLogState *xlog_state; Assert(logical_pg < sink->writer.pages_allocated); physical_block = sink->writer.pages[logical_pg]; buf = ReadBuffer(sink->index, physical_block); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(sink->index); page = GenericXLogRegisterBuffer(xlog_state, buf, 0); memcpy((char *)page + SizeOfPageHeaderData + pg_off, src, chunk); GenericXLogFinish(xlog_state); UnlockReleaseBuffer(buf); src += chunk; pos += chunk; remaining -= chunk; } } void merge_sink_finish(TpMergeSink *sink, TpSegmentHeader *header) { BlockNumber page_index_root; Assert(sink->writer.buffer_pos == SizeOfPageHeaderData); page_index_root = write_page_index_tracked( sink->index, sink->writer.pages, sink->writer.pages_allocated, &sink->page_index_pages, &sink->page_index_pages_allocated); header->page_index = page_index_root; header->num_pages = sink->writer.pages_allocated; merge_sink_write_at(sink, 0, header, sizeof(TpSegmentHeader)); tp_segment_writer_finish(&sink->writer); } TpSkipEntry merge_sink_write_posting_block( TpMergeSink *sink, TpBlockPosting *block, uint32 count) { TpSkipEntry skip; uint16 max_tf = 0; uint8 min_norm = 255; uint32 last_did = 0; uint32 i; Assert(count > 0 && count <= TP_BLOCK_SIZE); for (i = 0; i < count; i++) { if (block[i].doc_id > last_did) last_did = block[i].doc_id; if (block[i].frequency > max_tf) max_tf = block[i].frequency; if (block[i].fieldnorm < min_norm) min_norm = block[i].fieldnorm; } skip.last_doc_id = last_did; skip.doc_count = (uint8)count; skip.block_max_tf = max_tf; skip.block_max_norm = min_norm; skip.posting_offset = sink->current_offset; memset(skip.reserved, 0, sizeof(skip.reserved)); if (tp_compress_segments) { uint8 compressed[TP_MAX_COMPRESSED_BLOCK_SIZE]; uint32 compressed_size; compressed_size = tp_compress_block(block, count, compressed); skip.flags = TP_BLOCK_FLAG_DELTA; merge_sink_write(sink, compressed, compressed_size); } else { skip.flags = TP_BLOCK_FLAG_UNCOMPRESSED; merge_sink_write(sink, block, count * sizeof(TpBlockPosting)); } return skip; } /* ---------------------------------------------------------------- * Merge source operations * ---------------------------------------------------------------- */ #define TP_MERGE_STRING_OFFSETS_WINDOW 1024 static void merge_source_refill_string_offsets(TpMergeSource *source) { TpSegmentHeader *header = source->reader->header; uint32 count; Assert(source->reader->buffile != NULL); Assert(source->current_idx < source->num_terms); count = Min(TP_MERGE_STRING_OFFSETS_WINDOW, source->num_terms - source->current_idx); tp_segment_read( source->reader, header->dictionary_offset + sizeof(uint32) + (uint64)source->current_idx * sizeof(uint32), source->string_offsets, count * sizeof(uint32)); source->string_offsets_start = source->current_idx; source->string_offsets_count = count; } /* * Advance a merge source to its next term. * Returns false if source is exhausted. */ bool merge_source_advance(TpMergeSource *source) { TpSegmentHeader *header; if (source->exhausted) return false; /* Free previous term if any */ if (source->current_term) { pfree(source->current_term); source->current_term = NULL; } source->current_idx++; if (source->current_idx >= source->num_terms) { source->exhausted = true; return false; } header = source->reader->header; if (source->current_idx < source->string_offsets_start || source->current_idx >= source->string_offsets_start + source->string_offsets_count) merge_source_refill_string_offsets(source); /* Read the term at current index */ source->current_term = tp_segment_read_term_at_index( source->reader, source->reader->header, source->string_offsets, source->current_idx - source->string_offsets_start); /* Read the dictionary entry (version-aware) */ tp_segment_read_dict_entry( source->reader, header, source->current_idx, &source->current_entry); return true; } /* * Initialize a merge source for a segment. * Returns false if the segment is invalid. A segment without terms remains a * valid exhausted source because its document map must survive the merge. */ bool merge_source_init(TpMergeSource *source, Relation index, BlockNumber root) { TpSegmentHeader *header; TpDictionary dict_header; memset(source, 0, sizeof(TpMergeSource)); source->exhausted = true; /* Assume failure */ source->reader = tp_segment_open(index, root); if (!source->reader) return false; header = source->reader->header; if (header->num_terms == 0) { return true; } source->num_terms = header->num_terms; /* Read dictionary header */ tp_segment_read( source->reader, header->dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); /* Cache all string offsets for this segment */ source->string_offsets = palloc_extended( sizeof(uint32) * source->num_terms, MCXT_ALLOC_HUGE); tp_segment_read( source->reader, header->dictionary_offset + sizeof(dict_header.num_terms), source->string_offsets, sizeof(uint32) * source->num_terms); source->string_offsets_start = 0; source->string_offsets_count = source->num_terms; /* Position before first term (advance will move to index 0) */ source->current_idx = UINT32_MAX; /* Will wrap to 0 on advance */ source->exhausted = false; source->current_term = NULL; /* Advance to first term */ if (!merge_source_advance(source)) { tp_segment_close(source->reader); source->reader = NULL; pfree(source->string_offsets); source->string_offsets = NULL; return false; } return true; } /* * Initialize a merge source from a pre-opened reader. * Same as merge_source_init but takes a TpSegmentReader directly. * Does NOT close the reader on failure (caller owns it). */ bool merge_source_init_from_reader(TpMergeSource *source, TpSegmentReader *reader) { TpSegmentHeader *header; TpDictionary dict_header; memset(source, 0, sizeof(TpMergeSource)); source->exhausted = true; /* Assume failure */ if (!reader) return false; source->reader = reader; header = reader->header; if (header->num_terms == 0) return true; source->num_terms = header->num_terms; /* Read dictionary header */ tp_segment_read( source->reader, header->dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); /* Cache a bounded string-offset window for the BufFile source. */ source->string_offsets = palloc_extended( sizeof(uint32) * Min(TP_MERGE_STRING_OFFSETS_WINDOW, source->num_terms), MCXT_ALLOC_HUGE); source->string_offsets_start = 0; source->string_offsets_count = 0; /* Position before first term */ source->current_idx = UINT32_MAX; source->exhausted = false; source->current_term = NULL; /* Advance to first term */ if (!merge_source_advance(source)) { pfree(source->string_offsets); source->string_offsets = NULL; source->reader = NULL; /* Don't close, caller owns it */ return false; } return true; } /* * Close and cleanup a merge source. */ void merge_source_close(TpMergeSource *source) { if (source->current_term) { pfree(source->current_term); source->current_term = NULL; } if (source->string_offsets) { pfree(source->string_offsets); source->string_offsets = NULL; } if (source->reader) { tp_segment_close(source->reader); source->reader = NULL; } } /* * Find the source with the lexicographically smallest current term. * Returns -1 if all sources are exhausted. */ int merge_find_min_source(TpMergeSource *sources, int num_sources) { int min_idx = -1; const char *min_term = NULL; int i; for (i = 0; i < num_sources; i++) { if (sources[i].exhausted) continue; if (min_idx < 0 || strcmp(sources[i].current_term, min_term) < 0) { min_idx = i; min_term = sources[i].current_term; } } return min_idx; } /* * Add a segment reference to a merged term. * This records which segment has this term, without loading postings. */ void merged_term_add_segment_ref( TpMergedTerm *term, int segment_idx, TpDictEntry *entry) { /* Grow array if needed */ if (term->num_segment_refs >= term->segment_refs_capacity) { uint32 new_capacity = term->segment_refs_capacity == 0 ? 8 : term->segment_refs_capacity * 2; if (term->segment_refs == NULL) term->segment_refs = palloc( new_capacity * sizeof(TpTermSegmentRef)); else term->segment_refs = repalloc( term->segment_refs, new_capacity * sizeof(TpTermSegmentRef)); term->segment_refs_capacity = new_capacity; } term->segment_refs[term->num_segment_refs].segment_idx = segment_idx; term->segment_refs[term->num_segment_refs].entry = *entry; term->num_segment_refs++; } /* * Load the next block of postings for merge source. * Returns true if a block was loaded, false if no more blocks remain. */ static bool posting_source_load_block(TpPostingMergeSource *ps) { if (ps->current_block >= ps->block_count) return false; /* Read skip entry for current block (version-aware) */ tp_segment_read_skip_entry( ps->reader, ps->skip_index_offset, ps->current_block, &ps->skip_entry); /* * Ensure we have enough buffer space. We reuse the buffer between blocks, * only reallocating when a larger block is encountered. Old block data is * no longer needed since we process blocks sequentially. * * The NULL check ensures allocation on first call (when block_postings is * uninitialized) even if doc_count happens to equal block_capacity. */ if (ps->block_postings == NULL || ps->skip_entry.doc_count > ps->block_capacity) { if (ps->block_postings) pfree(ps->block_postings); ps->block_postings = palloc( ps->skip_entry.doc_count * sizeof(TpBlockPosting)); ps->block_capacity = ps->skip_entry.doc_count; } /* Read posting data for this block (handle compression) */ if (ps->skip_entry.flags == TP_BLOCK_FLAG_DELTA) { /* Compressed block - read and decompress */ uint8 compressed_buf[TP_MAX_COMPRESSED_BLOCK_SIZE]; tp_segment_read( ps->reader, ps->skip_entry.posting_offset, compressed_buf, TP_MAX_COMPRESSED_BLOCK_SIZE); tp_decompress_block( compressed_buf, ps->skip_entry.doc_count, 0, /* first_doc_id - deltas are relative within block */ ps->block_postings); } else { /* Uncompressed block - read directly */ tp_segment_read( ps->reader, ps->skip_entry.posting_offset, ps->block_postings, ps->skip_entry.doc_count * sizeof(TpBlockPosting)); } ps->current_in_block = 0; return true; } /* * Convert current block posting to output format. */ static void posting_source_convert_current(TpPostingMergeSource *ps) { TpBlockPosting *bp = &ps->block_postings[ps->current_in_block]; BlockNumber page; OffsetNumber offset; /* * Look up CTID from split arrays (needed for N-way merge ordering). * Use cached arrays if available, otherwise read from segment. */ if (ps->reader->cached_ctid_pages != NULL && bp->doc_id < ps->reader->cached_num_docs) { page = ps->reader->cached_ctid_pages[bp->doc_id]; offset = ps->reader->cached_ctid_offsets[bp->doc_id]; } else { TpSegmentHeader *header = ps->reader->header; tp_segment_read( ps->reader, header->ctid_pages_offset + (bp->doc_id * sizeof(BlockNumber)), &page, sizeof(BlockNumber)); tp_segment_read( ps->reader, header->ctid_offsets_offset + (bp->doc_id * sizeof(OffsetNumber)), &offset, sizeof(OffsetNumber)); } /* Build output posting info */ ItemPointerSet(&ps->current.ctid, page, offset); ps->current.old_doc_id = bp->doc_id; ps->current.frequency = bp->frequency; ps->current.fieldnorm = bp->fieldnorm; } /* * Initialize a posting merge source for streaming. */ void posting_source_init( TpPostingMergeSource *ps, TpSegmentReader *reader, TpDictEntry *entry) { memset(ps, 0, sizeof(TpPostingMergeSource)); ps->reader = reader; ps->skip_index_offset = entry->skip_index_offset; ps->block_count = entry->block_count; ps->current_block = 0; ps->current_in_block = 0; ps->block_postings = NULL; ps->block_capacity = 0; ps->exhausted = (entry->block_count == 0); if (!ps->exhausted) { if (posting_source_load_block(ps)) { posting_source_convert_current(ps); } else { ps->exhausted = true; } } } /* * Initialize a posting merge source for fast streaming (disjoint mode). * Skips posting_source_convert_current since CTID lookups are not needed. */ void posting_source_init_fast( TpPostingMergeSource *ps, TpSegmentReader *reader, TpDictEntry *entry) { memset(ps, 0, sizeof(TpPostingMergeSource)); ps->reader = reader; ps->skip_index_offset = entry->skip_index_offset; ps->block_count = entry->block_count; ps->current_block = 0; ps->current_in_block = 0; ps->block_postings = NULL; ps->block_capacity = 0; ps->exhausted = (entry->block_count == 0); if (!ps->exhausted) { if (!posting_source_load_block(ps)) ps->exhausted = true; } } /* * Free posting merge source resources. */ void posting_source_free(TpPostingMergeSource *ps) { if (ps->block_postings) { pfree(ps->block_postings); ps->block_postings = NULL; } } /* * Advance a posting merge source to the next posting. */ bool posting_source_advance(TpPostingMergeSource *ps) { if (ps->exhausted) return false; ps->current_in_block++; /* Move to next block if current exhausted */ while (ps->current_in_block >= ps->skip_entry.doc_count) { ps->current_block++; if (ps->current_block >= ps->block_count) { ps->exhausted = true; return false; } if (!posting_source_load_block(ps)) { ps->exhausted = true; return false; } } posting_source_convert_current(ps); return true; } /* * Advance a posting merge source without CTID conversion (disjoint mode). * Reads doc_id, frequency, fieldnorm directly from the block posting * array, skipping the expensive CTID lookups in convert_current. */ bool posting_source_advance_fast(TpPostingMergeSource *ps) { if (ps->exhausted) return false; ps->current_in_block++; /* Move to next block if current exhausted */ while (ps->current_in_block >= ps->skip_entry.doc_count) { ps->current_block++; if (ps->current_block >= ps->block_count) { ps->exhausted = true; return false; } if (!posting_source_load_block(ps)) { ps->exhausted = true; return false; } } return true; } /* * Find the posting source with the smallest current CTID. * Returns -1 if all sources are exhausted. */ int find_min_posting_source(TpPostingMergeSource *sources, int num_sources) { int min_idx = -1; ItemPointerData min_ctid; int i; for (i = 0; i < num_sources; i++) { ItemPointerData ctid; if (sources[i].exhausted) continue; /* Copy to avoid unaligned access from packed struct */ memcpy(&ctid, &sources[i].current.ctid, sizeof(ItemPointerData)); if (min_idx < 0 || ItemPointerCompare(&ctid, &min_ctid) < 0) { min_idx = i; memcpy(&min_ctid, &ctid, sizeof(ItemPointerData)); } } return min_idx; } /* TpMergeDocMapping is defined in merge_internal.h */ /* * Per-source state for the streaming N-way merge of docmaps. */ typedef struct TpDocmapMergeSource { BlockNumber *ctid_pages; /* CTID page numbers (4 bytes/doc) */ OffsetNumber *ctid_offsets; /* CTID tuple offsets (2 bytes/doc) */ uint8 *fieldnorms; /* Encoded fieldnorms (1 byte/doc) */ uint32 num_docs; /* Total docs in this source */ uint32 cursor; /* Current position in arrays */ bool owns_arrays; /* True if we allocated the arrays */ } TpDocmapMergeSource; /* * Build merged docmap using streaming N-way merge of sorted CTID arrays. * Also builds direct mapping arrays for fast old->new doc_id lookup. * * Each source segment maintains the invariant that doc_ids are in CTID * order. An N-way merge of these sorted streams produces the global * CTID order without needing a hash table, reducing memory from ~5.5GB * to ~2.5GB for 138M documents across 24 segments. */ TpDocMapBuilder * build_merged_docmap( TpMergeSource *sources, int num_sources, TpMergeDocMapping *mapping, bool disjoint_sources) { TpDocMapBuilder *docmap; TpDocmapMergeSource *msources; uint32 total_docs = 0; BlockNumber *out_pages; OffsetNumber *out_offsets; uint8 *out_fieldnorms; uint32 new_doc_id = 0; int i; /* Initialize mapping arrays */ mapping->num_sources = num_sources; mapping->old_to_new = (uint32 **)palloc0(num_sources * sizeof(uint32 *)); /* * Step 1: Load source CTID and fieldnorm arrays. * Reuse the reader's cached arrays when available to avoid * redundant copies. */ msources = palloc0(num_sources * sizeof(TpDocmapMergeSource)); for (i = 0; i < num_sources; i++) { TpSegmentHeader *header = sources[i].reader->header; TpDocmapMergeSource *ms = &msources[i]; ms->num_docs = header->num_docs; ms->cursor = 0; if (ms->num_docs == 0 || header->ctid_pages_offset == 0) { ms->num_docs = 0; /* Exclude from N-way merge */ continue; } if (!tp_document_count_fits((uint64)total_docs + ms->num_docs)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u documents", TP_MAX_GROWABLE_CAPACITY))); total_docs += ms->num_docs; mapping->old_to_new[i] = palloc_extended( ms->num_docs * sizeof(uint32), MCXT_ALLOC_HUGE); /* CTID arrays: use cached if available, else bulk-read */ if (sources[i].reader->cached_ctid_pages != NULL) { ms->ctid_pages = sources[i].reader->cached_ctid_pages; ms->ctid_offsets = sources[i].reader->cached_ctid_offsets; ms->owns_arrays = false; } else { ms->ctid_pages = palloc_extended( ms->num_docs * sizeof(BlockNumber), MCXT_ALLOC_HUGE); tp_segment_read( sources[i].reader, header->ctid_pages_offset, ms->ctid_pages, ms->num_docs * sizeof(BlockNumber)); ms->ctid_offsets = palloc_extended( ms->num_docs * sizeof(OffsetNumber), MCXT_ALLOC_HUGE); tp_segment_read( sources[i].reader, header->ctid_offsets_offset, ms->ctid_offsets, ms->num_docs * sizeof(OffsetNumber)); ms->owns_arrays = true; } /* Fieldnorms: always bulk-read (not cached by reader) */ ms->fieldnorms = palloc_extended(ms->num_docs * sizeof(uint8), MCXT_ALLOC_HUGE); tp_segment_read( sources[i].reader, header->fieldnorm_offset, ms->fieldnorms, ms->num_docs * sizeof(uint8)); } /* Step 2: Allocate output arrays (palloc(0) is valid in PG) */ out_pages = palloc_extended(total_docs * sizeof(BlockNumber), MCXT_ALLOC_HUGE); out_offsets = palloc_extended( total_docs * sizeof(OffsetNumber), MCXT_ALLOC_HUGE); out_fieldnorms = palloc_extended(total_docs * sizeof(uint8), MCXT_ALLOC_HUGE); /* * Step 3: Merge docmaps. * * When disjoint_sources is true, sources have non-overlapping * CTID ranges in source order, so we concatenate sequentially * instead of doing an N-way comparison. This eliminates the * per-doc CTID comparison overhead. */ if (disjoint_sources) { for (i = 0; i < num_sources; i++) { TpDocmapMergeSource *ms = &msources[i]; uint32 j; for (j = 0; j < ms->num_docs; j++) { if (!tp_segment_is_alive(sources[i].reader, j)) { mapping->old_to_new[i][j] = TP_MERGE_DOC_DEAD; continue; } mapping->old_to_new[i][j] = new_doc_id; out_pages[new_doc_id] = ms->ctid_pages[j]; out_offsets[new_doc_id] = ms->ctid_offsets[j]; out_fieldnorms[new_doc_id] = ms->fieldnorms[j]; new_doc_id++; } } } else { /* * N-way merge: each source's docs are already in CTID * order. Find the source with smallest current CTID via * linear scan (N is small, typically <= 24). */ while (new_doc_id < total_docs) { int min_src = -1; BlockNumber min_page = 0; OffsetNumber min_offset = 0; for (i = 0; i < num_sources; i++) { TpDocmapMergeSource *ms = &msources[i]; BlockNumber pg; OffsetNumber off; /* Skip past dead docs */ while (ms->cursor < ms->num_docs && !tp_segment_is_alive(sources[i].reader, ms->cursor)) { mapping->old_to_new[i][ms->cursor] = TP_MERGE_DOC_DEAD; ms->cursor++; } if (ms->cursor >= ms->num_docs) continue; pg = ms->ctid_pages[ms->cursor]; off = ms->ctid_offsets[ms->cursor]; if (min_src < 0 || pg < min_page || (pg == min_page && off < min_offset)) { min_src = i; min_page = pg; min_offset = off; } } if (min_src < 0) break; /* All sources exhausted */ { TpDocmapMergeSource *ms = &msources[min_src]; uint32 pos = ms->cursor; mapping->old_to_new[min_src][pos] = new_doc_id; out_pages[new_doc_id] = ms->ctid_pages[pos]; out_offsets[new_doc_id] = ms->ctid_offsets[pos]; out_fieldnorms[new_doc_id] = ms->fieldnorms[pos]; ms->cursor++; } new_doc_id++; } } /* Step 4: Package into a TpDocMapBuilder (finalized, no hash table) */ docmap = palloc0(sizeof(TpDocMapBuilder)); docmap->ctid_to_id = NULL; /* No hash table needed */ docmap->num_docs = new_doc_id; docmap->capacity = new_doc_id; docmap->finalized = true; docmap->ctid_pages = out_pages; docmap->ctid_offsets = out_offsets; docmap->fieldnorms = out_fieldnorms; /* * Compute merged total_tokens as Σ source.header.total_tokens * minus a fieldnorm-decoded approximation of dead-doc tokens. * For all-live sources the result is exact (matches the raw sum * maintained by the spill and build-context paths); for sources * with some dead docs the quantization error is bounded by the * dead contribution only, not by the whole segment. */ { uint64 merged_total_tokens = 0; for (i = 0; i < num_sources; i++) { TpDocmapMergeSource *ms = &msources[i]; uint64 src_total = sources[i].reader->header->total_tokens; uint64 dead_tokens = 0; for (uint32 d = 0; d < ms->num_docs; d++) { if (!tp_segment_is_alive(sources[i].reader, d)) dead_tokens += decode_fieldnorm(ms->fieldnorms[d]); } merged_total_tokens += (src_total > dead_tokens) ? (src_total - dead_tokens) : 0; } docmap->total_tokens = merged_total_tokens; } /* Free per-source arrays we allocated */ for (i = 0; i < num_sources; i++) { if (msources[i].owns_arrays) { pfree(msources[i].ctid_pages); pfree(msources[i].ctid_offsets); } if (msources[i].fieldnorms) pfree(msources[i].fieldnorms); } pfree(msources); return docmap; } /* * Free merge doc mapping arrays. */ void free_merge_doc_mapping(TpMergeDocMapping *mapping) { int i; for (i = 0; i < mapping->num_sources; i++) { if (mapping->old_to_new[i]) pfree(mapping->old_to_new[i]); } pfree((void *)mapping->old_to_new); } /* * Initialize N-way merge posting sources for a term. * Returns the number of sources initialized. */ TpPostingMergeSource * init_term_posting_sources( TpMergedTerm *term, TpMergeSource *sources, int *num_psources) { TpPostingMergeSource *psources; uint32 i; *num_psources = term->num_segment_refs; psources = palloc(sizeof(TpPostingMergeSource) * term->num_segment_refs); for (i = 0; i < term->num_segment_refs; i++) { TpTermSegmentRef *ref = &term->segment_refs[i]; TpMergeSource *source = &sources[ref->segment_idx]; posting_source_init(&psources[i], source->reader, &ref->entry); } return psources; } /* * Initialize posting sources in fast/disjoint mode (no CTID conversion). */ TpPostingMergeSource * init_term_posting_sources_fast( TpMergedTerm *term, TpMergeSource *sources, int *num_psources) { TpPostingMergeSource *psources; uint32 i; *num_psources = term->num_segment_refs; psources = palloc(sizeof(TpPostingMergeSource) * term->num_segment_refs); for (i = 0; i < term->num_segment_refs; i++) { TpTermSegmentRef *ref = &term->segment_refs[i]; TpMergeSource *source = &sources[ref->segment_idx]; posting_source_init_fast(&psources[i], source->reader, &ref->entry); } return psources; } /* * Free N-way merge posting sources for a term. */ static void free_term_posting_sources(TpPostingMergeSource *psources, int num_psources) { int i; for (i = 0; i < num_psources; i++) posting_source_free(&psources[i]); pfree(psources); } /* MergeTermBlockInfo is defined in merge_internal.h */ /* ---------------------------------------------------------------- * Unified merged segment writer * ---------------------------------------------------------------- */ void tp_validate_merged_terms(TpMergedTerm *terms, uint32 num_terms) { uint64 string_pos = 0; uint64 skip_entries = 0; uint32 i; if (!tp_dictionary_offsets_fit(num_terms)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment dictionary exceeds %u terms", TP_MAX_DICTIONARY_TERMS))); for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); uint64 term_postings = 0; uint64 blocks; uint32 ref; if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: merged string pool exceeds 4 " "GiB"))); string_pos += entry_size; for (ref = 0; ref < terms[i].num_segment_refs; ref++) { if (pg_add_u64_overflow( term_postings, terms[i].segment_refs[ref].entry.doc_freq, &term_postings)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: posting count overflow"))); } if (!tp_document_count_fits(term_postings)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: term exceeds %u postings", TP_MAX_GROWABLE_CAPACITY))); blocks = tp_posting_block_count(term_postings); if (blocks > TP_MAX_GROWABLE_CAPACITY - skip_entries) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u posting blocks", TP_MAX_GROWABLE_CAPACITY))); skip_entries += blocks; } } /* * Write a merged segment to pages via sink. * * Layout: [header] -> [dictionary] -> [postings] -> [skip index] -> * [fieldnorm] -> [ctid map] * * Also writes page index and backpatches header with * num_pages/page_index. Caller reads sink->writer.pages[0] for root. */ void write_merged_segment_to_sink( TpMergeSink *sink, TpMergedTerm *terms, uint32 num_terms, TpMergeSource *sources, int num_sources, uint32 target_level, uint64 total_tokens, bool disjoint_sources, BlockNumber next_segment) { TpSegmentHeader header; TpDictionary dict; TpDocMapBuilder *docmap; TpMergeDocMapping doc_mapping; MergeTermBlockInfo *term_blocks; uint32 *string_offsets; uint64 string_pos; uint32 i; /* Accumulated skip entries for all terms */ TpSkipEntry *all_skip_entries; uint32 skip_entries_count; uint32 skip_entries_capacity; if (!tp_dictionary_offsets_fit(num_terms)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment dictionary exceeds %u terms", TP_MAX_DICTIONARY_TERMS))); /* Build docmap and direct mapping arrays from source segments */ docmap = build_merged_docmap( sources, num_sources, &doc_mapping, disjoint_sources); /* * build_merged_docmap sets docmap->total_tokens from source * header.total_tokens minus dead-doc fieldnorm approximations; * see its comment for the exact-vs-approximate trade-off. The * caller's total_tokens parameter is ignored. */ total_tokens = docmap->total_tokens; /* Prepare header placeholder */ memset(&header, 0, sizeof(TpSegmentHeader)); header.magic = TP_SEGMENT_MAGIC; header.version = TP_SEGMENT_FORMAT_VERSION; header.created_at = GetCurrentTimestamp(); header.num_pages = 0; header.num_terms = num_terms; header.level = target_level; header.next_segment = next_segment; header.num_docs = docmap->num_docs; header.total_tokens = total_tokens; header.page_index = InvalidBlockNumber; /* Write placeholder header */ merge_sink_write(sink, &header, sizeof(TpSegmentHeader)); /* Dictionary immediately follows header */ header.dictionary_offset = sink->current_offset; /* Write dictionary header */ memset(&dict, 0, sizeof(dict)); dict.num_terms = num_terms; merge_sink_write(sink, &dict, offsetof(TpDictionary, string_offsets)); /* Calculate string offsets */ string_offsets = palloc_extended( num_terms * sizeof(uint32), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); string_pos = 0; for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: merged segment string pool " "exceeds the 4 GiB format limit"), errhint("The corpus vocabulary is too large for a " "single segment."))); string_offsets[i] = (uint32)string_pos; string_pos += entry_size; } /* Write string offsets array */ merge_sink_write(sink, string_offsets, num_terms * sizeof(uint32)); /* Write string pool */ header.strings_offset = sink->current_offset; for (i = 0; i < num_terms; i++) { uint32 length = terms[i].term_len; uint32 dict_offset = (uint32)((uint64)i * sizeof(TpDictEntry)); merge_sink_write(sink, &length, sizeof(uint32)); merge_sink_write(sink, terms[i].term, length); merge_sink_write(sink, &dict_offset, sizeof(uint32)); } /* Record entries offset - dict entries written after postings */ header.entries_offset = sink->current_offset; /* Write placeholder dict entries */ { TpDictEntry placeholder; memset(&placeholder, 0, sizeof(TpDictEntry)); for (i = 0; i < num_terms; i++) merge_sink_write(sink, &placeholder, sizeof(TpDictEntry)); } /* Postings start here */ header.postings_offset = sink->current_offset; /* Initialize per-term tracking and skip entry accumulator */ term_blocks = palloc_extended( num_terms * sizeof(MergeTermBlockInfo), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); skip_entries_capacity = 1024; skip_entries_count = 0; all_skip_entries = palloc(skip_entries_capacity * sizeof(TpSkipEntry)); /* * Helper macro: flush a full or partial block_buf to the sink. * Computes skip entry, optionally compresses, writes data, * and accumulates the skip entry. */ #define FLUSH_BLOCK(block_buf, block_count, num_blocks) \ do \ { \ TpSkipEntry skip_ = merge_sink_write_posting_block( \ sink, (block_buf), (block_count)); \ \ if (skip_entries_count >= skip_entries_capacity) \ { \ skip_entries_capacity = tp_grow_capacity( \ skip_entries_capacity, 1024, "posting blocks"); \ all_skip_entries = repalloc_huge( \ all_skip_entries, \ mul_size( \ (Size)skip_entries_capacity, \ sizeof(TpSkipEntry))); \ } \ all_skip_entries[skip_entries_count++] = skip_; \ (num_blocks)++; \ } while (0) /* * Streaming pass: for each term, stream postings one block at a * time and write immediately. * * When disjoint_sources is true, drain sources sequentially * (source 0 fully, then source 1, etc.) without CTID lookups. * Otherwise, use N-way CTID-comparison merge. */ for (i = 0; i < num_terms; i++) { TpPostingMergeSource *psources; int num_psources; TpBlockPosting block_buf[TP_BLOCK_SIZE]; uint32 block_count = 0; uint32 doc_count = 0; uint32 num_blocks = 0; /* Record where this term's postings start */ term_blocks[i].posting_offset = sink->current_offset; term_blocks[i].skip_entry_start = skip_entries_count; if (terms[i].num_segment_refs == 0) { term_blocks[i].doc_freq = 0; term_blocks[i].block_count = 0; continue; } if (disjoint_sources) { /* * Fast path: sequential drain. Sources have disjoint * CTID ranges, so drain source 0, then 1, etc. * Reads doc_id/frequency/fieldnorm directly from * the block posting array, skipping CTID lookups. */ psources = init_term_posting_sources_fast( &terms[i], sources, &num_psources); for (int src = 0; src < num_psources; src++) { while (!psources[src].exhausted) { TpBlockPosting *bp = &psources[src].block_postings [psources[src].current_in_block]; int seg_idx = terms[i].segment_refs[src].segment_idx; uint32 new_id = doc_mapping.old_to_new[seg_idx][bp->doc_id]; if (new_id == TP_MERGE_DOC_DEAD) { posting_source_advance_fast(&psources[src]); continue; } block_buf[block_count].doc_id = new_id; block_buf[block_count].frequency = bp->frequency; block_buf[block_count].fieldnorm = bp->fieldnorm; block_buf[block_count].reserved = 0; block_count++; doc_count++; posting_source_advance_fast(&psources[src]); if (block_count == TP_BLOCK_SIZE) { FLUSH_BLOCK(block_buf, block_count, num_blocks); block_count = 0; } } } } else { /* * Standard N-way merge: compare CTIDs across sources. */ psources = init_term_posting_sources( &terms[i], sources, &num_psources); while (true) { int min_idx = find_min_posting_source(psources, num_psources); if (min_idx < 0) break; { int src_idx = terms[i].segment_refs[min_idx].segment_idx; uint32 old_doc_id = psources[min_idx].current.old_doc_id; uint32 new_id = doc_mapping.old_to_new[src_idx][old_doc_id]; if (new_id == TP_MERGE_DOC_DEAD) { posting_source_advance(&psources[min_idx]); continue; } block_buf[block_count].doc_id = new_id; block_buf[block_count].frequency = psources[min_idx].current.frequency; block_buf[block_count].fieldnorm = psources[min_idx].current.fieldnorm; block_buf[block_count].reserved = 0; } block_count++; doc_count++; posting_source_advance(&psources[min_idx]); if (block_count == TP_BLOCK_SIZE) { FLUSH_BLOCK(block_buf, block_count, num_blocks); block_count = 0; } } } /* Write final partial block if any */ if (block_count > 0) FLUSH_BLOCK(block_buf, block_count, num_blocks); term_blocks[i].doc_freq = doc_count; term_blocks[i].block_count = num_blocks; free_term_posting_sources(psources, num_psources); /* Check for interrupt during long merges */ if ((i % 1000) == 0) CHECK_FOR_INTERRUPTS(); } #undef FLUSH_BLOCK /* Skip index starts here - after all postings */ header.skip_index_offset = sink->current_offset; /* Write all accumulated skip entries */ if (skip_entries_count > 0) { merge_sink_write( sink, all_skip_entries, skip_entries_count * sizeof(TpSkipEntry)); } pfree(all_skip_entries); /* Write fieldnorm table */ header.fieldnorm_offset = sink->current_offset; if (docmap->num_docs > 0) { merge_sink_write( sink, docmap->fieldnorms, docmap->num_docs * sizeof(uint8)); } /* Write CTID pages array */ header.ctid_pages_offset = sink->current_offset; if (docmap->num_docs > 0) { merge_sink_write( sink, docmap->ctid_pages, docmap->num_docs * sizeof(BlockNumber)); } /* Write CTID offsets array */ header.ctid_offsets_offset = sink->current_offset; if (docmap->num_docs > 0) { merge_sink_write( sink, docmap->ctid_offsets, docmap->num_docs * sizeof(OffsetNumber)); } /* Write alive bitset (all alive — dead docs already excluded) */ header.alive_bitset_offset = sink->current_offset; header.alive_count = docmap->num_docs; if (docmap->num_docs > 0) { uint32 bitset_size = tp_alive_bitset_size(docmap->num_docs); uint8 *bitset_data = palloc(bitset_size); tp_alive_bitset_init_data(bitset_data, docmap->num_docs); merge_sink_write(sink, bitset_data, bitset_size); pfree(bitset_data); } /* Finalize data_size */ header.data_size = sink->current_offset; /* Flush writer before dictionary and header backpatching. */ tp_segment_writer_flush(&sink->writer); sink->writer.buffer_pos = SizeOfPageHeaderData; /* Backpatch dict entries */ { TpDictEntry *dict_entries; dict_entries = palloc_extended( tp_dictionary_size(num_terms), MCXT_ALLOC_HUGE); for (i = 0; i < num_terms; i++) { dict_entries[i].skip_index_offset = header.skip_index_offset + ((uint64)term_blocks[i].skip_entry_start * sizeof(TpSkipEntry)); dict_entries[i].block_count = term_blocks[i].block_count; dict_entries[i].doc_freq = term_blocks[i].doc_freq; } merge_sink_write_at( sink, header.entries_offset, dict_entries, tp_dictionary_size(num_terms)); if (dict_entries) pfree(dict_entries); } merge_sink_finish(sink, &header); /* Cleanup */ if (string_offsets) pfree(string_offsets); if (term_blocks) pfree(term_blocks); free_merge_doc_mapping(&doc_mapping); tp_docmap_destroy(docmap); } bool tp_merge_segment_batch( Relation index, const BlockNumber *source_roots, uint32 num_sources, uint32 output_level, BlockNumber next_segment, TpMergedSegmentResult *result) { TpMergeSource *sources; TpMergedTerm *merged_terms = NULL; uint32 num_merged_terms = 0; uint32 merged_capacity = 0; uint64 total_tokens = 0; bool has_live_docs = false; MemoryContext merge_ctx; MemoryContext old_ctx; uint32 i; Assert(num_sources > 0); Assert(result != NULL); memset(result, 0, sizeof(*result)); result->root = InvalidBlockNumber; if (num_sources > PG_INT32_MAX) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("cannot merge %u segments in one batch", num_sources))); merge_ctx = AllocSetContextCreate( CurrentMemoryContext, "Segment Merge Batch", ALLOCSET_DEFAULT_SIZES); old_ctx = MemoryContextSwitchTo(merge_ctx); sources = palloc0(sizeof(TpMergeSource) * num_sources); for (i = 0; i < num_sources; i++) { uint64 source_tokens; if (!merge_source_init(&sources[i], index, source_roots[i])) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("could not open segment at block %u for merge", source_roots[i]))); source_tokens = sources[i].reader->header->total_tokens; if (pg_add_u64_overflow(total_tokens, source_tokens, &total_tokens)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment token count overflow while merging " "index \"%s\"", RelationGetRelationName(index)))); if (sources[i].reader->header->alive_count > 0) has_live_docs = true; } if (!has_live_docs) { for (i = 0; i < num_sources; i++) merge_source_close(&sources[i]); MemoryContextSwitchTo(old_ctx); MemoryContextDelete(merge_ctx); return false; } while (true) { int min_idx; const char *min_term; TpMergedTerm *current_merged; min_idx = merge_find_min_source(sources, (int)num_sources); if (min_idx < 0) break; min_term = sources[min_idx].current_term; if (num_merged_terms >= TP_MAX_DICTIONARY_TERMS) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment dictionary exceeds %u " "terms", TP_MAX_DICTIONARY_TERMS))); if (num_merged_terms >= merged_capacity) { merged_capacity = tp_grow_capacity(merged_capacity, 1024, "terms"); if (merged_terms == NULL) merged_terms = palloc_extended( mul_size((Size)merged_capacity, sizeof(TpMergedTerm)), MCXT_ALLOC_HUGE); else merged_terms = repalloc_huge( merged_terms, mul_size((Size)merged_capacity, sizeof(TpMergedTerm))); } current_merged = &merged_terms[num_merged_terms]; current_merged->term_len = strlen(min_term); current_merged->term = pstrdup(min_term); current_merged->segment_refs = NULL; current_merged->num_segment_refs = 0; current_merged->segment_refs_capacity = 0; current_merged->posting_offset = 0; current_merged->posting_count = 0; num_merged_terms++; for (i = 0; i < num_sources; i++) { if (sources[i].exhausted) continue; if (strcmp(sources[i].current_term, current_merged->term) == 0) { merged_term_add_segment_ref( current_merged, (int)i, &sources[i].current_entry); merge_source_advance(&sources[i]); } } CHECK_FOR_INTERRUPTS(); } tp_validate_merged_terms(merged_terms, num_merged_terms); tp_validate_merged_terms(merged_terms, num_merged_terms); { volatile TpMergeSink sink; memset((TpMergeSink *)&sink, 0, sizeof(TpMergeSink)); PG_TRY(); { BlockNumber new_segment; Buffer header_buf; Page header_page; TpSegmentHeader *header; merge_sink_init_pages((TpMergeSink *)&sink, index); if (sink.writer.pages_allocated == 0) elog(ERROR, "merge: failed to allocate segment pages"); new_segment = sink.writer.pages[0]; write_merged_segment_to_sink( (TpMergeSink *)&sink, merged_terms, num_merged_terms, sources, (int)num_sources, output_level, total_tokens, false, next_segment); header_buf = ReadBuffer(index, new_segment); LockBuffer(header_buf, BUFFER_LOCK_SHARE); header_page = BufferGetPage(header_buf); header = (TpSegmentHeader *)PageGetContents(header_page); if (header->magic != TP_SEGMENT_MAGIC || header->version != TP_SEGMENT_FORMAT_VERSION || header->level != output_level || header->next_segment != next_segment) { UnlockReleaseBuffer(header_buf); ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("merged segment header was not finalized " "before publication"))); } result->root = new_segment; result->num_pages = header->num_pages; result->num_docs = header->num_docs; result->total_tokens = header->total_tokens; result->data_size = header->data_size; UnlockReleaseBuffer(header_buf); } PG_CATCH(); { tp_discard_unpublished_pages( index, sink.writer.pages, sink.writer.pages_allocated); tp_discard_unpublished_pages( index, sink.page_index_pages, sink.page_index_pages_allocated); PG_RE_THROW(); } PG_END_TRY(); } for (i = 0; i < num_sources; i++) merge_source_close(&sources[i]); MemoryContextSwitchTo(old_ctx); MemoryContextDelete(merge_ctx); return true; } timescale-pg_textsearch-b63fe58/src/segment/merge.h000066400000000000000000000045671525762047700224760ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * merge.h - Segment merge for LSM-style compaction */ #pragma once #include "postgres.h" #include "segment/io.h" #include "segment/segment.h" #include "storage/block.h" #include "utils/rel.h" /* Forward declarations */ struct TpLocalIndexState; struct TpMergeSource; struct TpMergedTerm; typedef struct TpMergedSegmentResult { BlockNumber root; uint32 num_pages; uint32 num_docs; uint64 total_tokens; uint64 data_size; } TpMergedSegmentResult; /* * Merge sink: writes merged segment data to index pages. */ typedef struct TpMergeSink { uint64 current_offset; TpSegmentWriter writer; Relation index; /* Exact page-index allocations owned until publication succeeds. */ BlockNumber *page_index_pages; uint32 page_index_pages_allocated; } TpMergeSink; /* Sink initialization */ extern void merge_sink_init_pages(TpMergeSink *sink, Relation index); extern void merge_sink_write(TpMergeSink *sink, const void *data, Size size); extern void merge_sink_write_at( TpMergeSink *sink, uint64 offset, const void *data, uint64 size); extern void merge_sink_finish(TpMergeSink *sink, TpSegmentHeader *header); extern TpSkipEntry merge_sink_write_posting_block( TpMergeSink *sink, TpBlockPosting *block, uint32 count); /* * Write a merged segment to sink (pages or BufFile). * Unified function that replaces both write_merged_segment() and * write_merged_segment_to_buffile(). */ extern void write_merged_segment_to_sink( TpMergeSink *sink, struct TpMergedTerm *terms, uint32 num_terms, struct TpMergeSource *sources, int num_sources, uint32 target_level, uint64 total_tokens, bool disjoint_sources, BlockNumber next_segment); /* * Merge exactly the supplied immutable segments into one unpublished * current-format segment. The caller publishes or discards the result. */ extern bool tp_merge_segment_batch( Relation index, const BlockNumber *source_roots, uint32 num_sources, uint32 output_level, BlockNumber next_segment, TpMergedSegmentResult *result); /* * Return exactly one complete, unpublished merge output to the FSM. * Published segments must instead go through standby-safe deferred reclaim. */ extern void tp_discard_unpublished_segment(Relation index, BlockNumber root); timescale-pg_textsearch-b63fe58/src/segment/merge_internal.h000066400000000000000000000120101525762047700243500ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * merge_internal.h - Internal merge types and helpers * * Exposes merge internals needed by build_parallel.c for BufFile * merge. These types and functions are not part of the public API. */ #pragma once #include #include #include "segment/io.h" #include "segment/segment.h" /* * Merge source state - tracks current position in each source segment */ typedef struct TpMergeSource { TpSegmentReader *reader; /* Segment reader */ uint32 current_idx; /* Current term index in dictionary */ uint32 num_terms; /* Total terms in this segment */ char *current_term; /* Current term text (palloc'd) */ TpDictEntry current_entry; /* dictionary entry */ bool exhausted; /* True if no more terms */ uint32 *string_offsets; /* Cached string offsets array */ uint32 string_offsets_start; uint32 string_offsets_count; } TpMergeSource; /* * Reference to a segment that contains a particular term. */ typedef struct TpTermSegmentRef { int segment_idx; /* Index into sources array */ TpDictEntry entry; /* dict entry from that segment */ } TpTermSegmentRef; /* * Merged term info - tracks which segments have this term. */ typedef struct TpMergedTerm { char *term; /* Term text */ uint32 term_len; /* Term length */ TpTermSegmentRef *segment_refs; /* Which segments have this term */ uint32 num_segment_refs; /* Number of segment refs */ uint32 segment_refs_capacity; /* Allocated capacity */ uint64 posting_offset; /* Offset where postings start */ uint32 posting_count; /* Number of postings written */ } TpMergedTerm; /* * Current posting info during merge. */ typedef struct TpMergePostingInfo { ItemPointerData ctid; /* CTID for comparison ordering */ uint32 old_doc_id; /* Doc ID in source segment */ uint16 frequency; /* Term frequency */ uint8 fieldnorm; /* Encoded fieldnorm (1 byte) */ } TpMergePostingInfo; /* * Posting merge source - tracks position in one segment's posting * list for streaming N-way merge. */ typedef struct TpPostingMergeSource { TpSegmentReader *reader; /* Segment reader */ TpMergePostingInfo current; /* Current posting info */ bool exhausted; /* No more postings */ /* block iteration state */ uint64 skip_index_offset; /* Offset to skip entries */ uint32 block_count; /* Total blocks */ uint32 current_block; /* Current block index */ uint32 current_in_block; /* Position within current block */ TpSkipEntry skip_entry; /* Current block's skip entry */ TpBlockPosting *block_postings; /* Cached postings for block */ uint32 block_capacity; /* Allocated size */ } TpPostingMergeSource; /* * Mapping from (source_idx, old_doc_id) -> new_doc_id. */ typedef struct TpMergeDocMapping { uint32 **old_to_new; /* old_to_new[src_idx][old_doc_id] = new */ int num_sources; } TpMergeDocMapping; /* * Per-term block info for merge output. */ typedef struct MergeTermBlockInfo { uint64 posting_offset; /* Offset where postings were written */ uint32 block_count; /* Number of blocks for this term */ uint32 doc_freq; /* Document frequency */ uint32 skip_entry_start; /* Index into skip entries array */ } MergeTermBlockInfo; /* Forward declaration */ struct TpDocMapBuilder; /* * Merge source operations */ extern bool merge_source_advance(TpMergeSource *source); extern bool merge_source_init(TpMergeSource *source, Relation index, BlockNumber root); extern bool merge_source_init_from_reader(TpMergeSource *source, TpSegmentReader *reader); extern void merge_source_close(TpMergeSource *source); /* * Term merge operations */ extern int merge_find_min_source(TpMergeSource *sources, int num_sources); extern void merged_term_add_segment_ref( TpMergedTerm *term, int segment_idx, TpDictEntry *entry); extern void tp_validate_merged_terms(TpMergedTerm *terms, uint32 num_terms); /* * Posting merge operations */ extern void posting_source_init( TpPostingMergeSource *ps, TpSegmentReader *reader, TpDictEntry *entry); extern void posting_source_init_fast( TpPostingMergeSource *ps, TpSegmentReader *reader, TpDictEntry *entry); extern void posting_source_free(TpPostingMergeSource *ps); extern bool posting_source_advance(TpPostingMergeSource *ps); extern bool posting_source_advance_fast(TpPostingMergeSource *ps); extern int find_min_posting_source(TpPostingMergeSource *sources, int num_sources); /* * Docmap merge operations */ extern struct TpDocMapBuilder *build_merged_docmap( TpMergeSource *sources, int num_sources, TpMergeDocMapping *mapping, bool disjoint_sources); extern void free_merge_doc_mapping(TpMergeDocMapping *mapping); /* * Posting source initialization for a term */ extern TpPostingMergeSource *init_term_posting_sources( TpMergedTerm *term, TpMergeSource *sources, int *num_psources); extern TpPostingMergeSource *init_term_posting_sources_fast( TpMergedTerm *term, TpMergeSource *sources, int *num_psources); timescale-pg_textsearch-b63fe58/src/segment/pagemapper.h000066400000000000000000000034011525762047700235020ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * pagemapper.h - Segment logical-to-physical address translation * * Segments store data as a logical contiguous byte stream, but physically * the data is spread across multiple Postgres pages. This module provides * the translation between logical offsets and physical page locations. * * Logical address space: * [0 .. data_size) - contiguous byte offsets * * Physical storage: * Array of BlockNumbers (page_map), each page stores SEGMENT_DATA_PER_PAGE * bytes of logical data (page header is reserved for Postgres). */ #pragma once #include "postgres.h" #include "storage/bufpage.h" /* * Usable data bytes per segment data page. * We use the standard page header; special area is not used for data pages. */ #define SEGMENT_DATA_PER_PAGE (BLCKSZ - SizeOfPageHeaderData) /* * Convert a logical byte offset to logical page number. */ static inline uint32 tp_logical_page(uint64 logical_offset) { return (uint32)(logical_offset / SEGMENT_DATA_PER_PAGE); } /* * Convert a logical byte offset to offset within the logical page. */ static inline uint32 tp_page_offset(uint64 logical_offset) { return (uint32)(logical_offset % SEGMENT_DATA_PER_PAGE); } /* * Calculate bytes remaining on current page from given offset. */ static inline uint32 tp_bytes_remaining_on_page(uint64 logical_offset) { return SEGMENT_DATA_PER_PAGE - tp_page_offset(logical_offset); } /* * Check if a read of 'len' bytes starting at 'logical_offset' fits * within a single page. Useful for zero-copy access patterns. */ static inline bool tp_fits_on_page(uint64 logical_offset, uint32 len) { return tp_page_offset(logical_offset) + len <= SEGMENT_DATA_PER_PAGE; } timescale-pg_textsearch-b63fe58/src/segment/scan.c000066400000000000000000000573571525762047700223230ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * scan.c - Zero-copy scan execution for segments */ #include #include #include #include "segment/compression.h" #include "segment/dictionary.h" #include "segment/fieldnorm.h" #include "segment/io.h" #include "segment/segment.h" static void tp_segment_posting_iterator_init_at( TpSegmentPostingIterator *iter, TpSegmentReader *reader, const char *term, uint32 dict_entry_idx, bool force_copy) { memset(iter, 0, sizeof(*iter)); iter->reader = reader; iter->term = term; iter->dict_entry_idx = dict_entry_idx; iter->force_copy = force_copy; tp_segment_read_dict_entry( reader, reader->header, dict_entry_idx, &iter->dict_entry); iter->initialized = true; iter->finished = (iter->dict_entry.block_count == 0); } /* * Read a skip entry by block index. * Used by BMW scoring to pre-compute block max scores. */ void tp_segment_read_skip_entry( TpSegmentReader *reader, uint64 skip_index_offset, uint32 block_idx, TpSkipEntry *skip) { uint64 skip_offset; if (reader->segment_version <= TP_SEGMENT_FORMAT_VERSION_3) { TpSkipEntryV3 v3; skip_offset = skip_index_offset + (uint64)block_idx * sizeof(TpSkipEntryV3); tp_segment_read(reader, skip_offset, &v3, sizeof(TpSkipEntryV3)); /* Widen V3 fields to V4 */ skip->last_doc_id = v3.last_doc_id; skip->doc_count = v3.doc_count; skip->block_max_tf = v3.block_max_tf; skip->block_max_norm = v3.block_max_norm; skip->posting_offset = (uint64)v3.posting_offset; skip->flags = v3.flags; memcpy(skip->reserved, v3.reserved, sizeof(v3.reserved)); } else { skip_offset = skip_index_offset + (uint64)block_idx * sizeof(TpSkipEntry); tp_segment_read(reader, skip_offset, skip, sizeof(TpSkipEntry)); } } /* * Initialize iterator for a specific term in a segment. * Returns true if term found, false otherwise. */ bool tp_segment_posting_iterator_init( TpSegmentPostingIterator *iter, TpSegmentReader *reader, const char *term) { TpSegmentHeader *header; TpDictionary dict_header; int left, right, mid; char *term_buffer = NULL; uint32 buffer_size = 0; if (!reader || !reader->header) return false; header = reader->header; memset(iter, 0, sizeof(*iter)); iter->reader = reader; iter->term = term; iter->finished = true; if (header->num_terms == 0 || header->dictionary_offset == 0) return false; /* Read dictionary header */ tp_segment_read( reader, header->dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); /* Binary search for the term */ left = 0; right = dict_header.num_terms - 1; while (left <= right) { TpStringEntry string_entry; int cmp; uint32 string_offset_value; uint64 string_offset; mid = left + (right - left) / 2; /* Read string offset */ tp_segment_read( reader, header->dictionary_offset + sizeof(dict_header.num_terms) + (mid * sizeof(uint32)), &string_offset_value, sizeof(uint32)); string_offset = header->strings_offset + string_offset_value; /* Read string length */ tp_segment_read( reader, string_offset, &string_entry.length, sizeof(uint32)); if (string_entry.length > TP_MAX_TERM_LENGTH) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupt segment: term length %u exceeds " "maximum", string_entry.length))); /* Reallocate buffer if needed */ if (string_entry.length + 1 > buffer_size) { if (term_buffer) pfree(term_buffer); buffer_size = string_entry.length + 1; term_buffer = palloc(buffer_size); } /* Read term text */ tp_segment_read( reader, string_offset + sizeof(uint32), term_buffer, string_entry.length); term_buffer[string_entry.length] = '\0'; /* Compare terms */ cmp = strcmp(term, term_buffer); if (cmp == 0) { tp_segment_posting_iterator_init_at( iter, reader, term, mid, false); pfree(term_buffer); return true; } else if (cmp < 0) { right = mid - 1; } else { left = mid + 1; } } pfree(term_buffer); return false; } /* * Build candidates for dictionary terms with the prefix. Small expansions * return one iterator per term for ordered merging. Broad expansions consume * one iterator at a time into a document bitmap, bounding simultaneous pins. */ void tp_segment_prefix_candidates_init( TpSegmentReader *reader, const char *prefix, int prefix_length, uint32 max_iterators, TpSegmentPrefixCandidates *candidates) { TpSegmentHeader *header; TpDictionary dict_header; char *prefix_string; uint32 first; uint32 end; uint32 left; uint32 postings_since_interrupt = 0; uint32 right; memset(candidates, 0, sizeof(*candidates)); if (!reader || !reader->header || prefix_length <= 0) return; header = reader->header; if (header->num_terms == 0 || header->dictionary_offset == 0) return; tp_segment_read( reader, header->dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); if (dict_header.num_terms != header->num_terms) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupt segment: dictionary term count %u " "does not match segment term count %u", dict_header.num_terms, header->num_terms))); prefix_string = pnstrdup(prefix, prefix_length); left = 0; right = dict_header.num_terms; while (left < right) { uint32 mid = left + (right - left) / 2; char *term = tp_segment_read_term_at_index(reader, header, NULL, mid); int cmp = strcmp(term, prefix_string); pfree(term); if (cmp < 0) left = mid + 1; else right = mid; } first = left; end = first; while (end < dict_header.num_terms) { char *term = tp_segment_read_term_at_index(reader, header, NULL, end); if (strncmp(term, prefix_string, prefix_length) != 0) { pfree(term); break; } pfree(term); end++; } pfree(prefix_string); if (end - first <= max_iterators) { candidates->iterator_count = end - first; if (candidates->iterator_count == 0) return; candidates->iterators = palloc_array( TpSegmentPostingIterator, candidates->iterator_count); for (uint32 i = 0; i < candidates->iterator_count; i++) { TpSegmentPostingIterator *iter = &candidates->iterators[i]; tp_segment_posting_iterator_init_at( iter, reader, NULL, first + i, true); candidates->estimate = Min((uint64)header->num_docs, candidates->estimate + iter->dict_entry.doc_freq); } return; } candidates->doc_bitmap = palloc0(((Size)header->num_docs + 7) / 8); for (uint32 i = first; i < end; i++) { TpSegmentPostingIterator iter; TpSegmentPosting *posting; CHECK_FOR_INTERRUPTS(); tp_segment_posting_iterator_init_at(&iter, reader, NULL, i, false); while (tp_segment_posting_iterator_next(&iter, &posting)) { uint8 *byte; uint8 mask; if (posting->doc_id >= header->num_docs) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupt segment: posting document ID %u " "exceeds document count %u", posting->doc_id, header->num_docs))); if (++postings_since_interrupt == 4096) { CHECK_FOR_INTERRUPTS(); postings_since_interrupt = 0; } byte = &candidates->doc_bitmap[posting->doc_id >> 3]; mask = (uint8)(1 << (posting->doc_id & 7)); if ((*byte & mask) == 0) { *byte |= mask; candidates->estimate++; } } tp_segment_posting_iterator_free(&iter); } } /* * Load a block's postings for iteration. * Uses zero-copy access when block data fits within a single page and is * uncompressed. Compressed blocks are always decompressed into the fallback * buffer. CTIDs are looked up from segment-level cached arrays during * iteration. */ bool tp_segment_posting_iterator_load_block(TpSegmentPostingIterator *iter) { uint32 block_size; uint32 block_bytes; if (iter->current_block >= iter->dict_entry.block_count) return false; /* Release previous block access if any */ if (iter->has_block_access) { tp_segment_release_direct(&iter->block_access); iter->has_block_access = false; iter->block_postings = NULL; } /* Read skip entry: use cache if available, else read from disk */ if (iter->cached_skip_entries) iter->skip_entry = iter->cached_skip_entries[iter->current_block]; else tp_segment_read_skip_entry( iter->reader, iter->dict_entry.skip_index_offset, iter->current_block, &iter->skip_entry); block_size = iter->skip_entry.doc_count; block_bytes = block_size * sizeof(TpBlockPosting); /* Handle compressed blocks */ if (iter->skip_entry.flags == TP_BLOCK_FLAG_DELTA) { uint8 *compressed_buf; bool free_compressed = false; /* Use cached buffer if available, else palloc */ if (iter->compressed_buf_cache) compressed_buf = iter->compressed_buf_cache; else { compressed_buf = palloc(TP_MAX_COMPRESSED_BLOCK_SIZE); free_compressed = true; } tp_segment_read( iter->reader, iter->skip_entry.posting_offset, compressed_buf, TP_MAX_COMPRESSED_BLOCK_SIZE); /* Ensure fallback buffer is large enough */ if (block_size > iter->fallback_block_size) { if (iter->fallback_block) pfree(iter->fallback_block); iter->fallback_block = palloc(block_bytes); iter->fallback_block_size = block_size; } /* Decompress into fallback buffer */ tp_decompress_block( compressed_buf, block_size, 0, iter->fallback_block); if (free_compressed) pfree(compressed_buf); iter->block_postings = iter->fallback_block; } else { /* * Uncompressed block: try zero-copy direct access. * TpBlockPosting requires 4-byte alignment (due to uint32 doc_id). * If the data address is misaligned, fall back to copying. */ if (!iter->force_copy && tp_segment_get_direct( iter->reader, iter->skip_entry.posting_offset, block_bytes, &iter->block_access) && ((uintptr_t)iter->block_access.data % sizeof(uint32)) == 0) { /* Zero-copy: point directly into the page buffer */ iter->block_postings = (TpBlockPosting *)iter->block_access.data; iter->has_block_access = true; } else { /* Release direct access if we got it but it's misaligned */ if (iter->block_access.data != NULL) tp_segment_release_direct(&iter->block_access); /* Fallback: block spans page boundary, must copy */ if (block_size > iter->fallback_block_size) { if (iter->fallback_block) pfree(iter->fallback_block); iter->fallback_block = palloc(block_bytes); iter->fallback_block_size = block_size; } tp_segment_read( iter->reader, iter->skip_entry.posting_offset, iter->fallback_block, block_bytes); iter->block_postings = iter->fallback_block; } } iter->current_in_block = 0; return true; } /* * Get next posting from iterator. * Converts block posting to TpSegmentPosting for scoring compatibility. * Returns false when no more postings. */ bool tp_segment_posting_iterator_next( TpSegmentPostingIterator *iter, TpSegmentPosting **posting) { TpBlockPosting *bp; uint32 doc_id; if (iter->finished || !iter->initialized) return false; /* Load first block if needed */ if (iter->block_postings == NULL) { if (!tp_segment_posting_iterator_load_block(iter)) { /* load_block only fails on corrupted segment data */ if (iter->has_block_access) { tp_segment_release_direct(&iter->block_access); iter->has_block_access = false; } iter->finished = true; return false; } } /* Move to next block if current is exhausted */ while (iter->current_in_block >= iter->skip_entry.doc_count) { iter->current_block++; if (iter->current_block >= iter->dict_entry.block_count) { /* Release block access before finishing */ if (iter->has_block_access) { tp_segment_release_direct(&iter->block_access); iter->has_block_access = false; } iter->finished = true; return false; } if (!tp_segment_posting_iterator_load_block(iter)) { /* load_block only fails on corrupted segment data */ if (iter->has_block_access) { tp_segment_release_direct(&iter->block_access); iter->has_block_access = false; } iter->finished = true; return false; } } /* Get current posting from block */ bp = &iter->block_postings[iter->current_in_block]; doc_id = bp->doc_id; /* Always store doc_id for deferred CTID resolution */ iter->output_posting.doc_id = doc_id; /* * Look up CTID from segment-level cached arrays if available. * By default, CTIDs are not pre-loaded and will be resolved later * via tp_segment_lookup_ctid. */ if (iter->reader->cached_ctid_pages != NULL && doc_id < iter->reader->cached_num_docs) { ItemPointerData tmp; ItemPointerSet( &tmp, iter->reader->cached_ctid_pages[doc_id], iter->reader->cached_ctid_offsets[doc_id]); memcpy(&iter->output_posting.ctid, &tmp, sizeof(ItemPointerData)); } else { /* Mark CTID invalid - will be resolved later */ ItemPointerData tmp; ItemPointerSetInvalid(&tmp); memcpy(&iter->output_posting.ctid, &tmp, sizeof(ItemPointerData)); } /* Build output posting (fieldnorm is inline in bp) */ iter->output_posting.frequency = bp->frequency; iter->output_posting.doc_length = (uint16)decode_fieldnorm(bp->fieldnorm); *posting = &iter->output_posting; iter->current_in_block++; return true; } /* * Free iterator resources. */ void tp_segment_posting_iterator_free(TpSegmentPostingIterator *iter) { /* Release direct block access if active */ if (iter->has_block_access) { tp_segment_release_direct(&iter->block_access); iter->has_block_access = false; } /* Free fallback buffer if allocated */ if (iter->fallback_block) { pfree(iter->fallback_block); iter->fallback_block = NULL; } /* * Note: cached_skip_entries and compressed_buf_cache are borrowed * pointers owned by the BMW caller. Do NOT free them here. */ iter->cached_skip_entries = NULL; iter->compressed_buf_cache = NULL; iter->block_postings = NULL; } /* * Get current doc ID from iterator. * Returns UINT32_MAX if iterator is finished or not positioned. */ uint32 tp_segment_posting_iterator_current_doc_id(TpSegmentPostingIterator *iter) { if (iter->finished || !iter->initialized || iter->block_postings == NULL) return UINT32_MAX; if (iter->current_in_block >= iter->skip_entry.doc_count) return UINT32_MAX; return iter->block_postings[iter->current_in_block].doc_id; } /* * Seek iterator to target doc ID or the first doc ID >= target. * Returns true if a posting was found, false if exhausted. * * Uses binary search on skip entries (each has last_doc_id) to find * the right block, then linear scan within the block. This is the * core operation for WAND-style doc-ID ordered traversal. */ bool tp_segment_posting_iterator_seek( TpSegmentPostingIterator *iter, uint32 target_doc_id, TpSegmentPosting **posting) { uint32 block_count; int left, right, mid; uint32 target_block; uint32 step; TpSkipEntry skip; if (!iter->initialized || iter->finished) return false; block_count = iter->dict_entry.block_count; /* * Binary search skip entries to find block containing target_doc_id. * Each skip entry has last_doc_id = maximum doc ID in that block. * We want the first block where last_doc_id >= target_doc_id. */ left = 0; right = block_count - 1; /* * A target beyond the loaded block cannot occur in an earlier block. * Probe nearby blocks first because Boolean intersections seek * monotonically and usually advance only a few posting blocks. */ if (iter->block_postings != NULL && target_doc_id > iter->skip_entry.last_doc_id) { if (iter->current_block + 1 >= block_count) { iter->finished = true; return false; } left = iter->current_block + 1; right = left; step = 1; for (;;) { tp_segment_read_skip_entry( iter->reader, iter->dict_entry.skip_index_offset, right, &skip); if (skip.last_doc_id >= target_doc_id || right == (int)block_count - 1) break; left = right + 1; if (step >= block_count - 1 - right) right = block_count - 1; else right += step; step *= 2; } } while (left < right) { mid = left + (right - left) / 2; /* Read skip entry to get last_doc_id */ tp_segment_read_skip_entry( iter->reader, iter->dict_entry.skip_index_offset, mid, &skip); if (skip.last_doc_id < target_doc_id) { /* Target is past this block */ left = mid + 1; } else { /* Target might be in this block or earlier */ right = mid; } } target_block = left; /* Check if target is past all blocks */ if (target_block >= block_count) { iter->finished = true; return false; } /* Load the target block */ iter->current_block = target_block; iter->current_in_block = 0; iter->finished = false; if (!tp_segment_posting_iterator_load_block(iter)) { iter->finished = true; return false; } /* Linear scan within block to find target or first doc >= target */ while (iter->current_in_block < iter->skip_entry.doc_count) { TpBlockPosting *bp = &iter->block_postings[iter->current_in_block]; if (bp->doc_id >= target_doc_id) { /* Found it - convert to output posting */ iter->output_posting.doc_id = bp->doc_id; /* Resolve CTID if cached, otherwise leave invalid for later */ if (iter->reader->cached_ctid_pages != NULL && bp->doc_id < iter->reader->cached_num_docs) { ItemPointerData tmp; ItemPointerSet( &tmp, iter->reader->cached_ctid_pages[bp->doc_id], iter->reader->cached_ctid_offsets[bp->doc_id]); memcpy(&iter->output_posting.ctid, &tmp, sizeof(ItemPointerData)); } else { ItemPointerData tmp; ItemPointerSetInvalid(&tmp); memcpy(&iter->output_posting.ctid, &tmp, sizeof(ItemPointerData)); } iter->output_posting.frequency = bp->frequency; iter->output_posting.doc_length = (uint16)decode_fieldnorm( bp->fieldnorm); *posting = &iter->output_posting; return true; } iter->current_in_block++; } /* * Exhausted this block without finding target. * This shouldn't happen if last_doc_id was correct, but handle it * by moving to next block and trying next(). */ iter->current_block++; if (iter->current_block >= block_count) { iter->finished = true; return false; } /* Load next block and return first posting */ iter->current_in_block = 0; return tp_segment_posting_iterator_next(iter, posting); } /* * Sum doc_freq for a term across all segments. */ uint32 tp_segment_roots_get_doc_freq( Relation index, const BlockNumber *roots, uint32 root_count, const char *term) { TpSegmentReader *reader = NULL; uint32 doc_freq = 0; char *term_buffer = NULL; uint32 buffer_size = 0; for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentHeader *header; TpDictionary dict_header; int left, right; reader = tp_segment_open(index, roots[root_idx]); if (!reader) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("could not open BM25 segment %u", roots[root_idx]))); header = reader->header; if (header->num_terms == 0 || header->dictionary_offset == 0) { tp_segment_close(reader); continue; } tp_segment_read( reader, header->dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); left = 0; right = dict_header.num_terms - 1; while (left <= right) { TpStringEntry string_entry; int cmp; uint32 string_offset_value; uint64 string_offset; int mid = left + (right - left) / 2; tp_segment_read( reader, header->dictionary_offset + sizeof(dict_header.num_terms) + (mid * sizeof(uint32)), &string_offset_value, sizeof(uint32)); string_offset = header->strings_offset + string_offset_value; tp_segment_read( reader, string_offset, &string_entry.length, sizeof(uint32)); if (string_entry.length > TP_MAX_TERM_LENGTH) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupt segment: term length %u " "exceeds maximum", string_entry.length))); if (string_entry.length + 1 > buffer_size) { if (term_buffer) pfree(term_buffer); buffer_size = string_entry.length + 1; term_buffer = palloc(buffer_size); } tp_segment_read( reader, string_offset + sizeof(uint32), term_buffer, string_entry.length); term_buffer[string_entry.length] = '\0'; cmp = strcmp(term, term_buffer); if (cmp == 0) { TpDictEntry dict_entry; tp_segment_read_dict_entry(reader, header, mid, &dict_entry); doc_freq += dict_entry.doc_freq; break; } else if (cmp < 0) { right = mid - 1; } else { left = mid + 1; } } tp_segment_close(reader); } if (term_buffer) pfree(term_buffer); return doc_freq; } /* * Batch lookup doc_freq for multiple terms across explicit segment roots. * Opens each segment ONCE and looks up all terms, avoiding * O(terms * segments) segment opens. * * doc_freqs array should be pre-initialized (typically to 0 or memtable * counts). This function ADDS segment doc_freqs to existing values. */ void tp_batch_get_segment_roots_doc_freq( Relation index, const BlockNumber *roots, uint32 root_count, char **terms, int term_count, uint32 *doc_freqs) { char *term_buffer = NULL; uint32 buffer_size = 0; for (uint32 root_idx = 0; root_idx < root_count; root_idx++) { TpSegmentReader *reader; TpSegmentHeader *header; TpDictionary dict_header; int term_idx; /* Open segment ONCE for all terms */ reader = tp_segment_open(index, roots[root_idx]); if (!reader) ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("could not open BM25 segment %u", roots[root_idx]))); header = reader->header; if (header->num_terms == 0 || header->dictionary_offset == 0) { tp_segment_close(reader); continue; } /* Read dictionary header once per segment */ tp_segment_read( reader, header->dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); /* Look up each term in this segment */ for (term_idx = 0; term_idx < term_count; term_idx++) { const char *term = terms[term_idx]; int left = 0; int right = dict_header.num_terms - 1; /* Binary search for term in dictionary */ while (left <= right) { int mid = left + (right - left) / 2; uint32 string_offset_value; uint64 string_offset; uint32 string_length; int cmp; tp_segment_read( reader, header->dictionary_offset + sizeof(dict_header.num_terms) + (mid * sizeof(uint32)), &string_offset_value, sizeof(uint32)); string_offset = header->strings_offset + string_offset_value; tp_segment_read( reader, string_offset, &string_length, sizeof(uint32)); if (string_length > TP_MAX_TERM_LENGTH) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("corrupt segment: term length %u " "exceeds maximum", string_length))); if (string_length + 1 > buffer_size) { if (term_buffer) pfree(term_buffer); buffer_size = string_length + 1; term_buffer = palloc(buffer_size); } tp_segment_read( reader, string_offset + sizeof(uint32), term_buffer, string_length); term_buffer[string_length] = '\0'; cmp = strcmp(term, term_buffer); if (cmp == 0) { /* Found - read dict entry and add doc_freq */ TpDictEntry dict_entry; tp_segment_read_dict_entry( reader, header, mid, &dict_entry); doc_freqs[term_idx] += dict_entry.doc_freq; break; } else if (cmp < 0) { right = mid - 1; } else { left = mid + 1; } } } tp_segment_close(reader); } if (term_buffer) pfree(term_buffer); } timescale-pg_textsearch-b63fe58/src/segment/segment.c000066400000000000000000002140101525762047700230160ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * segment.c - Disk-based segment implementation */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "debug/dump.h" #include "debug/injection.h" #include "index/freepage.h" #include "index/metapage.h" #include "index/state.h" #include "segment/alive_bitset.h" #include "segment/compaction.h" #include "segment/compression.h" #include "segment/dictionary.h" #include "segment/docmap.h" #include "segment/fieldnorm.h" #include "segment/io.h" #include "segment/pagemapper.h" #include "segment/segment.h" /* External: compression GUC from mod.c */ extern bool tp_compress_segments; void tp_segment_log_dirty_buffer(Relation index, Buffer buffer) { START_CRIT_SECTION(); MarkBufferDirty(buffer); if (RelationNeedsWAL(index)) log_newpage_buffer(buffer, false); END_CRIT_SECTION(); } /* * Note: We previously had a global page map cache here, but it was removed * due to race conditions when multiple backends accessed it concurrently. * Since segments are small and page maps are not frequently re-read in the * same backend, the performance impact of removing the cache is minimal. */ /* * Version-aware dictionary entry reader. * V3 segments have 12-byte TpDictEntryV3; V4 have 16-byte TpDictEntry. */ void tp_segment_read_dict_entry( TpSegmentReader *reader, TpSegmentHeader *header, uint32 index, TpDictEntry *entry) { uint64 entry_offset; if (reader->segment_version <= TP_SEGMENT_FORMAT_VERSION_3) { TpDictEntryV3 v3; entry_offset = header->entries_offset + (uint64)index * sizeof(TpDictEntryV3); tp_segment_read(reader, entry_offset, &v3, sizeof(TpDictEntryV3)); /* Widen V3 fields to V4 */ entry->skip_index_offset = (uint64)v3.skip_index_offset; entry->block_count = v3.block_count; entry->doc_freq = v3.doc_freq; } else { entry_offset = header->entries_offset + (uint64)index * sizeof(TpDictEntry); tp_segment_read(reader, entry_offset, entry, sizeof(TpDictEntry)); } } bool tp_segment_read_next(Relation index, BlockNumber root, BlockNumber *next) { BlockNumber nblocks = RelationGetNumberOfBlocks(index); Buffer buffer; Page page; uint32 magic; uint32 version; Assert(next != NULL); if (root >= nblocks) return false; buffer = ReadBuffer(index, root); LockBuffer(buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(buffer); memcpy(&magic, PageGetContents(page), sizeof(magic)); memcpy(&version, (char *)PageGetContents(page) + sizeof(magic), sizeof(version)); if (magic != TP_SEGMENT_MAGIC || version < TP_SEGMENT_FORMAT_VERSION_3 || version > TP_SEGMENT_FORMAT_VERSION) { UnlockReleaseBuffer(buffer); ereport(ERROR, (errcode(ERRCODE_INDEX_CORRUPTED), errmsg("invalid segment header at block %u", root))); } if (version <= TP_SEGMENT_FORMAT_VERSION_3) *next = ((TpSegmentHeaderV3 *)PageGetContents(page))->next_segment; else if (version == TP_SEGMENT_FORMAT_VERSION_4) *next = ((TpSegmentHeaderV4 *)PageGetContents(page))->next_segment; else *next = ((TpSegmentHeader *)PageGetContents(page))->next_segment; UnlockReleaseBuffer(buffer); return true; } /* * Open segment for reading. * If load_ctids is true, preloads all CTID arrays into memory (expensive). * If load_ctids is false, skips CTID preloading - use tp_segment_lookup_ctid * for deferred resolution. */ static TpSegmentReader * tp_segment_open_internal( Relation index, BlockNumber root_block, bool load_ctids, bool header_only) { TpSegmentReader *reader; Buffer header_buf; Page header_page; TpSegmentHeader *header; BlockNumber page_index_block; Buffer index_buf; Page index_page; TpPageIndexSpecial *special; BlockNumber *page_entries; uint32 pages_loaded = 0; uint32 i; BlockNumber nblocks; /* * Validate root_block is within the relation. In Postgres, blocks are * allocated sequentially from 0 to nblocks-1, so any valid block number * must be < nblocks. This is the standard way to validate block numbers. */ nblocks = RelationGetNumberOfBlocks(index); if (root_block >= nblocks) return NULL; /* Allocate reader structure */ reader = palloc0(sizeof(TpSegmentReader)); reader->index = index; reader->root_block = root_block; reader->current_buffer = InvalidBuffer; reader->current_logical_page = UINT32_MAX; reader->ctid_pages_buffer = InvalidBuffer; reader->ctid_pages_logical_page = UINT32_MAX; reader->ctid_offsets_buffer = InvalidBuffer; reader->ctid_offsets_logical_page = UINT32_MAX; /* Read header from root block */ header_buf = ReadBuffer(index, root_block); LockBuffer(header_buf, BUFFER_LOCK_SHARE); header_page = BufferGetPage(header_buf); /* * Read raw magic and version first to determine format, then * read the correct header struct and widen to V4 if needed. */ { uint32 raw_magic; uint32 raw_version; memcpy(&raw_magic, PageGetContents(header_page), sizeof(uint32)); memcpy(&raw_version, (char *)PageGetContents(header_page) + sizeof(uint32), sizeof(uint32)); /* Validate magic before version dispatch */ if (raw_magic != TP_SEGMENT_MAGIC) { UnlockReleaseBuffer(header_buf); pfree(reader); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid segment header at block %u", root_block), errdetail( "magic=0x%08X, expected 0x%08X", raw_magic, TP_SEGMENT_MAGIC))); } reader->segment_version = raw_version; reader->header = palloc(sizeof(TpSegmentHeader)); if (raw_version <= TP_SEGMENT_FORMAT_VERSION_3) { /* V3: read legacy struct and widen to V4 */ TpSegmentHeaderV3 v3; memcpy(&v3, PageGetContents(header_page), sizeof(TpSegmentHeaderV3)); header = reader->header; header->magic = v3.magic; header->version = v3.version; header->created_at = v3.created_at; header->num_pages = v3.num_pages; header->data_size = (uint64)v3.data_size; header->level = v3.level; header->next_segment = v3.next_segment; header->dictionary_offset = (uint64)v3.dictionary_offset; header->strings_offset = (uint64)v3.strings_offset; header->entries_offset = (uint64)v3.entries_offset; header->postings_offset = (uint64)v3.postings_offset; header->skip_index_offset = (uint64)v3.skip_index_offset; header->fieldnorm_offset = (uint64)v3.fieldnorm_offset; header->ctid_pages_offset = (uint64)v3.ctid_pages_offset; header->ctid_offsets_offset = (uint64)v3.ctid_offsets_offset; header->num_terms = v3.num_terms; header->num_docs = v3.num_docs; header->total_tokens = v3.total_tokens; header->page_index = v3.page_index; /* V3 has no alive bitset */ header->alive_bitset_offset = 0; header->alive_count = header->num_docs; } else if (raw_version <= TP_SEGMENT_FORMAT_VERSION_4) { /* V4: read legacy struct and widen to V5 */ TpSegmentHeaderV4 v4; memcpy(&v4, PageGetContents(header_page), sizeof(TpSegmentHeaderV4)); header = reader->header; header->magic = v4.magic; header->version = v4.version; header->created_at = v4.created_at; header->num_pages = v4.num_pages; header->data_size = v4.data_size; header->level = v4.level; header->next_segment = v4.next_segment; header->dictionary_offset = v4.dictionary_offset; header->strings_offset = v4.strings_offset; header->entries_offset = v4.entries_offset; header->postings_offset = v4.postings_offset; header->skip_index_offset = v4.skip_index_offset; header->fieldnorm_offset = v4.fieldnorm_offset; header->ctid_pages_offset = v4.ctid_pages_offset; header->ctid_offsets_offset = v4.ctid_offsets_offset; header->num_terms = v4.num_terms; header->num_docs = v4.num_docs; header->total_tokens = v4.total_tokens; header->page_index = v4.page_index; /* V4 has no alive bitset */ header->alive_bitset_offset = 0; header->alive_count = header->num_docs; } else if (raw_version <= TP_SEGMENT_FORMAT_VERSION) { /* V5+: full header */ memcpy(reader->header, PageGetContents(header_page), sizeof(TpSegmentHeader)); header = reader->header; } else { UnlockReleaseBuffer(header_buf); pfree(reader->header); pfree(reader); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("unsupported segment format version %u " "at block %u", raw_version, root_block))); } } reader->num_pages = header->num_pages; reader->nblocks = nblocks; /* Get page index location from header */ page_index_block = header->page_index; /* Keep header buffer for later use */ reader->header_buffer = header_buf; LockBuffer( header_buf, BUFFER_LOCK_UNLOCK); /* Just unlock, don't release */ if (header_only) return reader; /* Always load page map from disk - no caching due to concurrency issues */ reader->page_map = palloc(sizeof(BlockNumber) * reader->num_pages); /* Read page index chain to build page map */ while (page_index_block != InvalidBlockNumber && pages_loaded < reader->num_pages) { index_buf = ReadBuffer(index, page_index_block); LockBuffer(index_buf, BUFFER_LOCK_SHARE); index_page = BufferGetPage(index_buf); /* Get special area with page index metadata */ special = (TpPageIndexSpecial *)PageGetSpecialPointer(index_page); /* Validate magic number and page type */ if (special->magic != TP_PAGE_INDEX_MAGIC || special->page_type != TP_PAGE_FILE_INDEX) { UnlockReleaseBuffer(index_buf); ReleaseBuffer(reader->header_buffer); pfree(reader->page_map); pfree(reader->header); pfree(reader); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid page index at block %u", page_index_block), errdetail( "magic=0x%08X (expected 0x%08X), " "page_type=%u (expected %u)", special->magic, TP_PAGE_INDEX_MAGIC, special->page_type, TP_PAGE_FILE_INDEX))); } /* Get pointer to page entries array */ page_entries = (BlockNumber *)((char *)index_page + SizeOfPageHeaderData); /* Copy page entries to our map with validation */ for (i = 0; i < special->num_entries && pages_loaded < reader->num_pages; i++) { BlockNumber page_block = page_entries[i]; /* Validate block number is within relation bounds */ if (page_block >= nblocks) { UnlockReleaseBuffer(index_buf); ReleaseBuffer(reader->header_buffer); pfree(reader->page_map); pfree(reader->header); pfree(reader); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid page block in segment page_map"), errdetail( "block %u at entry %u >= nblocks %u", page_block, pages_loaded, nblocks))); } reader->page_map[pages_loaded++] = page_block; } /* Move to next page in chain */ page_index_block = special->next_page; UnlockReleaseBuffer(index_buf); } if (pages_loaded != reader->num_pages) { /* Free allocated memory before erroring out */ if (reader->page_map) pfree(reader->page_map); pfree(reader); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("segment page index is incomplete"), errdetail( "Expected %u pages but only loaded %u pages", reader->num_pages, pages_loaded), errhint("The index may be corrupted and should be rebuilt"))); } /* * Optionally preload CTID arrays into memory for result lookup. * When load_ctids is false, callers should use tp_segment_lookup_ctid * for deferred resolution of individual CTIDs. */ reader->cached_ctid_pages = NULL; reader->cached_ctid_offsets = NULL; reader->cached_num_docs = 0; if (load_ctids && header->num_docs > 0 && header->ctid_pages_offset > 0) { reader->cached_num_docs = header->num_docs; /* Load CTID pages array (4 bytes per doc) */ reader->cached_ctid_pages = palloc( header->num_docs * sizeof(BlockNumber)); tp_segment_read( reader, header->ctid_pages_offset, reader->cached_ctid_pages, header->num_docs * sizeof(BlockNumber)); /* Load CTID offsets array (2 bytes per doc) */ reader->cached_ctid_offsets = palloc( header->num_docs * sizeof(OffsetNumber)); tp_segment_read( reader, header->ctid_offsets_offset, reader->cached_ctid_offsets, header->num_docs * sizeof(OffsetNumber)); } return reader; } /* Open only the version-normalized header, without loading the page map. */ TpSegmentReader * tp_segment_open_header(Relation index, BlockNumber root_block) { return tp_segment_open_internal(index, root_block, false, true); } TpSegmentReader * tp_segment_open_ex(Relation index, BlockNumber root_block, bool load_ctids) { return tp_segment_open_internal(index, root_block, load_ctids, false); } /* * Open segment for reading (default: skip CTID preloading). * This is the standard entry point for query execution. */ TpSegmentReader * tp_segment_open(Relation index, BlockNumber root_block) { return tp_segment_open_ex(index, root_block, false); } /* * Open segment from a BufFile temp file. * Used by parallel build leader to read worker segments. * * The BufFile contains a flat byte stream (no page boundaries). * The segment header starts at base_offset in the file. * Caller manages BufFile lifecycle (do not close in reader). */ TpSegmentReader * tp_segment_open_from_buffile(BufFile *file, uint64 base_offset) { TpSegmentReader *reader; TpSegmentHeader *header; reader = palloc0(sizeof(TpSegmentReader)); reader->index = NULL; reader->root_block = InvalidBlockNumber; reader->current_buffer = InvalidBuffer; reader->current_logical_page = UINT32_MAX; reader->ctid_pages_buffer = InvalidBuffer; reader->ctid_pages_logical_page = UINT32_MAX; reader->ctid_offsets_buffer = InvalidBuffer; reader->ctid_offsets_logical_page = UINT32_MAX; reader->header_buffer = InvalidBuffer; reader->page_map = NULL; reader->num_pages = 0; reader->nblocks = 0; /* Set BufFile fields */ reader->buffile = file; reader->buffile_base = base_offset; /* Read header from BufFile */ header = palloc(sizeof(TpSegmentHeader)); { int fileno; off_t offset; tp_buffile_decompose_offset(base_offset, &fileno, &offset); BufFileSeek(file, fileno, offset, SEEK_SET); } BufFileReadExact(file, header, sizeof(TpSegmentHeader)); if (header->magic != TP_SEGMENT_MAGIC) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid segment header in temp file " "at offset %" PRIu64, base_offset), errdetail( "magic=0x%08X, expected 0x%08X", header->magic, TP_SEGMENT_MAGIC))); reader->header = header; reader->segment_version = header->version; /* No CTID caches for temp file segments */ reader->cached_ctid_pages = NULL; reader->cached_ctid_offsets = NULL; reader->cached_num_docs = 0; return reader; } void tp_segment_enable_ctid_lookup_cache(TpSegmentReader *reader) { uint32 capacity; Assert(reader != NULL); if (reader->cached_ctid_pages != NULL || reader->lookup_ctid_pages != NULL || reader->header->num_docs == 0) return; /* Keep both arrays within roughly one PostgreSQL block. */ capacity = Min(reader->header->num_docs, (uint32)(BLCKSZ / (sizeof(BlockNumber) + sizeof(OffsetNumber)))); reader->lookup_ctid_pages = palloc(capacity * sizeof(BlockNumber)); reader->lookup_ctid_offsets = palloc(capacity * sizeof(OffsetNumber)); reader->lookup_ctid_capacity = capacity; } static void tp_segment_read_cached_page( TpSegmentReader *reader, uint64 logical_offset, void *dest, uint32 len, Buffer *cached_buffer, uint32 *cached_logical_page) { uint32 logical_page = (uint32)(logical_offset / SEGMENT_DATA_PER_PAGE); uint32 page_offset = (uint32)(logical_offset % SEGMENT_DATA_PER_PAGE); Page page; if (reader->buffile != NULL || page_offset + len > SEGMENT_DATA_PER_PAGE) { tp_segment_read(reader, logical_offset, dest, len); return; } if (*cached_logical_page != logical_page) { BlockNumber physical; if (BufferIsValid(*cached_buffer)) { ReleaseBuffer(*cached_buffer); *cached_buffer = InvalidBuffer; } if (logical_page >= reader->num_pages) elog(ERROR, "Invalid logical page %u (max %u), logical_offset=%" PRIu64, logical_page, reader->num_pages > 0 ? reader->num_pages - 1 : 0, logical_offset); physical = reader->page_map[logical_page]; if (physical >= reader->nblocks) elog(ERROR, "Invalid physical block %u for logical page %u (nblocks=%u)", physical, logical_page, reader->nblocks); *cached_buffer = ReadBuffer(reader->index, physical); *cached_logical_page = logical_page; } LockBuffer(*cached_buffer, BUFFER_LOCK_SHARE); page = BufferGetPage(*cached_buffer); memcpy(dest, (char *)page + SizeOfPageHeaderData + page_offset, len); LockBuffer(*cached_buffer, BUFFER_LOCK_UNLOCK); } /* * Look up a single CTID by doc_id. * Used for deferred CTID resolution when CTIDs weren't preloaded. */ void tp_segment_lookup_ctid( TpSegmentReader *reader, uint32 doc_id, ItemPointerData *ctid_out) { BlockNumber page; OffsetNumber offset; Assert(reader != NULL); Assert(ctid_out != NULL); if (doc_id >= reader->header->num_docs) { ItemPointerSetInvalid(ctid_out); return; } /* If CTIDs were preloaded, use the cache */ if (reader->cached_ctid_pages != NULL) { ItemPointerSet( ctid_out, reader->cached_ctid_pages[doc_id], reader->cached_ctid_offsets[doc_id]); return; } if (reader->lookup_ctid_pages != NULL) { if (reader->lookup_ctid_count == 0 || doc_id < reader->lookup_ctid_start || doc_id >= reader->lookup_ctid_start + reader->lookup_ctid_count) { reader->lookup_ctid_start = doc_id; reader->lookup_ctid_count = Min(reader->lookup_ctid_capacity, reader->header->num_docs - doc_id); tp_segment_read( reader, reader->header->ctid_pages_offset + (uint64)doc_id * sizeof(BlockNumber), reader->lookup_ctid_pages, reader->lookup_ctid_count * sizeof(BlockNumber)); tp_segment_read( reader, reader->header->ctid_offsets_offset + (uint64)doc_id * sizeof(OffsetNumber), reader->lookup_ctid_offsets, reader->lookup_ctid_count * sizeof(OffsetNumber)); } doc_id -= reader->lookup_ctid_start; ItemPointerSet( ctid_out, reader->lookup_ctid_pages[doc_id], reader->lookup_ctid_offsets[doc_id]); return; } /* Read page number (4 bytes) from ctid_pages array */ tp_segment_read_cached_page( reader, reader->header->ctid_pages_offset + doc_id * sizeof(BlockNumber), &page, sizeof(BlockNumber), &reader->ctid_pages_buffer, &reader->ctid_pages_logical_page); /* Read offset (2 bytes) from ctid_offsets array */ tp_segment_read_cached_page( reader, reader->header->ctid_offsets_offset + doc_id * sizeof(OffsetNumber), &offset, sizeof(OffsetNumber), &reader->ctid_offsets_buffer, &reader->ctid_offsets_logical_page); ItemPointerSet(ctid_out, page, offset); } void tp_segment_close(TpSegmentReader *reader) { if (!reader) return; /* * BufFile-backed readers don't use buffer manager. * Do NOT close the BufFile — caller manages its lifecycle. */ if (reader->buffile == NULL) { if (BufferIsValid(reader->current_buffer)) ReleaseBuffer(reader->current_buffer); if (BufferIsValid(reader->ctid_pages_buffer)) ReleaseBuffer(reader->ctid_pages_buffer); if (BufferIsValid(reader->ctid_offsets_buffer)) ReleaseBuffer(reader->ctid_offsets_buffer); if (BufferIsValid(reader->header_buffer)) ReleaseBuffer(reader->header_buffer); if (reader->page_map) pfree(reader->page_map); } if (reader->header) pfree(reader->header); /* Free CTID caches (fieldnorm is inline in postings, no cache needed) */ if (reader->cached_ctid_pages) pfree(reader->cached_ctid_pages); if (reader->cached_ctid_offsets) pfree(reader->cached_ctid_offsets); if (reader->lookup_ctid_pages) pfree(reader->lookup_ctid_pages); if (reader->lookup_ctid_offsets) pfree(reader->lookup_ctid_offsets); pfree(reader); } void tp_segment_read( TpSegmentReader *reader, uint64 logical_offset, void *dest, uint64 len) { char *dest_ptr = (char *)dest; uint64 bytes_read = 0; /* * BufFile fast path: flat byte stream, no page boundaries. * logical_offset maps directly to file position. */ if (reader->buffile != NULL) { int fileno; off_t offset; tp_buffile_decompose_offset( reader->buffile_base + logical_offset, &fileno, &offset); BufFileSeek(reader->buffile, fileno, offset, SEEK_SET); BufFileReadExact(reader->buffile, dest, len); return; } while (bytes_read < len) { uint32 logical_page = (uint32)(logical_offset / SEGMENT_DATA_PER_PAGE); uint32 page_offset = (uint32)(logical_offset % SEGMENT_DATA_PER_PAGE); uint32 to_read; Buffer buf; Page page; char *src; /* Calculate how much to read from this page */ to_read = (uint32) Min(len - bytes_read, (uint64)(SEGMENT_DATA_PER_PAGE - page_offset)); /* Check if we have the page in cache */ if (reader->current_logical_page != logical_page) { /* Release old buffer if any */ if (BufferIsValid(reader->current_buffer)) { ReleaseBuffer(reader->current_buffer); reader->current_buffer = InvalidBuffer; } /* Validate page number */ if (logical_page >= reader->num_pages) { elog(ERROR, "Invalid logical page %u (max %u), " "logical_offset=%" PRIu64 ", " "BLCKSZ=%d, reader->num_pages=%u", logical_page, reader->num_pages > 0 ? reader->num_pages - 1 : 0, logical_offset, BLCKSZ, reader->num_pages); } /* Validate physical block number */ { BlockNumber physical = reader->page_map[logical_page]; if (physical >= reader->nblocks) { elog(ERROR, "Invalid physical block %u for logical page %u " "(nblocks=%u)", physical, logical_page, reader->nblocks); } } /* Read the physical page */ buf = ReadBuffer(reader->index, reader->page_map[logical_page]); reader->current_buffer = buf; reader->current_logical_page = logical_page; } else { buf = reader->current_buffer; } /* Lock buffer for reading */ LockBuffer(buf, BUFFER_LOCK_SHARE); /* Segment payload starts after the page header. */ page = BufferGetPage(buf); src = (char *)page + SizeOfPageHeaderData + page_offset; memcpy(dest_ptr + bytes_read, src, to_read); /* Unlock but keep buffer pinned for potential reuse */ LockBuffer(buf, BUFFER_LOCK_UNLOCK); /* Advance pointers */ bytes_read += to_read; logical_offset += to_read; } } /* * Get direct access to data in a segment page (zero-copy) * Returns true if successful, false if data spans multiple pages * * This function creates its OWN buffer pin, independent of * reader->current_buffer. This is necessary because multiple iterators may * have active direct accesses while other code calls tp_segment_read() which * modifies reader->current_buffer. * * The buffer is locked with BUFFER_LOCK_SHARE and must be released by calling * tp_segment_release_direct(), which will both unlock AND release the pin. */ bool tp_segment_get_direct( TpSegmentReader *reader, uint64 logical_offset, uint32 len, TpSegmentDirectAccess *access) { uint32 logical_page = (uint32)(logical_offset / SEGMENT_DATA_PER_PAGE); uint32 page_offset = (uint32)(logical_offset % SEGMENT_DATA_PER_PAGE); Buffer buf; Page page; BlockNumber physical_block; /* Initialize access structure to invalid state */ access->buffer = InvalidBuffer; access->page = NULL; access->data = NULL; access->available = 0; /* Check if data spans pages - if so, can't do zero-copy */ if (page_offset + len > SEGMENT_DATA_PER_PAGE) return false; /* Validate logical page */ if (logical_page >= reader->num_pages) { elog(ERROR, "Invalid logical page %u (segment has %u pages)", logical_page, reader->num_pages); } /* Get physical block from page map */ physical_block = reader->page_map[logical_page]; /* * Always create a fresh buffer pin for direct access. * This ensures the buffer remains valid even if tp_segment_read() * is called and releases reader->current_buffer. */ buf = ReadBuffer(reader->index, physical_block); /* Lock buffer for reading */ LockBuffer(buf, BUFFER_LOCK_SHARE); /* Get page and data pointer */ page = BufferGetPage(buf); /* * Fill in access structure with our own buffer pin. * The caller MUST call tp_segment_release_direct() to release this pin. */ access->buffer = buf; access->page = page; access->data = (char *)page + SizeOfPageHeaderData + page_offset; access->available = SEGMENT_DATA_PER_PAGE - page_offset; return true; } /* * Release direct access to segment page * * Since tp_segment_get_direct() creates its own buffer pin, we must * both unlock AND release the buffer here. */ void tp_segment_release_direct(TpSegmentDirectAccess *access) { if (BufferIsValid(access->buffer)) { LockBuffer(access->buffer, BUFFER_LOCK_UNLOCK); ReleaseBuffer(access->buffer); access->buffer = InvalidBuffer; access->page = NULL; access->data = NULL; } } /* * Allocate a single page for segment. * Checks FSM for recycled pages first, then extends the relation if needed. * * The returned page is intentionally left uninitialized and unlogged. The * first writer to populate the page (tp_segment_writer_flush or * write_page_index_internal) is responsible for PageInit, laying out the * segment-specific header fields (e.g. pd_lower = BLCKSZ for content pages), * marking the buffer dirty, and emitting a WAL record for the new contents. * * Doing the page setup at content-write time — rather than here — avoids a * redundant alloc-time WAL image for every newly extended or recycled page. * Pages that are reached only by code that never writes them (e.g. an error * path that bails out before flushing) remain unmodified on disk; that is * safe because we never reference such pages from any persisted segment * structure. */ static BlockNumber allocate_segment_page(Relation index) { /* * Reuse a recyclable free page from the FSM (recycled from a * compaction/vacuum drain), extending the relation only when the * FSM offers none. tp_fsm_claim_or_extend_block skips any block * the non-crash-safe FSM offers that is still a live structure * page, so we never reinitialize a page another structure still * owns (issues #426, #427). */ return tp_fsm_claim_or_extend_block(index); } /* * Grow writer's page array if needed */ static void tp_segment_writer_grow_pages(TpSegmentWriter *writer) { if (writer->pages_allocated >= writer->pages_capacity) { uint32 new_capacity = writer->pages_capacity == 0 ? 8 : writer->pages_capacity * 2; if (writer->pages) { writer->pages = repalloc( writer->pages, new_capacity * sizeof(BlockNumber)); } else { writer->pages = palloc(new_capacity * sizeof(BlockNumber)); } writer->pages_capacity = new_capacity; } } /* * Allocate a new page for the writer via FSM/extend. */ static BlockNumber tp_segment_writer_allocate_page(TpSegmentWriter *writer) { BlockNumber new_page; tp_segment_writer_grow_pages(writer); new_page = allocate_segment_page(writer->index); writer->pages[writer->pages_allocated++] = new_page; tp_compaction_allocation_injection_point( TP_COMPACTION_ALLOCATION_OUTPUT_DATA); return new_page; } /* * Write page index (chain of BlockNumbers). * This function is also used by segment_merge.c for merged segments. */ static BlockNumber write_page_index_internal( Relation index, BlockNumber *pages, uint32 num_pages, BlockNumber **owned_pages, uint32 *owned_count) { BlockNumber index_root = InvalidBlockNumber; BlockNumber prev_block = InvalidBlockNumber; /* * Calculate how many index pages we need. * IMPORTANT: PageInit() aligns the special area to MAXALIGN, so we must * account for that when calculating available space. Using raw sizeof() * would give us 1 extra entry that overlaps the special area! */ uint32 entries_per_page = (BLCKSZ - SizeOfPageHeaderData - MAXALIGN(sizeof(TpPageIndexSpecial))) / sizeof(BlockNumber); uint32 num_index_pages = (num_pages + entries_per_page - 1) / entries_per_page; /* Allocate index pages incrementally */ BlockNumber *index_pages; uint32 i; Assert(owned_pages != NULL); Assert(owned_count != NULL); *owned_pages = NULL; *owned_count = 0; index_pages = palloc(num_index_pages * sizeof(BlockNumber)); *owned_pages = index_pages; for (i = 0; i < num_index_pages; i++) { index_pages[i] = allocate_segment_page(index); (*owned_count)++; tp_compaction_allocation_injection_point( TP_COMPACTION_ALLOCATION_PAGE_INDEX); } /* * Write index pages in reverse order (so we can chain them). * Each page i stores entries [i*entries_per_page, (i+1)*entries_per_page). * We iterate in reverse so we can set next_page pointers correctly. */ for (int i = num_index_pages - 1; i >= 0; i--) { Buffer buffer; Page page; BlockNumber *page_data; TpPageIndexSpecial *special; uint32 start_idx; uint32 entries_to_write; uint32 j; /* Calculate which entries this page should contain */ start_idx = i * entries_per_page; entries_to_write = Min(entries_per_page, num_pages - start_idx); buffer = ReadBuffer(index, index_pages[i]); LockBuffer(buffer, BUFFER_LOCK_EXCLUSIVE); page = BufferGetPage(buffer); /* Initialize page with special area */ PageInit(page, BLCKSZ, sizeof(TpPageIndexSpecial)); /* Set up special area */ special = (TpPageIndexSpecial *)PageGetSpecialPointer(page); special->magic = TP_PAGE_INDEX_MAGIC; special->version = TP_PAGE_INDEX_VERSION; special->page_type = TP_PAGE_FILE_INDEX; special->next_page = prev_block; special->num_entries = entries_to_write; special->flags = 0; /* Use the data area after the page header */ page_data = (BlockNumber *)((char *)page + SizeOfPageHeaderData); /* Fill with page numbers from pages[start_idx..start_idx+entries-1] */ for (j = 0; j < entries_to_write; j++) page_data[j] = pages[start_idx + j]; tp_segment_log_dirty_buffer(index, buffer); UnlockReleaseBuffer(buffer); prev_block = index_pages[i]; if (i == 0) index_root = index_pages[i]; } return index_root; } BlockNumber write_page_index(Relation index, BlockNumber *pages, uint32 num_pages) { volatile BlockNumber root = InvalidBlockNumber; BlockNumber *volatile owned_pages = NULL; volatile uint32 owned_count = 0; PG_TRY(); { root = write_page_index_internal( index, pages, num_pages, (BlockNumber **)&owned_pages, (uint32 *)&owned_count); } PG_CATCH(); { if (owned_count > 0) { tp_segment_free_pages(index, owned_pages, owned_count); IndexFreeSpaceMapVacuum(index); } if (owned_pages != NULL) pfree(owned_pages); PG_RE_THROW(); } PG_END_TRY(); if (owned_pages != NULL) pfree(owned_pages); return (BlockNumber)root; } BlockNumber write_page_index_tracked( Relation index, BlockNumber *pages, uint32 num_pages, BlockNumber **owned_pages, uint32 *owned_count) { return write_page_index_internal( index, pages, num_pages, owned_pages, owned_count); } /* * Segment Writer - Block-based storage with skip index * * Posting lists are organized into fixed-size blocks of 128 docs, * with a skip index that enables efficient skipping during query execution. * * Layout: * Header -> Dictionary -> Strings -> DictEntries -> * SkipIndex -> PostingBlocks -> Fieldnorms -> CTIDMap */ /* * Comparison function for sorting TpBlockPosting by doc_id ascending. * Used by tp_write_segment to sort the per-term posting array before * splitting into TP_BLOCK_SIZE blocks -- the memtable doesn't maintain * sort order under concurrent inserts (see comment at the qsort call). */ static int block_posting_cmp_by_doc_id(const void *a, const void *b) { uint32 da = ((const TpBlockPosting *)a)->doc_id; uint32 db = ((const TpBlockPosting *)b)->doc_id; if (da < db) return -1; if (da > db) return 1; return 0; } /* * Per-term block information for streaming format. * Updated for streaming layout: postings written before skip index. */ typedef struct TermBlockInfo { uint64 posting_offset; /* Absolute offset where postings were written */ uint32 block_count; /* Number of blocks for this term */ uint32 doc_freq; /* Document frequency */ uint32 skip_entry_start; /* Index into accumulated skip entries array */ } TermBlockInfo; static void validate_terms(TermInfo *terms, uint32 num_terms) { uint64 string_pos = 0; uint64 skip_entries = 0; uint32 i; if (!tp_dictionary_offsets_fit(num_terms)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment dictionary exceeds %u terms", TP_MAX_DICTIONARY_TERMS))); for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); uint64 blocks = tp_posting_block_count(terms[i].count); if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment string pool exceeds the " "4 GiB format limit"))); string_pos += entry_size; if (blocks > TP_MAX_GROWABLE_CAPACITY - skip_entries) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment exceeds %u posting blocks", TP_MAX_GROWABLE_CAPACITY))); skip_entries += blocks; } } /* * Write segment from a pre-built dictionary + docmap, e.g. produced * by walking the on-disk memtable chain via the chain source. * * Phase 4: this function no longer reads DSA memtable structures. * The caller is responsible for: * - Building `terms[]` (sorted lexicographically; each entry owns * its `term` plus parallel `ctids[]` / `freqs[]` arrays whose * length is `count`, and a `doc_freq` count of distinct docs). * - Building `docmap` and calling `tp_docmap_finalize` on it. * - Freeing both after the call (via `tp_free_dictionary` and * `tp_docmap_destroy`). The arrays inside TermInfo are presumed * to live in a memory context the caller drops as a whole. * * Layout: [header] -> [dictionary] -> [postings] -> [skip index] -> * [fieldnorm] -> [ctid map] * * This matches the merge format: postings written before skip index. */ BlockNumber tp_write_segment( Relation index, TermInfo *terms, uint32 num_terms, TpDocMapBuilder *docmap) { BlockNumber header_block; BlockNumber page_index_root; TpSegmentWriter writer; TpSegmentHeader header; TpDictionary dict; uint32 *string_offsets; uint64 string_pos; uint32 i; Buffer header_buf; Page header_page; TpSegmentHeader *existing_header; TermBlockInfo *term_blocks; /* Accumulated skip entries for all terms */ TpSkipEntry *all_skip_entries; uint32 skip_entries_count; uint32 skip_entries_capacity; /* Initialize the writer to avoid garbage values */ memset(&writer, 0, sizeof(TpSegmentWriter)); validate_terms(terms, num_terms); /* Initialize writer with incremental page allocation */ tp_segment_writer_init(&writer, index); if (writer.pages_allocated > 0) { header_block = writer.pages[0]; } else { elog(ERROR, "tp_write_segment: Failed to allocate first page"); } /* Initialize header */ memset(&header, 0, sizeof(TpSegmentHeader)); header.magic = TP_SEGMENT_MAGIC; header.version = TP_SEGMENT_FORMAT_VERSION; header.created_at = GetCurrentTimestamp(); header.num_pages = 0; header.num_terms = num_terms; header.level = 0; header.next_segment = InvalidBlockNumber; /* Dictionary immediately follows header */ header.dictionary_offset = sizeof(TpSegmentHeader); /* * Placeholder per-segment counts; header.num_docs and * header.total_tokens are overwritten below with the docmap's * per-segment values (Σ raw doc_length) before the final on-disk * patch. Build-context writes use the same raw sum; merge uses * source headers plus a dead-only fieldnorm correction so all * three paths produce raw-equivalent totals for the common case * of all-live sources. */ header.num_docs = 0; header.total_tokens = 0; /* Write placeholder header */ tp_segment_writer_write(&writer, &header, sizeof(TpSegmentHeader)); /* Write dictionary section */ dict.num_terms = num_terms; tp_segment_writer_write( &writer, &dict, offsetof(TpDictionary, string_offsets)); /* Build string offsets */ string_offsets = palloc_extended( num_terms * sizeof(uint32), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); string_pos = 0; for (i = 0; i < num_terms; i++) { uint64 entry_size = tp_string_pool_entry_size(terms[i].term_len); if (tp_string_pool_offset_overflows(string_pos, entry_size)) ereport(ERROR, (errcode(ERRCODE_PROGRAM_LIMIT_EXCEEDED), errmsg("pg_textsearch: segment string pool exceeds the " "4 GiB format limit"), errhint("The corpus vocabulary is too large for a " "single segment."))); string_offsets[i] = (uint32)string_pos; string_pos += entry_size; } /* Write string offsets array */ tp_segment_writer_write( &writer, string_offsets, num_terms * sizeof(uint32)); /* Write string pool */ header.strings_offset = writer.current_offset; for (i = 0; i < num_terms; i++) { uint32 length = terms[i].term_len; uint32 dict_offset = (uint32)((uint64)i * sizeof(TpDictEntry)); tp_segment_writer_write(&writer, &length, sizeof(uint32)); tp_segment_writer_write(&writer, terms[i].term, length); tp_segment_writer_write(&writer, &dict_offset, sizeof(uint32)); } /* Record entries offset - dict entries written after postings loop */ header.entries_offset = writer.current_offset; /* Write placeholder dict entries - we'll fill them in after streaming */ { TpDictEntry placeholder; memset(&placeholder, 0, sizeof(TpDictEntry)); for (i = 0; i < num_terms; i++) tp_segment_writer_write( &writer, &placeholder, sizeof(TpDictEntry)); } /* Postings start here - streaming format writes postings first */ header.postings_offset = writer.current_offset; /* Initialize per-term tracking and skip entry accumulator */ term_blocks = palloc_extended( num_terms * sizeof(TermBlockInfo), MCXT_ALLOC_HUGE | MCXT_ALLOC_ZERO); skip_entries_capacity = 1024; skip_entries_count = 0; all_skip_entries = palloc(skip_entries_capacity * sizeof(TpSkipEntry)); /* * Streaming pass: for each term, convert postings and write immediately. */ for (i = 0; i < num_terms; i++) { uint32 doc_count = terms[i].count; uint32 block_idx; uint32 num_blocks; TpBlockPosting *block_postings = NULL; /* Record where this term's postings start */ term_blocks[i].posting_offset = writer.current_offset; term_blocks[i].skip_entry_start = skip_entries_count; term_blocks[i].doc_freq = terms[i].doc_freq; if (doc_count == 0) { term_blocks[i].block_count = 0; continue; } num_blocks = (uint32)tp_posting_block_count(doc_count); term_blocks[i].block_count = num_blocks; /* Convert postings to block format */ block_postings = palloc(doc_count * sizeof(TpBlockPosting)); { uint32 j; for (j = 0; j < doc_count; j++) { uint32 doc_id = tp_docmap_lookup(docmap, &terms[i].ctids[j]); uint8 norm; if (doc_id == UINT32_MAX) elog(ERROR, "CTID (%u,%u) not found in docmap", ItemPointerGetBlockNumber(&terms[i].ctids[j]), ItemPointerGetOffsetNumber(&terms[i].ctids[j])); norm = tp_docmap_get_fieldnorm(docmap, doc_id); block_postings[j].doc_id = doc_id; block_postings[j].frequency = (uint16)terms[i].freqs[j]; block_postings[j].fieldnorm = norm; block_postings[j].reserved = 0; } } /* * Sort block_postings by doc_id. * * The memtable's posting list entries are appended in * insertion order. Under single-writer (CREATE INDEX) this * happens to match CTID order = doc_id order. But under * concurrent inserts, multiple backends interleave appends * to the same posting list in arbitrary thread-scheduling * order -- entries[] is NOT CTID-sorted, and so the * block_postings[] we just built is NOT doc_id-sorted * either. (posting.c sets posting_list->is_sorted = false * on every insert, but nothing was sorting on the read * side before this fix.) * * Writing unsorted block_postings produces segments that * silently violate the sorted-block invariant the entire * BMW machinery (binary search on block_last_doc_ids, * seek_term_to_doc, find_wand_pivot's smallest-first walk) * relies on. Symptoms: MS MARCO bucket-8 hangs in * seek_to_pivot / score_segment_multi_term_bmw (PR #360), * because every "advance to >= target" returns a doc whose * actual position in the block may be followed by smaller * doc_ids that should have been returned first. * * Sort here -- doc_id is monotonic with CTID after * tp_docmap_finalize, so this is equivalent to sorting by * CTID, which is the invariant the segment format * documents and the merge / scoring code assumes. */ qsort(block_postings, doc_count, sizeof(TpBlockPosting), block_posting_cmp_by_doc_id); /* Write posting blocks and build skip entries */ for (block_idx = 0; block_idx < num_blocks; block_idx++) { TpSkipEntry skip; uint32 block_start = block_idx * TP_BLOCK_SIZE; uint32 block_end = Min(block_start + TP_BLOCK_SIZE, doc_count); uint32 j; uint16 max_tf = 0; uint8 min_norm = 255; uint32 last_doc_id = 0; /* Calculate block stats */ for (j = block_start; j < block_end; j++) { if (block_postings[j].doc_id > last_doc_id) last_doc_id = block_postings[j].doc_id; if (block_postings[j].frequency > max_tf) max_tf = block_postings[j].frequency; if (block_postings[j].fieldnorm < min_norm) min_norm = block_postings[j].fieldnorm; } /* Build skip entry with actual posting offset */ skip.last_doc_id = last_doc_id; skip.doc_count = (uint8)(block_end - block_start); skip.block_max_tf = max_tf; skip.block_max_norm = min_norm; skip.posting_offset = writer.current_offset; memset(skip.reserved, 0, sizeof(skip.reserved)); /* Write posting block data (compressed or uncompressed) */ if (tp_compress_segments) { uint8 compressed_buf[TP_MAX_COMPRESSED_BLOCK_SIZE]; uint32 compressed_size; compressed_size = tp_compress_block( &block_postings[block_start], block_end - block_start, compressed_buf); skip.flags = TP_BLOCK_FLAG_DELTA; tp_segment_writer_write( &writer, compressed_buf, compressed_size); } else { skip.flags = TP_BLOCK_FLAG_UNCOMPRESSED; tp_segment_writer_write( &writer, &block_postings[block_start], (block_end - block_start) * sizeof(TpBlockPosting)); } /* Accumulate skip entry */ if (skip_entries_count >= skip_entries_capacity) { skip_entries_capacity = tp_grow_capacity( skip_entries_capacity, 1024, "posting blocks"); all_skip_entries = repalloc_huge( all_skip_entries, mul_size( (Size)skip_entries_capacity, sizeof(TpSkipEntry))); } all_skip_entries[skip_entries_count++] = skip; } pfree(block_postings); } /* Skip index starts here - after all postings */ header.skip_index_offset = writer.current_offset; /* Write all accumulated skip entries */ if (skip_entries_count > 0) { tp_segment_writer_write( &writer, all_skip_entries, skip_entries_count * sizeof(TpSkipEntry)); } pfree(all_skip_entries); /* Write fieldnorm table */ header.fieldnorm_offset = writer.current_offset; if (docmap->num_docs > 0) { tp_segment_writer_write( &writer, docmap->fieldnorms, docmap->num_docs * sizeof(uint8)); } /* Write CTID pages array */ header.ctid_pages_offset = writer.current_offset; if (docmap->num_docs > 0) { tp_segment_writer_write( &writer, docmap->ctid_pages, docmap->num_docs * sizeof(BlockNumber)); } /* Write CTID offsets array */ header.ctid_offsets_offset = writer.current_offset; if (docmap->num_docs > 0) { tp_segment_writer_write( &writer, docmap->ctid_offsets, docmap->num_docs * sizeof(OffsetNumber)); } /* Write alive bitset (all alive) */ header.alive_bitset_offset = writer.current_offset; header.alive_count = docmap->num_docs; if (docmap->num_docs > 0) { uint32 bitset_size = tp_alive_bitset_size(docmap->num_docs); uint8 *bitset_data = palloc(bitset_size); tp_alive_bitset_init_data(bitset_data, docmap->num_docs); tp_segment_writer_write(&writer, bitset_data, bitset_size); pfree(bitset_data); } /* Update num_docs and total_tokens to per-segment values */ header.num_docs = docmap->num_docs; header.total_tokens = docmap->total_tokens; /* Write page index */ tp_segment_writer_flush(&writer); /* * Mark buffer as empty to prevent tp_segment_writer_finish from flushing * again and overwriting our dict entry updates. */ writer.buffer_pos = SizeOfPageHeaderData; page_index_root = write_page_index(index, writer.pages, writer.pages_allocated); header.page_index = page_index_root; /* Update header with actual values */ header.data_size = writer.current_offset; header.num_pages = writer.pages_allocated; /* * Now write the dictionary entries with correct skip_index_offset values. * Do this BEFORE tp_segment_writer_finish so writer.pages is still valid. * * Each per-page batch of patches is logged as a GenericXLog delta record * against the page that the writer flush already wrote, so we don't pay * the cost of a full-page WAL image for what amounts to a small number * of 16-byte dict-entry updates per page. */ { Buffer dict_buf = InvalidBuffer; Page dict_page = NULL; GenericXLogState *xlog_state = NULL; uint32 entry_logical_page; uint32 current_page = UINT32_MAX; for (i = 0; i < num_terms; i++) { TpDictEntry entry; uint64 entry_offset; uint32 page_offset; BlockNumber physical_block; /* Build the entry */ entry.skip_index_offset = header.skip_index_offset + ((uint64)term_blocks[i].skip_entry_start * sizeof(TpSkipEntry)); entry.block_count = term_blocks[i].block_count; entry.doc_freq = term_blocks[i].doc_freq; /* Calculate where this entry is in the segment */ entry_offset = header.entries_offset + ((uint64)i * sizeof(TpDictEntry)); entry_logical_page = (uint32)(entry_offset / SEGMENT_DATA_PER_PAGE); page_offset = (uint32)(entry_offset % SEGMENT_DATA_PER_PAGE); /* Bounds check */ if (entry_logical_page >= writer.pages_allocated) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("dict entry %u logical page %u >= " "pages_allocated %u", i, entry_logical_page, writer.pages_allocated))); /* Read page if different from current */ if (entry_logical_page != current_page) { if (current_page != UINT32_MAX) { GenericXLogFinish(xlog_state); UnlockReleaseBuffer(dict_buf); } physical_block = writer.pages[entry_logical_page]; dict_buf = ReadBuffer(index, physical_block); LockBuffer(dict_buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(index); dict_page = GenericXLogRegisterBuffer(xlog_state, dict_buf, 0); current_page = entry_logical_page; } /* Write entry to working copy - handle page boundary spanning */ { uint32 bytes_on_this_page = SEGMENT_DATA_PER_PAGE - page_offset; if (bytes_on_this_page >= sizeof(TpDictEntry)) { /* Entry fits entirely on this page */ char *dest = (char *)dict_page + SizeOfPageHeaderData + page_offset; memcpy(dest, &entry, sizeof(TpDictEntry)); } else { /* Entry spans two pages */ char *dest = (char *)dict_page + SizeOfPageHeaderData + page_offset; char *src = (char *)&entry; /* Write first part to current working copy */ memcpy(dest, src, bytes_on_this_page); /* Commit current page's delta and move to next page */ GenericXLogFinish(xlog_state); UnlockReleaseBuffer(dict_buf); entry_logical_page++; if (entry_logical_page >= writer.pages_allocated) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("dict entry spans beyond allocated"))); physical_block = writer.pages[entry_logical_page]; dict_buf = ReadBuffer(index, physical_block); LockBuffer(dict_buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(index); dict_page = GenericXLogRegisterBuffer(xlog_state, dict_buf, 0); current_page = entry_logical_page; /* Write remaining part to next page's working copy */ dest = (char *)dict_page + SizeOfPageHeaderData; memcpy(dest, src + bytes_on_this_page, sizeof(TpDictEntry) - bytes_on_this_page); } } } /* Release last buffer */ if (current_page != UINT32_MAX) { GenericXLogFinish(xlog_state); UnlockReleaseBuffer(dict_buf); } } tp_segment_writer_finish(&writer); /* Flush to disk */ FlushRelationBuffers(index); /* * Update header on disk. The header page was already written and * WAL-logged in full during the writer flush above; this final patch * touches only the small TpSegmentHeader struct, so we use a * GenericXLog delta to avoid emitting another full-page image. */ { GenericXLogState *xlog_state; uint64 legacy_total_tokens; header_buf = ReadBuffer(index, header_block); LockBuffer(header_buf, BUFFER_LOCK_EXCLUSIVE); xlog_state = GenericXLogStart(index); header_page = GenericXLogRegisterBuffer(xlog_state, header_buf, 0); existing_header = (TpSegmentHeader *)PageGetContents(header_page); if (tp_injected_legacy_segment(&legacy_total_tokens)) { TpSegmentHeaderV4 legacy = { .magic = header.magic, .version = TP_SEGMENT_FORMAT_VERSION_4, .created_at = header.created_at, .num_pages = header.num_pages, .data_size = header.data_size, .level = header.level, .next_segment = header.next_segment, .dictionary_offset = header.dictionary_offset, .strings_offset = header.strings_offset, .entries_offset = header.entries_offset, .postings_offset = header.postings_offset, .skip_index_offset = header.skip_index_offset, .fieldnorm_offset = header.fieldnorm_offset, .ctid_pages_offset = header.ctid_pages_offset, .ctid_offsets_offset = header.ctid_offsets_offset, .num_terms = header.num_terms, .num_docs = header.num_docs, .total_tokens = legacy_total_tokens, .page_index = header.page_index, }; memcpy(existing_header, &legacy, sizeof(legacy)); } else { uint64 v5_total_tokens = tp_injected_v5_segment_total_len( header.total_tokens); existing_header->strings_offset = header.strings_offset; existing_header->entries_offset = header.entries_offset; existing_header->postings_offset = header.postings_offset; existing_header->skip_index_offset = header.skip_index_offset; existing_header->fieldnorm_offset = header.fieldnorm_offset; existing_header->ctid_pages_offset = header.ctid_pages_offset; existing_header->ctid_offsets_offset = header.ctid_offsets_offset; existing_header->alive_bitset_offset = header.alive_bitset_offset; existing_header->alive_count = header.alive_count; existing_header->num_docs = header.num_docs; existing_header->total_tokens = v5_total_tokens; existing_header->data_size = header.data_size; existing_header->num_pages = header.num_pages; existing_header->page_index = header.page_index; } GenericXLogFinish(xlog_state); UnlockReleaseBuffer(header_buf); } FlushRelationBuffers(index); /* Clean up writer-owned state. Caller frees terms[] and docmap. */ if (string_offsets) pfree(string_offsets); if (term_blocks) pfree(term_blocks); if (writer.pages) pfree(writer.pages); return header_block; } /* * Collect all pages belonging to a segment for later freeing. * This includes data pages (from page_map) and page index pages. * * Returns the total number of pages collected. * The caller must pfree the returned pages array when done. */ uint32 tp_segment_collect_pages( Relation index, BlockNumber root_block, BlockNumber **pages_out) { TpSegmentReader *reader; BlockNumber *all_pages; uint32 num_pages; uint32 capacity; BlockNumber page_index_block; uint32 i; *pages_out = NULL; reader = tp_segment_open(index, root_block); if (!reader) return 0; /* * Start with capacity for data pages. The +16 is just an optimization to * reduce reallocs for page index pages; the array grows dynamically below. */ capacity = reader->num_pages + 16; all_pages = palloc(sizeof(BlockNumber) * capacity); num_pages = 0; /* Collect all data pages from the page map */ for (i = 0; i < reader->num_pages; i++) all_pages[num_pages++] = reader->page_map[i]; /* Traverse and collect page index chain */ page_index_block = reader->header->page_index; while (page_index_block != InvalidBlockNumber) { Buffer index_buf; Page index_page; TpPageIndexSpecial *special; /* Grow array if needed */ if (num_pages >= capacity) { capacity *= 2; all_pages = repalloc(all_pages, sizeof(BlockNumber) * capacity); } /* Add this page index page */ all_pages[num_pages++] = page_index_block; /* Read the page to get next pointer */ index_buf = ReadBuffer(index, page_index_block); LockBuffer(index_buf, BUFFER_LOCK_SHARE); index_page = BufferGetPage(index_buf); special = (TpPageIndexSpecial *)PageGetSpecialPointer(index_page); /* Validate this is a page index page */ if (special->magic != TP_PAGE_INDEX_MAGIC) { UnlockReleaseBuffer(index_buf); break; } page_index_block = special->next_page; UnlockReleaseBuffer(index_buf); } tp_segment_close(reader); *pages_out = all_pages; return num_pages; } /* * Free pages belonging to a segment by recording them in the FSM. * Call this after the segment is no longer referenced (metapage updated). */ void tp_segment_free_pages(Relation index, BlockNumber *pages, uint32 num_pages) { uint32 i; for (i = 0; i < num_pages; i++) { if (pages[i] == 0) elog(ERROR, "attempted to free metapage (block 0)"); tp_record_free_index_page(index, pages[i]); } } /* * Unified segment dump function using DumpOutput abstraction. * This allows dumping to either StringInfo (SQL return) or FILE. */ void tp_dump_segment_to_output( Relation index, BlockNumber segment_root, DumpOutput *out) { TpSegmentHeader header; Buffer header_buf; Page header_page; uint32 terms_to_show; if (segment_root == InvalidBlockNumber) { dump_printf(out, "\nNo segments written yet\n"); return; } dump_printf( out, "\n========== Segment at block %u ==========\n", segment_root); /* Read the header page */ header_buf = ReadBuffer(index, segment_root); LockBuffer(header_buf, BUFFER_LOCK_SHARE); header_page = BufferGetPage(header_buf); /* Version-aware header read */ { uint32 raw_version; memcpy(&raw_version, (char *)PageGetContents(header_page) + sizeof(uint32), sizeof(uint32)); if (raw_version <= TP_SEGMENT_FORMAT_VERSION_3) { TpSegmentHeaderV3 v3; memcpy(&v3, PageGetContents(header_page), sizeof(TpSegmentHeaderV3)); header.magic = v3.magic; header.version = v3.version; header.created_at = v3.created_at; header.num_pages = v3.num_pages; header.data_size = (uint64)v3.data_size; header.level = v3.level; header.next_segment = v3.next_segment; header.dictionary_offset = (uint64)v3.dictionary_offset; header.strings_offset = (uint64)v3.strings_offset; header.entries_offset = (uint64)v3.entries_offset; header.postings_offset = (uint64)v3.postings_offset; header.skip_index_offset = (uint64)v3.skip_index_offset; header.fieldnorm_offset = (uint64)v3.fieldnorm_offset; header.ctid_pages_offset = (uint64)v3.ctid_pages_offset; header.ctid_offsets_offset = (uint64)v3.ctid_offsets_offset; header.alive_bitset_offset = 0; header.alive_count = v3.num_docs; header.num_terms = v3.num_terms; header.num_docs = v3.num_docs; header.total_tokens = v3.total_tokens; header.page_index = v3.page_index; } else if (raw_version <= TP_SEGMENT_FORMAT_VERSION_4) { TpSegmentHeaderV4 v4; memcpy(&v4, PageGetContents(header_page), sizeof(TpSegmentHeaderV4)); header.magic = v4.magic; header.version = v4.version; header.created_at = v4.created_at; header.num_pages = v4.num_pages; header.data_size = v4.data_size; header.level = v4.level; header.next_segment = v4.next_segment; header.dictionary_offset = v4.dictionary_offset; header.strings_offset = v4.strings_offset; header.entries_offset = v4.entries_offset; header.postings_offset = v4.postings_offset; header.skip_index_offset = v4.skip_index_offset; header.fieldnorm_offset = v4.fieldnorm_offset; header.ctid_pages_offset = v4.ctid_pages_offset; header.ctid_offsets_offset = v4.ctid_offsets_offset; header.alive_bitset_offset = 0; header.alive_count = v4.num_docs; header.num_terms = v4.num_terms; header.num_docs = v4.num_docs; header.total_tokens = v4.total_tokens; header.page_index = v4.page_index; } else { memcpy(&header, PageGetContents(header_page), sizeof(TpSegmentHeader)); } } /* Hex dump in full mode (file output) */ if (out->full_dump) { unsigned char *page_data = (unsigned char *)header_page; int i, j; int bytes_to_dump = BLCKSZ; /* Full page, no truncation */ dump_printf( out, "\n=== RAW PAGE DATA (%d bytes) ===\n", bytes_to_dump); for (i = 0; i < bytes_to_dump && i < BLCKSZ; i += 16) { dump_printf(out, "%04x: ", i); for (j = 0; j < 16 && (i + j) < bytes_to_dump; j++) dump_printf(out, "%02x ", page_data[i + j]); for (; j < 16; j++) dump_printf(out, " "); dump_printf(out, " |"); for (j = 0; j < 16 && (i + j) < bytes_to_dump; j++) { unsigned char c = page_data[i + j]; dump_printf(out, "%c", (c >= 32 && c < 127) ? c : '.'); } dump_printf(out, "|\n"); } } UnlockReleaseBuffer(header_buf); /* Header info */ dump_printf(out, "\n=== SEGMENT HEADER ===\n"); dump_printf( out, "Magic: 0x%08X (expected 0x%08X) %s\n", header.magic, TP_SEGMENT_MAGIC, header.magic == TP_SEGMENT_MAGIC ? "VALID" : "INVALID!"); dump_printf(out, "Version: %u\n", header.version); dump_printf(out, "Pages: %u\n", header.num_pages); dump_printf(out, "Data size: %" PRIu64 " bytes\n", header.data_size); dump_printf(out, "Level: %u\n", header.level); dump_printf(out, "Page index: block %u\n", header.page_index); dump_printf( out, "Alive: %u / %u docs\n", header.alive_count, header.num_docs); if (header.alive_bitset_offset > 0) dump_printf( out, "Alive bitset offset: %" PRIu64 " (%u bytes)\n", header.alive_bitset_offset, tp_alive_bitset_size(header.num_docs)); /* Corpus statistics */ dump_printf(out, "\n=== CORPUS STATISTICS ===\n"); dump_printf(out, "Terms: %u\n", header.num_terms); dump_printf(out, "Docs: %u\n", header.num_docs); dump_printf( out, "Total tokens: %llu\n", (unsigned long long)header.total_tokens); /* Section offsets */ dump_printf(out, "\n=== SECTION OFFSETS ===\n"); dump_printf( out, "Dictionary offset: %" PRIu64 "\n", header.dictionary_offset); dump_printf(out, "Strings offset: %" PRIu64 "\n", header.strings_offset); dump_printf(out, "Entries offset: %" PRIu64 "\n", header.entries_offset); dump_printf( out, "Skip index offset: %" PRIu64 "\n", header.skip_index_offset); dump_printf(out, "Postings offset: %" PRIu64 "\n", header.postings_offset); dump_printf( out, "Fieldnorm offset: %" PRIu64 "\n", header.fieldnorm_offset); dump_printf( out, "CTID pages offset: %" PRIu64 "\n", header.ctid_pages_offset); dump_printf( out, "CTID offsets offset: %" PRIu64 "\n", header.ctid_offsets_offset); /* Page layout summary */ if (header.data_size > 0) { dump_printf(out, "\n=== PAGE LAYOUT ===\n"); dump_printf( out, "Dictionary: pages %u-%u\n", (uint32)(header.dictionary_offset / SEGMENT_DATA_PER_PAGE), (uint32)((header.strings_offset - 1) / SEGMENT_DATA_PER_PAGE)); dump_printf( out, "Strings: pages %u-%u\n", (uint32)(header.strings_offset / SEGMENT_DATA_PER_PAGE), (uint32)((header.entries_offset - 1) / SEGMENT_DATA_PER_PAGE)); dump_printf( out, "Entries: pages %u-%u\n", (uint32)(header.entries_offset / SEGMENT_DATA_PER_PAGE), (uint32)((header.postings_offset - 1) / SEGMENT_DATA_PER_PAGE)); dump_printf( out, "Postings: pages %u-%u\n", (uint32)(header.postings_offset / SEGMENT_DATA_PER_PAGE), (uint32)((header.skip_index_offset - 1) / SEGMENT_DATA_PER_PAGE)); dump_printf( out, "Skip index: pages %u-%u\n", (uint32)(header.skip_index_offset / SEGMENT_DATA_PER_PAGE), (uint32)((header.data_size - 1) / SEGMENT_DATA_PER_PAGE)); } /* Dictionary dump */ if (header.num_terms > 0 && header.dictionary_offset > 0) { TpSegmentReader *reader; TpDictionary dict_header; uint32 *string_offsets; uint32 i; /* Validate offsets */ if (header.dictionary_offset >= header.data_size || header.strings_offset >= header.data_size || header.entries_offset >= header.data_size) { dump_printf(out, "\nERROR: Invalid offsets detected\n"); return; } dump_printf( out, "\n=== DICTIONARY TERMS (%u total) ===\n", header.num_terms); reader = tp_segment_open(index, segment_root); /* Read dictionary header */ tp_segment_read( reader, header.dictionary_offset, &dict_header, sizeof(dict_header.num_terms)); /* Read string offsets */ string_offsets = palloc(sizeof(uint32) * dict_header.num_terms); tp_segment_read( reader, header.dictionary_offset + sizeof(dict_header.num_terms), string_offsets, sizeof(uint32) * dict_header.num_terms); /* In full mode show all terms; otherwise limit */ terms_to_show = out->full_dump ? header.num_terms : Min(header.num_terms, 20); for (i = 0; i < terms_to_show; i++) { char *term_text; term_text = tp_segment_read_term_at_index( reader, &header, string_offsets, i); if (strlen(term_text) > 1024) { dump_printf(out, " [%u] ERROR: Invalid string length\n", i); pfree(term_text); continue; } { /* Block-based storage */ TpDictEntry entry; tp_segment_read_dict_entry(reader, &header, i, &entry); dump_printf( out, " [%04u] '%-30s' (docs=%4u, blocks=%4u)\n", i, term_text, entry.doc_freq, entry.block_count); /* Show blocks in full mode or for first few terms */ if ((out->full_dump || i < 5) && entry.block_count > 0) { uint32 j; uint32 blocks_to_show = out->full_dump ? entry.block_count : Min(entry.block_count, 3); for (j = 0; j < blocks_to_show; j++) { TpSkipEntry skip; uint32 k; uint32 postings_to_show; tp_segment_read_skip_entry( reader, entry.skip_index_offset, j, &skip); dump_printf( out, " Block %u: docs=%u, " "last_doc=%u, max_tf=%u, " "offset=%" PRIu64 "\n", j, skip.doc_count, skip.last_doc_id, skip.block_max_tf, skip.posting_offset); /* Show some postings from this block */ postings_to_show = out->full_dump ? skip.doc_count : Min(skip.doc_count, 3); if (postings_to_show > 0) { TpBlockPosting *block_postings; block_postings = palloc( sizeof(TpBlockPosting) * postings_to_show); tp_segment_read( reader, skip.posting_offset, block_postings, sizeof(TpBlockPosting) * postings_to_show); dump_printf(out, " Postings: "); for (k = 0; k < postings_to_show; k++) { dump_printf( out, "doc%u:%u ", block_postings[k].doc_id, block_postings[k].frequency); } if (skip.doc_count > postings_to_show) dump_printf( out, "... (%u more)", skip.doc_count - postings_to_show); dump_printf(out, "\n"); pfree(block_postings); } } if (entry.block_count > blocks_to_show) dump_printf( out, " ... (%u more blocks)\n", entry.block_count - blocks_to_show); } } pfree(term_text); } if (header.num_terms > terms_to_show) { dump_printf( out, " ... and %u more terms\n", header.num_terms - terms_to_show); } pfree(string_offsets); tp_segment_close(reader); } /* Dump fieldnorm table and CTID map */ if (header.num_docs > 0) { TpSegmentReader *reader; uint32 docs_to_show; uint32 i; reader = tp_segment_open(index, segment_root); /* Fieldnorm table */ dump_printf( out, "\n=== FIELDNORM TABLE (%u docs) ===\n", header.num_docs); docs_to_show = out->full_dump ? header.num_docs : Min(header.num_docs, 10); if (header.fieldnorm_offset > 0) { uint8 *fieldnorms = palloc(docs_to_show); tp_segment_read( reader, header.fieldnorm_offset, fieldnorms, docs_to_show); dump_printf(out, " Doc ID -> Length (encoded -> decoded):\n"); for (i = 0; i < docs_to_show; i++) { dump_printf( out, " [%04u] %3u -> %u\n", i, fieldnorms[i], decode_fieldnorm(fieldnorms[i])); } if (header.num_docs > docs_to_show) dump_printf( out, " ... and %u more docs\n", header.num_docs - docs_to_show); pfree(fieldnorms); } /* CTID map */ dump_printf(out, "\n=== CTID MAP (%u docs) ===\n", header.num_docs); if (header.ctid_pages_offset > 0) { BlockNumber *pages = palloc(sizeof(BlockNumber) * docs_to_show); OffsetNumber *offsets = palloc( sizeof(OffsetNumber) * docs_to_show); tp_segment_read( reader, header.ctid_pages_offset, pages, sizeof(BlockNumber) * docs_to_show); tp_segment_read( reader, header.ctid_offsets_offset, offsets, sizeof(OffsetNumber) * docs_to_show); dump_printf(out, " Doc ID -> CTID:\n"); for (i = 0; i < docs_to_show; i++) { dump_printf( out, " [%04u] (%u,%u)\n", i, pages[i], offsets[i]); } if (header.num_docs > docs_to_show) dump_printf( out, " ... and %u more docs\n", header.num_docs - docs_to_show); pfree(pages); pfree(offsets); } tp_segment_close(reader); } dump_printf(out, "\n========== End Segment Dump ==========\n"); } /* Segment writer helper functions */ void tp_segment_writer_init(TpSegmentWriter *writer, Relation index) { writer->index = index; writer->pages = NULL; writer->pages_allocated = 0; writer->pages_capacity = 0; writer->current_offset = 0; writer->buffer = palloc(BLCKSZ); writer->buffer_page = 0; writer->buffer_pos = SizeOfPageHeaderData; /* Skip page header */ /* Initialize reusable posting buffer */ writer->posting_buffer = NULL; writer->posting_buffer_size = 0; /* Allocate first page */ tp_segment_writer_allocate_page(writer); /* Initialize first page */ PageInit((Page)writer->buffer, BLCKSZ, 0); } void tp_segment_writer_write(TpSegmentWriter *writer, const void *data, Size len) { const char *src = (const char *)data; Size bytes_written = 0; while (bytes_written < len) { /* Calculate how much we can write to current page */ uint32 page_space = BLCKSZ - writer->buffer_pos; uint32 to_write = (uint32)Min((Size)page_space, len - bytes_written); /* Copy data to buffer */ memcpy(writer->buffer + writer->buffer_pos, src + bytes_written, to_write); writer->buffer_pos += to_write; writer->current_offset += to_write; bytes_written += to_write; /* If page is full, flush it */ if (writer->buffer_pos >= BLCKSZ) { tp_segment_writer_flush(writer); /* Move to next page if we have more data */ if (bytes_written < len) { writer->buffer_page++; /* Allocate a new page if needed */ if (writer->buffer_page >= writer->pages_allocated) { tp_segment_writer_allocate_page(writer); } /* Initialize new page */ PageInit((Page)writer->buffer, BLCKSZ, 0); writer->buffer_pos = SizeOfPageHeaderData; } } } } void tp_segment_writer_flush(TpSegmentWriter *writer) { Buffer buffer; Page page; BlockNumber block; if (writer->buffer_page >= writer->pages_allocated) return; /* Nothing to flush */ block = writer->pages[writer->buffer_page]; /* Write current buffer to disk */ buffer = ReadBuffer(writer->index, block); LockBuffer( buffer, BUFFER_LOCK_EXCLUSIVE); /* Need exclusive lock to modify */ page = BufferGetPage(buffer); /* * Lay out the standard page header before populating the content area. * allocate_segment_page intentionally returns uninitialized pages, so * the first writer to touch a buffer is responsible for PageInit and * for setting pd_lower to cover the custom segment content area. */ PageInit(page, BLCKSZ, 0); ((PageHeader)page)->pd_lower = BLCKSZ; /* * Copy only the data portion of the page, leaving the header we just * laid out in place. writer->buffer's first SizeOfPageHeaderData * bytes are scratch and must not overwrite the buffer manager's page * header on disk. */ memcpy((char *)page + SizeOfPageHeaderData, (char *)writer->buffer + SizeOfPageHeaderData, BLCKSZ - SizeOfPageHeaderData); tp_segment_log_dirty_buffer(writer->index, buffer); UnlockReleaseBuffer(buffer); } void tp_segment_writer_finish(TpSegmentWriter *writer) { /* Flush any remaining data */ if (writer->buffer_pos > SizeOfPageHeaderData) { tp_segment_writer_flush(writer); } pfree(writer->buffer); /* Free reusable posting buffer if allocated */ if (writer->posting_buffer) { pfree(writer->posting_buffer); writer->posting_buffer = NULL; writer->posting_buffer_size = 0; } } timescale-pg_textsearch-b63fe58/src/segment/segment.h000066400000000000000000000073361525762047700230360ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * segment.h - Disk-based segment structures */ #pragma once #include #include #include #include #include "compat.h" #include "constants.h" #include "segment/format.h" #include "storage/bufmgr.h" #include "storage/itemptr.h" #include "utils/timestamp.h" /* * BufFile segment size (1 GB). BufFile stores data across * multiple 1 GB physical files. Composite offsets encode both * fileno and position: fileno * TP_BUFFILE_SEG_SIZE + file_offset. */ #define TP_BUFFILE_SEG_SIZE ((uint64)(1024 * 1024 * 1024)) static inline uint64 tp_buffile_composite_offset(int fileno, off_t file_offset) { return (uint64)fileno * TP_BUFFILE_SEG_SIZE + (uint64)file_offset; } static inline void tp_buffile_decompose_offset(uint64 composite, int *fileno, off_t *offset) { *fileno = (int)(composite / TP_BUFFILE_SEG_SIZE); *offset = (off_t)(composite % TP_BUFFILE_SEG_SIZE); } /* * Segment posting - output format for iteration * 14 bytes, packed. Used when converting block postings for scoring. * * doc_id is segment-local, used for deferred CTID resolution in lazy loading. * When CTIDs are pre-loaded, ctid is set immediately. When lazy loading, * ctid is invalid and doc_id is used to look up CTID at result extraction. */ #ifdef _MSC_VER #pragma pack(push, 1) /* MSVC equivalent of TP_PACKED below */ #endif typedef struct TpSegmentPosting { ItemPointerData ctid; /* 6 bytes - heap tuple ID (may be invalid) */ uint32 doc_id; /* 4 bytes - segment-local doc ID */ uint16 frequency; /* 2 bytes - term frequency */ uint16 doc_length; /* 2 bytes - document length (from fieldnorm) */ } TP_PACKED TpSegmentPosting; #ifdef _MSC_VER #pragma pack(pop) #endif /* Version-aware struct size helpers */ static inline size_t tp_dict_entry_size(uint32 version) { return (version <= TP_SEGMENT_FORMAT_VERSION_3) ? sizeof(TpDictEntryV3) : sizeof(TpDictEntry); } static inline size_t tp_skip_entry_size(uint32 version) { return (version <= TP_SEGMENT_FORMAT_VERSION_3) ? sizeof(TpSkipEntryV3) : sizeof(TpSkipEntry); } /* * Document length - 12 bytes (padded to 16) */ typedef struct TpDocLength { ItemPointerData ctid; /* 6 bytes */ uint16 length; /* 2 bytes - total terms in doc */ uint32 reserved; /* 4 bytes padding */ } TpDocLength; /* Look up doc_freq for a term from explicit segment roots. */ extern uint32 tp_segment_roots_get_doc_freq( Relation index, const BlockNumber *roots, uint32 root_count, const char *term); /* Batch lookup doc_freq for multiple terms - opens each root once. */ extern void tp_batch_get_segment_roots_doc_freq( Relation index, const BlockNumber *roots, uint32 root_count, char **terms, int term_count, uint32 *doc_freqs); /* * Mark a segment buffer dirty and immediately emit a full-page WAL image when * WAL is required for the relation. Segment pages use custom layouts, so the * full image must not use REGBUF_STANDARD hole compression. * * This helper is intended for callers that have just populated an entire * segment page from scratch (e.g. tp_segment_writer_flush, * write_page_index_internal), where a full-page image is necessary anyway. * For small in-place updates to an already-written segment page (dict-entry * backpatches, header patches), prefer GenericXLog delta logging — it * produces much smaller WAL records. * * Callers must hold an exclusive buffer lock and must not perform any * ERROR-capable work between the final page modification and this call. */ extern void tp_segment_log_dirty_buffer(Relation index, Buffer buffer); timescale-pg_textsearch-b63fe58/src/segment/tombstone.c000066400000000000000000000415021525762047700233720ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * tombstone.c - Deferred-free tombstone chain (issue #380). */ #include #include #include #include #include #include "constants.h" #include "debug/injection.h" #include "index/freepage.h" #include "index/metapage.h" #include "index/state.h" #include "segment/compaction.h" #include "segment/io.h" #include "segment/tombstone.h" void tp_tombstone_page_init( Page page, FullTransactionId merged_fxid, BlockNumber next_page) { TpTombstonePage t; PageInit(page, BLCKSZ, 0); t = tp_tombstone_page(page); t->magic = TP_TOMBSTONE_MAGIC; t->version = TP_TOMBSTONE_VERSION; t->flags = 0; t->merged_fxid = merged_fxid; t->next_page = next_page; t->num_blocks = 0; /* * blocks[] lives in the page body; collapse the GenericXLog * page hole so the array is recorded and not zeroed on replay. * Same convention as the memtable page (src/memtable/page.c). */ ((PageHeader)page)->pd_lower = BLCKSZ; } bool tp_tombstone_page_is_valid(Page page) { TpTombstonePage t = tp_tombstone_page(page); return t->magic == TP_TOMBSTONE_MAGIC && t->version == TP_TOMBSTONE_VERSION && t->num_blocks <= TP_TOMBSTONE_CAPACITY; } static void tombstone_write_page( Relation index, BlockNumber block, const BlockNumber *blocks, uint32 start, uint32 count, FullTransactionId merged_fxid, BlockNumber next_page) { volatile Buffer buf = InvalidBuffer; GenericXLogState *volatile state = NULL; PG_TRY(); { Page page; TpTombstonePage t; buf = ReadBuffer(index, block); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); page = GenericXLogRegisterBuffer( (GenericXLogState *)state, buf, GENERIC_XLOG_FULL_IMAGE); tp_tombstone_page_init(page, merged_fxid, next_page); t = tp_tombstone_page(page); t->num_blocks = count; for (uint32 i = 0; i < count; i++) t->blocks[i] = blocks[start + i]; GenericXLogFinish((GenericXLogState *)state); state = NULL; UnlockReleaseBuffer(buf); buf = InvalidBuffer; } PG_CATCH(); { if (state != NULL) GenericXLogAbort((GenericXLogState *)state); if (BufferIsValid(buf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer(buf); } PG_RE_THROW(); } PG_END_TRY(); } static void tombstone_build_internal( Relation index, const BlockNumber *blocks, uint32 num_blocks, FullTransactionId merged_fxid, TpDetachedTombstoneBatch *batch) { BlockNumber next_page = InvalidBlockNumber; uint32 remaining = num_blocks; uint32 capacity; Assert(batch != NULL); memset(batch, 0, sizeof(*batch)); batch->head = InvalidBlockNumber; batch->tail = InvalidBlockNumber; if (num_blocks == 0) return; capacity = num_blocks / TP_TOMBSTONE_CAPACITY + (num_blocks % TP_TOMBSTONE_CAPACITY != 0); batch->owned_pages = palloc(sizeof(BlockNumber) * capacity); /* * Build the batch tail-first so each page's next_page points at * an already-decided successor. Active construction is always * detached, so the first page terminates at InvalidBlockNumber. * * Per-page chunking honors TP_TOMBSTONE_CAPACITY. We assign the * LAST chunk of `blocks` to the first page, walking backwards. */ while (remaining > 0) { uint32 chunk = Min(remaining, TP_TOMBSTONE_CAPACITY); uint32 start = remaining - chunk; BlockNumber blk = tp_fsm_claim_or_extend_block(index); Assert(batch->owned_count < capacity); batch->owned_pages[batch->owned_count++] = blk; tp_compaction_allocation_injection_point( TP_COMPACTION_ALLOCATION_TOMBSTONE); tombstone_write_page( index, blk, blocks, start, chunk, merged_fxid, next_page); if (batch->tail == InvalidBlockNumber) batch->tail = blk; batch->head = blk; next_page = blk; remaining = start; } } void tp_tombstone_build_detached( Relation index, const BlockNumber *blocks, uint32 num_blocks, FullTransactionId merged_fxid, TpDetachedTombstoneBatch *batch) { tombstone_build_internal(index, blocks, num_blocks, merged_fxid, batch); } void tp_tombstone_restamp_batch( Relation index, TpDetachedTombstoneBatch batch, FullTransactionId merged_fxid) { for (uint32 i = 0; i < batch.owned_count; i++) { volatile Buffer buf = InvalidBuffer; GenericXLogState *volatile state = NULL; PG_TRY(); { Page page; TpTombstonePage tombstone; buf = ReadBuffer(index, batch.owned_pages[i]); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); state = GenericXLogStart(index); page = GenericXLogRegisterBuffer( (GenericXLogState *)state, buf, 0); if (!tp_tombstone_page_is_valid(page)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: corrupt detached tombstone " "page %u in index \"%s\"", batch.owned_pages[i], RelationGetRelationName(index)))); tombstone = tp_tombstone_page(page); tombstone->merged_fxid = merged_fxid; GenericXLogFinish((GenericXLogState *)state); state = NULL; UnlockReleaseBuffer(buf); buf = InvalidBuffer; } PG_CATCH(); { if (state != NULL) GenericXLogAbort((GenericXLogState *)state); if (BufferIsValid(buf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer(buf); } PG_RE_THROW(); } PG_END_TRY(); } } Buffer tp_tombstone_attach_detached( GenericXLogState *state, Relation index, TpDetachedTombstoneBatch batch, BlockNumber old_head, bool allow_invalid_stamp) { volatile Buffer buf = InvalidBuffer; Assert(state != NULL); if (batch.owned_count == 0) { Assert(batch.head == InvalidBlockNumber); Assert(batch.tail == InvalidBlockNumber); return InvalidBuffer; } Assert(batch.head != InvalidBlockNumber); Assert(batch.tail != InvalidBlockNumber); PG_TRY(); { Page page; TpTombstonePage t; buf = ReadBuffer(index, batch.tail); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); page = GenericXLogRegisterBuffer(state, buf, 0); if (!tp_tombstone_page_is_valid(page)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: corrupt detached tombstone tail " "page %u in index \"%s\"", batch.tail, RelationGetRelationName(index)))); t = tp_tombstone_page(page); if (t->next_page != InvalidBlockNumber) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: detached tombstone tail page %u " "is already attached", batch.tail))); if (!allow_invalid_stamp && !FullTransactionIdIsValid(t->merged_fxid)) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("detached tombstone batch has no publication " "reclaim stamp"))); t->next_page = old_head; } PG_CATCH(); { if (BufferIsValid(buf)) { if (InterruptHoldoffCount == 0) HOLD_INTERRUPTS(); UnlockReleaseBuffer(buf); } PG_RE_THROW(); } PG_END_TRY(); return (Buffer)buf; } void tp_tombstone_discard_detached(Relation index, TpDetachedTombstoneBatch batch) { for (uint32 i = 0; i < batch.owned_count; i++) tp_record_free_index_page(index, batch.owned_pages[i]); if (batch.owned_count > 0) IndexFreeSpaceMapVacuum(index); if (batch.owned_pages != NULL) pfree(batch.owned_pages); } /* * Read pending_free_head from the metapage under a SHARE lock. */ BlockNumber tp_tombstone_read_head(Relation index) { Buffer buf; Page page; TpIndexMetaPage metap; BlockNumber head; buf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); metap = (TpIndexMetaPage)PageGetContents(page); /* v6/v7 pages predate the field; treat as empty. */ if (metap->version < TP_METAPAGE_VERSION_V8) head = InvalidBlockNumber; else head = metap->pending_free_head; UnlockReleaseBuffer(buf); return head; } /* * WAL-unlink the tombstone at `victim` whose predecessor is * `prev` (InvalidBlockNumber => the metapage is the predecessor). * `victim_next` is the victim's next_page (the new link target). * Upgrades the metapage to v8 first when the predecessor is the * metapage (so the field exists to write). */ static void tombstone_unlink( Relation index, BlockNumber prev, BlockNumber victim, BlockNumber victim_next) { GenericXLogState *state; Buffer buf; Page page; (void)victim; state = GenericXLogStart(index); if (prev == InvalidBlockNumber) { TpIndexMetaPage metap; buf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); page = GenericXLogRegisterBuffer(state, buf, 0); tp_metapage_upgrade_to_current(index, page); metap = (TpIndexMetaPage)PageGetContents(page); metap->pending_free_head = victim_next; } else { TpTombstonePage t; buf = ReadBuffer(index, prev); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); page = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE); t = tp_tombstone_page(page); t->next_page = victim_next; } GenericXLogFinish(state); UnlockReleaseBuffer(buf); } uint32 tp_tombstone_drain( Relation index, struct TpLocalIndexState *state, FullTransactionId horizon, bool own_lock) { uint32 freed = 0; /* own_lock=true needs `state` to acquire and release the lock. */ Assert(!own_lock || state != NULL); for (;;) { BlockNumber nblocks; BlockNumber prev = InvalidBlockNumber; BlockNumber cur; BlockNumber victim = InvalidBlockNumber; BlockNumber victim_prev = InvalidBlockNumber; BlockNumber victim_next = InvalidBlockNumber; BlockNumber *unstamped_pages = NULL; uint32 unstamped_count = 0; uint32 unstamped_capacity = 0; BlockNumber *victim_blocks = NULL; uint32 victim_count = 0; FullTransactionId victim_fxid = InvalidFullTransactionId; bool corrupt = false; BlockNumber corrupt_at = InvalidBlockNumber; BlockNumber corrupt_prev = InvalidBlockNumber; CHECK_FOR_INTERRUPTS(); if (own_lock) tp_acquire_index_lock(state, LW_EXCLUSIVE); /* * Re-read under the lock each iteration: the lock is dropped * between iterations and the relation can shrink. This only * keeps the block bounds checks below honest; it is not what * makes the frees safe (see below). */ nblocks = RelationGetNumberOfBlocks(index); /* Walk the chain to find the first past-horizon tombstone. */ cur = tp_tombstone_read_head(index); while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpTombstonePage t; if (cur == 0 || cur >= nblocks) { corrupt = true; corrupt_at = cur; corrupt_prev = prev; break; } buf = ReadBuffer(index, cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_tombstone_page_is_valid(page)) { UnlockReleaseBuffer(buf); corrupt = true; corrupt_at = cur; corrupt_prev = prev; break; } t = tp_tombstone_page(page); if (!FullTransactionIdIsValid(t->merged_fxid)) { /* * Collect the whole contiguous unstamped run; * restamping one page per chain walk is quadratic * in the number of tombstones. */ BlockNumber next = t->next_page; if (unstamped_count == unstamped_capacity) { unstamped_capacity = unstamped_capacity == 0 ? 8 : unstamped_capacity * 2; unstamped_pages = unstamped_pages == NULL ? palloc(sizeof(BlockNumber) * unstamped_capacity) : repalloc( unstamped_pages, sizeof(BlockNumber) * unstamped_capacity); } unstamped_pages[unstamped_count++] = cur; UnlockReleaseBuffer(buf); cur = next; continue; } if (unstamped_count > 0) { /* Run ended; restamp it before considering a victim. */ UnlockReleaseBuffer(buf); break; } if (FullTransactionIdPrecedes(t->merged_fxid, horizon)) { uint32 k; victim = cur; victim_prev = prev; victim_next = t->next_page; victim_count = t->num_blocks; victim_fxid = t->merged_fxid; victim_blocks = palloc( sizeof(BlockNumber) * Max(victim_count, 1)); for (k = 0; k < victim_count; k++) { BlockNumber b = t->blocks[k]; if (b == 0 || b >= nblocks || b == cur) { corrupt = true; corrupt_at = cur; corrupt_prev = prev; break; } victim_blocks[k] = b; } UnlockReleaseBuffer(buf); if (corrupt) { pfree(victim_blocks); victim_blocks = NULL; victim = InvalidBlockNumber; } break; } prev = cur; cur = t->next_page; UnlockReleaseBuffer(buf); } if (unstamped_count > 0) { TpDetachedTombstoneBatch batch; memset(&batch, 0, sizeof(batch)); batch.head = unstamped_pages[0]; batch.tail = unstamped_pages[unstamped_count - 1]; batch.owned_pages = unstamped_pages; batch.owned_count = unstamped_count; tp_tombstone_restamp_batch( index, batch, ReadNextFullTransactionId()); pfree(unstamped_pages); if (own_lock) tp_release_index_lock(state); continue; } /* * Self-heal an already-corrupt chain instead of wedging every * writer (issue #427). An older page-reuse bug (or a crash * with a stale FSM) can leave a chain node that no longer * validates, or a valid node that lists an impossible block. * We cannot trust such a node's next_page, so drop it and the * unverifiable remainder by pointing its predecessor — a * still-valid tombstone page, or the metapage head when the * corruption is at the head — at InvalidBlockNumber. The * still-drainable prefix ahead of the corruption is preserved; * the dropped tail leaks pages that a REINDEX reclaims. We * never free the corrupt node's listed blocks, so if the block * was double-owned by a live structure, healing cannot corrupt * that structure. */ if (corrupt) { tombstone_unlink( index, corrupt_prev, corrupt_at, InvalidBlockNumber); if (own_lock) tp_release_index_lock(state); ereport(WARNING, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: recovered from corrupt tombstone " "page %u in index \"%s\"", corrupt_at, RelationGetRelationName(index)), errdetail( "Dropped the deferred-free chain from that " "page onward; leaked index pages are " "reclaimed by REINDEX."))); break; } if (victim == InvalidBlockNumber) { if (own_lock) tp_release_index_lock(state); break; /* nothing left to drain */ } tp_log_page_reuse_conflict( index, victim_count > 0 ? victim_blocks[0] : victim, victim_fxid); /* Unlink first (corruption-safe; a crash here only leaks). */ tombstone_unlink(index, victim_prev, victim, victim_next); if (own_lock) tp_release_index_lock(state); TP_INJECTION_POINT(TP_INJECTION_TOMBSTONE_AFTER_UNLINK); /* * Free only after unlink, but without the per-index lock. The * maintenance lock keeps another drain out, and no published * structure references this batch now. Freeing before unlink is * unsafe: a still-chained tombstone's blocks could be claimed from * the FSM, then freed again by a later drain. * * No CHECK_FOR_INTERRUPTS here: cancellation part-way strands the * remaining pages until REINDEX. */ { uint32 k; for (k = 0; k < victim_count; k++) tp_record_free_index_page(index, victim_blocks[k]); tp_record_free_index_page(index, victim); freed += victim_count + 1; } if (victim_blocks) pfree(victim_blocks); } return freed; } uint64 tp_pending_free_block_count(Relation index) { uint64 total = 0; BlockNumber nblocks = RelationGetNumberOfBlocks(index); BlockNumber cur = tp_tombstone_read_head(index); /* * Caller must hold the per-index LWLock in shared mode so a concurrent * drain/enqueue can't recycle a tombstone page mid-walk (see * tp_pending_free_pages in dump.c). Under that lock an invalid page * means real corruption, so ERROR rather than return a short count. */ while (cur != InvalidBlockNumber) { Buffer buf; Page page; TpTombstonePage t; BlockNumber next; CHECK_FOR_INTERRUPTS(); if (cur == 0 || cur >= nblocks) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: corrupt tombstone page %u " "in index \"%s\"", cur, RelationGetRelationName(index)))); buf = ReadBuffer(index, cur); LockBuffer(buf, BUFFER_LOCK_SHARE); page = BufferGetPage(buf); if (!tp_tombstone_page_is_valid(page)) { UnlockReleaseBuffer(buf); ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("pg_textsearch: corrupt tombstone page %u " "in index \"%s\"", cur, RelationGetRelationName(index)))); } t = tp_tombstone_page(page); total += t->num_blocks; next = t->next_page; UnlockReleaseBuffer(buf); cur = next; } return total; } timescale-pg_textsearch-b63fe58/src/segment/tombstone.h000066400000000000000000000117571525762047700234100ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * tombstone.h - Deferred-free tombstone chain for displaced * segment pages (issue #380). * * A tombstone page parks the block numbers of segment pages that a * merge displaced, together with the merge's transaction horizon. * The pages are returned to the index FSM only once the horizon is * past GetOldestNonRemovableTransactionId, which (with * hot_standby_feedback=on) is past every standby snapshot. This * defers physical reuse until no standby query can still be reading * the old segment through those pages. */ #pragma once #include #include #include #include #include #include #include #include "constants.h" /* * On-disk tombstone page, stored at PageGetContents(page). The * blocks[] array lives in the page body, so tp_tombstone_page_init * sets pd_lower = BLCKSZ (the GenericXLog page-hole guard). */ typedef struct TpTombstonePageData { uint32 magic; /* TP_TOMBSTONE_MAGIC */ uint16 version; /* TP_TOMBSTONE_VERSION */ uint16 flags; /* reserved, 0 */ FullTransactionId merged_fxid; /* reclaim horizon for this page */ BlockNumber next_page; /* next tombstone, or Invalid */ uint32 num_blocks; /* used entries in blocks[] */ BlockNumber blocks[FLEXIBLE_ARRAY_MEMBER]; } TpTombstonePageData; typedef TpTombstonePageData *TpTombstonePage; typedef struct TpDetachedTombstoneBatch { BlockNumber head; BlockNumber tail; /* Exact container allocations owned until publication succeeds. */ BlockNumber *owned_pages; uint32 owned_count; } TpDetachedTombstoneBatch; static inline TpTombstonePage tp_tombstone_page(Page page) { return (TpTombstonePage)PageGetContents(page); } /* Max block entries that fit on one tombstone page. */ #define TP_TOMBSTONE_CAPACITY \ ((uint32)((BLCKSZ - MAXALIGN(SizeOfPageHeaderData) - \ offsetof(TpTombstonePageData, blocks)) / \ sizeof(BlockNumber))) extern void tp_tombstone_page_init( Page page, FullTransactionId merged_fxid, BlockNumber next_page); extern bool tp_tombstone_page_is_valid(Page page); /* * Read metap->pending_free_head (SHARE-locked). Returns * InvalidBlockNumber for v6/v7 metapages (the field predates v8). */ extern BlockNumber tp_tombstone_read_head(Relation index); /* * Build an unreachable batch of tombstone pages. The tail initially * links to InvalidBlockNumber; neither the metapage nor an existing * tombstone page is changed. */ extern void tp_tombstone_build_detached( Relation index, const BlockNumber *blocks, uint32 num_blocks, FullTransactionId merged_fxid, TpDetachedTombstoneBatch *batch); /* * Replace the reclaim stamp on every tracked container page. Ordinary * compaction calls this while the batch is detached; parallel VACUUM calls it * after the provisionally stamped batch becomes reachable. */ extern void tp_tombstone_restamp_batch( Relation index, TpDetachedTombstoneBatch batch, FullTransactionId merged_fxid); /* * Register the detached tail in the caller's GenericXLog publication record * and link it to old_head. A parallel replacement may allow an invalid * provisional stamp. Returns the still-locked tail buffer, which the caller * must release after GenericXLogFinish. */ extern Buffer tp_tombstone_attach_detached( GenericXLogState *state, Relation index, TpDetachedTombstoneBatch batch, BlockNumber old_head, bool allow_invalid_stamp); /* * Return only an unreachable batch's container pages to the FSM. * The displaced source blocks listed in those pages remain untouched. */ extern void tp_tombstone_discard_detached(Relation index, TpDetachedTombstoneBatch batch); /* * Drain past-horizon tombstones. For each tombstone whose * merged_fxid < `horizon`, WAL-unlink it then tp_record_free_index_page * its listed blocks and its own page. * * `own_lock` selects locking discipline: * - true (vacuum path): take the per-index LWLock EXCLUSIVE once * per drained tombstone, held across the unlink and that * tombstone's page frees (see tp_tombstone_drain). Readers wait * for one chain walk, unlink, and up to TP_TOMBSTONE_CAPACITY * frees. `state` must be non-NULL. * - false (merge path): caller already holds the per-index lock * EXCLUSIVE end-to-end; `state` is ignored. * * Returns the number of index pages returned to the FSM (listed * blocks + tombstone pages). Caller runs IndexFreeSpaceMapVacuum * when the return value is > 0. */ struct TpLocalIndexState; extern uint32 tp_tombstone_drain( Relation index, struct TpLocalIndexState *state, FullTransactionId horizon, bool own_lock); /* * Total displaced blocks currently parked (debug/observability). * Caller must hold the per-index LWLock in shared mode. */ extern uint64 tp_pending_free_block_count(Relation index); timescale-pg_textsearch-b63fe58/src/types/000077500000000000000000000000001525762047700207145ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/src/types/array.c000066400000000000000000000036621525762047700222050ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * array.c - Text array helpers for BM25 indexing * * Provides utilities for flattening text-like array columns * (text[], varchar[], bpchar[]) into a single text value for * tokenization by the BM25 index build, insert, and scoring * paths. */ #include #include #include #include #include #include "types/array.h" /* * Is this type OID a text-like array (text[], varchar[], bpchar[])? */ bool tp_is_text_array_type(Oid typid) { return typid == TEXTARRAYOID || typid == 1015 || /* varchar[] */ typid == 1014; /* bpchar[] */ } /* * Does the index's first key use a text-array operator class? * * Uses the opclass input type rather than the key type so domains over * text[] are recognized. */ bool tp_index_key_is_text_array(Relation index) { if (IndexRelationGetNumberOfKeyAttributes(index) < 1) return false; return tp_is_text_array_type(index->rd_opcintype[0]); } /* * Flatten a text array into a single space-separated text value. * NULL elements are skipped. Returns empty text for empty arrays. * Works for text[], varchar[], and bpchar[] (same binary format). */ text * tp_flatten_text_array(Datum array_datum) { ArrayType *arr; Datum *elems; bool *nulls; int nelems; StringInfoData buf; arr = DatumGetArrayTypeP(array_datum); deconstruct_array( arr, TEXTOID, -1, false, TYPALIGN_INT, &elems, &nulls, &nelems); initStringInfo(&buf); for (int i = 0; i < nelems; i++) { text *elem; int len; if (nulls[i]) continue; elem = DatumGetTextPP(elems[i]); len = VARSIZE_ANY_EXHDR(elem); if (len == 0) continue; if (buf.len > 0) appendStringInfoChar(&buf, ' '); appendBinaryStringInfo(&buf, VARDATA_ANY(elem), len); } return cstring_to_text_with_len(buf.data, buf.len); } timescale-pg_textsearch-b63fe58/src/types/array.h000066400000000000000000000012231525762047700222010ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * array.h - Text array helpers for BM25 indexing */ #pragma once #include #include /* Check if a type OID is a text-like array */ extern bool tp_is_text_array_type(Oid typid); /* * Does the index's first key yield a text-like array? Reads the * index tuple descriptor, so plain columns and expressions are * handled alike. */ extern bool tp_index_key_is_text_array(Relation index); /* Flatten a text array into a single space-separated text */ extern text *tp_flatten_text_array(Datum array_datum); timescale-pg_textsearch-b63fe58/src/types/query.c000066400000000000000000001053031525762047700222270ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * query.c - bm25query data type implementation */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "constants.h" #include "index/metapage.h" #include "index/resolve.h" #include "index/source.h" #include "index/state.h" #include "memtable/cache_source.h" #include "memtable/chain_source.h" #include "planner/hooks.h" #include "scoring/bm25.h" #include "scoring/snapshot.h" #include "segment/fieldnorm.h" #include "segment/graph_snapshot.h" #include "segment/io.h" #include "segment/segment.h" #include "types/array.h" #include "types/query.h" #include "types/vector.h" /* * Cache for per-query IDF values to avoid repeated segment lookups. * * When the <@> operator is called per-row (e.g., ORDER BY text <@> query), * we need to calculate IDF for each query term. Without caching, this * requires opening all segments for EVERY row, which is catastrophically * slow with 1.86M rows and multiple segments. * * The cache is stored in fn_extra and persists for the duration of the * query. IDF values are computed once on first call and reused. * * Cached terms are capped: the cache is keyed by index state, not by * query text, so a per-row query value would otherwise accumulate the * whole corpus vocabulary for the life of the statement. Terms past * the cap are simply recomputed. */ #define TP_MAX_CACHED_IDF_TERMS 1024 typedef struct TermIdfEntry { char *term; /* null-terminated term string */ uint32 doc_freq; /* unified doc frequency (memtable + segments) */ float4 idf; /* cached IDF value */ } TermIdfEntry; typedef struct QueryScoreCache { Oid index_oid; /* index this cache is for */ uint64 snapshot_serial; int32 total_docs; /* total docs at cache time */ float4 avg_doc_len; /* avg doc length at cache time */ int num_terms; /* number of cached terms */ int terms_capacity; MemoryContext context; TermIdfEntry *terms; } QueryScoreCache; /* Local helper functions */ static float4 find_term_frequency_in_arrays( char **doc_terms, int32 *doc_frequencies, int doc_term_count, char *query_lexeme); /* * Look up cached IDF for a term. Returns -1.0 if not found in cache. */ static float4 lookup_cached_idf(QueryScoreCache *cache, const char *term, uint32 *doc_freq) { int i; if (!cache) return -1.0f; for (i = 0; i < cache->num_terms; i++) { if (strcmp(cache->terms[i].term, term) == 0) { if (doc_freq) *doc_freq = cache->terms[i].doc_freq; return cache->terms[i].idf; } } return -1.0f; } /* * Add a term's IDF to the cache. */ static void cache_term_idf( QueryScoreCache *cache, const char *term, uint32 doc_freq, float4 idf) { int new_capacity; if (!cache) return; if (cache->num_terms >= TP_MAX_CACHED_IDF_TERMS) return; if (cache->num_terms == cache->terms_capacity) { new_capacity = cache->terms_capacity == 0 ? 16 : cache->terms_capacity * 2; if (new_capacity > TP_MAX_CACHED_IDF_TERMS) new_capacity = TP_MAX_CACHED_IDF_TERMS; if (cache->terms == NULL) cache->terms = MemoryContextAlloc( cache->context, sizeof(TermIdfEntry) * new_capacity); else cache->terms = repalloc( cache->terms, sizeof(TermIdfEntry) * new_capacity); cache->terms_capacity = new_capacity; } cache->terms[cache->num_terms].term = MemoryContextStrdup(cache->context, term); cache->terms[cache->num_terms].doc_freq = doc_freq; cache->terms[cache->num_terms].idf = idf; cache->num_terms++; } /* * Check if the cache is valid for the current index state. */ static bool cache_is_valid(QueryScoreCache *cache, Oid index_oid, uint64 snapshot_serial) { if (!cache) return false; if (cache->index_oid != index_oid) return false; if (cache->snapshot_serial != snapshot_serial) return false; return true; } static bool query_cache_misses_document_term( QueryScoreCache *cache, TSVector query, char **doc_terms, int32 *doc_frequencies, int doc_term_count) { WordEntry *entries; char *lexemes; entries = ARRPTR(query); lexemes = STRPTR(query); for (int i = 0; i < query->size; i++) { char *term; float4 idf; uint32 doc_freq; float4 tf; term = pnstrdup(lexemes + entries[i].pos, entries[i].len); idf = lookup_cached_idf(cache, term, &doc_freq); if (idf >= 0.0f) { /* * A cached term cannot force the full sources open, so its * frequency in the document does not matter here. Checking * the cache first keeps the common all-cached row off the * O(doc_term_count) scan that scoring repeats anyway. */ pfree(term); continue; } tf = find_term_frequency_in_arrays( doc_terms, doc_frequencies, doc_term_count, term); pfree(term); if (tf != 0.0f) return true; } return false; } PG_FUNCTION_INFO_V1(tpquery_in); PG_FUNCTION_INFO_V1(tpquery_out); PG_FUNCTION_INFO_V1(tpquery_recv); PG_FUNCTION_INFO_V1(tpquery_send); PG_FUNCTION_INFO_V1(to_tpquery_text); PG_FUNCTION_INFO_V1(to_tpquery_text_index); PG_FUNCTION_INFO_V1(bm25_text_bm25query_score); PG_FUNCTION_INFO_V1(bm25_text_text_score); PG_FUNCTION_INFO_V1(bm25_textarray_bm25query_score); PG_FUNCTION_INFO_V1(bm25_textarray_text_score); PG_FUNCTION_INFO_V1(tpquery_eq); PG_FUNCTION_INFO_V1(bm25_get_current_score); /* * bm25_get_current_score - stub function for ORDER BY optimization * * Returns the cached BM25 score from the most recent index scan tuple. * The planner replaces resjunk ORDER BY expressions with calls to this * function, avoiding expensive re-computation of scores already computed * by the index scan. */ Datum bm25_get_current_score(PG_FUNCTION_ARGS pg_attribute_unused()) { PG_RETURN_FLOAT8(tp_get_cached_score()); } /* * tpquery input function * Formats: * "query_text" - simple query without index (InvalidOid) * "index_name:query_text" - query with index name (resolved to OID) * Note: If query_text contains a colon, use to_tpquery() instead */ Datum tpquery_in(PG_FUNCTION_ARGS) { char *str = PG_GETARG_CSTRING(0); char *colon; TpQuery *result; /* Check for index name prefix (format: "index_name:query") */ colon = strchr(str, ':'); if (colon && colon != str) { /* Found colon and it's not at the start - extract index name */ int index_name_len = colon - str; char *index_name = palloc(index_name_len + 1); char *query_text = colon + 1; /* Skip past : */ /* Copy the index name */ memcpy(index_name, str, index_name_len); index_name[index_name_len] = '\0'; /* Create query with index name (resolves to OID) */ result = create_tpquery_from_name(query_text, index_name); pfree(index_name); } else { /* No index name prefix - create without index */ result = create_tpquery(str, InvalidOid); } PG_RETURN_POINTER(result); } /* * tpquery output function * Converts OID back to index name for display */ Datum tpquery_out(PG_FUNCTION_ARGS) { TpQuery *tpquery = (TpQuery *)PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); StringInfo str = makeStringInfo(); if (OidIsValid(tpquery->index_oid)) { /* Format with index name: "index_name:query_text" */ char *index_name = get_rel_name(tpquery->index_oid); char *query_text = get_tpquery_text(tpquery); if (index_name) appendStringInfo(str, "%s:%s", index_name, query_text); else /* Index was dropped - show OID for debugging */ appendStringInfo( str, "[oid=%u]:%s", tpquery->index_oid, query_text); } else { /* Format without index: just the query text */ char *query_text = get_tpquery_text(tpquery); appendStringInfo(str, "%s", query_text); } PG_RETURN_CSTRING(str->data); } /* * tpquery receive function (binary input) * Binary format v1: version (1 byte) + index_oid (4 bytes) + query_text_len * (4 bytes) + query_text * Binary format v2: version (1 byte) + flags (1 byte) + index_oid (4 bytes) + * query_text_len (4 bytes) + query_text */ Datum tpquery_recv(PG_FUNCTION_ARGS) { StringInfo buf = (StringInfo)PG_GETARG_POINTER(0); TpQuery *result; uint8 version; uint8 flags = 0; Oid index_oid; int32 query_text_len; char *query_text; bool explicit_index; /* Read and validate version */ version = pq_getmsgbyte(buf); if (version != 1 && version != 2) ereport(ERROR, (errcode(ERRCODE_DATA_EXCEPTION), errmsg("unsupported bm25query binary format version %u", version), errhint("Expected version 1 or 2. This may indicate data " "from " "an incompatible pg_textsearch version."))); /* Read flags for v2+ */ if (version >= 2) flags = pq_getmsgbyte(buf); index_oid = pq_getmsgint(buf, sizeof(Oid)); query_text_len = pq_getmsgint(buf, sizeof(int32)); /* Validate length to prevent unbounded memory allocation */ if (query_text_len < 0 || query_text_len > 1000000) /* 1MB limit */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("invalid query text length: %d", query_text_len))); query_text = palloc(query_text_len + 1); pq_copymsgbytes(buf, query_text, query_text_len); query_text[query_text_len] = '\0'; explicit_index = (flags & TPQUERY_FLAG_EXPLICIT_INDEX) != 0; result = create_tpquery_explicit(query_text, index_oid, explicit_index); pfree(query_text); PG_RETURN_POINTER(result); } /* * tpquery send function (binary output) * Binary format v2: version (1 byte) + flags (1 byte) + index_oid (4 bytes) + * query_text_len (4 bytes) + query_text */ Datum tpquery_send(PG_FUNCTION_ARGS) { TpQuery *tpquery = (TpQuery *)PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); StringInfoData buf; char *query_text; pq_begintypsend(&buf); pq_sendint8(&buf, TPQUERY_VERSION); /* Planner-only hints never cross the type's binary I/O boundary. */ pq_sendint8(&buf, tpquery->flags & TPQUERY_FLAG_EXPLICIT_INDEX); pq_sendint32(&buf, tpquery->index_oid); pq_sendint32(&buf, tpquery->query_text_len); query_text = get_tpquery_text(tpquery); pq_sendbytes(&buf, query_text, tpquery->query_text_len); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } /* * Create a tpquery from text (no index - InvalidOid) */ Datum to_tpquery_text(PG_FUNCTION_ARGS) { text *input_text = PG_GETARG_TEXT_PP(0); char *query_text = text_to_cstring(input_text); TpQuery *result = create_tpquery(query_text, InvalidOid); pfree(query_text); PG_RETURN_POINTER(result); } /* * Create a tpquery from text with index name (resolves to OID) */ Datum to_tpquery_text_index(PG_FUNCTION_ARGS) { text *input_text = PG_GETARG_TEXT_PP(0); text *index_text = PG_GETARG_TEXT_PP(1); char *query_text = text_to_cstring(input_text); char *index_name = text_to_cstring(index_text); TpQuery *result = create_tpquery_from_name(query_text, index_name); pfree(query_text); pfree(index_name); PG_RETURN_POINTER(result); } /* * Find the first child index of a partitioned index via pg_inherits. * Returns InvalidOid if no children found. */ static Oid find_first_child_index(Oid parent_index_oid) { Relation inhrel; SysScanDesc scan; HeapTuple tuple; ScanKeyData skey; Oid child_oid = InvalidOid; inhrel = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit( &skey, Anum_pg_inherits_inhparent, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(parent_index_oid)); scan = systable_beginscan( inhrel, InheritsParentIndexId, true, NULL, 1, &skey); tuple = systable_getnext(scan); if (tuple != NULL) { Form_pg_inherits inhform = (Form_pg_inherits)GETSTRUCT(tuple); child_oid = inhform->inhrelid; } systable_endscan(scan); table_close(inhrel, AccessShareLock); return child_oid; } /* * Find the first child BM25 index for an inheritance parent index. * This handles hypertables where child indexes don't have pg_inherits * relationship to the parent index - we find via the table hierarchy. * * Returns InvalidOid if no matching child index found. */ static Oid find_first_child_bm25_index(Oid parent_index_oid, const char *indexed_colname) { Relation inhrel; SysScanDesc scan; HeapTuple tuple; ScanKeyData skey; HeapTuple idx_tuple; Form_pg_index idx_form; Oid parent_table_oid; Oid bm25_am_oid; Oid result = InvalidOid; if (!indexed_colname) return InvalidOid; /* Look up BM25 access method OID */ bm25_am_oid = get_am_oid("bm25", true); if (!OidIsValid(bm25_am_oid)) return InvalidOid; /* Get the table this index is on */ idx_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(parent_index_oid)); if (!HeapTupleIsValid(idx_tuple)) return InvalidOid; idx_form = (Form_pg_index)GETSTRUCT(idx_tuple); parent_table_oid = idx_form->indrelid; ReleaseSysCache(idx_tuple); /* Find first child table */ inhrel = table_open(InheritsRelationId, AccessShareLock); ScanKeyInit( &skey, Anum_pg_inherits_inhparent, BTEqualStrategyNumber, F_OIDEQ, ObjectIdGetDatum(parent_table_oid)); scan = systable_beginscan( inhrel, InheritsParentIndexId, true, NULL, 1, &skey); while ((tuple = systable_getnext(scan)) != NULL && result == InvalidOid) { Form_pg_inherits inhform = (Form_pg_inherits)GETSTRUCT(tuple); Oid child_table_oid = inhform->inhrelid; Relation child_table; List *child_indexes; ListCell *lc; child_table = table_open(child_table_oid, AccessShareLock); child_indexes = RelationGetIndexList(child_table); foreach (lc, child_indexes) { Oid child_idx_oid = lfirst_oid(lc); HeapTuple child_idx_tuple; HeapTuple child_class_tuple; Form_pg_index child_idx_form; Form_pg_class child_class_form; AttrNumber child_attnum; char *child_colname; child_idx_tuple = SearchSysCache1( INDEXRELID, ObjectIdGetDatum(child_idx_oid)); if (!HeapTupleIsValid(child_idx_tuple)) continue; child_idx_form = (Form_pg_index)GETSTRUCT(child_idx_tuple); /* * Match by column name instead of raw attnum. * Dropped columns can cause attnum drift between * parent and child tables. */ if (child_idx_form->indnatts < 1) { ReleaseSysCache(child_idx_tuple); continue; } child_attnum = child_idx_form->indkey.values[0]; child_colname = get_attname(child_table_oid, child_attnum, true); if (!child_colname || strcmp(child_colname, indexed_colname) != 0) { ReleaseSysCache(child_idx_tuple); continue; } /* Check it's a BM25 index */ child_class_tuple = SearchSysCache1(RELOID, ObjectIdGetDatum(child_idx_oid)); if (!HeapTupleIsValid(child_class_tuple)) { ReleaseSysCache(child_idx_tuple); continue; } child_class_form = (Form_pg_class)GETSTRUCT(child_class_tuple); if (child_class_form->relam == bm25_am_oid) { result = child_idx_oid; ReleaseSysCache(child_class_tuple); ReleaseSysCache(child_idx_tuple); break; } ReleaseSysCache(child_class_tuple); ReleaseSysCache(child_idx_tuple); } list_free(child_indexes); table_close(child_table, AccessShareLock); } systable_endscan(scan); table_close(inhrel, AccessShareLock); return result; } /* * Helper: Validate query and get index * Returns opened index relation, sets index_oid_out if provided * * For partitioned indexes (created on partitioned tables), this function * finds the first child index to use for text config. The actual corpus * stats are aggregated from all children later. */ static Relation validate_and_open_index(TpQuery *query, Oid *index_oid_out) { Oid index_oid = get_tpquery_index_oid(query); Relation index_rel; char relkind; if (!OidIsValid(index_oid)) { /* No index resolved - return ERROR */ ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("text <@> tpquery operator requires index"), errhint("Use to_tpquery(text, index_name) for standalone " "scoring"))); } /* * Enforce the invoking role's SELECT privilege on the indexed relation * and column before reading any index-derived data. Standalone scoring * opens the index directly, bypassing the executor's range-table * permission checks. */ tp_check_index_read_privilege(index_oid); /* * Check if this is a partitioned index. Partitioned indexes don't have * storage - they're just templates. We need to use a child index for * text config, but aggregate stats from all children. */ relkind = get_rel_relkind(index_oid); if (relkind == RELKIND_PARTITIONED_INDEX) { Oid child_index_oid = find_first_child_index(index_oid); if (!OidIsValid(child_index_oid)) { ereport(ERROR, (errcode(ERRCODE_OBJECT_NOT_IN_PREREQUISITE_STATE), errmsg("partitioned index has no child partitions"), errdetail( "The index \"%s\" is a partitioned index with " "no partition indexes.", get_rel_name(index_oid)))); } /* Use the child index for text config access */ index_rel = index_open(child_index_oid, AccessShareLock); } else { /* Open the index relation directly */ index_rel = index_open(index_oid, AccessShareLock); } if (index_oid_out) *index_oid_out = index_oid; return index_rel; } /* * Helper: Find term frequency in document by linear scan of the * (terms, frequencies) arrays returned by tp_tokenize_text. */ static float4 find_term_frequency_in_arrays( char **doc_terms, int32 *doc_frequencies, int doc_term_count, char *query_lexeme) { int i; for (i = 0; i < doc_term_count; i++) { if (strcmp(doc_terms[i], query_lexeme) == 0) return (float4)doc_frequencies[i]; } return 0.0f; /* Term not found */ } /* * Helper: Calculate BM25 score for a single term */ static float4 calculate_term_score( float4 tf, float4 idf, float4 doc_length, float4 avg_doc_len, float4 k1, float4 b, int query_freq) { double numerator_d; double denominator_d; float4 term_score; /* BM25 formula: IDF * tf*(k1+1) / (tf + k1*(1-b+b*dl/avgdl)) */ numerator_d = (double)tf * ((double)k1 + 1.0); if (avg_doc_len > 0.0f) { denominator_d = (double)tf + (double)k1 * (1.0 - (double)b + (double)b * ((double)doc_length / (double)avg_doc_len)); } else { denominator_d = (double)tf + (double)k1; } term_score = (float4)((double)idf * (numerator_d / denominator_d) * (double)query_freq); return term_score; } static QueryScoreCache * tp_standalone_prepare_cache( FunctionCallInfo fcinfo, Oid index_oid, TpScoringSnapshot *snapshot, TpIndexMetaPage *metap_out, Oid *text_config_oid, int32 *total_docs, float4 *avg_doc_len) { TpIndexMetaPage metap; QueryScoreCache *cache; Assert(snapshot != NULL); Assert(metap_out != NULL); Assert(text_config_oid != NULL); Assert(total_docs != NULL); Assert(avg_doc_len != NULL); metap = &snapshot->graph->metapage; *metap_out = metap; *text_config_oid = metap->text_config_oid; *total_docs = snapshot->total_docs; *avg_doc_len = snapshot->avg_doc_len; cache = (QueryScoreCache *)fcinfo->flinfo->fn_extra; if (!cache_is_valid(cache, index_oid, snapshot->serial)) { MemoryContext cache_context; if (cache != NULL) MemoryContextDelete(cache->context); cache_context = AllocSetContextCreate( fcinfo->flinfo->fn_mcxt, "pg_textsearch standalone score cache", ALLOCSET_DEFAULT_SIZES); cache = (QueryScoreCache *) MemoryContextAllocZero(cache_context, sizeof(QueryScoreCache)); cache->index_oid = index_oid; cache->snapshot_serial = snapshot->serial; cache->total_docs = *total_docs; cache->avg_doc_len = *avg_doc_len; cache->num_terms = 0; cache->context = cache_context; fcinfo->flinfo->fn_extra = cache; } return cache; } /* * BM25 scoring function for text <@> bm25query operations * * This operator is called per-row when scoring documents, so we use fn_extra * to cache IDF values across rows. Without caching, each row would require * opening all segments for each query term - catastrophic for large indexes. */ Datum bm25_text_bm25query_score(PG_FUNCTION_ARGS) { text *text_arg = PG_GETARG_TEXT_PP(0); TpQuery *query = (TpQuery *)PG_DETOAST_DATUM(PG_GETARG_DATUM(1)); char *query_text = get_tpquery_text(query); Oid index_oid; /* * Reassigned inside PG_TRY and read by PG_CATCH, so they must survive * the longjmp. */ Relation volatile index_rel = NULL; TpIndexMetaPage metap = NULL; TpScoringSnapshot *scoring; TpSegmentGraphSnapshot *segment_snapshot; Oid text_config_oid; char **doc_terms = NULL; int32 *doc_frequencies = NULL; int doc_term_count = 0; int raw_doc_length; Datum query_tsvector_datum; TSVector query_tsvector; WordEntry *query_entries; char *query_lexemes_start; TpLocalIndexState *index_state; TpDataSource *volatile memtable_src = NULL; float4 avg_doc_len; int32 total_docs; float8 result = 0.0; int q_i; float4 doc_length; int query_term_count; QueryScoreCache *cache; bool is_partitioned; char *indexed_colname = NULL; /* Get index OID from query */ index_oid = get_tpquery_index_oid(query); /* * Validate query and open index. For partitioned indexes, this opens * the first child index for text config access. */ index_rel = validate_and_open_index(query, &index_oid); /* Check if this is a partitioned index */ is_partitioned = (get_rel_relkind(index_oid) == RELKIND_PARTITIONED_INDEX); /* * Get the indexed column name for child index matching. * We use the column name rather than the raw attnum because * dropped columns can cause attnum drift between parent and * child tables (e.g., TimescaleDB hypertables). */ { HeapTuple idx_tuple; Form_pg_index idx_form; Oid lookup_oid; lookup_oid = index_oid; idx_tuple = SearchSysCache1(INDEXRELID, ObjectIdGetDatum(lookup_oid)); if (HeapTupleIsValid(idx_tuple)) { idx_form = (Form_pg_index)GETSTRUCT(idx_tuple); if (idx_form->indnatts >= 1) { AttrNumber attnum = idx_form->indkey.values[0]; indexed_colname = get_attname(idx_form->indrelid, attnum, true); } ReleaseSysCache(idx_tuple); } } PG_TRY(); { /* * Surface the pre-v1.3 upgrade data-loss marker to standalone * scoring callers too. This path does not go through * tp_beginscan, so without this the WARNING would only reach * index-scan (ORDER BY ... LIMIT) workloads. The per-session * throttle is shared, so at most one warning per index fires. */ tp_warn_if_pending_docid(index_rel); if (is_partitioned) index_state = tp_get_local_index_state( RelationGetRelid(index_rel)); else index_state = tp_get_local_index_state(index_oid); if (!index_state) ereport(ERROR, (errcode(ERRCODE_INTERNAL_ERROR), errmsg("could not get index state for index OID %u", RelationGetRelid(index_rel)))); scoring = tp_scoring_snapshot_get(index_rel, NULL, 0, NULL); segment_snapshot = scoring->graph; /* * If a storage-less inheritance parent was selected, switch to its * first physical child and use its scoring snapshot. */ if (!is_partitioned && scoring->total_docs == 0 && indexed_colname != NULL) { Oid first_child_idx = find_first_child_bm25_index(index_oid, indexed_colname); if (OidIsValid(first_child_idx)) { TpLocalIndexState *child_state = tp_get_local_index_state( first_child_idx); if (child_state && child_state->shared) { Relation old_rel = index_rel; metap = NULL; index_rel = index_open(first_child_idx, AccessShareLock); index_close(old_rel, AccessShareLock); index_state = child_state; scoring = tp_scoring_snapshot_get(index_rel, NULL, 0, NULL); segment_snapshot = scoring->graph; } } } cache = tp_standalone_prepare_cache( fcinfo, index_oid, scoring, &metap, &text_config_oid, &total_docs, &avg_doc_len); /* Tokenize the query text to get query terms (always small) */ query_tsvector_datum = DirectFunctionCall2Coll( to_tsvector_byid, InvalidOid, ObjectIdGetDatum(text_config_oid), PointerGetDatum(cstring_to_text(query_text))); query_tsvector = DatumGetTSVector(query_tsvector_datum); query_entries = ARRPTR(query_tsvector); query_lexemes_start = STRPTR(query_tsvector); /* Cache-hit rows need no memtable source or dictionary lookups. */ raw_doc_length = tp_tokenize_text( text_arg, text_config_oid, &doc_terms, &doc_frequencies, &doc_term_count); if (query_cache_misses_document_term( cache, query_tsvector, doc_terms, doc_frequencies, doc_term_count)) { memtable_src = tp_memtable_source_create_for_snapshot( index_state, index_rel, &segment_snapshot->memtable, scoring->recovery, NULL, 0); } /* * Calculate document length with fieldnorm quantization. * We use the same encode/decode as segments to ensure the operator * produces identical scores to index scans. This quantization is * a deliberate approximation following Lucene's SmallFloat scheme. */ doc_length = (float4)decode_fieldnorm( encode_fieldnorm(raw_doc_length)); query_term_count = query_tsvector->size; /* Calculate BM25 score for each query term */ for (q_i = 0; q_i < query_term_count; q_i++) { char *query_lexeme_raw = query_lexemes_start + query_entries[q_i].pos; int lexeme_len = query_entries[q_i].len; char *query_lexeme; float4 idf; float4 tf; /* term frequency in document */ float4 term_score; int query_freq; /* Create properly null-terminated string from tsvector lexeme */ query_lexeme = palloc(lexeme_len + 1); memcpy(query_lexeme, query_lexeme_raw, lexeme_len); query_lexeme[lexeme_len] = '\0'; if (query_entries[q_i].haspos) query_freq = (int32) POSDATALEN(query_tsvector, &query_entries[q_i]); else query_freq = 1; /* Find term frequency in the document */ tf = find_term_frequency_in_arrays( doc_terms, doc_frequencies, doc_term_count, query_lexeme); if (tf == 0.0f) { pfree(query_lexeme); continue; /* Query term not in document */ } /* * Get IDF from cache if available. The cache avoids repeated * segment opens which was causing catastrophic performance * (opening all segments for each term for each row). */ { uint32 cached_doc_freq = 0; idf = lookup_cached_idf(cache, query_lexeme, &cached_doc_freq); if (idf < 0.0f) { /* Cache miss - calculate IDF and cache it */ uint32 unified_doc_freq = 0; uint32 memtable_doc_freq = 0; uint32 segment_doc_freq = 0; /* * Phase 4 of issue #374: query the memtable via the * chain source built above (DSA posting lists are dead * code in the live path). */ if (memtable_src != NULL) { TpPostingData *postings = tp_source_get_postings( memtable_src, query_lexeme); if (postings != NULL) { memtable_doc_freq = (uint32)postings->count; tp_source_free_postings(memtable_src, postings); } } Assert(segment_snapshot->roots_captured); segment_doc_freq = tp_segment_roots_get_doc_freq( index_rel, segment_snapshot->roots, segment_snapshot->root_count, query_lexeme); unified_doc_freq = memtable_doc_freq + segment_doc_freq; if (unified_doc_freq == 0) { pfree(query_lexeme); continue; } /* Calculate IDF and cache it */ idf = tp_calculate_idf(unified_doc_freq, total_docs); cache_term_idf(cache, query_lexeme, unified_doc_freq, idf); } } /* Calculate BM25 term score */ term_score = calculate_term_score( tf, idf, doc_length, avg_doc_len, metap->k1, metap->b, query_freq); /* Accumulate the score */ result += term_score; /* Free the null-terminated string copy */ pfree(query_lexeme); } /* Clean up */ if (memtable_src != NULL) { tp_source_close(memtable_src); memtable_src = NULL; } metap = NULL; index_close(index_rel, AccessShareLock); index_rel = NULL; } PG_CATCH(); { if (memtable_src != NULL) tp_source_close(memtable_src); if (index_rel) index_close(index_rel, AccessShareLock); PG_RE_THROW(); } PG_END_TRY(); /* Return negative score for PostgreSQL ASC ordering compatibility */ PG_RETURN_FLOAT8((result > 0) ? -result : result); } /* * tpquery equality function */ Datum tpquery_eq(PG_FUNCTION_ARGS) { TpQuery *a = (TpQuery *)PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); TpQuery *b = (TpQuery *)PG_DETOAST_DATUM(PG_GETARG_DATUM(1)); /* Compare index OIDs */ if (a->index_oid != b->index_oid) PG_RETURN_BOOL(false); /* Compare query text lengths */ if (a->query_text_len != b->query_text_len) PG_RETURN_BOOL(false); /* Compare query texts */ { char *query_text_a = get_tpquery_text(a); char *query_text_b = get_tpquery_text(b); if (strcmp(query_text_a, query_text_b) != 0) PG_RETURN_BOOL(false); } PG_RETURN_BOOL(true); } /* * Utility function to create a tpquery with resolved index OID and flags. */ TpQuery * create_tpquery_explicit( const char *query_text, Oid index_oid, bool explicit_index) { TpQuery *result; int query_text_len = strlen(query_text); int total_size; /* Calculate total size */ total_size = offsetof(TpQuery, data) + query_text_len + 1; result = (TpQuery *)palloc0(total_size); SET_VARSIZE(result, total_size); result->version = TPQUERY_VERSION; result->flags = explicit_index ? TPQUERY_FLAG_EXPLICIT_INDEX : 0; result->index_oid = index_oid; result->query_text_len = query_text_len; /* Copy query text */ memcpy(result->data, query_text, query_text_len); result->data[query_text_len] = '\0'; return result; } /* * Utility function to create a tpquery with resolved index OID. * Index is marked as NOT explicit (for implicit resolution). */ TpQuery * create_tpquery(const char *query_text, Oid index_oid) { return create_tpquery_explicit(query_text, index_oid, false); } /* * Create a tpquery from index name (resolves name to OID). * Index is marked as EXPLICIT since user provided the name. * * Partitioned indexes are allowed - they will be resolved to the * appropriate partition index at scan time. */ TpQuery * create_tpquery_from_name(const char *query_text, const char *index_name) { Oid index_oid = InvalidOid; bool explicit_index = false; if (index_name != NULL) { index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("index \"%s\" does not exist", index_name))); } explicit_index = true; } return create_tpquery_explicit(query_text, index_oid, explicit_index); } /* * Get index OID from tpquery (returns InvalidOid if unresolved) */ Oid get_tpquery_index_oid(TpQuery *tpquery) { return tpquery->index_oid; } /* * Get query text from tpquery */ char * get_tpquery_text(TpQuery *tpquery) { return TPQUERY_TEXT_PTR(tpquery); } /* * Check if tpquery has a resolved index */ bool tpquery_has_index(TpQuery *tpquery) { return OidIsValid(tpquery->index_oid); } /* * Check if tpquery has an explicitly specified index. * Returns true if the user specified an index via to_bm25query(text, * index_name) or bm25query('index_name:text'). * Returns false for implicitly resolved indexes. */ bool tpquery_is_explicit_index(TpQuery *tpquery) { /* Version 1 didn't have flags, treat as non-explicit */ if (tpquery->version < 2) return false; return (tpquery->flags & TPQUERY_FLAG_EXPLICIT_INDEX) != 0; } /* * Make a private planner copy carrying scan-local LIMIT information. The * hint is an optional trailer after the existing query string, so values * produced by older versions remain valid and their textual/binary form is * unchanged. It is deliberately ignored by output, equality, and scoring. */ TpQuery * tpquery_copy_with_seed_hint(TpQuery *tpquery, int64 k, double selectivity) { Size old_size = VARSIZE_ANY(tpquery); Size new_size = old_size + sizeof(TpQuerySeedHint); TpQuerySeedHint hint; TpQuery *copy = (TpQuery *)palloc(new_size); memcpy(copy, tpquery, old_size); SET_VARSIZE(copy, new_size); copy->flags |= TPQUERY_FLAG_SEED_HINT; memset(&hint, 0, sizeof(hint)); hint.magic = TPQUERY_SEED_HINT_MAGIC; hint.k = k; hint.selectivity = selectivity; memcpy((char *)copy + old_size, &hint, sizeof(hint)); return copy; } bool tpquery_get_seed_hint(TpQuery *tpquery, int64 *k, double *selectivity) { Size base = offsetof(TpQuery, data) + tpquery->query_text_len + 1; TpQuerySeedHint hint; if ((tpquery->flags & TPQUERY_FLAG_SEED_HINT) == 0 || VARSIZE_ANY(tpquery) < base + sizeof(hint)) return false; memcpy(&hint, (char *)tpquery + base, sizeof(hint)); if (hint.magic != TPQUERY_SEED_HINT_MAGIC || hint.k <= 0 || hint.k > INT_MAX || !isfinite(hint.selectivity)) return false; *k = hint.k; *selectivity = hint.selectivity; return true; } /* * Scoring function for text <@> text operations. * * This function should never actually be called because the planner hook * transforms all `text <@> text` expressions to `text <@> bm25query`. * We keep this as a fallback that errors with a helpful message. */ Datum bm25_text_text_score(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("no BM25 index found for text <@> text expression"), errdetail("Create a BM25 index on the column or use ORDER BY."), errhint("SELECT col <@> to_bm25query('q', 'idx') AS score"))); PG_RETURN_NULL(); /* never reached */ } /* * Standalone scoring for text[] <@> bm25query. * * Flattens the array elements into a single space-separated text * value, then delegates to bm25_text_bm25query_score(). */ Datum bm25_textarray_bm25query_score(PG_FUNCTION_ARGS) { Datum array_datum = PG_GETARG_DATUM(0); text *flattened = tp_flatten_text_array(array_datum); /* * Replace the first argument with the flattened text, * then delegate to the scalar scoring function. */ fcinfo->args[0].value = PointerGetDatum(flattened); fcinfo->args[0].isnull = false; return bm25_text_bm25query_score(fcinfo); } /* * Error stub for text[] <@> text when planner rewrite fails. */ Datum bm25_textarray_text_score(PG_FUNCTION_ARGS) { ereport(ERROR, (errcode(ERRCODE_UNDEFINED_OBJECT), errmsg("no BM25 index found for text[] <@> text " "expression"), errdetail( "Create a BM25 index on the column or " "use ORDER BY."), errhint("SELECT col <@> to_bm25query('q', 'idx') " "AS score"))); PG_RETURN_NULL(); /* never reached */ } timescale-pg_textsearch-b63fe58/src/types/query.h000066400000000000000000000054521525762047700222400ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * query.h - bm25query data type interface */ #pragma once #include #include /* * bm25query binary format version * * Version history: * 0: Pre-0.0.6 format with index_name string (no longer supported) * 1: 0.0.6+ format with index_oid * 2: 0.5.0+ adds explicit_index flag */ #define TPQUERY_VERSION 2 /* * Flags for TpQuery */ #define TPQUERY_FLAG_EXPLICIT_INDEX 0x01 /* Index was explicitly specified */ #define TPQUERY_FLAG_SEED_HINT 0x02 /* Planner-only filtered-seed hint */ #define TPQUERY_SEED_HINT_MAGIC 0x54505348U /* "TPSH" */ typedef struct TpQuerySeedHint { uint32 magic; int64 k; double selectivity; } TpQuerySeedHint; /* * tpquery data type structure * Represents a query for BM25 scoring with optional index reference * * The index can be specified by OID (resolved at creation time) or left * unresolved (InvalidOid) for later resolution by planner hooks. */ typedef struct TpQuery { int32 vl_len_; /* varlena header (must be first) */ uint8 version; /* binary format version */ uint8 flags; /* flags (TPQUERY_FLAG_*) */ Oid index_oid; /* resolved index OID (InvalidOid if * unresolved) */ int32 query_text_len; /* length of query text */ char data[FLEXIBLE_ARRAY_MEMBER]; /* payload: query text only */ } TpQuery; /* Macro for accessing query text */ #define TPQUERY_TEXT_PTR(x) (((TpQuery *)(x))->data) /* Function declarations */ PGDLLEXPORT Datum tpquery_in(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpquery_out(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpquery_recv(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpquery_send(PG_FUNCTION_ARGS); /* Constructor functions */ PGDLLEXPORT Datum to_tpquery_text(PG_FUNCTION_ARGS); PGDLLEXPORT Datum to_tpquery_text_index(PG_FUNCTION_ARGS); /* Operator functions */ PGDLLEXPORT Datum bm25_text_bm25query_score(PG_FUNCTION_ARGS); PGDLLEXPORT Datum bm25_text_text_score(PG_FUNCTION_ARGS); PGDLLEXPORT Datum bm25_textarray_bm25query_score(PG_FUNCTION_ARGS); PGDLLEXPORT Datum bm25_textarray_text_score(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpquery_eq(PG_FUNCTION_ARGS); /* Utility functions */ TpQuery *create_tpquery(const char *query_text, Oid index_oid); TpQuery *create_tpquery_explicit( const char *query_text, Oid index_oid, bool explicit_index); TpQuery * create_tpquery_from_name(const char *query_text, const char *index_name); Oid get_tpquery_index_oid(TpQuery *tpquery); char *get_tpquery_text(TpQuery *tpquery); bool tpquery_has_index(TpQuery *tpquery); bool tpquery_is_explicit_index(TpQuery *tpquery); TpQuery * tpquery_copy_with_seed_hint(TpQuery *tpquery, int64 k, double selectivity); bool tpquery_get_seed_hint(TpQuery *tpquery, int64 *k, double *selectivity); timescale-pg_textsearch-b63fe58/src/types/vector.c000066400000000000000000000566051525762047700223760ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * vector.c - bm25vector data type implementation */ #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include #include "access/am.h" #include "constants.h" #include "index/metapage.h" #include "index/resolve.h" #include "types/vector.h" #if SIZEOF_VOID_P == 4 #define CURSOR_END(cursor) ((const uint8 *)UINT32_MAX) #else #define CURSOR_END(cursor) ((const uint8 *)(cursor + UINT32_MAX)) #endif /* --------------------------------------------------------------------- * Legacy v1 detection * * pg_textsearch < 1.2.0 used a different on-disk bm25vector layout * (no magic, int32 freq + int32 lexeme_len per entry, MAXALIGN * padding). Storing bm25vector values in user tables was always * uncommon — the type is overwhelmingly used transiently for * query/insert paths — so we no longer support reading legacy v1 * values. Detection of a non-v2 value produces a clear * REINDEX-or-recompute error instead of silently misinterpreting * the bytes. * --------------------------------------------------------------------- */ /* --------------------------------------------------------------------- * Varint helpers (LEB128 base-128 little-endian) * --------------------------------------------------------------------- */ size_t tpvector_varint_encode(uint32 v, uint8 *out) { size_t n = 0; while (v >= 0x80) { out[n++] = (uint8)(v | 0x80); v >>= 7; } out[n++] = (uint8)v; return n; } uint32 tpvector_varint_decode(const uint8 **cursor, const uint8 *end) { uint32 result = 0; int shift = 0; const uint8 *p = *cursor; while (p < end && (*p & 0x80) != 0) { result |= ((uint32)(*p & 0x7F)) << shift; shift += 7; p++; if (shift >= 32) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector varint exceeds 32 bits"))); } if (p >= end) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector varint truncated"))); result |= ((uint32)*p) << shift; p++; *cursor = p; return result; } size_t tpvector_varint_size(uint32 v) { size_t n = 1; while (v >= 0x80) { v >>= 7; n++; } return n; } /* --------------------------------------------------------------------- * Local helpers * --------------------------------------------------------------------- */ typedef struct { const char *lexeme; int32 frequency; } LexemeFreqPair; static int strcmp_wrapper(const void *a, const void *b) { const LexemeFreqPair *pair_a = (const LexemeFreqPair *)a; const LexemeFreqPair *pair_b = (const LexemeFreqPair *)b; return strcmp(pair_a->lexeme, pair_b->lexeme); } #ifndef min #define min(a, b) ((a) < (b) ? (a) : (b)) #endif PG_FUNCTION_INFO_V1(tpvector_in); PG_FUNCTION_INFO_V1(tpvector_out); PG_FUNCTION_INFO_V1(tpvector_recv); PG_FUNCTION_INFO_V1(tpvector_send); PG_FUNCTION_INFO_V1(tpvector_eq); PG_FUNCTION_INFO_V1(to_tpvector); /* --------------------------------------------------------------------- * Format detection and v1 → v2 conversion * --------------------------------------------------------------------- */ bool tpvector_is_v2(const void *raw) { const char *bytes = (const char *)raw; if (raw == NULL || VARSIZE(raw) < (Size)(sizeof(int32) + 4)) return false; bytes += sizeof(int32); /* skip vl_len_ */ return bytes[0] == TPVECTOR_V2_MAGIC0 && bytes[1] == TPVECTOR_V2_MAGIC1 && bytes[2] == TPVECTOR_V2_MAGIC2 && bytes[3] == TPVECTOR_V2_MAGIC3; } /* * Validate a v2 TpVector's entry stream against the varlena's * declared length. Walks each entry, decoding varint freq + lex_len * and ensuring the lexeme bytes stay in bounds. Errors on any * inconsistency. * * Called from tpvector_canonicalize for every v2 input so that * operator entry points (tpvector_out, tpvector_eq, etc.) can * iterate entries without re-checking bounds at every read. The * pre-v2 code had per-entry checks scattered throughout each * operator; this centralizes them. */ static void tpvector_validate_v2(TpVector *v) { const uint8 *cursor; const uint8 *end; int i; if (VARSIZE(v) < sizeof(TpVector)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("v2 bm25vector too small: %zu", (Size)VARSIZE(v)))); if (v->version != TPVECTOR_VERSION) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("unsupported bm25vector version: %u", v->version))); if (v->index_name_len < 0 || v->index_name_len > TP_MAX_INDEX_NAME_LENGTH) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid index name length in bm25vector: %d", v->index_name_len))); if (v->entry_count < 0) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid entry count in bm25vector: %d", v->entry_count))); cursor = (const uint8 *)TPVECTOR_ENTRIES_PTR(v); end = (const uint8 *)v + VARSIZE(v); if ((const char *)cursor > (const char *)end) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector header overruns varlena"))); for (i = 0; i < v->entry_count; i++) { uint32 freq; uint32 lex_len; /* Keep validation of long streams cancellable */ if (i > 0 && (i % 1000) == 0) CHECK_FOR_INTERRUPTS(); if (cursor >= end) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("v2 bm25vector entry %d truncated", i))); freq = tpvector_varint_decode(&cursor, end); lex_len = tpvector_varint_decode(&cursor, end); (void)freq; if (cursor + lex_len > end) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("v2 bm25vector entry %d lexeme " "extends beyond buffer", i))); /* * Lexemes are rendered verbatim into a cstring by * tpvector_out; an embedded NUL would truncate the text * output at that byte. A NUL can never come from tokenized * text or the text input parser (both NUL-terminated), so a * lexeme carrying one can only arrive via crafted binary * input. Reject it here rather than emit a malformed value. */ if (memchr(cursor, '\0', lex_len) != NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("v2 bm25vector entry %d lexeme " "contains an embedded NUL byte", i))); cursor += lex_len; } } TpVector * tpvector_canonicalize(TpVector *raw) { if (raw == NULL) return NULL; if (!tpvector_is_v2(raw)) ereport(ERROR, (errcode(ERRCODE_FEATURE_NOT_SUPPORTED), errmsg("bm25vector value uses unsupported legacy format"), errdetail( "Persisted bm25vector values from " "pg_textsearch versions prior to 1.2.0 cannot " "be read by this version."), errhint("Recompute the value via " "to_bm25vector(text, 'index_name') from the " "source text, or DROP and recreate the column."))); tpvector_validate_v2(raw); return raw; } void tpvector_validate_v2_buffer(const char *bytes, uint32 len) { TpVector *v; if (bytes == NULL) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector buffer is NULL"))); if (len < sizeof(TpVector)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector buffer too small: %u bytes, need at " "least %zu", len, sizeof(TpVector)))); v = (TpVector *)bytes; if (v->magic[0] != TPVECTOR_V2_MAGIC0 || v->magic[1] != TPVECTOR_V2_MAGIC1 || v->magic[2] != TPVECTOR_V2_MAGIC2 || v->magic[3] != TPVECTOR_V2_MAGIC3) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector buffer has wrong magic bytes"))); if ((uint32)VARSIZE(v) != len) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("bm25vector varlena length %u does not match buffer " "length %u", (uint32)VARSIZE(v), len))); tpvector_validate_v2(v); } /* --------------------------------------------------------------------- * Entry iteration (v2) * --------------------------------------------------------------------- */ void tpvector_entry_decode(const TpVectorEntry *entry, TpVectorEntryView *out) { const uint8 *cursor = (const uint8 *)entry; const uint8 *end = CURSOR_END(cursor); out->frequency = tpvector_varint_decode(&cursor, end); out->lexeme_len = tpvector_varint_decode(&cursor, end); out->lexeme = (const char *)cursor; } TpVectorEntry * get_tpvector_first_entry(TpVector *vec) { if (!vec || vec->entry_count == 0) return NULL; return TPVECTOR_ENTRIES_PTR(vec); } TpVectorEntry * get_tpvector_next_entry(TpVectorEntry *current) { const uint8 *cursor; const uint8 *end; uint32 freq; uint32 lex_len; if (!current) return NULL; cursor = (const uint8 *)current; end = CURSOR_END(cursor); freq = tpvector_varint_decode(&cursor, end); lex_len = tpvector_varint_decode(&cursor, end); (void)freq; cursor += lex_len; return (TpVectorEntry *)cursor; } /* * Decode the entry pointed at by `entry` into `*out` and return * the pointer to the next entry in the stream — in one varint * pass, not two. This is the hot-path iterator: the legacy * pattern of `tpvector_entry_decode(e, &v); e = * get_tpvector_next_entry(e);` re-decodes both varints just to * advance the cursor, and shows up clearly in per-record profiles * on read-heavy workloads against the on-disk memtable chain. * * The returned pointer is `(uint8 *)entry + 2 varints + lex_len`, * computed from the cursor that `tpvector_varint_decode` already * advanced — no extra work. */ TpVectorEntry * tpvector_entry_decode_advance( const TpVectorEntry *entry, TpVectorEntryView *out) { const uint8 *cursor = (const uint8 *)entry; const uint8 *end = CURSOR_END(cursor); out->frequency = tpvector_varint_decode(&cursor, end); out->lexeme_len = tpvector_varint_decode(&cursor, end); out->lexeme = (const char *)cursor; return (TpVectorEntry *)(cursor + out->lexeme_len); } /* --------------------------------------------------------------------- * tpvector_in: text input * --------------------------------------------------------------------- */ Datum tpvector_in(PG_FUNCTION_ARGS) { /* * Copy the input: parsing below writes NUL terminators into the string * in place. The argument cstring must not be mutated, or a re-evaluated * const-folded literal would see a truncated buffer. */ char *str = pstrdup(PG_GETARG_CSTRING(0)); char *colon_pos; char *index_name; char *entries_str; TpVector *result; int index_name_len; int entry_count = 0; char **lexemes = NULL; int32 *frequencies = NULL; char *ptr; char *end_ptr; int i = 0; colon_pos = strchr(str, ':'); if (colon_pos == NULL) ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("invalid input syntax for type tpvector: \"%s\"", str), errhint("Expected format: " "\"index_name:{lexeme:freq,...}\""))); index_name_len = colon_pos - str; index_name = palloc(index_name_len + 1); memcpy(index_name, str, index_name_len); index_name[index_name_len] = '\0'; entries_str = colon_pos + 1; if (!entries_str || strlen(entries_str) < 2 || entries_str[0] != '{' || entries_str[strlen(entries_str) - 1] != '}') ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("invalid tpvector format: \"%s\"", str), errhint("Entries must be enclosed in braces: " "{lexeme:freq,...}"))); entries_str++; end_ptr = entries_str + strlen(entries_str) - 1; *end_ptr = '\0'; if (strlen(entries_str) > 0) { entry_count = 1; for (ptr = entries_str; *ptr; ptr++) { if (*ptr == ',') entry_count++; } } if (entry_count > 0) { lexemes = palloc(entry_count * sizeof(char *)); frequencies = palloc(entry_count * sizeof(int32)); ptr = entries_str; for (i = 0; i < entry_count; i++) { char *comma_pos = strchr(ptr, ','); char *entry_colon_pos = strchr(ptr, ':'); char *lexeme; int lexeme_len; char *freq_str; if (!entry_colon_pos || (comma_pos && entry_colon_pos > comma_pos)) ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("invalid entry format in tpvector: \"%s\"", ptr))); lexeme_len = entry_colon_pos - ptr; lexeme = palloc(lexeme_len + 1); memcpy(lexeme, ptr, lexeme_len); lexeme[lexeme_len] = '\0'; lexemes[i] = lexeme; freq_str = entry_colon_pos + 1; if (comma_pos) { char saved = *comma_pos; *comma_pos = '\0'; frequencies[i] = pg_strtoint32(freq_str); *comma_pos = saved; ptr = comma_pos + 1; } else { frequencies[i] = pg_strtoint32(freq_str); } /* * v2 stores frequency as an unsigned varint; negative * input is meaningless for BM25 and would silently * clamp to zero. Reject explicitly so callers see a * parse error rather than a silent normalization. */ if (frequencies[i] < 0) ereport(ERROR, (errcode(ERRCODE_INVALID_TEXT_REPRESENTATION), errmsg("negative frequency in tpvector entry: " "%d", frequencies[i]))); } } result = create_tpvector_from_strings( index_name, entry_count, (const char **)lexemes, frequencies); pfree(index_name); if (lexemes) { for (i = 0; i < entry_count; i++) pfree(lexemes[i]); pfree(lexemes); } if (frequencies) pfree(frequencies); PG_RETURN_POINTER(result); } /* --------------------------------------------------------------------- * tpvector_out: text output (canonicalized) * --------------------------------------------------------------------- */ Datum tpvector_out(PG_FUNCTION_ARGS) { TpVector *tpvec; char *index_name = NULL; StringInfoData result; TpVectorEntry *entry; int i; tpvec = (TpVector *)PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); if (!tpvec) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("null tpvector passed to tpvector_out"))); tpvec = tpvector_canonicalize(tpvec); if (VARSIZE(tpvec) < sizeof(TpVector)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid tpvector structure"))); index_name = get_tpvector_index_name(tpvec); initStringInfo(&result); appendStringInfo(&result, "%s:{", index_name); entry = TPVECTOR_ENTRIES_PTR(tpvec); for (i = 0; i < tpvec->entry_count; i++) { TpVectorEntryView v; /* Keep long outputs cancellable; entry_count is uncapped */ if (i > 0 && (i % 1000) == 0) CHECK_FOR_INTERRUPTS(); /* Decode this entry and advance in a single varint pass */ entry = tpvector_entry_decode_advance(entry, &v); if (i > 0) appendStringInfoChar(&result, ','); appendBinaryStringInfo(&result, v.lexeme, (int)v.lexeme_len); appendStringInfo(&result, ":%u", v.frequency); } appendStringInfoChar(&result, '}'); pfree(index_name); PG_RETURN_CSTRING(result.data); } /* --------------------------------------------------------------------- * tpvector_recv: binary receive (accepts v1 + v2) * --------------------------------------------------------------------- */ Datum tpvector_recv(PG_FUNCTION_ARGS) { StringInfo buf = (StringInfo)PG_GETARG_POINTER(0); int total_size; TpVector *result; total_size = pq_getmsgint(buf, 4); if (total_size < (int)sizeof(TpVector) || (Size)total_size > MaxAllocSize) ereport(ERROR, (errcode(ERRCODE_INVALID_BINARY_REPRESENTATION), errmsg("invalid bm25vector binary size: %d", total_size))); result = (TpVector *)palloc(total_size); SET_VARSIZE(result, total_size); pq_copymsgbytes( buf, (char *)result + sizeof(int32), total_size - sizeof(int32)); /* * Defer to tpvector_canonicalize for magic-check + validation. * Legacy v1 inputs (no magic, pre-1.2.0) are rejected with a * clear error; v2 inputs have their entry stream bounds-checked * via tpvector_validate_v2. */ PG_RETURN_POINTER(tpvector_canonicalize(result)); } /* --------------------------------------------------------------------- * tpvector_send: binary send (always emits v2) * --------------------------------------------------------------------- */ Datum tpvector_send(PG_FUNCTION_ARGS) { TpVector *tpvec; StringInfoData buf; int total_size; tpvec = (TpVector *)PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); tpvec = tpvector_canonicalize(tpvec); pq_begintypsend(&buf); total_size = VARSIZE(tpvec); pq_sendint32(&buf, total_size); pq_sendbytes( &buf, (char *)tpvec + sizeof(int32), total_size - sizeof(int32)); PG_RETURN_BYTEA_P(pq_endtypsend(&buf)); } /* --------------------------------------------------------------------- * tpvector_eq: equality * --------------------------------------------------------------------- */ Datum tpvector_eq(PG_FUNCTION_ARGS) { TpVector *vec1 = (TpVector *)PG_DETOAST_DATUM(PG_GETARG_DATUM(0)); TpVector *vec2 = (TpVector *)PG_DETOAST_DATUM(PG_GETARG_DATUM(1)); char *index_name1; char *index_name2; bool result = true; int i; vec1 = tpvector_canonicalize(vec1); vec2 = tpvector_canonicalize(vec2); index_name1 = get_tpvector_index_name(vec1); index_name2 = get_tpvector_index_name(vec2); if (strcmp(index_name1, index_name2) != 0) { pfree(index_name1); pfree(index_name2); PG_RETURN_BOOL(false); } if (vec1->entry_count != vec2->entry_count) { pfree(index_name1); pfree(index_name2); PG_RETURN_BOOL(false); } { TpVectorEntry *e1 = get_tpvector_first_entry(vec1); for (i = 0; i < vec1->entry_count && result && e1; i++) { TpVectorEntryView v1; TpVectorEntry *e2 = get_tpvector_first_entry(vec2); bool found_match = false; int j; /* Outer scan is O(n*m); keep it cancellable */ if (i > 0 && (i % 1000) == 0) CHECK_FOR_INTERRUPTS(); tpvector_entry_decode(e1, &v1); for (j = 0; j < vec2->entry_count && e2; j++) { TpVectorEntryView v2; if (j > 0 && (j % 1000) == 0) CHECK_FOR_INTERRUPTS(); tpvector_entry_decode(e2, &v2); if (v1.lexeme_len == v2.lexeme_len && v1.frequency == v2.frequency && memcmp(v1.lexeme, v2.lexeme, v1.lexeme_len) == 0) { found_match = true; break; } e2 = get_tpvector_next_entry(e2); } if (!found_match) result = false; e1 = get_tpvector_next_entry(e1); } } pfree(index_name1); pfree(index_name2); PG_RETURN_BOOL(result); } /* --------------------------------------------------------------------- * Accessors * --------------------------------------------------------------------- */ char * get_tpvector_index_name(TpVector *tpvec) { int name_len; char *index_name; char *name_ptr; unsigned int vector_total_size; if (!tpvec) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("null tpvector passed to get_tpvector_index_name"))); vector_total_size = VARSIZE(tpvec); if (vector_total_size < sizeof(TpVector)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid tpvector size: %d", vector_total_size))); name_len = tpvec->index_name_len; if (name_len < 0 || name_len > TP_MAX_INDEX_NAME_LENGTH) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("invalid index name length: %d", name_len))); name_ptr = TPVECTOR_INDEX_NAME_PTR(tpvec); if (name_ptr < (char *)tpvec || name_ptr + name_len > (char *)tpvec + vector_total_size) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("index name data extends beyond vector bounds"))); index_name = palloc(name_len + 1); memcpy(index_name, name_ptr, name_len); index_name[name_len] = '\0'; return index_name; } TpVector * create_tpvector_from_strings( const char *index_name, int entry_count, const char **lexemes, const int32 *frequencies) { int index_name_len; int total_size; int i; TpVector *result; LexemeFreqPair *pairs = NULL; if (!index_name) ereport(ERROR, (errcode(ERRCODE_NULL_VALUE_NOT_ALLOWED), errmsg("null index name to create_tpvector_from_strings"))); index_name_len = strlen(index_name); if (index_name_len == 0) ereport(ERROR, (errcode(ERRCODE_INVALID_PARAMETER_VALUE), errmsg("empty index name in create_tpvector_from_strings"))); if (entry_count > 0 && lexemes && frequencies) { pairs = palloc(entry_count * sizeof(LexemeFreqPair)); for (i = 0; i < entry_count; i++) { pairs[i].lexeme = lexemes[i]; pairs[i].frequency = frequencies[i]; } if (entry_count > 1) qsort(pairs, entry_count, sizeof(LexemeFreqPair), strcmp_wrapper); } total_size = MAXALIGN(sizeof(TpVector)) + MAXALIGN(index_name_len + 1); for (i = 0; i < entry_count; i++) { const char *lex = pairs ? pairs[i].lexeme : (lexemes ? lexemes[i] : NULL); int32 freq = pairs ? pairs[i].frequency : (frequencies ? frequencies[i] : 0); int lex_len; if (!lex) continue; lex_len = strlen(lex); total_size += tpvector_varint_size((uint32)Max(freq, 0)) + tpvector_varint_size((uint32)lex_len) + lex_len; } result = (TpVector *)palloc0(total_size); SET_VARSIZE(result, total_size); result->magic[0] = TPVECTOR_V2_MAGIC0; result->magic[1] = TPVECTOR_V2_MAGIC1; result->magic[2] = TPVECTOR_V2_MAGIC2; result->magic[3] = TPVECTOR_V2_MAGIC3; result->version = TPVECTOR_VERSION; result->index_name_len = index_name_len; result->entry_count = entry_count; memcpy(TPVECTOR_INDEX_NAME_PTR(result), index_name, index_name_len); *((char *)TPVECTOR_INDEX_NAME_PTR(result) + index_name_len) = '\0'; if (pairs) { uint8 *entry_ptr = (uint8 *)TPVECTOR_ENTRIES_PTR(result); for (i = 0; i < entry_count; i++) { uint32 freq = (uint32)Max(pairs[i].frequency, 0); int lex_len = strlen(pairs[i].lexeme); entry_ptr += tpvector_varint_encode(freq, entry_ptr); entry_ptr += tpvector_varint_encode((uint32)lex_len, entry_ptr); memcpy(entry_ptr, pairs[i].lexeme, lex_len); entry_ptr += lex_len; } pfree(pairs); } return result; } /* --------------------------------------------------------------------- * to_tpvector: build a v2 vector from text + index name * --------------------------------------------------------------------- */ Datum to_tpvector(PG_FUNCTION_ARGS) { text *input_text = PG_GETARG_TEXT_PP(0); text *index_name_text = PG_GETARG_TEXT_PP(1); char *index_name; Oid index_oid; Oid text_config_oid; Relation index_rel = NULL; TpIndexMetaPage metap = NULL; int i; char **lexemes; int32 *frequencies; int entry_count; TpVector *result; index_name = text_to_cstring(index_name_text); index_oid = tp_resolve_index_name_shared(index_name); if (!OidIsValid(index_oid)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("index \"%s\" not found", index_name))); index_rel = index_open(index_oid, AccessShareLock); if (!RelationIsValid(index_rel)) ereport(ERROR, (errcode(ERRCODE_UNDEFINED_TABLE), errmsg("could not open index \"%s\"", index_name))); metap = tp_get_metapage(index_rel); text_config_oid = metap->text_config_oid; if (!OidIsValid(text_config_oid)) ereport(ERROR, (errcode(ERRCODE_DATA_CORRUPTED), errmsg("index \"%s\" has no text search configuration", index_name))); pfree(metap); index_close(index_rel, AccessShareLock); (void)tp_tokenize_text( input_text, text_config_oid, &lexemes, &frequencies, &entry_count); result = create_tpvector_from_strings( index_name, entry_count, (const char **)lexemes, frequencies); if (lexemes) { for (i = 0; i < entry_count; i++) pfree(lexemes[i]); pfree(lexemes); } if (frequencies) pfree(frequencies); pfree(index_name); PG_RETURN_POINTER(result); } timescale-pg_textsearch-b63fe58/src/types/vector.h000066400000000000000000000142061525762047700223720ustar00rootroot00000000000000/* * Copyright (c) 2025-2026 Tiger Data, Inc. * Licensed under the PostgreSQL License. See LICENSE for details. * * vector.h - bm25vector data type interface * * Wire / on-disk format (v2, introduced in pg_textsearch 1.2.0). * * Each value carries a 4-byte ASCII magic ("BM25") immediately * after the varlena header, then a 1-byte version + 3 reserved * bytes, then the index_name and entries payload. Per-entry data * is variable-length (varint freq + varint lexeme_len + raw * lexeme bytes); typical entries are ~8 bytes for an English * stemmed lexeme with frequency 1. * * v2 layout: * int32 vl_len_, char magic[4], uint8 version, uint8 reserved[3], * int32 index_name_len, int32 entry_count, data... * per-entry (variable-length, no padding): * varint frequency (1 byte for freq <= 127) * varint lexeme_len (1 byte for lex_len <= 127) * char lexeme[lexeme_len] * * Legacy (pre-1.2.0) values use a different layout. They are * detected via the absence of the v2 magic and rejected by * tpvector_canonicalize() with a clear REINDEX-or-recompute * error, rather than silently misinterpreted. bm25vector values * are almost always transient — the storage path through user * tables was uncommon — so the cost of converting legacy values * isn't worth carrying. Users with persisted v1 data must * recompute via to_bm25vector() from the source text. * * Internal code only ever sees validated v2 values: operator * entry points (tpvector_recv, tpvector_eq, tpvector_send, * tpvector_out) canonicalize at the boundary, which both checks * the magic and bounds-validates the entry stream. * * Entry access pattern: entries are byte streams, so the legacy * `entry->frequency` / `entry->lexeme_len` / `entry->lexeme` field * accesses are no longer possible. Use TpVectorEntryView (a decoded * snapshot) and the iterator helpers below. */ #pragma once #include #include /* v2 format constants */ #define TPVECTOR_V2_MAGIC0 'B' #define TPVECTOR_V2_MAGIC1 'M' #define TPVECTOR_V2_MAGIC2 '2' #define TPVECTOR_V2_MAGIC3 '5' #define TPVECTOR_VERSION 2 /* * Opaque entry handle. v2 entries are variable-length byte * sequences; field access goes through tpvector_entry_decode(). */ typedef struct TpVectorEntry TpVectorEntry; /* * Decoded snapshot of a single entry. Fields are normal C types, * `lexeme` points into the underlying TpVector buffer (not * separately allocated; do not free). */ typedef struct TpVectorEntryView { uint32 frequency; uint32 lexeme_len; const char *lexeme; } TpVectorEntryView; /* tpvector data type structure (v2 layout) */ typedef struct TpVector { int32 vl_len_; /* varlena header (must be first) */ char magic[4]; /* "BM25" — discriminates v2 from v1 */ uint8 version; /* TPVECTOR_VERSION (2) */ uint8 reserved[3]; /* zero; future flags / format bits */ int32 index_name_len; /* length of index name */ int32 entry_count; /* number of term/frequency pairs */ char data[FLEXIBLE_ARRAY_MEMBER]; /* payload: index name + entries */ } TpVector; /* Macros for accessing tpvector variable-length data (v2). */ #define TPVECTOR_INDEX_NAME_PTR(x) (((TpVector *)(x))->data) #define TPVECTOR_ENTRIES_PTR(x) \ ((TpVectorEntry *)(((TpVector *)(x))->data + \ MAXALIGN(((TpVector *)(x))->index_name_len + 1))) /* Function declarations */ PGDLLEXPORT Datum tpvector_in(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpvector_out(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpvector_recv(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpvector_send(PG_FUNCTION_ARGS); PGDLLEXPORT Datum to_tpvector(PG_FUNCTION_ARGS); PGDLLEXPORT Datum tpvector_eq(PG_FUNCTION_ARGS); /* Constructor */ TpVector *create_tpvector_from_strings( const char *index_name, int entry_count, const char **lexemes, const int32 *frequencies); char *get_tpvector_index_name(TpVector *tpvec); /* * Entry iteration. Pattern: * * TpVectorEntry *e = get_tpvector_first_entry(vec); * for (int i = 0; i < vec->entry_count; i++) { * TpVectorEntryView v; * e = tpvector_entry_decode_advance(e, &v); * // use v.frequency, v.lexeme_len, v.lexeme * } * * The decode-only / advance-only variants are kept for callers * that need either operation independently (e.g. validators); * the hot path should use the combined `_advance` form, which * decodes the varints exactly once. */ TpVectorEntry *get_tpvector_first_entry(TpVector *vec); TpVectorEntry *get_tpvector_next_entry(TpVectorEntry *current); void tpvector_entry_decode(const TpVectorEntry *entry, TpVectorEntryView *out); TpVectorEntry *tpvector_entry_decode_advance( const TpVectorEntry *entry, TpVectorEntryView *out); /* * Format detection + canonicalization. * * tpvector_is_v2(raw) returns true if `raw` carries the v2 magic. * tpvector_canonicalize(raw) returns a v2 TpVector — either `raw` * itself if already v2, or a freshly palloc'd v2 copy converted * from v1. */ bool tpvector_is_v2(const void *raw); TpVector *tpvector_canonicalize(TpVector *raw); /* * Full validation of a raw byte buffer as a v2 TpVector. * * Checks that `len` is large enough to cover the TpVector header, * that the magic and version bytes match, that the embedded * varlena length equals `len`, and that the entry stream decodes * fully within the buffer (each varint freq + varint lexeme_len + * lexeme bytes stays in bounds). On any inconsistency, raises an * ereport(ERROR, errcode=ERRCODE_DATA_CORRUPTED). * * After a successful call the buffer is safe to iterate with * get_tpvector_first_entry / get_tpvector_next_entry without per- * entry bounds checks. Use this at trust boundaries where the * bytes come from on-disk memtable pages, network input, or other * untrusted sources. */ void tpvector_validate_v2_buffer(const char *bytes, uint32 len); /* * Varint helpers (internal but exposed for adjacent code). * * Encoding: little-endian base-128. Each byte's high bit indicates * whether more bytes follow. Values 0..127 take 1 byte. */ uint32 tpvector_varint_decode(const uint8 **cursor, const uint8 *end); size_t tpvector_varint_encode(uint32 v, uint8 *out); size_t tpvector_varint_size(uint32 v); timescale-pg_textsearch-b63fe58/test/000077500000000000000000000000001525762047700177405ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/test/README.md000066400000000000000000000156241525762047700212270ustar00rootroot00000000000000# Testing ## Quick Start Choose the target that matches the environment: ```bash make installcheck # installed extension, existing PostgreSQL server make test-local # install and run against a temporary local cluster make test-all # SQL regression plus the standard shell suite ``` The extension must be built for the selected PostgreSQL installation. Shell targets assume it is already installed. The `preload` SQL regression checks maintenance without the extension in the current database, including after `CREATE EXTENSION` and `DROP EXTENSION` in the same session. `make test-local` runs `make install` before `initdb`, so the invoking user must be non-root and able to write to the selected PostgreSQL installation prefix. ## Test Targets The Makefile defines these entry points: | Target | Coverage | | --- | --- | | `make test` | SQL regression suite | | `make installcheck` | Standard PGXS regression run | | `make test-local` | Install and test in a temporary cluster on port 55433 | | `make test-shell` | Concurrency, recovery, segment, CIC, multi-index, and reindex | | `make test-all` | `make test` plus `make test-shell` | | `make test-compression` | Standalone scalar/AVX2 decoder and dispatch coverage | | `make test-concurrency` | Multi-backend concurrency and VACUUM/merge races | | `make test-memtable-stale-tail` | Corrupt-tail rejection and REINDEX repair | | `make test-recovery` | Crash, shutdown-spill, reclaim, and compaction recovery | | `make test-segment` | Multi-backend segments and parallel-build bulk reads | | `make test-stress` | Long-running stress workload | | `make test-replication` | Basic physical replication | | `make test-replication-extended` | Extended physical replication and WAL audit | | `make test-logical-replication` | Logical replication | | `make test-cic` | `CREATE INDEX CONCURRENTLY` | | `make test-multi-index` | Multi-index, user, and schema behavior | | `make test-reindex` | Rebuild, rollback, and prepared-transaction caches | | `make test-cross-database-registry` | Database isolation and DROP cleanup | | `make test-drop-rollback` | Multi-backend DROP rollback and deferred cleanup | | `make test-injection-sql` | Behavior-specific SQL regressions requiring injection points | | `make test-injection-shell` | Crash and concurrency injection-point tests | | `make test-chinese` | Optional zhparser regression | The compression harness checks all supported widths, counts, and input alignments through runtime dispatch and the scalar fallback. It also rebuilds with the CPU probe forced to report no AVX2 support; no backend is required. `test/scripts/parallel_build_bulkread.sh` starts a private 16MB-buffer cluster and builds an index with two workers on a heap large enough for a bulk-read strategy. It checks complete, duplicate-free results and asserts worker buffer reuse through `pg_stat_io`, catching scans that fill shared buffers instead of reusing the bulk-read ring. `make test-shell` is the standard shell suite. Replication and stress targets are separate because they create additional PostgreSQL instances or run for an extended period. ## Injection Points Injection-point tests require PostgreSQL configured with `--enable-injection-points`. The Makefile detects this through `enable_injection_points` and, when it is set, adds the injection regressions to `REGRESS` and installs the `pg_textsearch_test` helper extension as part of `make install`. On ordinary packaged PostgreSQL builds, these tests are omitted. Scripts that mix injection-point cases with ordinary ones — such as `standby_reclaim.sh`, `compaction_recovery.sh`, and `vacuum_concurrent_merge.sh` — skip only the cases that need a deterministic pause, so they stay useful on packaged builds. `nonblocking_compaction.sh` needs injection points throughout and skips entirely. `nonblocking_spill.sh` also races force-merge suffix truncation with spills in both lock-acquisition orders. It pauses between suffix inspection and truncation, checks writer-gate exclusion, and verifies segment page references and exact ranked results before and after crash recovery. CI builds PostgreSQL 17, 18, and 19 with injection points enabled and caches the installed prefixes. The sanitizer builds use the same configure option. `memtable_stale_tail_injection` pauses normal and oversized writers after reading the tail, advances it from another session, then requires both writers to retry successfully. `make test-memtable-stale-tail` covers a corrupt tail and REINDEX repair without requiring injection points. `tombstone_bounds_injection` corrupts head and next-page links at block zero, EOF, and past EOF. It checks diagnostic errors and drain recovery while a pinned horizon keeps the valid chain prefix parked. ## Sanitizers Pull-request CI builds PostgreSQL 17.2 and 18.1 and pg_textsearch with Clang AddressSanitizer and UndefinedBehaviorSanitizer. It runs SQL regression; concurrency, duplicate-read, and VACUUM/merge race tests; crash, deferred reclaim, and shutdown-spill recovery tests; multi-backend segment tests; `CREATE INDEX CONCURRENTLY`; multi-index, user, and schema tests; and standby segment-reclaim coverage. The nightly stress workflow enables leak detection. There is no local `make sanitizer` target; [`.github/workflows/ci.yml`](https://github.com/timescale/pg_textsearch/blob/main/.github/workflows/ci.yml) is the canonical reproduction recipe. The standalone [`sanitizer-build-and-test.yml`](https://github.com/timescale/pg_textsearch/blob/main/.github/workflows/sanitizer-build-and-test.yml) workflow runs on `main`; pull-request sanitizer coverage is consolidated in `.github/workflows/ci.yml`. ## Adding SQL Tests 1. Add `test/sql/.sql`. 2. Add `` to `REGRESS` in `Makefile`. 3. Run `make installcheck REGRESS=`. 4. Inspect `test/results/.out` and `test/regression.diffs`. 5. Copy the reviewed output to `test/expected/.out`. 6. Rerun `make installcheck REGRESS=`. Keep fixtures deterministic and include only output that the test intends to verify. Use temporary tables for single-session tests that require immediate VACUUM cleanup or page reuse. Their reclaim horizon is session-local, so unrelated transactions and standby feedback cannot retain dead tuples or pages. Keep shared-buffer, WAL, and standby-reclaim coverage in permanent-table tests such as `standby_reclaim.sh`. The single-session `segment_reclaim_injection.sql` fixture is temporary so detaching its injected hold restores a horizon unaffected by other backends. ## Debugging Failures Inspect `test/regression.diffs` first, then compare the generated file under `test/results/` with its counterpart under `test/expected/`. Rerun a single SQL test with: ```bash make installcheck REGRESS= ``` For `make test-local`, PostgreSQL logs are written to `tmp_check_shared/data/logfile` while the temporary cluster exists. Shell tests print their commands and diagnostics directly; rerun the corresponding individual target to isolate a failure. timescale-pg_textsearch-b63fe58/test/expected/000077500000000000000000000000001525762047700215415ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/test/expected/abort.out000066400000000000000000000202631525762047700234040ustar00rootroot00000000000000-- -- Test transaction abort scenarios -- -- Verify that pg_textsearch hooks handle aborted transactions gracefully. -- The post_parse_analyze_hook fires on every SQL statement including -- ROLLBACK. When a transaction is in an aborted state, catalog lookups -- are forbidden, so the hook must skip its work. -- CREATE EXTENSION pg_textsearch; -- Scenario 1: Basic abort with no pg_textsearch objects -- This is the original bug report (#247): ROLLBACK after an error in a -- transaction that has nothing to do with BM25 should not crash. BEGIN; CREATE TABLE abort_test1 (id serial PRIMARY KEY); SELECT 1/0; ERROR: division by zero ROLLBACK; -- Scenario 2: Abort with a BM25 index present (created before the xact) CREATE TABLE abort_test2 (id serial PRIMARY KEY, body text); CREATE INDEX abort_test2_idx ON abort_test2 USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation abort_test2_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 BEGIN; INSERT INTO abort_test2 (body) VALUES ('hello world'); SELECT 1/0; ERROR: division by zero ROLLBACK; -- Verify the index is still usable after the abort INSERT INTO abort_test2 (body) VALUES ('hello world'); SELECT id, ROUND((body <@> to_bm25query('hello', 'abort_test2_idx'))::numeric, 4) AS score FROM abort_test2 ORDER BY body <@> to_bm25query('hello', 'abort_test2_idx') LIMIT 5; id | score ----+--------- 2 | -0.1823 (1 row) -- Scenario 3: Abort during CREATE INDEX after charging its runtime cache SELECT bm25_cache_global_estimated_bytes() AS abort_create_baseline \gset BEGIN; CREATE TABLE abort_test3 (id serial PRIMARY KEY, body text); INSERT INTO abort_test3 (body) VALUES ('test document'); CREATE INDEX abort_test3_idx ON abort_test3 USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation abort_test3_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- INSERT only appends the durable chain. A cold build must read it to charge -- the derived shared-memory cache before abort cleanup is exercised. INSERT INTO abort_test3 (body) VALUES ('charged cache tail'); SELECT result = 'OK' AND estimated_bytes > 0 AS create_abort_cache_charged FROM bm25_cache_cold_build('abort_test3_idx') \gset \echo create_abort_cache_charged=:create_abort_cache_charged create_abort_cache_charged=t SELECT 1/0; ERROR: division by zero ROLLBACK; -- Table and index should not exist after rollback SELECT count(*) FROM pg_class WHERE relname = 'abort_test3'; count ------- 0 (1 row) SELECT bm25_cache_global_estimated_bytes() = :abort_create_baseline AS create_abort_accounting_restored \gset \echo create_abort_accounting_restored=:create_abort_accounting_restored create_abort_accounting_restored=t -- Scenario 4: Abort with DDL (DROP TABLE that has a BM25 index) CREATE TABLE abort_test4 (id serial PRIMARY KEY, body text); CREATE INDEX abort_test4_idx ON abort_test4 USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation abort_test4_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO abort_test4 (body) VALUES ('still here'); BEGIN; DROP TABLE abort_test4; SELECT 1/0; ERROR: division by zero ROLLBACK; -- Table should still exist after the aborted DROP SET client_min_messages = warning; SELECT count(*) FROM abort_test4; count ------- 1 (1 row) RESET client_min_messages; -- Scenario 5: Multiple errors in sequence BEGIN; SELECT 1/0; ERROR: division by zero SELECT 2/0; ERROR: current transaction is aborted, commands ignored until end of transaction block ROLLBACK; -- Scenario 6: SAVEPOINT and error within subtransaction BEGIN; INSERT INTO abort_test2 (body) VALUES ('before savepoint'); SAVEPOINT sp1; SELECT 1/0; ERROR: division by zero ROLLBACK TO sp1; INSERT INTO abort_test2 (body) VALUES ('after rollback to savepoint'); COMMIT; SELECT id, body FROM abort_test2 ORDER BY body <@> to_bm25query('savepoint', 'abort_test2_idx'); id | body ----+----------------------------- 3 | before savepoint 4 | after rollback to savepoint (2 rows) -- Scenario 7: Error in a BM25 query BEGIN; INSERT INTO abort_test2 (body) VALUES ('scenario seven'); -- Force an error via division by zero in the same statement SELECT (body <@> to_bm25query('scenario', 'abort_test2_idx'))::int / 0 FROM abort_test2; ERROR: division by zero ROLLBACK; -- Scenario 8: Abort after BM25 query succeeds, then later stmt fails BEGIN; SELECT id FROM abort_test2 ORDER BY body <@> to_bm25query('hello', 'abort_test2_idx') LIMIT 3; id ---- 2 (1 row) SELECT 1/0; ERROR: division by zero ROLLBACK; -- Scenario 9: SAVEPOINT rollback of CREATE INDEX with a charged cache -- Exercises SubXactCallback cleanup of cache dshash tables and accounting. BEGIN; CREATE TABLE abort_subxact1 (id serial PRIMARY KEY, body text); SELECT bm25_cache_global_estimated_bytes() AS abort_subxact_create_baseline \gset SAVEPOINT sp1; CREATE INDEX abort_subxact1_idx ON abort_subxact1 USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation abort_subxact1_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Append durable records, then read them into the shared-memory cache. INSERT INTO abort_subxact1 (body) VALUES ('memtable data one'); INSERT INTO abort_subxact1 (body) VALUES ('memtable data two'); SELECT result = 'OK' AND estimated_bytes > 0 AS subxact_create_abort_cache_charged FROM bm25_cache_cold_build('abort_subxact1_idx') \gset \echo subxact_create_abort_cache_charged=:subxact_create_abort_cache_charged subxact_create_abort_cache_charged=t ROLLBACK TO sp1; SELECT bm25_cache_global_estimated_bytes() = :abort_subxact_create_baseline AS subxact_create_abort_accounting_restored \gset \echo subxact_create_abort_accounting_restored=:subxact_create_abort_accounting_restored subxact_create_abort_accounting_restored=t -- Table should still be usable without the index INSERT INTO abort_subxact1 (body) VALUES ('after rollback'); COMMIT; SELECT count(*) FROM abort_subxact1; count ------- 1 (1 row) DROP TABLE abort_subxact1; -- Scenario 10: Nested savepoints with RELEASE (promotes subxact state) BEGIN; CREATE TABLE abort_subxact2 (id serial PRIMARY KEY, body text); SAVEPOINT sp1; SAVEPOINT sp2; CREATE INDEX abort_subxact2_idx ON abort_subxact2 USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation abort_subxact2_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 RELEASE SAVEPOINT sp2; -- Index alive, state promoted from sp2 to sp1 INSERT INTO abort_subxact2 (body) VALUES ('after release'); ROLLBACK TO sp1; -- Index (promoted to sp1) should be cleaned up COMMIT; SELECT count(*) FROM abort_subxact2; count ------- 0 (1 row) DROP TABLE abort_subxact2; -- Scenario 11: SAVEPOINT abort followed by full transaction abort -- Validates no double-free between subxact and xact cleanup BEGIN; CREATE TABLE abort_subxact3 (id serial PRIMARY KEY, body text); SAVEPOINT sp1; CREATE INDEX abort_subxact3_idx ON abort_subxact3 USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation abort_subxact3_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO abort_subxact3 (body) VALUES ('will be rolled back'); ROLLBACK TO sp1; -- Subxact cleanup already ran; now abort the whole transaction ROLLBACK; -- Table should not exist after full rollback SELECT count(*) FROM pg_class WHERE relname = 'abort_subxact3'; count ------- 0 (1 row) -- Cleanup DROP TABLE abort_test2; DROP TABLE abort_test4; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/aerodocs.out000066400000000000000000000442751525762047700241050ustar00rootroot00000000000000-- Aerodynamics Documents pg_textsearch Test - Real Cranfield Dataset -- This test validates pg_textsearch implementation using a substantial subset of the Cranfield collection -- Contains real aerodynamics documents from the Cranfield dataset for realistic testing -- Tests both dataset loading and pg_textsearch scoring functionality with the <@> operator -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none -- Enable score logging for testing SET pg_textsearch.log_scores = true; \set ON_ERROR_STOP on \echo 'Testing pg_textsearch with sample aerodynamics documents...' Testing pg_textsearch with sample aerodynamics documents... -- Create tables for aerodynamics documents CREATE TABLE aerodocs_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Expected rankings table removed - we test against actual output \echo 'Loading subset of Cranfield documents for testing...' Loading subset of Cranfield documents for testing... INSERT INTO aerodocs_documents (doc_id, title, author, content) VALUES (1, 'experimental investigation of the aerodynamics of a wing in a slipstream .', 'brenckman,m.', 'experimental investigation of the aerodynamics of a wing in a slipstream . an experimental study of a wing in a propeller slipstream was made in order to determine the spanwise distribution of the lift increase due to slipstream at different angles of attack of the wing and at different free stream to slipstream velocity ratios . the results were intended in part as an evaluation basis for different theoretical treatments of this problem . the comparative span loading curves, together with supporting evidence, showed that a substantial part of the lift increment produced by the slipstream was due to a /destalling/ or boundary-layer-control effect . the integrated remaining lift increment, after subtracting this destalling lift, was found to agree well with a potential flow theory . an empirical evaluation of the destalling effects was made for the specific configuration of the experiment .'), (2, 'simple shear flow past a flat plate in an incompressible fluid of small viscosity .', 'ting-yili', 'in the study of high-speed viscous flow past a two-dimensional body it is usually necessary to consider a curved shock wave emitting from the nose or leading edge of the body . consequently, there exists an inviscid rotational flow region between the shock wave and the boundary layer . such a situation arises, for instance, in the study of the hypersonic viscous flow past a flat plate . the situation is somewhat different from prandtl''s classical boundary-layer problem . in prandtl''s original problem the inviscid free stream outside the boundary layer is irrotational while in a hypersonic boundary-layer problem the inviscid free stream must be considered as rotational . the possible effects of vorticity have been recently discussed by ferri and libby . in the present paper, the simple shear flow past a flat plate in a fluid of small viscosity is investigated . it can be shown that this problem can again be treated by the boundary-layer approximation, the only novel feature being that the free stream has a constant vorticity . the discussion here is restricted to two-dimensional incompressible steady flow .'), (3, 'the boundary layer in simple shear flow past a flat plate .', 'm. b. glauert', 'the boundary-layer equations are presented for steady incompressible flow with no pressure gradient .'), (4, 'approximate solutions of the incompressible laminar boundary layer equations for a plate in shear flow .', 'yen,k.t.', 'the two-dimensional steady boundary-layer problem for a flat plate in a shear flow of incompressible fluid is considered . solutions for the boundary- layer thickness, skin friction, and the velocity distribution in the boundary layer are obtained by the karman-pohlhausen technique . comparison with the boundary layer of a uniform flow has also been made to show the effect of vorticity .'), (5, 'one-dimensional transient heat conduction into a double-layer slab subjected to a linear heat input for a small time internal .', 'wasserman,b.', 'analytic solutions are presented for the transient heat conduction in composite slabs exposed at one surface to a triangular heat rate . this type of heating rate may occur, for example, during aerodynamic heating .'), (6, 'one-dimensional transient heat flow in a multilayer slab .', 'campbell,w.f.', 'in a recent contribution to the readers forum wassermann gave analytic solutions for the temperature in a double layer slab, with a triangular heat rate input at one face, insulated at the other, and with no thermal resistance at the interface . his solutions were for the three particular cases.. i propose here to give the general solution to this problem, to indicate briefly how it is obtained using the method of reference 2, and to point out that the solutions given by wassermann are incomplete for times longer than the duration of the heat input .'), (7, 'the effect of controlled three-dimensional roughness on boundary layer transition at supersonic speeds .', 'van driest,e.r. and mccauley,w.d.', 'experiments were performed in the 12-in. supersonic wind tunnel of the jet propulsion laboratory of the california institute of technology to investigate the effect of three-dimensional roughness elements (spheres) on boundary-layer transition on a tained at local mach numbers of 1.90, 2.71, and 3.67 by varying trip size, position, spacing, and reynolds number per inch . the results indicate that (1) transition from laminar to turbulent flow induced by three-dimensional roughness elements begins when the double row of spiral vortices trailing each element contaminates and breaks down the surrounding field of vorticity, (2) transition appears rather suddenly, becoming more violent with increasing roughness height relative to the boundary-layer thickness, (3) after the breakdown of the vorticity field, the strength of the spiral vortices may still persist in the sublayer of the ensuing turbulent flow, (4) lateral spacing of roughness elements has little effect upon the initial breakdown (contamination) of the laminar flow, and (5) the trip reynolds number where u and v are the velocity and kinematic viscosity at the outer edge of the boundary layer and k is roughness height, such that transition occurs at the roughness position, varies as the position reynolds number to the one-fourth power, viz., where x is trip position .'), (8, 'measurements of the effect of two-dimensional and three-dimensional roughness elements on boundary layer transition .', 'klebanoff,p.s.', 'in his study of the effect of roughness on transition, h. l. dryden found, on the basis of available data, that the effect of a two-dimensional roughness element such as a /trip wire/ could be represented reasonably well in terms of a functional relation between and, where is the reynolds number of transition based on distance from the leading edge, is the height of the roughness element, and is the boundary-layer displacement thickness at the position of the element . at his suggestion some additional data were obtained, primarily to extend the range to higher values of, during the course of an investigation of transition on a flat plate conducted at the national bureau of standards . after the results on the two- dimensional roughness elements were obtained, it appeared to be desirable to see whether a row of three-dimensional roughness elements would behave in the same way .'), (9, 'transition studies and skin friction measurements on an insulated flat plate at a mach number of 5.8 .', 'korkegi,r.h.', 'an investigation of transition and skin friction on an insulated flat plate, 5 by 26 in., was made in the galcit 5 by 5 in. hypersonic wind tunnel at a nominal mach number of 5.8 . the phosphorescent lacquer technique was used for transition detection and was found to be in good agreement with total-head rake measurements along the plate surface and pitot boundary- layer surveys . it was found that the boundary layer was laminar at reynolds numbers of at least 5 x 10 . transverse contamination caused by the turbulent boundary layer on the tunnel sidewall originated far downstream of the flat plate leading edge at reynolds numbers of 1.5 to 2 x 10, and spread at a uniform angle of 5 compared to 9 degree in low-speed flow . the effect of two-dimensional and local disturbances was investigated . the technique of air injection into the boundary layer as a means of hastening transition was extensively used . although the onset of transition occurred at reynolds numbers as low as 10, a fully developed turbulent boundary layer was not obtained at reynolds numbers much below 2 x 10 regardless of the amount of air injected . a qualitative discussion of these results is given with emphasis on the possibility of a greater stability of the laminar boundary layer in hypersonic flow than at lower speeds . direct skin-friction measurements were made by means of the floating element technique, over a range of reynolds numbers verified as being laminar over the complete range . with air injection, turbulent shear was obtained only for reynolds numbers greater than 2 x 10, this value being in good agreement with earlier results of this investigation . the turbulent skin-friction coefficient was found to be approximately 0.40 of that for incompressible flow for a constant value of r, and 0.46 for an effective reynolds number between 5 and 6 x 10 .'), (10, 'the theory of the impact tube at low pressure .', 'chambre,p.l. and schaaf,s.a.', 'a theoretical analysis has been made for an impact tube of the relation between free-stream mach number and the impact and free-stream pressures and densities for extremely low pressures . it is shown that the results differ appreciably from the corresponding continuum relations .'); -- Insert sample queries INSERT INTO cranfield_queries (query_id, query_text) VALUES (1, 'aerodynamic flow analysis'), (2, 'boundary layer turbulent'), (3, 'wing design optimization'); -- Expected rankings are now captured in the test's expected output file -- Create pg_textsearch index for Cranfield documents with text_config CREATE INDEX cranfield_tapir_idx ON aerodocs_documents USING bm25(full_text) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation cranfield_tapir_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 10 documents, avg_length=87.80 \echo 'pg_textsearch index created. Running validation tests...' pg_textsearch index created. Running validation tests... -- Disable sequential scans to ensure index usage SET enable_seqscan = off; -- Test 0: Simple single-term validation \echo 'Test 0: Single-term query validation' Test 0: Single-term query validation SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'aerodynamic')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'aerodynamic' ASC LIMIT 3; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.0623 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-1.9061 doc_id | title_preview | score --------+--------------------------------------------------------------+--------- 1 | experimental investigation of the aerodynamics of a wing in | -2.0623 5 | one-dimensional transient heat conduction into a double-laye | -1.9061 (2 rows) -- Validate single-term BM25 scoring SELECT validate_bm25_scoring('aerodocs_documents', 'full_text', 'cranfield_tapir_idx', 'aerodynamic', 'english', 1.2, 0.75) as aerodynamic_valid; aerodynamic_valid ------------------- f (1 row) -- Test 1: Basic search functionality \echo 'Test 1: Basic search with <@> operator' Test 1: Basic search with <@> operator SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'aerodynamic flow')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'aerodynamic flow' ASC LIMIT 5; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.4522 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-1.9061 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6714 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.6700 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.6595 doc_id | title_preview | score --------+--------------------------------------------------------------+--------- 1 | experimental investigation of the aerodynamics of a wing in | -2.4522 5 | one-dimensional transient heat conduction into a double-laye | -1.9061 2 | simple shear flow past a flat plate in an incompressible flu | -0.6714 3 | the boundary layer in simple shear flow past a flat plate . | -0.6700 4 | approximate solutions of the incompressible laminar boundary | -0.6595 (5 rows) -- Test 2: Multi-term search \echo 'Test 2: Multi-term search' Test 2: Multi-term search SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'boundary layer turbulent')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'boundary layer turbulent' ASC LIMIT 5; NOTICE: BM25 index scan: tid=(1,3), BM25_score=-2.8904 NOTICE: BM25 index scan: tid=(1,1), BM25_score=-2.4853 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.9987 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.9264 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.8921 doc_id | title_preview | score --------+--------------------------------------------------------------+--------- 9 | transition studies and skin friction measurements on an insu | -2.8904 7 | the effect of controlled three-dimensional roughness on boun | -2.4853 4 | approximate solutions of the incompressible laminar boundary | -0.9987 3 | the boundary layer in simple shear flow past a flat plate . | -0.9264 2 | simple shear flow past a flat plate in an incompressible flu | -0.8921 (5 rows) -- Test 3: Top-10 results for first query \echo 'Test 3: Top-10 results for first query' Test 3: Top-10 results for first query WITH pgts_results AS ( SELECT doc_id, ROUND((full_text <@> 'aerodynamic flow analysis')::numeric, 4) as pgts_score, ROW_NUMBER() OVER (ORDER BY full_text <@> 'aerodynamic flow analysis' ASC) as pgts_rank FROM aerodocs_documents ) SELECT br.doc_id, br.pgts_rank, br.pgts_score as score, LEFT(d.title, 50) as title_preview FROM pgts_results br JOIN aerodocs_documents d ON br.doc_id = d.doc_id WHERE br.pgts_rank <= 10 ORDER BY br.pgts_rank; NOTICE: BM25 index scan: tid=(1,4), BM25_score=-2.6263 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.4522 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-1.9061 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6714 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.6700 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.6595 NOTICE: BM25 index scan: tid=(1,1), BM25_score=-0.5203 NOTICE: BM25 index scan: tid=(1,3), BM25_score=-0.4725 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-0.4496 doc_id | pgts_rank | score | title_preview --------+-----------+---------+---------------------------------------------------- 10 | 1 | -2.6263 | the theory of the impact tube at low pressure . 1 | 2 | -2.4522 | experimental investigation of the aerodynamics of 5 | 3 | -1.9061 | one-dimensional transient heat conduction into a d 2 | 4 | -0.6714 | simple shear flow past a flat plate in an incompre 3 | 5 | -0.6700 | the boundary layer in simple shear flow past a fla 4 | 6 | -0.6595 | approximate solutions of the incompressible lamina 7 | 7 | -0.5203 | the effect of controlled three-dimensional roughne 9 | 8 | -0.4725 | transition studies and skin friction measurements 6 | 9 | -0.4496 | one-dimensional transient heat flow in a multilaye (9 rows) -- Test 4: Performance test with multiple queries \echo 'Test 4: Performance test with top-k search' Test 4: Performance test with top-k search SELECT q.query_id, LEFT(q.query_text, 40) || '...' as query_preview, COUNT(d.doc_id) as total_docs, COUNT(CASE WHEN d.full_text <@> to_bm25query(q.query_text, 'cranfield_tapir_idx') < 0 THEN 1 END) as matching_docs FROM cranfield_queries q CROSS JOIN aerodocs_documents d WHERE q.query_id <= 5 -- Test first 5 queries for performance GROUP BY q.query_id, q.query_text ORDER BY q.query_id; query_id | query_preview | total_docs | matching_docs ----------+------------------------------+------------+--------------- 1 | aerodynamic flow analysis... | 10 | 9 2 | boundary layer turbulent... | 10 | 9 3 | wing design optimization... | 10 | 1 (3 rows) -- Test 5: Index usage verification \echo 'Test 5: Verify index usage with EXPLAIN' Test 5: Verify index usage with EXPLAIN SET jit = off; EXPLAIN (COSTS OFF) SELECT doc_id, ROUND((full_text <@> 'supersonic aircraft design')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'supersonic aircraft design' ASC LIMIT 10; QUERY PLAN ----------------------------------------------------------------------------------------------- Limit -> Index Scan using cranfield_tapir_idx on aerodocs_documents Order By: (full_text <@> 'cranfield_tapir_idx:supersonic aircraft design'::bm25query) (3 rows) -- Reset settings SET enable_seqscan = on; \echo 'Cranfield pg_textsearch validation completed successfully.' Cranfield pg_textsearch validation completed successfully. \echo 'All tests demonstrate proper pg_textsearch functionality with the <@> operator.' All tests demonstrate proper pg_textsearch functionality with the <@> operator. -- Clean up DROP TABLE aerodocs_documents CASCADE; DROP TABLE cranfield_queries CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/aerodocs_1.out000066400000000000000000000442751525762047700243250ustar00rootroot00000000000000-- Aerodynamics Documents pg_textsearch Test - Real Cranfield Dataset -- This test validates pg_textsearch implementation using a substantial subset of the Cranfield collection -- Contains real aerodynamics documents from the Cranfield dataset for realistic testing -- Tests both dataset loading and pg_textsearch scoring functionality with the <@> operator -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none -- Enable score logging for testing SET pg_textsearch.log_scores = true; \set ON_ERROR_STOP on \echo 'Testing pg_textsearch with sample aerodynamics documents...' Testing pg_textsearch with sample aerodynamics documents... -- Create tables for aerodynamics documents CREATE TABLE aerodocs_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Expected rankings table removed - we test against actual output \echo 'Loading subset of Cranfield documents for testing...' Loading subset of Cranfield documents for testing... INSERT INTO aerodocs_documents (doc_id, title, author, content) VALUES (1, 'experimental investigation of the aerodynamics of a wing in a slipstream .', 'brenckman,m.', 'experimental investigation of the aerodynamics of a wing in a slipstream . an experimental study of a wing in a propeller slipstream was made in order to determine the spanwise distribution of the lift increase due to slipstream at different angles of attack of the wing and at different free stream to slipstream velocity ratios . the results were intended in part as an evaluation basis for different theoretical treatments of this problem . the comparative span loading curves, together with supporting evidence, showed that a substantial part of the lift increment produced by the slipstream was due to a /destalling/ or boundary-layer-control effect . the integrated remaining lift increment, after subtracting this destalling lift, was found to agree well with a potential flow theory . an empirical evaluation of the destalling effects was made for the specific configuration of the experiment .'), (2, 'simple shear flow past a flat plate in an incompressible fluid of small viscosity .', 'ting-yili', 'in the study of high-speed viscous flow past a two-dimensional body it is usually necessary to consider a curved shock wave emitting from the nose or leading edge of the body . consequently, there exists an inviscid rotational flow region between the shock wave and the boundary layer . such a situation arises, for instance, in the study of the hypersonic viscous flow past a flat plate . the situation is somewhat different from prandtl''s classical boundary-layer problem . in prandtl''s original problem the inviscid free stream outside the boundary layer is irrotational while in a hypersonic boundary-layer problem the inviscid free stream must be considered as rotational . the possible effects of vorticity have been recently discussed by ferri and libby . in the present paper, the simple shear flow past a flat plate in a fluid of small viscosity is investigated . it can be shown that this problem can again be treated by the boundary-layer approximation, the only novel feature being that the free stream has a constant vorticity . the discussion here is restricted to two-dimensional incompressible steady flow .'), (3, 'the boundary layer in simple shear flow past a flat plate .', 'm. b. glauert', 'the boundary-layer equations are presented for steady incompressible flow with no pressure gradient .'), (4, 'approximate solutions of the incompressible laminar boundary layer equations for a plate in shear flow .', 'yen,k.t.', 'the two-dimensional steady boundary-layer problem for a flat plate in a shear flow of incompressible fluid is considered . solutions for the boundary- layer thickness, skin friction, and the velocity distribution in the boundary layer are obtained by the karman-pohlhausen technique . comparison with the boundary layer of a uniform flow has also been made to show the effect of vorticity .'), (5, 'one-dimensional transient heat conduction into a double-layer slab subjected to a linear heat input for a small time internal .', 'wasserman,b.', 'analytic solutions are presented for the transient heat conduction in composite slabs exposed at one surface to a triangular heat rate . this type of heating rate may occur, for example, during aerodynamic heating .'), (6, 'one-dimensional transient heat flow in a multilayer slab .', 'campbell,w.f.', 'in a recent contribution to the readers forum wassermann gave analytic solutions for the temperature in a double layer slab, with a triangular heat rate input at one face, insulated at the other, and with no thermal resistance at the interface . his solutions were for the three particular cases.. i propose here to give the general solution to this problem, to indicate briefly how it is obtained using the method of reference 2, and to point out that the solutions given by wassermann are incomplete for times longer than the duration of the heat input .'), (7, 'the effect of controlled three-dimensional roughness on boundary layer transition at supersonic speeds .', 'van driest,e.r. and mccauley,w.d.', 'experiments were performed in the 12-in. supersonic wind tunnel of the jet propulsion laboratory of the california institute of technology to investigate the effect of three-dimensional roughness elements (spheres) on boundary-layer transition on a tained at local mach numbers of 1.90, 2.71, and 3.67 by varying trip size, position, spacing, and reynolds number per inch . the results indicate that (1) transition from laminar to turbulent flow induced by three-dimensional roughness elements begins when the double row of spiral vortices trailing each element contaminates and breaks down the surrounding field of vorticity, (2) transition appears rather suddenly, becoming more violent with increasing roughness height relative to the boundary-layer thickness, (3) after the breakdown of the vorticity field, the strength of the spiral vortices may still persist in the sublayer of the ensuing turbulent flow, (4) lateral spacing of roughness elements has little effect upon the initial breakdown (contamination) of the laminar flow, and (5) the trip reynolds number where u and v are the velocity and kinematic viscosity at the outer edge of the boundary layer and k is roughness height, such that transition occurs at the roughness position, varies as the position reynolds number to the one-fourth power, viz., where x is trip position .'), (8, 'measurements of the effect of two-dimensional and three-dimensional roughness elements on boundary layer transition .', 'klebanoff,p.s.', 'in his study of the effect of roughness on transition, h. l. dryden found, on the basis of available data, that the effect of a two-dimensional roughness element such as a /trip wire/ could be represented reasonably well in terms of a functional relation between and, where is the reynolds number of transition based on distance from the leading edge, is the height of the roughness element, and is the boundary-layer displacement thickness at the position of the element . at his suggestion some additional data were obtained, primarily to extend the range to higher values of, during the course of an investigation of transition on a flat plate conducted at the national bureau of standards . after the results on the two- dimensional roughness elements were obtained, it appeared to be desirable to see whether a row of three-dimensional roughness elements would behave in the same way .'), (9, 'transition studies and skin friction measurements on an insulated flat plate at a mach number of 5.8 .', 'korkegi,r.h.', 'an investigation of transition and skin friction on an insulated flat plate, 5 by 26 in., was made in the galcit 5 by 5 in. hypersonic wind tunnel at a nominal mach number of 5.8 . the phosphorescent lacquer technique was used for transition detection and was found to be in good agreement with total-head rake measurements along the plate surface and pitot boundary- layer surveys . it was found that the boundary layer was laminar at reynolds numbers of at least 5 x 10 . transverse contamination caused by the turbulent boundary layer on the tunnel sidewall originated far downstream of the flat plate leading edge at reynolds numbers of 1.5 to 2 x 10, and spread at a uniform angle of 5 compared to 9 degree in low-speed flow . the effect of two-dimensional and local disturbances was investigated . the technique of air injection into the boundary layer as a means of hastening transition was extensively used . although the onset of transition occurred at reynolds numbers as low as 10, a fully developed turbulent boundary layer was not obtained at reynolds numbers much below 2 x 10 regardless of the amount of air injected . a qualitative discussion of these results is given with emphasis on the possibility of a greater stability of the laminar boundary layer in hypersonic flow than at lower speeds . direct skin-friction measurements were made by means of the floating element technique, over a range of reynolds numbers verified as being laminar over the complete range . with air injection, turbulent shear was obtained only for reynolds numbers greater than 2 x 10, this value being in good agreement with earlier results of this investigation . the turbulent skin-friction coefficient was found to be approximately 0.40 of that for incompressible flow for a constant value of r, and 0.46 for an effective reynolds number between 5 and 6 x 10 .'), (10, 'the theory of the impact tube at low pressure .', 'chambre,p.l. and schaaf,s.a.', 'a theoretical analysis has been made for an impact tube of the relation between free-stream mach number and the impact and free-stream pressures and densities for extremely low pressures . it is shown that the results differ appreciably from the corresponding continuum relations .'); -- Insert sample queries INSERT INTO cranfield_queries (query_id, query_text) VALUES (1, 'aerodynamic flow analysis'), (2, 'boundary layer turbulent'), (3, 'wing design optimization'); -- Expected rankings are now captured in the test's expected output file -- Create pg_textsearch index for Cranfield documents with text_config CREATE INDEX cranfield_tapir_idx ON aerodocs_documents USING bm25(full_text) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation cranfield_tapir_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 10 documents, avg_length=87.80 \echo 'pg_textsearch index created. Running validation tests...' pg_textsearch index created. Running validation tests... -- Disable sequential scans to ensure index usage SET enable_seqscan = off; -- Test 0: Simple single-term validation \echo 'Test 0: Single-term query validation' Test 0: Single-term query validation SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'aerodynamic')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'aerodynamic' ASC LIMIT 3; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.0424 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-1.8948 doc_id | title_preview | score --------+--------------------------------------------------------------+--------- 1 | experimental investigation of the aerodynamics of a wing in | -2.0424 5 | one-dimensional transient heat conduction into a double-laye | -1.8948 (2 rows) -- Validate single-term BM25 scoring SELECT validate_bm25_scoring('aerodocs_documents', 'full_text', 'cranfield_tapir_idx', 'aerodynamic', 'english', 1.2, 0.75) as aerodynamic_valid; aerodynamic_valid ------------------- t (1 row) -- Test 1: Basic search functionality \echo 'Test 1: Basic search with <@> operator' Test 1: Basic search with <@> operator SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'aerodynamic flow')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'aerodynamic flow' ASC LIMIT 5; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.4269 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-1.8948 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.6700 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6668 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.6595 doc_id | title_preview | score --------+--------------------------------------------------------------+--------- 1 | experimental investigation of the aerodynamics of a wing in | -2.4269 5 | one-dimensional transient heat conduction into a double-laye | -1.8948 3 | the boundary layer in simple shear flow past a flat plate . | -0.6700 2 | simple shear flow past a flat plate in an incompressible flu | -0.6668 4 | approximate solutions of the incompressible laminar boundary | -0.6595 (5 rows) -- Test 2: Multi-term search \echo 'Test 2: Multi-term search' Test 2: Multi-term search SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'boundary layer turbulent')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'boundary layer turbulent' ASC LIMIT 5; NOTICE: BM25 index scan: tid=(1,3), BM25_score=-2.8603 NOTICE: BM25 index scan: tid=(1,1), BM25_score=-2.4678 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.9987 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.9264 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.8852 doc_id | title_preview | score --------+--------------------------------------------------------------+--------- 9 | transition studies and skin friction measurements on an insu | -2.8603 7 | the effect of controlled three-dimensional roughness on boun | -2.4678 4 | approximate solutions of the incompressible laminar boundary | -0.9987 3 | the boundary layer in simple shear flow past a flat plate . | -0.9264 2 | simple shear flow past a flat plate in an incompressible flu | -0.8852 (5 rows) -- Test 3: Top-10 results for first query \echo 'Test 3: Top-10 results for first query' Test 3: Top-10 results for first query WITH pgts_results AS ( SELECT doc_id, ROUND((full_text <@> 'aerodynamic flow analysis')::numeric, 4) as pgts_score, ROW_NUMBER() OVER (ORDER BY full_text <@> 'aerodynamic flow analysis' ASC) as pgts_rank FROM aerodocs_documents ) SELECT br.doc_id, br.pgts_rank, br.pgts_score as score, LEFT(d.title, 50) as title_preview FROM pgts_results br JOIN aerodocs_documents d ON br.doc_id = d.doc_id WHERE br.pgts_rank <= 10 ORDER BY br.pgts_rank; NOTICE: BM25 index scan: tid=(1,4), BM25_score=-2.6263 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.4269 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-1.8948 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.6700 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6668 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.6595 NOTICE: BM25 index scan: tid=(1,1), BM25_score=-0.5170 NOTICE: BM25 index scan: tid=(1,3), BM25_score=-0.4662 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-0.4424 doc_id | pgts_rank | score | title_preview --------+-----------+---------+---------------------------------------------------- 10 | 1 | -2.6263 | the theory of the impact tube at low pressure . 1 | 2 | -2.4269 | experimental investigation of the aerodynamics of 5 | 3 | -1.8948 | one-dimensional transient heat conduction into a d 3 | 4 | -0.6700 | the boundary layer in simple shear flow past a fla 2 | 5 | -0.6668 | simple shear flow past a flat plate in an incompre 4 | 6 | -0.6595 | approximate solutions of the incompressible lamina 7 | 7 | -0.5170 | the effect of controlled three-dimensional roughne 9 | 8 | -0.4662 | transition studies and skin friction measurements 6 | 9 | -0.4424 | one-dimensional transient heat flow in a multilaye (9 rows) -- Test 4: Performance test with multiple queries \echo 'Test 4: Performance test with top-k search' Test 4: Performance test with top-k search SELECT q.query_id, LEFT(q.query_text, 40) || '...' as query_preview, COUNT(d.doc_id) as total_docs, COUNT(CASE WHEN d.full_text <@> to_bm25query(q.query_text, 'cranfield_tapir_idx') < 0 THEN 1 END) as matching_docs FROM cranfield_queries q CROSS JOIN aerodocs_documents d WHERE q.query_id <= 5 -- Test first 5 queries for performance GROUP BY q.query_id, q.query_text ORDER BY q.query_id; query_id | query_preview | total_docs | matching_docs ----------+------------------------------+------------+--------------- 1 | aerodynamic flow analysis... | 10 | 9 2 | boundary layer turbulent... | 10 | 9 3 | wing design optimization... | 10 | 1 (3 rows) -- Test 5: Index usage verification \echo 'Test 5: Verify index usage with EXPLAIN' Test 5: Verify index usage with EXPLAIN SET jit = off; EXPLAIN (COSTS OFF) SELECT doc_id, ROUND((full_text <@> 'supersonic aircraft design')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'supersonic aircraft design' ASC LIMIT 10; QUERY PLAN ----------------------------------------------------------------------------------------------- Limit -> Index Scan using cranfield_tapir_idx on aerodocs_documents Order By: (full_text <@> 'cranfield_tapir_idx:supersonic aircraft design'::bm25query) (3 rows) -- Reset settings SET enable_seqscan = on; \echo 'Cranfield pg_textsearch validation completed successfully.' Cranfield pg_textsearch validation completed successfully. \echo 'All tests demonstrate proper pg_textsearch functionality with the <@> operator.' All tests demonstrate proper pg_textsearch functionality with the <@> operator. -- Clean up DROP TABLE aerodocs_documents CASCADE; DROP TABLE cranfield_queries CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/basic.out000066400000000000000000000056141525762047700233610ustar00rootroot00000000000000-- Basic functionality tests for pg_textsearch extension -- Test extension creation CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Verify library version GUC is set SELECT current_setting('pg_textsearch.library_version') IS NOT NULL AS has_version; has_version ------------- t (1 row) -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Test bm25vector type exists SELECT pg_typeof('my_index:{database:2,system:1}'::bm25vector); pg_typeof ------------ bm25vector (1 row) -- Test bm25vector input/output SELECT 'my_index:{database:2,system:1}'::bm25vector; bm25vector -------------------------------- my_index:{database:2,system:1} (1 row) -- Test empty bm25vector SELECT 'my_index:{}'::bm25vector; bm25vector ------------- my_index:{} (1 row) -- Test bm25query type SELECT pg_typeof('search terms'::bm25query); pg_typeof ----------- bm25query (1 row) -- Test bm25query input/output SELECT 'search terms'::bm25query; bm25query -------------- search terms (1 row) -- Test to_bm25query functions SELECT to_bm25query('hello world'); to_bm25query -------------- hello world (1 row) SELECT to_bm25query('test query', 'my_index'); ERROR: index "my_index" does not exist -- Test bm25 access method exists SELECT amname FROM pg_am WHERE amname = 'bm25'; amname -------- bm25 (1 row) -- Test creating a bm25 index with text_config CREATE TABLE test_docs (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX test_tapir_idx ON test_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation test_tapir_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Verify index was created SELECT indexrelid::regclass FROM pg_index WHERE indrelid = 'test_docs'::regclass AND indexrelid::regclass::text LIKE '%tapir%'; indexrelid ---------------- test_tapir_idx (1 row) -- Test bm25query with new operators INSERT INTO test_docs (content) VALUES ('hello world example'), ('database system design'), ('the quick brown fox'), ('jumped over lazy dog'), ('sphinx of black quartz'); -- Test text <@> bm25query operator with explicit index for score projection SELECT content, content <@> to_bm25query('hello', 'test_tapir_idx') as score FROM test_docs ORDER BY score LIMIT 1; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-1.3863 content | score ---------------------+--------------------- hello world example | -1.3862943649291992 (1 row) -- Test non-existent index error \set VERBOSITY terse \set ON_ERROR_STOP off SELECT content FROM test_docs ORDER BY content <@> to_bm25query('hello', 'wrong_index') LIMIT 1; ERROR: index "wrong_index" does not exist \set ON_ERROR_STOP on \set VERBOSITY default -- Clean up DROP TABLE test_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/binary_io.out000066400000000000000000000064421525762047700242530ustar00rootroot00000000000000-- Test binary I/O for bm25query type (COPY BINARY) CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create a table and index for testing CREATE TABLE binary_io_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO binary_io_docs (content) VALUES ('hello world test'), ('database search query'); CREATE INDEX binary_io_idx ON binary_io_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation binary_io_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=3.00 -- Create a table with bm25query column CREATE TABLE query_export (id SERIAL, q bm25query); -- Insert some queries INSERT INTO query_export (q) VALUES (to_bm25query('hello world', 'binary_io_idx')), (to_bm25query('database search', 'binary_io_idx')); -- Show original data before COPY SELECT id, q::text AS original_query FROM query_export ORDER BY id; id | original_query ----+------------------------------- 1 | binary_io_idx:hello world 2 | binary_io_idx:database search (2 rows) -- Test COPY TO BINARY (exercises tpquery_send) -- Use \copy (client-side) to avoid /tmp permission issues \copy query_export TO '/tmp/query_export.bin' WITH (FORMAT binary) -- Test COPY FROM BINARY (exercises tpquery_recv) CREATE TABLE query_import (id int, q bm25query); \copy query_import FROM '/tmp/query_export.bin' WITH (FORMAT binary) -- Verify data was imported correctly SELECT COUNT(*) AS imported_count FROM query_import; imported_count ---------------- 2 (1 row) -- Verify the queries are equivalent SELECT e.id, e.q::text AS exported, i.q::text AS imported, e.q = i.q AS queries_match FROM query_export e JOIN query_import i ON e.id = i.id ORDER BY e.id; id | exported | imported | queries_match ----+-------------------------------+-------------------------------+--------------- 1 | binary_io_idx:hello world | binary_io_idx:hello world | t 2 | binary_io_idx:database search | binary_io_idx:database search | t (2 rows) -- ============================================================================= -- Test bm25vector binary I/O (exercises vector.c recv/send functions) -- ============================================================================= -- Create table with bm25vector column for binary copy test CREATE TABLE vector_export (id serial, v bm25vector); INSERT INTO vector_export (v) VALUES ('binary_io_idx:{hello:1,world:2}'::bm25vector), ('binary_io_idx:{database:3,search:1}'::bm25vector); -- Export via COPY BINARY (exercises bm25vector_send) \copy vector_export TO '/tmp/vector_export.bin' WITH (FORMAT binary) -- Import via COPY BINARY (exercises bm25vector_recv) CREATE TABLE vector_import (id int, v bm25vector); \copy vector_import FROM '/tmp/vector_export.bin' WITH (FORMAT binary) -- Verify round-trip SELECT COUNT(*) AS vector_imported_count FROM vector_import; vector_imported_count ----------------------- 2 (1 row) -- Clean up \! rm -f /tmp/query_export.bin /tmp/vector_export.bin DROP TABLE query_export; DROP TABLE query_import; DROP TABLE vector_export; DROP TABLE vector_import; DROP TABLE binary_io_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/bmw.out000066400000000000000000001326201525762047700230630ustar00rootroot00000000000000-- BMW (Block-Max WAND) Algorithm Tests -- Tests the top-k optimization for both single-term and multi-term queries -- Setup: Create extension CREATE EXTENSION pg_textsearch; -- Force index scan so ORDER BY uses the BM25 index (small tables -- would otherwise prefer seq scan, returning non-matching rows) SET enable_seqscan = off; \set ECHO none -- ============================================================ -- SECTION 1: Basic BMW correctness -- ============================================================ CREATE TABLE bmw_basic (id SERIAL PRIMARY KEY, content TEXT); -- Insert documents with known term frequencies INSERT INTO bmw_basic (content) VALUES ('apple apple apple'), -- id=1, tf=3 ('apple apple'), -- id=2, tf=2 ('apple'), -- id=3, tf=1 ('apple apple apple apple'), -- id=4, tf=4 ('banana'), -- id=5, no apple ('apple banana'), -- id=6, tf=1 ('apple apple banana banana'); -- id=7, tf=2 CREATE INDEX bmw_basic_idx ON bmw_basic USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_basic_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 7 documents, avg_length=2.43 -- Test: Single-term BMW should return docs in score order SELECT id, content <@> 'apple'::bm25query as score FROM bmw_basic ORDER BY score LIMIT 5; id | score ----+---------------------- 4 | -0.31600049138069153 1 | -0.3106285035610199 2 | -0.30041441321372986 3 | -0.27344056963920593 7 | -0.24154625833034515 (5 rows) DROP TABLE bmw_basic; -- ============================================================ -- SECTION 2: Empty and edge result counts -- ============================================================ CREATE TABLE bmw_empty (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_empty (content) VALUES ('cat dog bird'), ('fish whale shark'), ('lion tiger bear'); CREATE INDEX bmw_empty_idx ON bmw_empty USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_empty_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.00 -- Test: Query term not in corpus (empty results) SELECT id FROM bmw_empty ORDER BY content <@> 'elephant'::bm25query LIMIT 10; id ---- (0 rows) -- Test: Only 1 document matches SELECT id FROM bmw_empty ORDER BY content <@> 'whale'::bm25query LIMIT 10; id ---- 2 (1 row) -- Test: Fewer results than LIMIT (3 docs match, LIMIT 10) SELECT id FROM bmw_empty ORDER BY content <@> 'cat fish lion'::bm25query LIMIT 10; id ---- 1 2 3 (3 rows) DROP TABLE bmw_empty; -- ============================================================ -- SECTION 3: Tie-breaking with identical scores -- ============================================================ CREATE TABLE bmw_ties (id SERIAL PRIMARY KEY, content TEXT); -- Create documents with IDENTICAL scores (same tf, same doc length) INSERT INTO bmw_ties (content) VALUES ('unique term here'), -- id=1 ('exact same length'), -- id=2, same length as id=1 ('exact same length'), -- id=3, same length ('exact same length'), -- id=4, same length ('exact same length'); -- id=5, same length CREATE INDEX bmw_ties_idx ON bmw_ties USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_ties_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=2.00 -- Test: Documents with identical scores should be ordered by CTID (id) -- All "exact same length" docs have tf=1 and same doc_len SELECT id, content <@> 'exact'::bm25query as score FROM bmw_ties ORDER BY content <@> 'exact'::bm25query LIMIT 10; id | score ----+---------------------- 2 | -0.28768208622932434 3 | -0.28768208622932434 4 | -0.28768208622932434 5 | -0.28768208622932434 (4 rows) DROP TABLE bmw_ties; -- ============================================================ -- SECTION 4: LIMIT edge cases -- ============================================================ CREATE TABLE bmw_limits (id SERIAL PRIMARY KEY, content TEXT); -- Create 20 documents with term "test" INSERT INTO bmw_limits (content) SELECT 'test document number ' || i FROM generate_series(1, 20) i; CREATE INDEX bmw_limits_idx ON bmw_limits USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_limits_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 20 documents, avg_length=4.00 -- Test: LIMIT 1 (minimum) SELECT id FROM bmw_limits ORDER BY content <@> 'test'::bm25query LIMIT 1; id ---- 1 (1 row) -- Test: LIMIT equals exact document count SELECT COUNT(*) FROM ( SELECT id FROM bmw_limits ORDER BY content <@> 'test'::bm25query LIMIT 20 ) x; count ------- 20 (1 row) -- Test: LIMIT exceeds document count SELECT COUNT(*) FROM ( SELECT id FROM bmw_limits ORDER BY content <@> 'test'::bm25query LIMIT 100 ) x; count ------- 20 (1 row) DROP TABLE bmw_limits; -- ============================================================ -- SECTION 5: Multi-term query edge cases -- ============================================================ CREATE TABLE bmw_multi (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_multi (content) VALUES ('alpha beta gamma'), -- id=1, has all 3 ('alpha beta'), -- id=2, has 2 ('alpha'), -- id=3, has 1 ('beta gamma'), -- id=4, has 2 ('delta epsilon'), -- id=5, has none ('alpha alpha alpha beta'), -- id=6, high tf for alpha ('gamma gamma gamma gamma'); -- id=7, high tf for gamma CREATE INDEX bmw_multi_idx ON bmw_multi USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_multi_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 7 documents, avg_length=2.57 -- Test: 2-term query SELECT id, content <@> 'alpha beta'::bm25query as score FROM bmw_multi ORDER BY score LIMIT 5; id | score ----+--------------------- 6 | -1.2767734229564667 2 | -1.265801191329956 1 | -1.0772775411605835 3 | -0.7671522498130798 4 | -0.632900595664978 (5 rows) -- Test: 3-term query SELECT id, content <@> 'alpha beta gamma'::bm25query as score FROM bmw_multi ORDER BY score LIMIT 5; id | score ----+--------------------- 1 | -1.851189374923706 4 | -1.5422469973564148 6 | -1.2767734229564667 7 | -1.2762757539749146 2 | -1.265801191329956 (5 rows) -- Test: Query with non-existent term (partial match) SELECT id, content <@> 'alpha nonexistent'::bm25query as score FROM bmw_multi ORDER BY score LIMIT 5; id | score ----+--------------------- 6 | -0.80795818567276 3 | -0.7671522498130798 2 | -0.632900595664978 1 | -0.5386387705802917 (4 rows) -- Test: Query where all terms are non-existent SELECT id FROM bmw_multi ORDER BY content <@> 'xxx yyy zzz'::bm25query LIMIT 5; id ---- (0 rows) DROP TABLE bmw_multi; -- ============================================================ -- SECTION 6: 8-term boundary (max for multi-term BMW) -- ============================================================ CREATE TABLE bmw_8term (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_8term (content) VALUES ('one two three four five six seven eight nine ten'), ('one two three four'), ('five six seven eight'), ('one three five seven nine'), ('two four six eight ten'); CREATE INDEX bmw_8term_idx ON bmw_8term USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_8term_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=5.60 -- Test: Exactly 8 terms (should use multi-term BMW) SELECT id, content <@> 'one two three four five six seven eight'::bm25query as score FROM bmw_8term ORDER BY score LIMIT 5; id | score ----+--------------------- 1 | -3.2631139755249023 2 | -2.4413371086120605 3 | -2.4413371086120605 4 | -2.254817247390747 5 | -2.254817247390747 (5 rows) -- Test: 9+ terms (should fall back to exhaustive, but still work) SELECT id, content <@> 'one two three four five six seven eight nine'::bm25query as score FROM bmw_8term ORDER BY score LIMIT 5; id | score ----+--------------------- 1 | -3.9256308674812317 4 | -3.170417845249176 2 | -2.4413371086120605 3 | -2.4413371086120605 5 | -2.254817247390747 (5 rows) DROP TABLE bmw_8term; -- ============================================================ -- SECTION 7: Document length extremes -- ============================================================ CREATE TABLE bmw_lengths (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_lengths (content) VALUES ('x'), -- id=1, very short ('word'), -- id=2, short (repeat('word ', 100)), -- id=3, medium (100 words) (repeat('word ', 1000)), -- id=4, long (1000 words) ('word ' || repeat('filler ', 500)); -- id=5, word + filler CREATE INDEX bmw_lengths_idx ON bmw_lengths USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_lengths_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=122.60 -- Test: Query should rank by BM25 (considers doc length) -- Short docs with term should score higher than long docs SELECT id, length(content) as doc_chars, content <@> 'word'::bm25query as score FROM bmw_lengths ORDER BY content <@> 'word'::bm25query LIMIT 5; id | doc_chars | score ----+-----------+--------------------- 4 | 5000 | -0.6276808977127075 3 | 500 | -0.6266049146652222 2 | 4 | -0.4841128885746002 5 | 3505 | -0.2028166949748993 (4 rows) DROP TABLE bmw_lengths; -- ============================================================ -- SECTION 8: High term frequency -- ============================================================ CREATE TABLE bmw_highfreq (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_highfreq (content) VALUES (repeat('keyword ', 1)), -- id=1, tf=1 (repeat('keyword ', 10)), -- id=2, tf=10 (repeat('keyword ', 50)), -- id=3, tf=50 (repeat('keyword ', 100)), -- id=4, tf=100 (repeat('keyword ', 200)); -- id=5, tf=200 CREATE INDEX bmw_highfreq_idx ON bmw_highfreq USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_highfreq_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=72.20 -- Test: High TF documents should rank higher (with saturation) SELECT id, content <@> 'keyword'::bm25query as score FROM bmw_highfreq ORDER BY content <@> 'keyword'::bm25query LIMIT 5; id | score ----+---------------------- 5 | -0.1887885183095932 4 | -0.1886022686958313 3 | -0.187954381108284 2 | -0.18362723290920258 1 | -0.1458514928817749 (5 rows) -- Verify: Order should be 5,4,3,2,1 (highest tf first) -- But scores should show diminishing returns due to BM25 saturation DROP TABLE bmw_highfreq; -- ============================================================ -- SECTION 9: Segment + Memtable combination -- ============================================================ CREATE TABLE bmw_hybrid (id SERIAL PRIMARY KEY, content TEXT); -- Insert batch 1 (will go to segment via CREATE INDEX) INSERT INTO bmw_hybrid (content) SELECT 'segment document ' || i || ' with searchterm' FROM generate_series(1, 500) i; CREATE INDEX bmw_hybrid_idx ON bmw_hybrid USING bm25(content) WITH (text_config='english', compaction='manual'); NOTICE: BM25 index build started for relation bmw_hybrid_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=4.00 -- Batch 2: insert and spill to create a second segment INSERT INTO bmw_hybrid (content) SELECT 'more segment data ' || i || ' searchterm' FROM generate_series(501, 1000) i; SELECT bm25_spill_index('bmw_hybrid_idx'); bm25_spill_index ------------------ 13 (1 row) -- Insert recent data (stays in memtable) INSERT INTO bmw_hybrid (content) VALUES ('fresh memtable searchterm searchterm searchterm'), -- High relevance ('another memtable searchterm searchterm'); -- Test: BMW should find results from both segment and memtable -- The memtable docs with higher tf should rank at top SELECT id, content <@> 'searchterm'::bm25query as score FROM bmw_hybrid ORDER BY content <@> 'searchterm'::bm25query LIMIT 10; id | score ------+------------------------ 1001 | -0.0007437646854668856 1002 | -0.0006856626714579761 1 | -0.0004986796411685646 2 | -0.0004986796411685646 3 | -0.0004986796411685646 4 | -0.0004986796411685646 551 | -0.0004986796411685646 501 | -0.0004986796411685646 502 | -0.0004986796411685646 503 | -0.0004986796411685646 (10 rows) DROP TABLE bmw_hybrid; -- ============================================================ -- SECTION 10: Validate BMW equals exhaustive (LARGE SCALE) -- ============================================================ -- CRITICAL: Must use >128 docs to test block skipping! -- Block size is 128 docs, so we need multiple blocks for skip testing. CREATE TABLE bmw_validate (id SERIAL PRIMARY KEY, content TEXT); -- Create 1000 diverse documents - spans ~8 blocks -- Varying term frequencies to create score diversity INSERT INTO bmw_validate (content) SELECT CASE (i % 10) WHEN 0 THEN 'apple apple apple banana cherry' -- high apple tf WHEN 1 THEN 'apple apple banana' WHEN 2 THEN 'cherry cherry cherry' WHEN 3 THEN 'apple cherry' WHEN 4 THEN 'banana banana cherry cherry' WHEN 5 THEN 'apple' WHEN 6 THEN 'banana apple cherry' WHEN 7 THEN 'cherry banana' WHEN 8 THEN 'apple apple apple apple' -- highest apple tf WHEN 9 THEN 'banana' END || ' document number ' || i FROM generate_series(1, 1000) i; -- Force to segment with low memory limit CREATE INDEX bmw_validate_idx ON bmw_validate USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_validate_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1000 documents, avg_length=5.80 -- Test: BMW (LIMIT) should match exhaustive (subquery LIMIT) -- Single-term WITH bmw AS ( SELECT id, content <@> 'apple'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'apple'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple'::bm25query ) x LIMIT 20 ) SELECT 'single-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result -------------+-------- single-term | PASS (1 row) -- Multi-term WITH bmw AS ( SELECT id, content <@> 'apple banana'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'apple banana'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana'::bm25query ) x LIMIT 20 ) SELECT 'multi-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result ------------+-------- multi-term | PASS (1 row) -- Three-term WITH bmw AS ( SELECT id, content <@> 'apple banana cherry'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana cherry'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'apple banana cherry'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana cherry'::bm25query ) x LIMIT 20 ) SELECT 'three-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result ------------+-------- three-term | PASS (1 row) DROP TABLE bmw_validate; -- ============================================================ -- SECTION 11: Score monotonicity with multiple blocks -- ============================================================ -- Use 500 docs to span multiple blocks (~4 blocks) CREATE TABLE bmw_monotonic (id SERIAL PRIMARY KEY, content TEXT); -- Create varying scores by varying term frequency INSERT INTO bmw_monotonic (content) SELECT CASE WHEN i % 50 = 0 THEN repeat('target ', 5) -- high tf every 50 docs WHEN i % 10 = 0 THEN repeat('target ', 3) -- medium tf every 10 docs ELSE 'target other words here' -- low tf END || ' doc' || i FROM generate_series(1, 500) i; CREATE INDEX bmw_monotonic_idx ON bmw_monotonic USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_monotonic_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=3.14 -- Test: Scores must be monotonically descending (more negative) -- This tests that BMW correctly orders results from multiple blocks WITH ranked AS ( SELECT id, content <@> 'target'::bm25query as score, LAG(content <@> 'target'::bm25query) OVER (ORDER BY content <@> 'target'::bm25query) as prev_score FROM bmw_monotonic ORDER BY content <@> 'target'::bm25query LIMIT 50 ) SELECT 'monotonic-multiblock' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM ranked WHERE prev_score IS NOT NULL AND score < prev_score; test | result ----------------------+-------- monotonic-multiblock | PASS (1 row) DROP TABLE bmw_monotonic; -- ============================================================ -- SECTION 12: Block boundary stress test -- ============================================================ -- Test with 640 docs = exactly 5 blocks of 128 docs each -- Place high-scoring docs at specific positions to test block skipping CREATE TABLE bmw_blocks (id SERIAL PRIMARY KEY, content TEXT); -- Insert 640 base documents (5 blocks worth) INSERT INTO bmw_blocks (content) SELECT 'baseline searchword document ' || i FROM generate_series(1, 640) i; -- Insert high-scoring docs at specific block positions: -- These should all appear in top-10 regardless of which block they're in INSERT INTO bmw_blocks (content) VALUES ('searchword searchword searchword searchword searchword'), -- id=641, tf=5 ('searchword searchword searchword searchword'), -- id=642, tf=4 ('searchword searchword searchword'); -- id=643, tf=3 CREATE INDEX bmw_blocks_idx ON bmw_blocks USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_blocks_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 643 documents, avg_length=4.00 -- Test: High-tf docs (641,642,643) must be in top results -- This validates block-max scoring finds them across multiple blocks SELECT id, content <@> 'searchword'::bm25query as score FROM bmw_blocks ORDER BY content <@> 'searchword'::bm25query LIMIT 10; id | score -----+------------------------ 641 | -0.0013297571567818522 642 | -0.001314413733780384 643 | -0.0012896135449409485 1 | -0.0007766990456730127 2 | -0.0007766990456730127 3 | -0.0007766990456730127 4 | -0.0007766990456730127 5 | -0.0007766990456730127 6 | -0.0007766990456730127 7 | -0.0007766990456730127 (10 rows) -- Validation: top 3 must be our high-tf docs WITH top3 AS ( SELECT id FROM bmw_blocks ORDER BY content <@> 'searchword'::bm25query LIMIT 3 ) SELECT 'block-boundary' as test, CASE WHEN COUNT(*) = 3 AND 641 IN (SELECT id FROM top3) AND 642 IN (SELECT id FROM top3) AND 643 IN (SELECT id FROM top3) THEN 'PASS' ELSE 'FAIL' END as result FROM top3; test | result ----------------+-------- block-boundary | PASS (1 row) DROP TABLE bmw_blocks; -- ============================================================ -- SECTION 13: Repeated query terms -- ============================================================ CREATE TABLE bmw_repeat (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_repeat (content) VALUES ('cat cat cat'), ('cat dog'), ('dog dog dog'); CREATE INDEX bmw_repeat_idx ON bmw_repeat USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_repeat_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=2.67 -- Test: Repeated terms in query should boost relevance SELECT id, content <@> 'cat cat'::bm25query as score FROM bmw_repeat ORDER BY score LIMIT 5; id | score ----+--------------------- 1 | -1.43861985206604 2 | -1.0470967292785645 (2 rows) DROP TABLE bmw_repeat; -- ============================================================ -- SECTION 14: Unicode and special characters -- ============================================================ CREATE TABLE bmw_unicode (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_unicode (content) VALUES ('cafe with accent'), ('naïve approach'), ('北京 beijing china'), ('emoji test'), ('simple ascii text'); CREATE INDEX bmw_unicode_idx ON bmw_unicode USING bm25(content) WITH (text_config='simple'); -- simple config for unicode NOTICE: BM25 index build started for relation bmw_unicode_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=2.60 -- Test: Should handle various character sets SELECT id, content <@> 'beijing'::bm25query as score FROM bmw_unicode ORDER BY score LIMIT 5; id | score ----+-------------------- 3 | -1.304211139678955 (1 row) SELECT id, content <@> 'simple'::bm25query as score FROM bmw_unicode ORDER BY score LIMIT 5; id | score ----+-------------------- 5 | -1.304211139678955 (1 row) DROP TABLE bmw_unicode; -- ============================================================ -- SECTION 15: Top-K scattered across blocks -- ============================================================ -- This is the CRITICAL test for BMW: best docs are spread across -- different blocks, forcing the algorithm to check multiple blocks. CREATE TABLE bmw_scattered (id SERIAL PRIMARY KEY, content TEXT); -- Create 1000 docs where high-scoring docs are deliberately scattered: -- - Every 100th doc (100,200,300...) has tf=4 (highest) -- - Every 50th doc has tf=3 -- - Every 10th doc has tf=2 -- - Others have tf=1 -- This ensures top-10 must come from DIFFERENT blocks INSERT INTO bmw_scattered (content) SELECT CASE WHEN i % 100 = 0 THEN 'scattered scattered scattered scattered' WHEN i % 50 = 0 THEN 'scattered scattered scattered' WHEN i % 10 = 0 THEN 'scattered scattered' ELSE 'scattered filler words here' END || ' doc' || i FROM generate_series(1, 1000) i; CREATE INDEX bmw_scattered_idx ON bmw_scattered USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_scattered_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1000 documents, avg_length=3.93 -- The top-10 should include docs 100,200,300,400,500,600,700,800,900,1000 -- These are spread across 8 different blocks (1000/128 = ~8 blocks) SELECT id, content <@> 'scattered'::bm25query as score FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query LIMIT 10; id | score ------+------------------------ 100 | -0.0008074695360846817 200 | -0.0008074695360846817 300 | -0.0008074695360846817 400 | -0.0008074695360846817 500 | -0.0008074695360846817 600 | -0.0008074695360846817 700 | -0.0008074695360846817 800 | -0.0008074695360846817 900 | -0.0008074695360846817 1000 | -0.0008074695360846817 (10 rows) -- Validation: Verify top 10 includes the high-tf docs from different blocks WITH top10 AS ( SELECT id FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query LIMIT 10 ) SELECT 'scattered-blocks' as test, CASE WHEN (SELECT COUNT(*) FROM top10 WHERE id IN (100,200,300,400,500,600,700,800,900,1000)) >= 8 THEN 'PASS' ELSE 'FAIL' END as result; test | result ------------------+-------- scattered-blocks | PASS (1 row) -- Also validate with exhaustive comparison WITH bmw AS ( SELECT id, content <@> 'scattered'::bm25query as score FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'scattered'::bm25query as score FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query ) x LIMIT 10 ) SELECT 'scattered-equals-exhaustive' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result -----------------------------+-------- scattered-equals-exhaustive | PASS (1 row) DROP TABLE bmw_scattered; -- ============================================================ -- SECTION 16: Multi-term with scattered high-scorers -- ============================================================ -- Tests that multi-term BMW also works with scattered results CREATE TABLE bmw_multi_scattered (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_multi_scattered (content) SELECT CASE WHEN i % 100 = 0 THEN 'alpha alpha alpha beta beta gamma' WHEN i % 50 = 0 THEN 'alpha alpha beta beta' WHEN i % 10 = 0 THEN 'alpha beta' ELSE 'alpha filler content' END || ' doc' || i FROM generate_series(1, 1000) i; CREATE INDEX bmw_multi_scattered_idx ON bmw_multi_scattered USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_multi_scattered_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1000 documents, avg_length=3.96 -- Multi-term query - top results should come from scattered positions WITH bmw AS ( SELECT id, content <@> 'alpha beta gamma'::bm25query as score FROM bmw_multi_scattered ORDER BY content <@> 'alpha beta gamma'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'alpha beta gamma'::bm25query as score FROM bmw_multi_scattered ORDER BY content <@> 'alpha beta gamma'::bm25query ) x LIMIT 20 ) SELECT 'multiterm-scattered' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result ---------------------+-------- multiterm-scattered | PASS (1 row) DROP TABLE bmw_multi_scattered; -- ============================================================ -- SECTION 17: Multiple segments -- ============================================================ CREATE TABLE bmw_multiseg (id SERIAL PRIMARY KEY, content TEXT); -- Batch 1 -> segment 1 (via CREATE INDEX) INSERT INTO bmw_multiseg (content) SELECT 'first segment batch ' || i || ' targetword' FROM generate_series(1, 300) i; CREATE INDEX bmw_multiseg_idx ON bmw_multiseg USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_multiseg_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 300 documents, avg_length=5.00 -- Batch 2 -> segment 2 INSERT INTO bmw_multiseg (content) SELECT 'second segment batch ' || i || ' targetword targetword' FROM generate_series(301, 600) i; SELECT bm25_spill_index('bmw_multiseg_idx'); bm25_spill_index ------------------ 10 (1 row) -- Batch 3 -> memtable (not spilled) INSERT INTO bmw_multiseg (content) SELECT 'third batch ' || i || ' targetword targetword targetword' FROM generate_series(601, 700) i; -- Test: Should find best results across all segments + memtable -- Third batch has highest tf, should rank first SELECT id, content <@> 'targetword'::bm25query as score FROM bmw_multiseg ORDER BY content <@> 'targetword'::bm25query LIMIT 10; id | score -----+------------------------ 601 | -0.0011030653258785605 602 | -0.0011030653258785605 603 | -0.0011030653258785605 604 | -0.0011030653258785605 605 | -0.0011030653258785605 606 | -0.0011030653258785605 607 | -0.0011030653258785605 608 | -0.0011030653258785605 609 | -0.0011030653258785605 610 | -0.0011030653258785605 (10 rows) -- Verify: Results should include docs from batch 3 (id > 600) at top -- Also verify total count is correct (use to_bm25query for count) SELECT COUNT(*) FROM ( SELECT 1 FROM bmw_multiseg ORDER BY content <@> to_bm25query('targetword', 'bmw_multiseg_idx') ) sub; count ------- 700 (1 row) DROP TABLE bmw_multiseg; -- ============================================================ -- SECTION 18: Partial last block -- ============================================================ -- Test with 150 docs = 1 full block (128) + 1 partial block (22) -- This tests off-by-one errors in block iteration CREATE TABLE bmw_partial (id SERIAL PRIMARY KEY, content TEXT); -- 150 docs with varying tf to create score diversity INSERT INTO bmw_partial (content) SELECT CASE WHEN i <= 5 THEN 'partial partial partial partial' -- tf=4, in block 0 WHEN i = 130 THEN 'partial partial partial' -- tf=3, in partial block 1 WHEN i = 145 THEN 'partial partial partial partial partial' -- tf=5, in partial block 1 ELSE 'partial filler doc' || i END FROM generate_series(1, 150) i; CREATE INDEX bmw_partial_idx ON bmw_partial USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_partial_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 150 documents, avg_length=3.05 -- Test: Should find high-tf docs from BOTH blocks -- Doc 145 (tf=5) should be #1, docs 1-5 (tf=4) should follow SELECT validate_bm25_scoring('bmw_partial', 'content', 'bmw_partial_idx', 'partial', 'english', 1.2, 0.75) as partial_block_valid; partial_block_valid --------------------- t (1 row) -- Verify doc 145 is top result (highest tf in partial block) SELECT 'partial-block-top' as test, CASE WHEN (SELECT id FROM bmw_partial ORDER BY content <@> to_bm25query('partial', 'bmw_partial_idx') LIMIT 1) = 145 THEN 'PASS' ELSE 'FAIL' END as result; test | result -------------------+-------- partial-block-top | PASS (1 row) DROP TABLE bmw_partial; -- ============================================================ -- SECTION 19: Term only in memtable (not in segments) -- ============================================================ -- Tests the case where segment iterators return "not found" -- but memtable has data for the term CREATE TABLE bmw_memonly (id SERIAL PRIMARY KEY, content TEXT); -- Insert docs that will go to segment (common term) INSERT INTO bmw_memonly (content) SELECT 'segment content common word doc' || i FROM generate_series(1, 200) i; CREATE INDEX bmw_memonly_idx ON bmw_memonly USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_memonly_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 200 documents, avg_length=5.00 -- Force spill SELECT bm25_spill_index('bmw_memonly_idx'); bm25_spill_index ------------------ (1 row) -- Now insert docs with a NEW term that only exists in memtable INSERT INTO bmw_memonly (content) VALUES ('uniquememterm uniquememterm uniquememterm'), -- tf=3 ('uniquememterm uniquememterm'), -- tf=2 ('uniquememterm'); -- tf=1 -- Test: Query for memtable-only term should work SELECT id, content <@> 'uniquememterm'::bm25query as score FROM bmw_memonly ORDER BY content <@> 'uniquememterm'::bm25query LIMIT 5; id | score -----+-------------------- 201 | -6.978553295135498 202 | -6.716518878936768 203 | -6.036530494689941 (3 rows) -- Verify: Should return exactly 3 docs, in tf order (201, 202, 203) SELECT 'memtable-only-term' as test, CASE WHEN (SELECT COUNT(*) FROM ( SELECT 1 FROM bmw_memonly ORDER BY content <@> to_bm25query('uniquememterm', 'bmw_memonly_idx') ) sub) = 3 THEN 'PASS' ELSE 'FAIL' END as result; test | result --------------------+-------- memtable-only-term | PASS (1 row) -- Test: Multi-term with one term in segment, one in memtable only SELECT id, content <@> 'common uniquememterm'::bm25query as score FROM bmw_memonly ORDER BY content <@> 'common uniquememterm'::bm25query LIMIT 5; id | score -----+----------------------- 201 | -6.978553295135498 202 | -6.716518878936768 203 | -6.036530494689941 1 | -0.017242640256881714 2 | -0.017242640256881714 (5 rows) DROP TABLE bmw_memonly; -- ============================================================ -- SECTION 20: Sparse posting lists (few postings per block) -- ============================================================ -- When a term has very few postings spread across many doc IDs, -- each "block" may have only 1-2 postings. This tests block-max -- effectiveness when block_max ≈ actual score. CREATE TABLE bmw_sparse (id SERIAL PRIMARY KEY, content TEXT); -- Create 500 docs where only every 50th doc has the target term -- This creates ~10 postings spread across what would be 4 blocks INSERT INTO bmw_sparse (content) SELECT CASE WHEN i % 50 = 0 THEN 'sparse rare term here doc' || i ELSE 'common filler words only doc' || i END FROM generate_series(1, 500) i; CREATE INDEX bmw_sparse_idx ON bmw_sparse USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_sparse_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=4.00 -- Test: Should find all 10 sparse docs SELECT 'sparse-count' as test, CASE WHEN (SELECT COUNT(*) FROM ( SELECT 1 FROM bmw_sparse ORDER BY content <@> to_bm25query('sparse', 'bmw_sparse_idx') ) sub) = 10 THEN 'PASS' ELSE 'FAIL' END as result; test | result --------------+-------- sparse-count | PASS (1 row) -- Test: BMW should match exhaustive for sparse terms WITH bmw AS ( SELECT id, content <@> 'sparse'::bm25query as score FROM bmw_sparse ORDER BY content <@> 'sparse'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'sparse'::bm25query as score FROM bmw_sparse ORDER BY content <@> 'sparse'::bm25query ) x LIMIT 10 ) SELECT 'sparse-equals-exhaustive' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result --------------------------+-------- sparse-equals-exhaustive | PASS (1 row) DROP TABLE bmw_sparse; -- ============================================================ -- SECTION 21: Multi-term with different block structures -- ============================================================ -- Term A has many postings (many blocks), term B has few (1 block) -- Tests WAND traversal when iterators have very different lengths CREATE TABLE bmw_asymmetric (id SERIAL PRIMARY KEY, content TEXT); -- 500 docs: "common" in all, "rare" in only 5 specific docs INSERT INTO bmw_asymmetric (content) SELECT CASE WHEN i IN (50, 150, 250, 350, 450) THEN 'common rare both terms doc' || i ELSE 'common only single term doc' || i END FROM generate_series(1, 500) i; CREATE INDEX bmw_asymmetric_idx ON bmw_asymmetric USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_asymmetric_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=4.00 -- Test: "common rare" should find only the 5 docs with both terms at top -- (they have higher combined score than single-term docs) SELECT validate_bm25_scoring('bmw_asymmetric', 'content', 'bmw_asymmetric_idx', 'common rare', 'english', 1.2, 0.75) as asymmetric_valid; asymmetric_valid ------------------ t (1 row) -- Verify top 5 are the dual-term docs (they should have best scores) WITH top5 AS ( SELECT id FROM bmw_asymmetric ORDER BY content <@> 'common rare'::bm25query LIMIT 5 ) SELECT 'asymmetric-top5' as test, CASE WHEN (SELECT COUNT(*) FROM top5 WHERE id IN (50, 150, 250, 350, 450)) = 5 THEN 'PASS' ELSE 'FAIL' END as result; test | result -----------------+-------- asymmetric-top5 | PASS (1 row) DROP TABLE bmw_asymmetric; -- ============================================================ -- SECTION 22: Threshold exactly equals block_max -- ============================================================ -- Create scenario where block_max score equals the threshold exactly -- Tests boundary condition in score < threshold vs score <= threshold CREATE TABLE bmw_threshold (id SERIAL PRIMARY KEY, content TEXT); -- Create docs with IDENTICAL scores to force threshold edge case -- All docs have tf=1 and same length -> identical BM25 scores INSERT INTO bmw_threshold (content) SELECT 'threshold test word' FROM generate_series(1, 200); CREATE INDEX bmw_threshold_idx ON bmw_threshold USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_threshold_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 200 documents, avg_length=3.00 -- Test: With 200 identical-score docs, LIMIT 10 should return 10 -- and they should be ordered by CTID (id 1-10) SELECT 'threshold-boundary' as test, CASE WHEN (SELECT COUNT(*) FROM ( SELECT id FROM bmw_threshold ORDER BY content <@> to_bm25query('threshold', 'bmw_threshold_idx') LIMIT 10 ) x) = 10 THEN 'PASS' ELSE 'FAIL' END as result; test | result --------------------+-------- threshold-boundary | PASS (1 row) -- Verify CTID ordering for tied scores (should be ids 1-10) SELECT 'threshold-tiebreak' as test, CASE WHEN (SELECT array_agg(id ORDER BY id) FROM ( SELECT id FROM bmw_threshold ORDER BY content <@> to_bm25query('threshold', 'bmw_threshold_idx') LIMIT 10 ) x) = ARRAY[1,2,3,4,5,6,7,8,9,10] THEN 'PASS' ELSE 'FAIL' END as result; test | result --------------------+-------- threshold-tiebreak | PASS (1 row) DROP TABLE bmw_threshold; -- ============================================================ -- SECTION 23: All iterators exhaust simultaneously -- ============================================================ -- Create a scenario where multiple term iterators all finish -- at exactly the same document CREATE TABLE bmw_simul (id SERIAL PRIMARY KEY, content TEXT); -- 100 docs where all have both terms - iterators move in lockstep INSERT INTO bmw_simul (content) SELECT 'alpha beta both terms doc' || i FROM generate_series(1, 100) i; -- Add one doc with only alpha at end INSERT INTO bmw_simul (content) VALUES ('alpha only final'); CREATE INDEX bmw_simul_idx ON bmw_simul USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_simul_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 101 documents, avg_length=3.98 -- Test: Should handle simultaneous exhaustion correctly WITH bmw AS ( SELECT id, content <@> 'alpha beta'::bm25query as score FROM bmw_simul ORDER BY content <@> 'alpha beta'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'alpha beta'::bm25query as score FROM bmw_simul ORDER BY content <@> 'alpha beta'::bm25query ) x LIMIT 20 ) SELECT 'simultaneous-exhaust' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result ----------------------+-------- simultaneous-exhaust | PASS (1 row) DROP TABLE bmw_simul; -- ============================================================ -- SECTION 24: Document length variance within blocks -- ============================================================ -- When a block contains a mix of very short and very long docs, -- the block-max upper bound must use the shortest doc's length -- (lowest fieldnorm) to produce a valid BM25 upper bound. -- This test catches bugs where the longest doc's fieldnorm is -- used instead, making the upper bound too tight and causing -- BMW to incorrectly skip blocks with high-scoring short docs. CREATE TABLE bmw_length_variance (id SERIAL PRIMARY KEY, content TEXT); -- Create 500 docs with high length variance within each 128-doc block. -- Short docs (3 words) are interleaved with long docs (100+ words). -- A rare term "unicorn" appears only in a few short docs, scattered -- across blocks. These short docs should score highest for "unicorn" -- because BM25 penalizes longer documents. INSERT INTO bmw_length_variance (content) SELECT CASE -- Rare term in short docs at positions that span multiple blocks WHEN i IN (10, 140, 270, 400) THEN 'unicorn magic sparkle doc' || i -- Long docs with padding (pushes block fieldnorm high) WHEN i % 3 = 0 THEN 'common words here doc' || i || ' ' || repeat('padding ', 100) -- Medium docs WHEN i % 3 = 1 THEN 'common words here with more content doc' || i -- Short docs without the rare term ELSE 'common words here doc' || i END FROM generate_series(1, 500) i; CREATE INDEX bmw_length_variance_idx ON bmw_length_variance USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_length_variance_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=36.34 -- Single-term: BMW must match exhaustive for "unicorn" WITH bmw AS ( SELECT id, content <@> 'unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'unicorn'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'unicorn'::bm25query ) x LIMIT 10 ) SELECT 'length-variance-single-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result -----------------------------+-------- length-variance-single-term | PASS (1 row) -- Multi-term: "common unicorn" -- the rare "unicorn" term has high IDF, -- so short docs with "unicorn" should dominate. BMW must not skip their -- blocks just because long docs in the same block deflate the upper bound. -- Verify the unicorn docs are top-4 for "common unicorn" WITH top4 AS ( SELECT id FROM bmw_length_variance ORDER BY content <@> 'common unicorn'::bm25query LIMIT 4 ) SELECT 'length-variance-top4' as test, CASE WHEN (SELECT COUNT(*) FROM top4 WHERE id IN (10, 140, 270, 400)) = 4 THEN 'PASS' ELSE 'FAIL' END as result; test | result ----------------------+-------- length-variance-top4 | PASS (1 row) -- BMW vs exhaustive with enough LIMIT to avoid tie boundary WITH bmw AS ( SELECT id, content <@> 'common unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'common unicorn'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'common unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'common unicorn'::bm25query ) x LIMIT 20 ) SELECT 'length-variance-bmw-exhaustive' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result --------------------------------+-------- length-variance-bmw-exhaustive | PASS (1 row) DROP TABLE bmw_length_variance; -- ============================================================ -- Cleanup -- ============================================================ DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/bmw_skip_advance.out000066400000000000000000000113711525762047700255710ustar00rootroot00000000000000-- Coverage / regression test for the BMW block_max_skip_advance path -- (issue #355). -- -- The fix in src/scoring/bmw.c changed block_max_skip_advance() to -- pick the pivot term whose current block ends *soonest* as the -- term to seek forward, instead of the highest-max_score term. The -- old selection could pick a term whose cur_doc_id was already past -- min_block_end + 1, making seek_term_to_doc a no-op and spinning -- the outer WAND loop forever on certain MS MARCO data -- distributions. -- -- A deterministic SQL workload that triggers the *old* infinite loop -- has been elusive to construct synthetically: real-world repro -- requires the ~8.8M-passage MS MARCO corpus with concurrent-insert -- segment topology. Internal HorizonDB tests at Microsoft are the -- real validation signal for the bug. -- -- This test is a coverage-focused smoke test: it builds a workload -- that reliably enters block_max_skip_advance many times (verified -- via pg_textsearch.log_bmw_stats: seeks_performed > 0), exercising -- the patched code path. statement_timeout ensures any future -- regression that reintroduces a no-op-seek hang is caught quickly -- rather than burning the whole test run. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; SET statement_timeout = '30s'; CREATE TABLE bmw_skip_advance (id serial PRIMARY KEY, content text); -- Anchor docs: short with high TF on two query terms, raise the -- top-k threshold so most subsequent blocks fail block_max upper -- bound and force the WAND loop into block_max_skip_advance. INSERT INTO bmw_skip_advance (content) SELECT 'alpha alpha alpha alpha alpha beta beta beta beta beta' FROM generate_series(1, 20); -- Bulk: each doc omits exactly one of the four query terms in -- rotation. The four per-term posting lists end up misaligned, so -- per-term block boundaries land at different doc_ids -- which is -- exactly the data condition for the WAND skip-advance path. INSERT INTO bmw_skip_advance (content) SELECT CASE i % 4 WHEN 0 THEN 'beta gamma delta ' WHEN 1 THEN 'alpha gamma delta ' WHEN 2 THEN 'alpha beta delta ' ELSE 'alpha beta gamma ' END || array_to_string(ARRAY( SELECT 'filler' || g FROM generate_series(1, 80) g), ' ') FROM generate_series(1, 5000) i; CREATE INDEX bmw_skip_advance_idx ON bmw_skip_advance USING bm25(content) WITH (text_config = 'simple'); NOTICE: BM25 index build started for relation bmw_skip_advance_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5020 documents, avg_length=82.71 -- This query exercises block_max_skip_advance heavily. Asserting it -- terminates and returns 10 rows is the regression signal -- any -- future change that reintroduces the no-op-seek would cause this to -- hang and statement_timeout would fire. SELECT count(*) AS got_results FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'alpha beta gamma delta', 'bmw_skip_advance_idx') LIMIT 10 ) sub; got_results ------------- 10 (1 row) -- Same query under BMW stats logging, to exercise the -- seeks_performed / blocks_skipped accounting in the patched -- function. (The stats LOG line itself is filtered out of pg_regress -- diff comparison via :v variable; we just want the code path -- exercised.) SET pg_textsearch.log_bmw_stats = on; SELECT count(*) AS got_results FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'alpha beta gamma delta', 'bmw_skip_advance_idx') LIMIT 10 ) sub; got_results ------------- 10 (1 row) SET pg_textsearch.log_bmw_stats = off; -- Now exercise the *memtable* scoring paths (single-term and -- multi-term), where the CHECK_FOR_INTERRUPTS calls added by this -- patch sit. Above this point everything was in segments after -- CREATE INDEX. Inserting fresh docs here without forcing a spill -- leaves them in the memtable, so the next queries traverse both -- segments (via BMW skip-advance) and the memtable. INSERT INTO bmw_skip_advance (content) SELECT 'alpha beta gamma delta extra_memtable_token' FROM generate_series(1, 100); -- Single-term query against memtable (score_memtable_single_term). SELECT count(*) FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'extra_memtable_token', 'bmw_skip_advance_idx') LIMIT 5 ) sub; count ------- 5 (1 row) -- Multi-term query against memtable (score_memtable_multi_term). SELECT count(*) FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'alpha extra_memtable_token', 'bmw_skip_advance_idx') LIMIT 5 ) sub; count ------- 5 (1 row) DROP TABLE bmw_skip_advance; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/boolean_queries.out000066400000000000000000000424101525762047700254470ustar00rootroot00000000000000CREATE EXTENSION pg_textsearch; SET default_text_search_config = 'pg_catalog.english'; CREATE TABLE boolean_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_docs VALUES (1, 'refund approved'), (2, 'refund fraud'), (3, 'fraud refund investigation'), (4, 'billing refund request'), (5, 'billing question'), (8, ''), (9, 'the'); SET client_min_messages = WARNING; CREATE INDEX boolean_docs_body_idx ON boolean_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; INSERT INTO boolean_docs VALUES (6, 'refund fraud followup'), (7, 'refund suspected fraud'); SET enable_seqscan = off; SET pg_textsearch.memtable_cache_enabled = off; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!fraud') ORDER BY id; id ---- 1 4 5 8 9 (5 rows) RESET pg_textsearch.memtable_cache_enabled; EXPLAIN (COSTS OFF) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & fraud'); QUERY PLAN ----------------------------------------------------------- Index Scan using boolean_docs_body_idx on boolean_docs Index Cond: (body @@ '''refund'' & ''fraud'''::tsquery) (2 rows) CREATE FUNCTION pg_temp.first_plan_child(query text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE plan json; BEGIN EXECUTE 'EXPLAIN (FORMAT JSON, COSTS OFF) ' || query INTO plan; RETURN plan->0->'Plan'->'Plans'->0->>'Node Type'; END $$; CREATE TEMP TABLE boolean_other (id integer); INSERT INTO boolean_other SELECT generate_series(1, 9); \pset format unaligned SET enable_sort = off; SELECT pg_temp.first_plan_child($query$ SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund') ORDER BY body <@> to_bm25query('refund', 'boolean_docs_body_idx') LIMIT 1 $query$) = 'Sort' AS unsupported_path_falls_back; unsupported_path_falls_back t (1 row) SET enable_nestloop = off; SELECT pg_temp.first_plan_child($query$ SELECT d.id FROM boolean_docs d JOIN boolean_other other ON other.id = d.id WHERE d.body @@ to_tsquery('english', 'refund') ORDER BY d.body <@> to_bm25query('refund', 'boolean_docs_body_idx') LIMIT 1 $query$) = 'Sort' AS unsupported_join_path_falls_back; unsupported_join_path_falls_back t (1 row) SELECT ( SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'fraud') ORDER BY body <@> to_bm25query('refund', 'boolean_docs_body_idx') LIMIT 1 ) = 2 AS combined_query_ranks_matches; combined_query_ranks_matches t (1 row) RESET enable_nestloop; RESET enable_sort; \pset format aligned SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & fraud') ORDER BY id; id ---- 2 3 6 7 (4 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund | fraud') ORDER BY id; id ---- 1 2 3 4 6 7 (6 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & !fraud') ORDER BY id; id ---- 1 4 (2 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'billing & (refund | !question)') ORDER BY id; id ---- 4 (1 row) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund <-> fraud') ORDER BY id; id ---- 2 6 (2 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!fraud') ORDER BY id; id ---- 1 4 5 8 9 (5 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund | !fraud') ORDER BY id; id ---- 1 2 3 4 5 6 7 8 9 (9 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!(refund <-> fraud)') ORDER BY id; id ---- 1 3 4 5 7 8 9 (7 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund:*') ORDER BY id; id ---- 1 2 3 4 6 7 (6 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund:A') ORDER BY id; id ---- (0 rows) SET plan_cache_mode = force_generic_plan; PREPARE boolean_scan(tsquery) AS SELECT id FROM boolean_docs WHERE body @@ $1 ORDER BY id; EXPLAIN (COSTS OFF) EXECUTE boolean_scan(to_tsquery('english', 'refund & !fraud')); QUERY PLAN -------------------------------------------------------------- Sort Sort Key: id -> Index Scan using boolean_docs_body_idx on boolean_docs Index Cond: (body @@ $1) (4 rows) EXECUTE boolean_scan(to_tsquery('english', 'refund & fraud')); id ---- 2 3 6 7 (4 rows) EXECUTE boolean_scan(to_tsquery('english', '!fraud')); id ---- 1 4 5 8 9 (5 rows) EXECUTE boolean_scan(to_tsquery('english', 'refund <-> fraud')); id ---- 2 6 (2 rows) EXECUTE boolean_scan(to_tsquery('english', 'refund:*')); id ---- 1 2 3 4 6 7 (6 rows) SET default_text_search_config = 'pg_catalog.simple'; EXECUTE boolean_scan(NULL); id ---- (0 rows) EXECUTE boolean_scan(to_tsquery('simple', 'refund & fraud')); ERROR: BM25 index "boolean_docs_body_idx" cannot evaluate text search configuration "simple" DETAIL: The index uses text search configuration "english". HINT: Set default_text_search_config to match the index or use a different index. DEALLOCATE boolean_scan; RESET plan_cache_mode; RESET enable_seqscan; EXPLAIN (COSTS OFF) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('simple', 'refund & fraud'); QUERY PLAN ------------------------------------------------------- Seq Scan on boolean_docs Filter: (body @@ '''refund'' & ''fraud'''::tsquery) (2 rows) SET enable_seqscan = off; SET default_text_search_config = 'pg_catalog.english'; SET client_min_messages = WARNING; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '') ORDER BY id; id ---- (0 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'the') ORDER BY id; id ---- (0 rows) RESET client_min_messages; UPDATE boolean_docs SET body = 'billing fraud question' WHERE id = 5; DELETE FROM boolean_docs WHERE id = 1; VACUUM boolean_docs; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!fraud') ORDER BY id; id ---- 4 8 9 (3 rows) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & !fraud') ORDER BY id; id ---- 4 (1 row) RESET enable_seqscan; RESET default_text_search_config; DROP TABLE boolean_docs; CREATE TABLE boolean_empty_docs ( id integer PRIMARY KEY, body text NOT NULL ); SET client_min_messages = WARNING; CREATE INDEX boolean_empty_docs_body_idx ON boolean_empty_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; INSERT INTO boolean_empty_docs VALUES (1, ''), (2, 'the'); SET enable_seqscan = off; SET pg_textsearch.memtable_cache_enabled = off; SELECT id FROM boolean_empty_docs WHERE body @@ to_tsquery('english', '!missing') ORDER BY id; id ---- 1 2 (2 rows) RESET pg_textsearch.memtable_cache_enabled; SELECT bm25_spill_index('boolean_empty_docs_body_idx') IS NOT NULL AS spilled_empty_docs; spilled_empty_docs -------------------- t (1 row) INSERT INTO boolean_empty_docs VALUES (3, ''), (4, 'the'); SELECT bm25_spill_index('boolean_empty_docs_body_idx') IS NOT NULL AS spilled_more_empty_docs; spilled_more_empty_docs ------------------------- t (1 row) SELECT bm25_force_merge('boolean_empty_docs_body_idx'); bm25_force_merge ------------------ (1 row) SELECT id FROM boolean_empty_docs WHERE body @@ to_tsquery('english', '!missing') ORDER BY id; id ---- 1 2 3 4 (4 rows) RESET enable_seqscan; DROP TABLE boolean_empty_docs; CREATE TABLE boolean_segment_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_segment_docs VALUES (1, 'alpha beta'), (2, 'alpha'), (3, 'beta'); SET client_min_messages = WARNING; CREATE INDEX boolean_segment_docs_body_idx ON boolean_segment_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; INSERT INTO boolean_segment_docs VALUES (4, 'gamma delta'), (5, 'gamma'), (6, 'delta'), (7, 'alpha gamma'); SELECT bm25_spill_index('boolean_segment_docs_body_idx') IS NOT NULL AS spilled_segment_docs; spilled_segment_docs ---------------------- t (1 row) INSERT INTO boolean_segment_docs VALUES (8, 'alpha delta'), (9, 'beta gamma'), (10, 'epsilon'); DELETE FROM boolean_segment_docs WHERE id = 2; VACUUM boolean_segment_docs; SET enable_seqscan = off; SELECT array_agg(id ORDER BY id) AS nested_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', '(alpha & beta) | (gamma & delta)'); nested_matches ---------------- {1,4} (1 row) SELECT array_agg(id ORDER BY id) AS anchored_not_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha & !gamma'); anchored_not_matches ---------------------- {1,8} (1 row) SELECT array_agg(id ORDER BY id) AS duplicate_term_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha | alpha'); duplicate_term_matches ------------------------ {1,7,8} (1 row) SELECT count(*) AS missing_and_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha & missing'); missing_and_matches --------------------- 0 (1 row) SELECT array_agg(id ORDER BY id) AS mixed_source_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha | epsilon'); mixed_source_matches ---------------------- {1,7,8,10} (1 row) SELECT array_agg(id ORDER BY id) AS pure_negative_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', '!missing'); pure_negative_matches ----------------------- {1,3,4,5,6,7,8,9,10} (1 row) SELECT array_agg(id ORDER BY id) AS phrase_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha <-> beta'); phrase_matches ---------------- {1} (1 row) RESET enable_seqscan; DROP TABLE boolean_segment_docs; SET default_text_search_config = 'pg_catalog.simple'; CREATE TABLE boolean_prefix_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_prefix_docs VALUES (1, 'cat'), (2, 'cater'), (3, 'dog'); SET client_min_messages = WARNING; SET pg_textsearch.compress_segments = off; CREATE INDEX boolean_prefix_docs_body_idx ON boolean_prefix_docs USING bm25(body) WITH (text_config = 'simple'); RESET client_min_messages; INSERT INTO boolean_prefix_docs VALUES (4, 'cattle'), (5, 'catalog'), (6, 'catch'); \pset format unaligned SELECT bm25_spill_index('boolean_prefix_docs_body_idx') IS NOT NULL AS spilled_prefix_docs; spilled_prefix_docs t (1 row) RESET pg_textsearch.compress_segments; INSERT INTO boolean_prefix_docs VALUES (7, 'catfish'), (8, 'category'), (9, 'bird'), (10, 'cat cater'); SET enable_seqscan = off; SELECT array_agg(id ORDER BY id) AS prefix_matches FROM boolean_prefix_docs WHERE body @@ to_tsquery('simple', 'cat:*'); prefix_matches {1,2,4,5,6,7,8,10} (1 row) \pset format aligned RESET enable_seqscan; DROP TABLE boolean_prefix_docs; CREATE TABLE boolean_broad_prefix_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_broad_prefix_docs SELECT id, CASE WHEN id <= 257 THEN format('broadprefix%s', lpad(id::text, 4, '0')) ELSE 'unrelated' END FROM generate_series(1, 258) AS id; SET client_min_messages = WARNING; CREATE INDEX boolean_broad_prefix_docs_body_idx ON boolean_broad_prefix_docs USING bm25(body) WITH (text_config = 'simple'); RESET client_min_messages; SET enable_seqscan = off; \pset format unaligned SELECT count(*) AS broad_prefix_matches FROM boolean_broad_prefix_docs WHERE body @@ to_tsquery('simple', 'broadprefix:*'); broad_prefix_matches 257 (1 row) \pset format aligned RESET enable_seqscan; DROP TABLE boolean_broad_prefix_docs; SET default_text_search_config = 'pg_catalog.english'; CREATE TABLE boolean_seek_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_seek_docs SELECT id, concat_ws(' ', CASE WHEN id % 2 = 0 THEN 'common' END, CASE WHEN id % 997 = 0 OR id = 5001 THEN 'anchor' END) FROM generate_series(1, 5001) AS id; SET client_min_messages = WARNING; CREATE INDEX boolean_seek_docs_body_idx ON boolean_seek_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; ANALYZE boolean_seek_docs; \pset format unaligned SELECT pg_temp.first_plan_child($query$ SELECT id FROM boolean_seek_docs WHERE body @@ to_tsquery('english', 'common') LIMIT 1 $query$) = 'Seq Scan' AS boolean_limit_avoids_materializing_index; boolean_limit_avoids_materializing_index t (1 row) \pset format aligned SET enable_seqscan = off; \pset format unaligned SELECT array_agg(id ORDER BY id) AS galloping_seek_matches FROM boolean_seek_docs WHERE body @@ to_tsquery('english', 'anchor & common'); galloping_seek_matches {1994,3988} (1 row) \pset format aligned RESET enable_seqscan; DROP TABLE boolean_seek_docs; CREATE TABLE boolean_memtable_stream_docs ( id integer PRIMARY KEY, body text NOT NULL ); SET client_min_messages = WARNING; CREATE INDEX boolean_memtable_stream_docs_body_idx ON boolean_memtable_stream_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO boolean_memtable_stream_docs SELECT id, concat_ws(' ', CASE WHEN id % 2 = 0 THEN 'common' END, CASE WHEN id % 997 = 0 OR id = 5001 THEN 'anchor' END, CASE WHEN id = 5001 THEN 'alternative' END, 'ubiquitous') FROM generate_series(1, 5001) AS id; SET enable_seqscan = off; \pset format unaligned SELECT array_agg(id ORDER BY id) AS memtable_selective_and_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery('english', 'anchor & common'); memtable_selective_and_matches {1994,3988} (1 row) SELECT array_agg(id ORDER BY id) AS memtable_nested_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery( 'english', 'anchor & (common | alternative)'); memtable_nested_matches {1994,3988,5001} (1 row) SELECT array_agg(id ORDER BY id) AS memtable_anchored_not_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery('english', 'anchor & !common'); memtable_anchored_not_matches {997,2991,4985,5001} (1 row) SELECT count(*) AS memtable_nested_not_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery( 'english', '(alternative | !common) & ubiquitous'); memtable_nested_not_matches 2501 (1 row) \pset format aligned SET pg_textsearch.memtable_cache_enabled = off; \pset format unaligned SELECT array_agg(id ORDER BY id) AS chain_selective_and_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery('english', 'anchor & common'); chain_selective_and_matches {1994,3988} (1 row) SELECT count(*) AS chain_nested_not_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery( 'english', '(alternative | !common) & ubiquitous'); chain_nested_not_matches 2501 (1 row) \pset format aligned RESET pg_textsearch.memtable_cache_enabled; RESET enable_seqscan; RESET pg_textsearch.bulk_load_threshold; RESET pg_textsearch.memtable_pages_threshold; DROP TABLE boolean_memtable_stream_docs; CREATE TABLE boolean_many_term_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_many_term_docs SELECT document, query.body FROM generate_series(1, 1000) AS document CROSS JOIN ( SELECT string_agg(format('term%s', term), ' ') AS body FROM generate_series(1, 201) AS term ) AS query; SET pg_textsearch.compress_segments = off; SET client_min_messages = WARNING; CREATE INDEX boolean_many_term_docs_body_idx ON boolean_many_term_docs USING bm25(body) WITH (text_config = 'simple'); RESET client_min_messages; RESET pg_textsearch.compress_segments; CREATE FUNCTION pg_temp.boolean_and_query(term_count integer) RETURNS tsquery LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT string_agg(format('term%s', term), ' & ')::tsquery FROM generate_series(1, term_count) AS term $$; CREATE FUNCTION pg_temp.boolean_repeated_or_query(term_count integer) RETURNS tsquery LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT string_agg('term1', ' | ')::tsquery FROM generate_series(1, term_count) $$; SET default_text_search_config = 'pg_catalog.simple'; SELECT pg_temp.boolean_and_query(64)::text AS query \gset exact_limit_ SELECT pg_temp.boolean_and_query(201)::text AS query \gset oversized_ SELECT pg_temp.boolean_repeated_or_query(65)::text AS query \gset repeated_ \pset format unaligned SELECT pg_temp.first_plan_child(format( 'SELECT count(*) FROM boolean_many_term_docs WHERE body @@ %L::tsquery', :'oversized_query' )) = 'Seq Scan' AS oversized_boolean_falls_back; oversized_boolean_falls_back t (1 row) SELECT pg_temp.first_plan_child(format( 'SELECT count(*) FROM boolean_many_term_docs WHERE body @@ %L::tsquery', :'repeated_query' )) = 'Seq Scan' AS repeated_boolean_falls_back; repeated_boolean_falls_back t (1 row) SET enable_seqscan = off; SELECT pg_temp.first_plan_child(format( 'SELECT count(*) FROM boolean_many_term_docs WHERE body @@ %L::tsquery', :'exact_limit_query' )) = 'Index Scan' AS exact_operand_limit_uses_index; exact_operand_limit_uses_index t (1 row) SELECT count(*) = 1000 AS exact_operand_limit_succeeds FROM boolean_many_term_docs WHERE body @@ :'exact_limit_query'::tsquery; exact_operand_limit_succeeds t (1 row) RESET enable_seqscan; \pset format aligned DROP TABLE boolean_many_term_docs; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/build_progress.out000066400000000000000000000175371525762047700253320ustar00rootroot00000000000000-- Test BM25 build-progress lifecycle. CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Progress notices should be emitted only when an index is actually created. CREATE TABLE progress_noop_docs (content text); CREATE INDEX progress_noop_idx ON progress_noop_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation progress_noop_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE INDEX IF NOT EXISTS progress_noop_idx ON progress_noop_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: relation "progress_noop_idx" already exists, skipping CREATE TABLE progress_empty_partitioned ( id integer, content text ) PARTITION BY RANGE (id); CREATE INDEX progress_empty_partitioned_idx ON progress_empty_partitioned USING bm25(content) WITH (text_config = 'english'); CREATE TABLE progress_actual_docs (content text); CREATE TABLE progress_collision_docs (id integer); CREATE INDEX progress_actual_idx ON progress_collision_docs(id); CREATE FUNCTION progress_drop_collision() RETURNS event_trigger AS $$ BEGIN IF current_setting('progress_test.drop_collision', true) = 'on' THEN PERFORM set_config('progress_test.drop_collision', 'off', true); EXECUTE 'DROP INDEX progress_actual_idx'; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER progress_drop_collision_trigger ON ddl_command_start WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION progress_drop_collision(); SET progress_test.drop_collision = on; CREATE INDEX IF NOT EXISTS progress_actual_idx ON progress_actual_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation progress_actual_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 RESET progress_test.drop_collision; SELECT indrelid = 'progress_actual_docs'::regclass AS actual_index_created FROM pg_index WHERE indexrelid = 'progress_actual_idx'::regclass; actual_index_created ---------------------- t (1 row) -- A failed build must not leave progress active for later implicit builds. CREATE TABLE progress_abort_docs ( id integer, content text ) PARTITION BY RANGE (id); CREATE TABLE progress_abort_part PARTITION OF progress_abort_docs FOR VALUES FROM (0) TO (10); CREATE FUNCTION progress_fail_build_end() RETURNS event_trigger AS $$ BEGIN IF current_setting('progress_test.fail_build_end', true) = 'on' THEN PERFORM set_config('progress_test.fail_build_end', 'off', true); RAISE EXCEPTION 'forced progress abort'; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER progress_fail_build_end_trigger ON ddl_command_end WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION progress_fail_build_end(); SET progress_test.fail_build_end = on; \set VERBOSITY terse CREATE INDEX progress_abort_idx ON progress_abort_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation progress_abort_part_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 ERROR: forced progress abort \set VERBOSITY default RESET progress_test.fail_build_end; REINDEX INDEX progress_actual_idx; NOTICE: BM25 index build started for relation progress_actual_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Nested index DDL must not replace an outer partitioned build's progress. CREATE TABLE progress_nested_docs ( id integer, content text ) PARTITION BY RANGE (id); CREATE TABLE progress_nested_part_a PARTITION OF progress_nested_docs FOR VALUES FROM (0) TO (10); CREATE TABLE progress_nested_part_b PARTITION OF progress_nested_docs FOR VALUES FROM (10) TO (20); CREATE TABLE progress_nested_side (content text); CREATE TABLE progress_nested_reindex_side (content text); INSERT INTO progress_nested_docs VALUES (1, 'apple'), (11, 'apple banana'), (12, 'apple banana cherry'); INSERT INTO progress_nested_side VALUES ('kiwi kiwi kiwi kiwi'); INSERT INTO progress_nested_reindex_side VALUES ('plum plum plum plum plum'); CREATE INDEX progress_nested_reindex_idx ON progress_nested_reindex_side USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation progress_nested_reindex_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=5.00 CREATE FUNCTION progress_create_nested_index() RETURNS event_trigger AS $$ DECLARE action text := current_setting('progress_test.nested_action', true); BEGIN IF action = 'create' THEN PERFORM set_config('progress_test.nested_action', 'running', true); EXECUTE 'CREATE INDEX progress_nested_side_idx ' 'ON progress_nested_side USING bm25(content) ' 'WITH (text_config = ''english'')'; ELSIF action = 'reindex' THEN PERFORM set_config('progress_test.nested_action', 'running', true); EXECUTE 'REINDEX INDEX progress_nested_reindex_idx'; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER progress_create_nested_trigger ON ddl_command_end WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION progress_create_nested_index(); SET progress_test.nested_action = 'create'; CREATE INDEX progress_nested_idx ON progress_nested_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation progress_nested_part_a_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build started for relation progress_nested_side_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=4.00 NOTICE: BM25 index build completed: 3 documents across 2 partitions, avg_length=2.00 RESET progress_test.nested_action; CREATE TABLE progress_reindex_docs ( id integer, content text ) PARTITION BY RANGE (id); CREATE TABLE progress_reindex_part_a PARTITION OF progress_reindex_docs FOR VALUES FROM (0) TO (10); CREATE TABLE progress_reindex_part_b PARTITION OF progress_reindex_docs FOR VALUES FROM (10) TO (20); INSERT INTO progress_reindex_docs VALUES (1, 'apple'), (11, 'apple banana'), (12, 'apple banana cherry'); SET progress_test.nested_action = reindex; CREATE INDEX progress_reindex_outer_idx ON progress_reindex_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation progress_reindex_part_a_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build started for relation progress_nested_reindex_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=5.00 NOTICE: BM25 index build completed: 3 documents across 2 partitions, avg_length=2.00 RESET progress_test.nested_action; DROP EVENT TRIGGER progress_create_nested_trigger; DROP FUNCTION progress_create_nested_index(); DROP EVENT TRIGGER progress_fail_build_end_trigger; DROP FUNCTION progress_fail_build_end(); DROP EVENT TRIGGER progress_drop_collision_trigger; DROP FUNCTION progress_drop_collision(); DROP TABLE progress_noop_docs; DROP TABLE progress_empty_partitioned; DROP TABLE progress_actual_docs; DROP TABLE progress_collision_docs; DROP TABLE progress_abort_docs; DROP TABLE progress_nested_docs; DROP TABLE progress_nested_side; DROP TABLE progress_nested_reindex_side; DROP TABLE progress_reindex_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/bulk_load.out000066400000000000000000000050171525762047700242310ustar00rootroot00000000000000-- Test bulk load spill threshold -- Exercises state.c bulk_load_spill_check path CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Set a very low bulk load threshold to trigger automatic spill SET pg_textsearch.bulk_load_threshold = 100; -- Create table and index CREATE TABLE bulk_load_test (id serial PRIMARY KEY, content text); CREATE INDEX bulk_load_idx ON bulk_load_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bulk_load_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert enough distinct terms in one transaction to exceed threshold -- With threshold=100, this should trigger auto-spill at commit BEGIN; INSERT INTO bulk_load_test (content) SELECT 'uniqueterm' || i || ' extra words for length variation' FROM generate_series(1, 200) AS i; COMMIT; -- Verify the data is searchable (spill should have happened) SELECT count(*) FROM ( SELECT 1 FROM bulk_load_test ORDER BY content <@> to_bm25query('uniqueterm1', 'bulk_load_idx') ) sub; count ------- 1 (1 row) -- Check index summary shows segment data SELECT bm25_summarize_index('bulk_load_idx') IS NOT NULL AS has_summary; has_summary ------------- t (1 row) -- ============================================================================= -- Test 2: Double bulk load spill with pre-existing L0 segment -- Exercises state.c segment chain linking in tp_bulk_load_spill_check -- ============================================================================= -- First, create a segment by spilling the existing memtable SELECT bm25_spill_index('bulk_load_idx'); bm25_spill_index ------------------ (1 row) -- Now insert more data in a single transaction with low threshold -- This triggers bulk_load_spill_check with a pre-existing L0 chain BEGIN; INSERT INTO bulk_load_test (content) SELECT 'secondbatch' || i || ' more content for spill' FROM generate_series(1, 200) AS i; COMMIT; -- Verify both batches are searchable SELECT count(*) FROM ( SELECT 1 FROM bulk_load_test ORDER BY content <@> to_bm25query('uniqueterm1', 'bulk_load_idx') ) sub; count ------- 1 (1 row) SELECT count(*) FROM ( SELECT 1 FROM bulk_load_test ORDER BY content <@> to_bm25query('secondbatch1', 'bulk_load_idx') ) sub; count ------- 1 (1 row) -- Reset threshold SET pg_textsearch.bulk_load_threshold = 100000; -- Clean up DROP TABLE bulk_load_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/cache_apply.out000066400000000000000000000114101525762047700245370ustar00rootroot00000000000000-- Unit coverage for the in-memory memtable cache apply protocol -- (cache.c; see ARCHITECTURE.md, "Memtable Cache"). -- -- Drives bm25_cache_cold_build / bm25_cache_apply_to_tail directly -- so we can observe (result, records_applied, cursor_seq, -- estimated_bytes) deltas across each step without going through -- the normal reader path. -- -- Records come from real heap inserts so we exercise the -- production write path end-to-end. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE cache_apply_t (id int, body text); CREATE INDEX cache_apply_idx ON cache_apply_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_apply_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- ---------- empty chain ---------- -- Cold build over an empty chain returns OK with 0 records and -- leaves the cursor unseeded (head_blkno was Invalid). SELECT result, records_applied FROM bm25_cache_cold_build('cache_apply_idx'); result | records_applied --------+----------------- OK | 0 (1 row) -- apply_to_tail on an empty/uninitialized cache returns -- NOT_INITIALIZED so the caller knows to cold-build first. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | records_applied -----------------+----------------- NOT_INITIALIZED | 0 (1 row) -- ---------- cold build over real records ---------- INSERT INTO cache_apply_t SELECT g, 'shared term ' || g FROM generate_series(1, 5) g; -- Cold-build picks up every record. records_applied = 5, -- cursor_seq = 5, estimated_bytes > 0. SELECT result, records_applied, cursor_seq, (estimated_bytes > 0) AS bytes_nonzero FROM bm25_cache_cold_build('cache_apply_idx'); result | records_applied | cursor_seq | bytes_nonzero --------+-----------------+------------+--------------- OK | 5 | 5 | t (1 row) -- Re-running cold_build now returns RETRY (cache already built); -- cursor_seq stays at 5 and no new records get applied. SELECT result, records_applied FROM bm25_cache_cold_build('cache_apply_idx'); result | records_applied --------+----------------- RETRY | 0 (1 row) -- apply_to_tail with no new records is a no-op: OK / 0 records. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | records_applied --------+----------------- OK | 0 (1 row) -- ---------- incremental catchup ---------- INSERT INTO cache_apply_t SELECT g, 'newer term ' || g FROM generate_series(6, 10) g; -- apply_to_tail picks up the 5 new records. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | records_applied --------+----------------- OK | 5 (1 row) -- A second apply_to_tail is again a no-op. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | records_applied --------+----------------- OK | 0 (1 row) -- ---------- spill invalidates the cache ---------- -- Bump the spill generation directly to exercise the DROPPED -- defensive branch in tp_cache_apply_to_tail. Real spill paths -- (tp_spill_finalize) bump the generation AND immediately drop -- the cache's dshash tables via tp_cache_clear, so the next -- apply_to_tail lands on NOT_INITIALIZED instead of DROPPED. -- The helper isolates the stale-cursor-without-cleared-tables -- case so the safety net stays exercised. SELECT (bm25_cache_bump_spill_generation('cache_apply_idx') > 0) AS gen_advanced; gen_advanced -------------- t (1 row) -- apply_to_tail sees the stale generation token and DROPs the -- cache. cursor_seq drops back to 0 because tp_cache_clear -- resets it. SELECT result, cursor_seq FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | cursor_seq ---------+------------ DROPPED | 0 (1 row) -- The cache is now back at the cold state: NOT_INITIALIZED. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | records_applied -----------------+----------------- NOT_INITIALIZED | 0 (1 row) -- Cold-build re-bootstraps with the current generation, so the -- next apply_to_tail returns OK with 0 new records. SELECT result, (records_applied >= 0) AS records_ok FROM bm25_cache_cold_build('cache_apply_idx'); result | records_ok --------+------------ OK | t (1 row) SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); result | records_applied --------+----------------- OK | 0 (1 row) DROP INDEX cache_apply_idx; DROP TABLE cache_apply_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/cache_memory_cap.out000066400000000000000000000335121525762047700255540ustar00rootroot00000000000000-- Integration coverage for the memory cap and the -- cross-index eviction path -- (see ARCHITECTURE.md, "Memtable Cache"). -- -- We exercise the registry-wide accounting counter and the -- `tp_cache_evict_largest` argmax routine directly through -- internal SQL scaffolds. End-to-end cap-driven eviction -- (where `apply_to_tail` itself triggers eviction because -- `pg_textsearch.memory_limit` was exceeded) cannot be -- exercised from a regression test because `memory_limit` -- is PGC_SIGHUP, so we focus on the deterministic primitives -- that the cap protocol composes from. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_cache_enabled = on; CREATE TABLE cache_cap_a (id int, body text); CREATE TABLE cache_cap_b (id int, body text); CREATE INDEX cache_cap_a_idx ON cache_cap_a USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_cap_a_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE INDEX cache_cap_b_idx ON cache_cap_b USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_cap_b_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- ---------- empty registry: nothing to evict ---------- -- Eviction with no caches populated returns nothing_found. -- `bm25_cache_evict_largest('cache_cap_a_idx')` excludes -- cache_cap_a_idx from the argmax, but cache_cap_b_idx has -- zero estimated_bytes so still no eligible victim. SELECT bm25_cache_evict_largest('cache_cap_a_idx') AS empty_evict; empty_evict --------------- nothing_found (1 row) SELECT bm25_cache_global_estimated_bytes() AS empty_global_bytes; empty_global_bytes -------------------- 0 (1 row) -- ---------- populate index A heavily, B lightly ---------- INSERT INTO cache_cap_a SELECT g, 'apple banana cherry doc ' || g FROM generate_series(1, 80) g; INSERT INTO cache_cap_b SELECT g, 'kiwi ' || g FROM generate_series(1, 5) g; -- Drive the apply protocol so the dshash tables are -- populated and per-index estimated_bytes is bumped. -- We check the apply outcome was "applied" (vs ABORT) and -- that bytes is non-zero. SELECT result, (estimated_bytes > 0) AS a_bytes_nonzero FROM bm25_cache_cold_build('cache_cap_a_idx'); result | a_bytes_nonzero --------+----------------- OK | t (1 row) SELECT result, (estimated_bytes > 0) AS b_bytes_nonzero FROM bm25_cache_cold_build('cache_cap_b_idx'); result | b_bytes_nonzero --------+----------------- OK | t (1 row) -- After both caches built, global counter must equal the -- sum of per-index estimated_bytes (we already proved both -- are > 0; here we just assert that global > 0). SELECT bm25_cache_global_estimated_bytes() > 0 AS post_build_global_nonzero; post_build_global_nonzero --------------------------- t (1 row) -- ---------- cross-index eviction ---------- -- evict_largest called with caller = cache_cap_b_idx should -- pick cache_cap_a_idx as victim (a has more bytes than b) -- and evict it. SELECT bm25_cache_evict_largest('cache_cap_b_idx') AS evict_a_from_b; evict_a_from_b ---------------- evicted (1 row) -- After evicting A, the global counter should equal B's -- bytes only. We check by re-opening B's cache source -- without rebuilding (open is idempotent for a populated -- cache: `catchup_cache` returns OK with zero new records). -- Easier observable: global counter dropped strictly. -- Snapshot pre-evict (we recomputed above to be > 0) and -- post-evict, then verify post < pre via a CTE. WITH post AS ( SELECT bm25_cache_global_estimated_bytes() AS bytes ) SELECT bytes > 0 AS post_evict_b_still_present FROM post; post_evict_b_still_present ---------------------------- t (1 row) -- A second call to evict_largest('cache_cap_b_idx') now -- finds no eligible victim: the only other index -- (cache_cap_a_idx) was just drained to zero, and B is -- excluded. SELECT bm25_cache_evict_largest('cache_cap_b_idx') AS evict_again_no_victim; evict_again_no_victim ----------------------- nothing_found (1 row) -- And evicting with caller = cache_cap_a_idx drains B as -- the sole remaining candidate. SELECT bm25_cache_evict_largest('cache_cap_a_idx') AS evict_b_from_a; evict_b_from_a ---------------- evicted (1 row) -- All caches drained: global must be zero. SELECT bm25_cache_global_estimated_bytes() AS final_global_bytes; final_global_bytes -------------------- 0 (1 row) -- ---------- rebuild + drop equivalence ---------- -- After eviction the caches are empty but the cache_source -- chooser will rebuild them on the next query, so end-to-end -- search still works. SELECT count(*) AS a_hits FROM ( SELECT id FROM cache_cap_a ORDER BY body <@> to_bm25query('apple', 'cache_cap_a_idx') LIMIT 100 ) q; a_hits -------- 80 (1 row) SELECT count(*) AS b_hits FROM ( SELECT id FROM cache_cap_b ORDER BY body <@> to_bm25query('kiwi', 'cache_cap_b_idx') LIMIT 100 ) q; b_hits -------- 5 (1 row) -- Remove the original test indexes before taking a baseline for the -- REINDEX accounting regression. The baseline is intentionally -- relative because the global counter belongs to shared memory. DROP INDEX cache_cap_a_idx; DROP INDEX cache_cap_b_idx; DROP TABLE cache_cap_a; DROP TABLE cache_cap_b; SELECT bm25_cache_global_estimated_bytes() AS reindex_original_baseline \gset -- Keep one populated cache alive across both REINDEX cycles. This -- proves each replaced index releases only its own accounting rather -- than relying on the global counter reaching zero. CREATE TABLE cache_reindex_control (id int, body text); CREATE INDEX cache_reindex_control_idx ON cache_reindex_control USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_reindex_control_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO cache_reindex_control SELECT g, 'control retained ' || g FROM generate_series(1, 10) g; SELECT count(*) AS retained_control_hits FROM ( SELECT 1 FROM cache_reindex_control ORDER BY body <@> to_bm25query('control', 'cache_reindex_control_idx') ) q; retained_control_hits ----------------------- 10 (1 row) SELECT bm25_cache_global_estimated_bytes() - :reindex_original_baseline AS retained_control_bytes \gset SELECT :retained_control_bytes > 0 AS retained_control_charged; retained_control_charged -------------------------- t (1 row) -- Exercise the parallel REINDEX completion path separately. The -- analyzed row count and worker settings match parallel_build.sql so -- both the initial build and REINDEX request one parallel worker. SET min_parallel_table_scan_size = 0; SET maintenance_work_mem = '256MB'; SET max_parallel_maintenance_workers = 1; CREATE TABLE cache_reindex_parallel (id int, body text); INSERT INTO cache_reindex_parallel SELECT g, 'parallel database document ' || g FROM generate_series(1, 100000) g; ANALYZE cache_reindex_parallel; CREATE INDEX cache_reindex_parallel_idx ON cache_reindex_parallel USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_reindex_parallel_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 1 of 1 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=4.00 -- Add an unflushed runtime tail after the parallel build, then force -- that tail into the cache so REINDEX replaces a charged entry. INSERT INTO cache_reindex_parallel SELECT g, 'parallel database tail ' || g FROM generate_series(100001, 100025) g; SELECT count(*) AS parallel_reindex_hits FROM ( SELECT 1 FROM cache_reindex_parallel ORDER BY body <@> to_bm25query('database', 'cache_reindex_parallel_idx') ) q; parallel_reindex_hits ----------------------- 100025 (1 row) SELECT bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS parallel_runtime_cache_charged; parallel_runtime_cache_charged -------------------------------- t (1 row) REINDEX INDEX cache_reindex_parallel_idx; NOTICE: BM25 index build started for relation cache_reindex_parallel_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 1 of 1 requested workers NOTICE: BM25 index build completed: 100025 documents, avg_length=4.00 -- Finalization itself must drain the old cache. Checking before DROP -- distinguishes in-place REINDEX cleanup from eventual relation teardown. SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS parallel_reindex_drained_before_drop \gset \echo parallel_reindex_drained_before_drop=:parallel_reindex_drained_before_drop parallel_reindex_drained_before_drop=t DROP TABLE cache_reindex_parallel; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS parallel_reindex_released, bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS parallel_reindex_exceeds_retained; parallel_reindex_released | parallel_reindex_exceeds_retained ---------------------------+----------------------------------- t | f (1 row) -- Cycle 1: segment spill, unflushed tail, cache-building scan, -- REINDEX, and drop. CREATE TABLE cache_reindex_test (id int, body text); CREATE INDEX cache_reindex_test_idx ON cache_reindex_test USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_reindex_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO cache_reindex_test SELECT g, 'databases original ' || g FROM generate_series(1, 50) g; SELECT bm25_spill_index('cache_reindex_test_idx'); bm25_spill_index ------------------ 2 (1 row) INSERT INTO cache_reindex_test SELECT g, 'databases tail ' || g FROM generate_series(51, 75) g; SELECT count(*) FROM ( SELECT 1 FROM cache_reindex_test ORDER BY body <@> to_bm25query('databases', 'cache_reindex_test_idx') ) q; count ------- 75 (1 row) REINDEX INDEX cache_reindex_test_idx; NOTICE: BM25 index build started for relation cache_reindex_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 75 documents, avg_length=3.00 SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS serial_reindex_drained_before_drop \gset \echo serial_reindex_drained_before_drop=:serial_reindex_drained_before_drop serial_reindex_drained_before_drop=t DROP TABLE cache_reindex_test; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_1_released, bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_1_exceeds_retained; reindex_cycle_1_released | reindex_cycle_1_exceeds_retained --------------------------+---------------------------------- t | f (1 row) -- Repeat the complete cycle to show that each registry replacement -- leaks another cache charge on the unfixed code. CREATE TABLE cache_reindex_test (id int, body text); CREATE INDEX cache_reindex_test_idx ON cache_reindex_test USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_reindex_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO cache_reindex_test SELECT g, 'databases original ' || g FROM generate_series(1, 50) g; SELECT bm25_spill_index('cache_reindex_test_idx'); bm25_spill_index ------------------ 2 (1 row) INSERT INTO cache_reindex_test SELECT g, 'databases tail ' || g FROM generate_series(51, 75) g; SELECT count(*) FROM ( SELECT 1 FROM cache_reindex_test ORDER BY body <@> to_bm25query('databases', 'cache_reindex_test_idx') ) q; count ------- 75 (1 row) REINDEX INDEX cache_reindex_test_idx; NOTICE: BM25 index build started for relation cache_reindex_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 75 documents, avg_length=3.00 DROP TABLE cache_reindex_test; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_2_released, bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_2_exceeds_retained; reindex_cycle_2_released | reindex_cycle_2_exceeds_retained --------------------------+---------------------------------- t | f (1 row) DROP TABLE cache_reindex_control; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline AS control_drop_restored_baseline, bm25_cache_global_estimated_bytes() > :reindex_original_baseline AS control_drop_exceeds_baseline; control_drop_restored_baseline | control_drop_exceeds_baseline --------------------------------+------------------------------- t | f (1 row) RESET pg_textsearch.memtable_cache_enabled; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/cache_source.out000066400000000000000000000074711525762047700247260ustar00rootroot00000000000000-- Unit coverage for the in-memory memtable cache TpDataSource -- (cache_source.c; see ARCHITECTURE.md, "Memtable Cache"). -- -- Drives bm25_test_cache_source directly so we can observe the -- TpDataSource interface (open / total_docs / get_postings / -- get_doc_freq / get_doc_length / close) backed by the cache -- rather than the chain, without going through the query -- planner. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE cache_source_t (id int, body text); CREATE INDEX cache_source_idx ON cache_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- ---------- empty memtable ---------- -- An empty memtable yields a NULL cache source, just like the -- chain source does. SELECT bm25_test_cache_source('cache_source_idx', 'empty'); bm25_test_cache_source ------------------------ ok:null (1 row) -- ---------- populate and serve ---------- INSERT INTO cache_source_t SELECT g, 'shared term ' || g FROM generate_series(1, 5) g; -- Opening a cache source over a 5-doc memtable reports -- total_docs = 5 and a non-zero total_len. We assert -- structurally to avoid pinning the tokenizer's exact length -- output. SELECT split_part(bm25_test_cache_source('cache_source_idx', 'open'), ',', 1) AS total_docs_clause; total_docs_clause ------------------- ok:total_docs=5 (1 row) SELECT (split_part( split_part(bm25_test_cache_source('cache_source_idx', 'open'), ',', 2), '=', 2)::bigint > 0) AS total_len_positive; total_len_positive -------------------- t (1 row) -- get_postings via the cache: 'shared' appears in every -- document. We snapshot doc_freq = 5 (all five docs) and -- count = 5 (one CTID per occurrence). Tokenizer stemming may -- shape the lexeme, so we look up the stemmed form 'share'. SELECT bm25_test_cache_source('cache_source_idx', 'lookup:share'); bm25_test_cache_source ------------------------ ok:df=5,count=5 (1 row) -- A term that doesn't exist returns 'miss:df=0'. SELECT bm25_test_cache_source('cache_source_idx', 'lookup:nonexistentterm'); bm25_test_cache_source ------------------------ miss:df=0 (1 row) -- get_doc_length via the cache: doc 1 is at heap block 0, -- offset 1. We don't pin the exact length (depends on tokens -- after stemming), only that the lookup succeeds. SELECT (split_part( split_part(bm25_test_cache_source( 'cache_source_idx', 'doclen:0'), ':', 2), '=', 2)::int >= 0) AS doc_length_lookup_ok; doc_length_lookup_ok ---------------------- t (1 row) -- ---------- end-to-end equivalence (chain vs cache) ---------- -- With the cache GUC off the read path uses chain_source; with -- it on the read path uses cache_source. Either way the BM25 -- ranking must be identical because the cache is derived state -- over the same chain. SET pg_textsearch.memtable_cache_enabled = off; SELECT id, body FROM cache_source_t ORDER BY body <@> to_bm25query('shared', 'cache_source_idx') LIMIT 3; id | body ----+--------------- 1 | shared term 1 2 | shared term 2 3 | shared term 3 (3 rows) SET pg_textsearch.memtable_cache_enabled = on; SELECT id, body FROM cache_source_t ORDER BY body <@> to_bm25query('shared', 'cache_source_idx') LIMIT 3; id | body ----+--------------- 1 | shared term 1 2 | shared term 2 3 | shared term 3 (3 rows) RESET pg_textsearch.memtable_cache_enabled; DROP INDEX cache_source_idx; DROP TABLE cache_source_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/cache_spill.out000066400000000000000000000116321525762047700245430ustar00rootroot00000000000000-- Integration coverage for the spill -> cache-invalidate -> rebuild -- cycle (cache_source.c + log.c; see ARCHITECTURE.md, -- "Memtable Cache"). -- -- tp_spill_finalize (a) bumps the per-index spill_generation atomic -- and (b) drops the cache's dshash tables via tp_cache_clear, so a -- query that runs after a spill rebuilds a fresh cache from -- whatever chain pages exist post-spill (zero in the typical case, -- since spill resets the chain head/tail to InvalidBlockNumber). -- -- We exercise both with the cache GUC explicitly enabled so the -- chooser routes reads through cache_source. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_cache_enabled = on; CREATE TABLE cache_spill_t (id int, body text); CREATE INDEX cache_spill_idx ON cache_spill_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cache_spill_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- ---------- populate + open cache, pre-spill ---------- INSERT INTO cache_spill_t SELECT g, 'alpha term ' || g FROM generate_series(1, 10) g; -- Cache source opens against the populated memtable. total_docs -- should be 10; we check structurally. SELECT split_part(bm25_test_cache_source('cache_spill_idx', 'open'), ',', 1) AS open_docs; open_docs ------------------ ok:total_docs=10 (1 row) -- The term 'alpha' is in every doc so doc_freq = 10. SELECT bm25_test_cache_source('cache_spill_idx', 'lookup:alpha'); bm25_test_cache_source ------------------------ ok:df=10,count=10 (1 row) -- End-to-end query through the chooser returns all 10 rows. SELECT count(*) AS pre_spill_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; pre_spill_hits ---------------- 10 (1 row) -- ---------- force a spill ---------- -- bm25_spill_index drives tp_do_spill -> tp_spill_finalize, which -- bumps spill_generation and clears the cache. SELECT bm25_spill_index('cache_spill_idx') > 0 AS spill_wrote_docs; spill_wrote_docs ------------------ t (1 row) -- After spill the chain is empty (head/tail reset). The cache -- source constructor returns NULL on empty memtable. SELECT bm25_test_cache_source('cache_spill_idx', 'empty'); bm25_test_cache_source ------------------------ ok:null (1 row) -- The query still returns 10 rows: cache_source returns NULL, -- chain source returns NULL, the scoring path falls through to -- segments only and finds all 10 docs in the new segment. SELECT count(*) AS post_spill_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; post_spill_hits ----------------- 10 (1 row) -- ---------- populate again post-spill, cache rebuilds cold ---------- INSERT INTO cache_spill_t SELECT g, 'beta term ' || g FROM generate_series(11, 15) g; -- New cache source opens over the 5 fresh chain records (the -- alpha docs are in the segment now, not the chain). SELECT split_part(bm25_test_cache_source('cache_spill_idx', 'open'), ',', 1) AS post_spill_open_docs; post_spill_open_docs ---------------------- ok:total_docs=5 (1 row) -- 'beta' is unique to the new chain records. SELECT bm25_test_cache_source('cache_spill_idx', 'lookup:beta'); bm25_test_cache_source ------------------------ ok:df=5,count=5 (1 row) -- 'alpha' is in the segment but not the new chain; cache lookup -- misses for it. SELECT bm25_test_cache_source('cache_spill_idx', 'lookup:alpha'); bm25_test_cache_source ------------------------ miss:df=0 (1 row) -- End-to-end queries see both alpha (10, segment) and beta (5, -- cache) with no duplicates. SELECT count(*) AS alpha_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; alpha_hits ------------ 10 (1 row) SELECT count(*) AS beta_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('beta', 'cache_spill_idx') LIMIT 100 ) q; beta_hits ----------- 5 (1 row) -- ---------- equivalence with cache disabled ---------- -- Same queries with the chooser falling back to the chain must -- return identical counts. SET pg_textsearch.memtable_cache_enabled = off; SELECT count(*) AS alpha_hits_chain FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; alpha_hits_chain ------------------ 10 (1 row) SELECT count(*) AS beta_hits_chain FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('beta', 'cache_spill_idx') LIMIT 100 ) q; beta_hits_chain ----------------- 5 (1 row) RESET pg_textsearch.memtable_cache_enabled; DROP INDEX cache_spill_idx; DROP TABLE cache_spill_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/catalog_stats.out000066400000000000000000000232721525762047700251300ustar00rootroot00000000000000-- Test that BM25 indexes report correct catalog statistics -- (pg_class.reltuples, pg_class.relpages, pg_stat_user_indexes) CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================ -- heap_tuples vs index_tuples for partial indexes -- -- IndexBuildResult.heap_tuples updates pg_class.reltuples for -- the HEAP relation. For a partial index, this must be the -- total heap rows scanned, not just rows that passed the -- predicate. Otherwise the planner underestimates table size. -- ============================================================ CREATE TABLE stats_partial ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO stats_partial (content, category) SELECT 'document ' || i || ' about databases', CASE WHEN i <= 10 THEN 'tech' ELSE 'other' END FROM generate_series(1, 100) i; -- Record heap reltuples before index build ANALYZE stats_partial; SELECT reltuples AS heap_reltuples_before FROM pg_class WHERE oid = 'stats_partial'::regclass; heap_reltuples_before ----------------------- 100 (1 row) -- Build partial index (only 10 of 100 rows match) CREATE INDEX stats_partial_idx ON stats_partial USING bm25 (content) WITH (text_config='english') WHERE category = 'tech'; NOTICE: BM25 index build started for relation stats_partial_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 10 documents, avg_length=3.00 -- heap reltuples must still reflect ~100, not 10 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_partial'::regclass; heap_tuples_ok ---------------- t (1 row) -- index reltuples should reflect ~10 (indexed docs) SELECT reltuples BETWEEN 5 AND 15 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_partial_idx'::regclass; index_tuples_ok ----------------- t (1 row) DROP TABLE stats_partial CASCADE; -- ============================================================ -- heap_tuples for non-partial index (baseline sanity check) -- ============================================================ CREATE TABLE stats_full ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO stats_full (content) SELECT 'document ' || i || ' about databases' FROM generate_series(1, 100) i; CREATE INDEX stats_full_idx ON stats_full USING bm25 (content) WITH (text_config='english'); NOTICE: BM25 index build started for relation stats_full_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100 documents, avg_length=3.00 -- Both heap and index should reflect ~100 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_full'::regclass; heap_tuples_ok ---------------- t (1 row) SELECT reltuples >= 90 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_full_idx'::regclass; index_tuples_ok ----------------- t (1 row) DROP TABLE stats_full CASCADE; -- ============================================================ -- heap_tuples for expression index -- ============================================================ CREATE TABLE stats_expr ( id SERIAL PRIMARY KEY, data JSONB ); INSERT INTO stats_expr (data) SELECT ('{"content": "document ' || i || ' about databases"}')::jsonb FROM generate_series(1, 100) i; CREATE INDEX stats_expr_idx ON stats_expr USING bm25 ((data->>'content')) WITH (text_config='english'); NOTICE: BM25 index build started for relation stats_expr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100 documents, avg_length=3.00 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_expr'::regclass; heap_tuples_ok ---------------- t (1 row) SELECT reltuples >= 90 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_expr_idx'::regclass; index_tuples_ok ----------------- t (1 row) DROP TABLE stats_expr CASCADE; -- ============================================================ -- heap_tuples for expression + partial combined -- ============================================================ CREATE TABLE stats_expr_partial ( id SERIAL PRIMARY KEY, data JSONB, active BOOLEAN ); INSERT INTO stats_expr_partial (data, active) SELECT ('{"text": "document ' || i || ' about databases"}')::jsonb, (i <= 20) FROM generate_series(1, 100) i; CREATE INDEX stats_ep_idx ON stats_expr_partial USING bm25 ((data->>'text')) WITH (text_config='english') WHERE active = true; NOTICE: BM25 index build started for relation stats_ep_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 20 documents, avg_length=3.00 -- heap must reflect ~100 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_expr_partial'::regclass; heap_tuples_ok ---------------- t (1 row) -- index should reflect ~20 SELECT reltuples BETWEEN 15 AND 25 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_ep_idx'::regclass; index_tuples_ok ----------------- t (1 row) DROP TABLE stats_expr_partial CASCADE; -- ============================================================ -- pg_relation_size: index should have nonzero size after build -- ============================================================ CREATE TABLE stats_size ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO stats_size (content) SELECT 'document ' || i || ' about databases and search' FROM generate_series(1, 50) i; CREATE INDEX stats_size_idx ON stats_size USING bm25 (content) WITH (text_config='english'); NOTICE: BM25 index build started for relation stats_size_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=4.00 SELECT pg_relation_size('stats_size_idx') > 0 AS has_size; has_size ---------- t (1 row) DROP TABLE stats_size CASCADE; -- ============================================================ -- Stats after VACUUM: reltuples should decrease after deletes -- ============================================================ -- Isolate physical cleanup from other backends and standby feedback. CREATE TEMP TABLE stats_vacuum ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO stats_vacuum (content) SELECT 'document ' || i || ' about databases' FROM generate_series(1, 100) i; CREATE INDEX stats_vacuum_idx ON stats_vacuum USING bm25 (content) WITH (text_config='english'); NOTICE: BM25 index build started for relation stats_vacuum_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100 documents, avg_length=3.00 -- Delete half the rows and vacuum DELETE FROM stats_vacuum WHERE id > 50; VACUUM stats_vacuum; -- Index reltuples should reflect roughly 50 after vacuum SELECT reltuples BETWEEN 40 AND 60 AS index_tuples_after_vacuum FROM pg_class WHERE oid = 'stats_vacuum_idx'::regclass; index_tuples_after_vacuum --------------------------- t (1 row) -- Second delete + vacuum; reltuples must continue tracking the -- live count (regression test: reporting metap->total_docs - only -- this-round deaths would leave reltuples stuck at the first -- round's value). DELETE FROM stats_vacuum WHERE id > 25; VACUUM stats_vacuum; SELECT reltuples BETWEEN 15 AND 35 AS index_tuples_after_second_vacuum FROM pg_class WHERE oid = 'stats_vacuum_idx'::regclass; index_tuples_after_second_vacuum ---------------------------------- t (1 row) -- No-deletes maintenance round; reltuples must stay at the live -- count, not reset to the cumulative insert count. VACUUM stats_vacuum; SELECT reltuples BETWEEN 15 AND 35 AS index_tuples_after_noop_vacuum FROM pg_class WHERE oid = 'stats_vacuum_idx'::regclass; index_tuples_after_noop_vacuum -------------------------------- t (1 row) DROP TABLE stats_vacuum CASCADE; -- ============================================================ -- Stats after REINDEX: should reflect current row count -- ============================================================ -- An unrelated snapshot must not keep the deleted rows recently dead. CREATE TEMP TABLE stats_reindex ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO stats_reindex (content, category) SELECT 'document ' || i || ' about databases', CASE WHEN i <= 30 THEN 'tech' ELSE 'other' END FROM generate_series(1, 100) i; CREATE INDEX stats_reindex_idx ON stats_reindex USING bm25 (content) WITH (text_config='english') WHERE category = 'tech'; NOTICE: BM25 index build started for relation stats_reindex_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 30 documents, avg_length=3.00 -- Delete some matching rows DELETE FROM stats_reindex WHERE category = 'tech' AND id > 20; REINDEX INDEX stats_reindex_idx; NOTICE: BM25 index build started for relation stats_reindex_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 20 documents, avg_length=3.00 -- heap reltuples should still be ~90 (100 - 10 deleted) SELECT reltuples >= 80 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_reindex'::regclass; heap_tuples_ok ---------------- t (1 row) -- index reltuples should reflect ~20 remaining tech rows SELECT reltuples BETWEEN 15 AND 25 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_reindex_idx'::regclass; index_tuples_ok ----------------- t (1 row) DROP TABLE stats_reindex CASCADE; -- ============================================================ -- Cleanup -- ============================================================ DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/chain_source.out000066400000000000000000000134041525762047700247360ustar00rootroot00000000000000-- Unit-level coverage for the on-disk memtable chain -- TpDataSource implementation (memtable v2, issue #374), driven -- through the bm25_test_chain_source() scaffold. Each case -- writes records through tp_memtable_append() with real TpVector -- payloads, then opens a chain source and verifies the read-back -- through the TpDataSource interface (get_postings, -- get_doc_length, total_docs, total_len). CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE chain_source_t (id int, body text); CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Empty chain: source create returns NULL. SELECT bm25_test_chain_source('chain_source_idx', 'empty_chain'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Single doc, single term: count=1, doc_freq=1, freq matches. SELECT bm25_test_chain_source('chain_source_idx', 'one_doc_one_term'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Multiple docs share one term: count=N, all ctids match. SELECT bm25_test_chain_source( 'chain_source_idx', 'multi_doc_shared_term'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- One doc, three distinct terms: each term round-trips. SELECT bm25_test_chain_source('chain_source_idx', 'multi_term'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- A non-NULL zero-length query-term list requests document -- metadata without materializing any term postings. \pset tuples_only on SELECT bm25_test_chain_source('chain_source_idx', 'documents_only'); OK \pset tuples_only off DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Records spill across multiple memtable pages: chain walk -- aggregates all docs into one term entry. SELECT bm25_test_chain_source('chain_source_idx', 'multi_page_chain'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- get_postings for an absent term returns NULL. SELECT bm25_test_chain_source('chain_source_idx', 'term_not_found'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- get_doc_length: present ctids return their length; absent -- ctids return -1. SELECT bm25_test_chain_source('chain_source_idx', 'doc_length_lookup'); bm25_test_chain_source ------------------------ OK (1 row) DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation chain_source_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Fragment round-trip: write a single record whose TpVector -- payload exceeds TP_MEMTABLE_PAGE_MAX_VECTOR_LEN, then read it -- back through the chain source. Verifies that the writer -- splits the payload across a fragment head page + N -- continuation pages and that the reader correctly reassembles -- the bytes before decoding term entries. SELECT bm25_test_chain_source('chain_source_idx', 'fragment_roundtrip'); bm25_test_chain_source ------------------------ OK (1 row) -- Clean up DROP EXTENSION pg_textsearch CASCADE; NOTICE: drop cascades to index chain_source_idx timescale-pg_textsearch-b63fe58/test/expected/chinese.out000066400000000000000000000071651525762047700237210ustar00rootroot00000000000000-- End-to-end Chinese full-text search test. -- -- pg_textsearch tokenizes both documents and queries through the index's -- text_config, so Chinese support is a matter of choosing a Chinese-aware -- text search configuration. This test pairs pg_textsearch with zhparser -- (https://github.com/amutu/zhparser), an SCWS dictionary-based Chinese -- word segmenter, and verifies indexing and ranked retrieval end to end. -- -- Requires the zhparser extension (and its SCWS library) to be installed. -- This test is not part of the default REGRESS schedule; run it via -- `make test-chinese` in an environment where zhparser is available. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE EXTENSION IF NOT EXISTS zhparser; -- Pin zhparser's segmentation GUCs to their defaults so tokenization is -- deterministic regardless of any cluster-level configuration. SET zhparser.multi_short = off; SET zhparser.multi_duality = off; SET zhparser.multi_zmain = off; SET zhparser.multi_zall = off; SET zhparser.seg_with_duality = off; -- CREATE EXTENSION zhparser also registers the zhparser parser. Define a -- configuration that maps zhparser's content token types (nouns, verbs, -- adjectives, idioms, interjections, set phrases). The configuration -- must be schema-qualified so the bm25 index build can resolve it. CREATE TEXT SEARCH CONFIGURATION public.chinese (PARSER = zhparser); ALTER TEXT SEARCH CONFIGURATION public.chinese ADD MAPPING FOR n, v, a, i, e, l WITH simple; -- zhparser segments Chinese into dictionary words (deterministic for a -- given SCWS dictionary): 机器学习 splits into 机器 + 学习. COLLATE "C" -- keeps the ordering byte-stable across locales. SELECT alias, token FROM ts_debug('public.chinese', '机器学习') ORDER BY token COLLATE "C"; alias | token -------+------- v | 学习 n | 机器 (2 rows) CREATE TABLE zh_docs (id int PRIMARY KEY, content text); INSERT INTO zh_docs VALUES (1, 'PostgreSQL 是一个强大的开源关系数据库'), -- database (2, '机器学习是人工智能的一个重要分支'), -- machine learning (3, '全文检索能够快速查找文档内容'), -- full-text search (4, '自然语言处理研究人类语言'), -- natural language (5, '深度学习推动人工智能技术发展'); -- shares 学习 only CREATE INDEX zh_docs_bm25 ON zh_docs USING bm25 (content) WITH (text_config = 'public.chinese'); NOTICE: BM25 index build started for relation zh_docs_bm25 NOTICE: Using text search configuration: public.chinese NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=5.60 -- Force the bm25 index scan so we exercise indexed retrieval, not -- standalone scoring. SET enable_seqscan = off; -- Query 机器学习 (words 机器 + 学习): doc 2 contains both, doc 5 contains -- only 学习, so doc 2 outranks doc 5 and no other doc matches. SELECT id, content FROM zh_docs ORDER BY content <@> to_bm25query('机器学习', 'zh_docs_bm25') LIMIT 2; id | content ----+---------------------------------- 2 | 机器学习是人工智能的一个重要分支 5 | 深度学习推动人工智能技术发展 (2 rows) -- Query 数据库: only doc 1 contains that word. SELECT id, content FROM zh_docs ORDER BY content <@> to_bm25query('数据库', 'zh_docs_bm25') LIMIT 1; id | content ----+--------------------------------------- 1 | PostgreSQL 是一个强大的开源关系数据库 (1 row) RESET enable_seqscan; DROP TABLE zh_docs; DROP TEXT SEARCH CONFIGURATION public.chinese; DROP EXTENSION zhparser; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/compaction.out000066400000000000000000000517241525762047700244370ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_textsearch; \pset format unaligned SET client_min_messages = warning; SET pg_textsearch.segments_per_level = 64; CREATE TABLE compaction_step (id serial PRIMARY KEY, body text); CREATE TABLE compaction_full (id serial PRIMARY KEY, body text); CREATE INDEX compaction_step_idx ON compaction_step USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE INDEX compaction_full_idx ON compaction_full USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE INDEX compaction_btree_idx ON compaction_step(id); CREATE TABLE compaction_partitioned (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE compaction_partitioned_leaf PARTITION OF compaction_partitioned FOR VALUES FROM (0) TO (10); CREATE INDEX compaction_partitioned_idx ON compaction_partitioned USING bm25(body) WITH (text_config = 'english'); -- New indexes expose all levels and have no compaction debt. SELECT array_length( bm25_level_counts('compaction_step_idx'::regclass), 1) = 8 AS has_all_levels; has_all_levels t (1 row) SELECT bm25_needs_compaction('compaction_step_idx'::regclass) AS new_index_needs_compaction; new_index_needs_compaction f (1 row) -- Runtime spill publishes under the per-index lock, then applies inline -- compaction under a separately acquired maintenance lock. CREATE TABLE compaction_inline (id serial PRIMARY KEY, body text); CREATE INDEX compaction_inline_idx ON compaction_inline USING bm25(body) WITH (text_config = 'english'); SET pg_textsearch.segments_per_level = 2; BEGIN; DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO compaction_inline (body) VALUES (format('inline spill document %s filler', n)); PERFORM bm25_spill_index('compaction_inline_idx'); END LOOP; END $$; SELECT bm25_level_counts('compaction_inline_idx'::regclass) = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AS inline_policy_compacts_after_spill; inline_policy_compacts_after_spill t (1 row) SELECT NOT EXISTS ( SELECT 1 FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'object' AND classid = 'pg_am'::regclass AND objid = 'compaction_inline_idx'::regclass AND objsubid = 3 AND mode = 'ExclusiveLock' AND granted ) AS inline_policy_releases_maintenance_lock; inline_policy_releases_maintenance_lock t (1 row) COMMIT; DROP TABLE compaction_inline CASCADE; SET pg_textsearch.segments_per_level = 64; -- Non-bm25 relations are rejected. SELECT bm25_level_counts('compaction_btree_idx'::regclass); ERROR: "compaction_btree_idx" is not a bm25 index SELECT bm25_level_counts('compaction_step'::regclass); ERROR: "compaction_step" is not a bm25 index -- Partitioned parents have no storage; their physical leaves remain usable. SELECT bm25_level_counts('compaction_partitioned_idx'::regclass); ERROR: "compaction_partitioned_idx" is a partitioned bm25 index HINT: Use a physical partition index instead. SELECT bm25_compact('compaction_partitioned_idx'::regclass); ERROR: "compaction_partitioned_idx" is a partitioned bm25 index HINT: Use a physical partition index instead. SELECT bm25_compact_step('compaction_partitioned_idx'::regclass); ERROR: "compaction_partitioned_idx" is a partitioned bm25 index HINT: Use a physical partition index instead. SELECT bm25_needs_compaction('compaction_partitioned_idx'::regclass); ERROR: "compaction_partitioned_idx" is a partitioned bm25 index HINT: Use a physical partition index instead. SELECT array_length(bm25_level_counts(i.inhrelid), 1) = 8 AS partition_leaf_counts_supported FROM pg_inherits i WHERE i.inhparent = 'compaction_partitioned_idx'::regclass; partition_leaf_counts_supported t (1 row) SELECT bm25_compact(i.inhrelid) FROM pg_inherits i WHERE i.inhparent = 'compaction_partitioned_idx'::regclass; bm25_compact (1 row) SELECT NOT bm25_compact_step(i.inhrelid) AND NOT bm25_needs_compaction(i.inhrelid) AS partition_leaf_compaction_supported FROM pg_inherits i WHERE i.inhparent = 'compaction_partitioned_idx'::regclass; partition_leaf_compaction_supported t (1 row) -- Construct identical four-segment layouts without inline compaction. DO $$ DECLARE n integer; BEGIN FOR n IN 1..4 LOOP INSERT INTO compaction_step (body) SELECT format('step batch %s document %s filler', n, i) FROM generate_series(1, 20) i; INSERT INTO compaction_full (body) SELECT format('full batch %s document %s filler', n, i) FROM generate_series(1, 20) i; PERFORM bm25_spill_index('compaction_step_idx'); PERFORM bm25_spill_index('compaction_full_idx'); END LOOP; END $$; SET pg_textsearch.segments_per_level = 2; SELECT bm25_level_counts('compaction_step_idx'::regclass) = ARRAY[4, 0, 0, 0, 0, 0, 0, 0] AND bm25_needs_compaction('compaction_step_idx'::regclass) AS step_starts_with_debt; step_starts_with_debt t (1 row) UPDATE pg_catalog.pg_class SET reloptions = array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'compaction_step_idx'::regclass; SELECT d.oid AS db_oid, c.oid AS index_oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace) AS spc_oid, pg_relation_filenode(c.oid) AS relfilenumber, c.relowner AS owner_oid FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'compaction_step_idx'::regclass \gset target_ -- Stale physical identities are rejected without touching the live index. SELECT NOT bm25_compact_step_if_current( :target_index_oid::oid, 0::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AND NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, 0::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AND NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, 0::oid, :target_owner_oid::oid) AND NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, 0::oid) AS stale_step_targets_rejected; stale_step_targets_rejected t (1 row) -- Permanent-index mutators reject read-only transactions. BEGIN READ ONLY; SELECT bm25_compact('compaction_step_idx'::regclass); ERROR: cannot execute bm25 index compaction in a read-only transaction ROLLBACK; BEGIN READ ONLY; SELECT bm25_compact_step('compaction_step_idx'::regclass); ERROR: cannot execute bm25 index compaction in a read-only transaction ROLLBACK; -- In this layout one pass happens to merge exactly one batch. A pass is -- not limited to one batch in general; it may span several, and may pull -- in further batches to unblock a destination level. SELECT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AS one_batch_ran; one_batch_ran t (1 row) SELECT bm25_level_counts('compaction_step_idx'::regclass) = ARRAY[2, 1, 0, 0, 0, 0, 0, 0] AND bm25_needs_compaction('compaction_step_idx'::regclass) AS one_batch_only; one_batch_only t (1 row) -- Repeated steps converge, then report that no batch ran. CREATE TEMP TABLE compaction_step_result (steps integer); DO $$ DECLARE step_count integer := 0; BEGIN WHILE bm25_needs_compaction('compaction_step_idx'::regclass) LOOP step_count := step_count + 1; IF step_count > 100 THEN RAISE EXCEPTION 'bm25_compact_step did not terminate'; END IF; IF NOT bm25_compact_step('compaction_step_idx'::regclass) THEN RAISE EXCEPTION 'bm25_needs_compaction disagrees with step'; END IF; END LOOP; INSERT INTO compaction_step_result VALUES (step_count); END $$; SELECT steps = 2 AS step_split_cascade FROM compaction_step_result; step_split_cascade t (1 row) SELECT NOT bm25_needs_compaction('compaction_step_idx'::regclass) AND NOT bm25_compact_step('compaction_step_idx'::regclass) AS step_converged; step_converged t (1 row) -- Whole-index compaction reaches the same terminal layout. SELECT bm25_compact('compaction_full_idx'::regclass); bm25_compact (1 row) SELECT NOT bm25_needs_compaction('compaction_full_idx'::regclass) AND bm25_level_counts('compaction_step_idx'::regclass) = bm25_level_counts('compaction_full_idx'::regclass) AS full_compaction_converged; full_compaction_converged t (1 row) SELECT count(*) = 80 AS step_preserves_documents FROM ( SELECT 1 FROM compaction_step ORDER BY body <@> to_bm25query('filler', 'compaction_step_idx') ) ranked; step_preserves_documents t (1 row) SELECT count(*) = 80 AS full_preserves_documents FROM ( SELECT 1 FROM compaction_full ORDER BY body <@> to_bm25query('filler', 'compaction_full_idx') ) ranked; full_preserves_documents t (1 row) -- Background workers validate mode and physical identity without writing. CREATE TABLE compaction_background_target (id integer, body text); CREATE INDEX compaction_background_target_idx ON compaction_background_target USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..4 LOOP INSERT INTO compaction_background_target (body) SELECT format('background batch %s document %s', n, i) FROM generate_series(1, 20) i; PERFORM bm25_spill_index('compaction_background_target_idx'); END LOOP; END $$; UPDATE pg_catalog.pg_class SET reloptions = pg_catalog.array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'compaction_background_target_idx'::regclass; SELECT d.oid AS db_oid, c.oid AS index_oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace) AS spc_oid, pg_relation_filenode(c.oid) AS relfilenumber, c.relowner AS owner_oid FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'compaction_background_target_idx'::regclass \gset background_ SELECT bm25_background_target_is_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid) AS background_target_is_current; background_target_is_current t (1 row) ALTER INDEX compaction_background_target_idx SET (compaction = 'manual'); SELECT NOT bm25_background_target_is_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid) AS manual_target_is_not_current; manual_target_is_not_current t (1 row) SELECT NOT bm25_compact_step_if_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid) AND bm25_level_counts('compaction_background_target_idx'::regclass) = ARRAY[4, 0, 0, 0, 0, 0, 0, 0] AS manual_target_is_not_compacted; manual_target_is_not_compacted t (1 row) -- Inspection is public, but only the index owner may mutate it. DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'compaction_user') THEN EXECUTE 'REASSIGN OWNED BY compaction_user TO CURRENT_USER'; EXECUTE 'DROP OWNED BY compaction_user CASCADE'; DROP ROLE compaction_user; END IF; END $$; CREATE ROLE compaction_user LOGIN; GRANT USAGE ON SCHEMA public TO compaction_user; SET ROLE compaction_user; SELECT array_length( bm25_level_counts('compaction_step_idx'::regclass), 1) = 8 AS nonowner_can_inspect; nonowner_can_inspect t (1 row) SELECT bm25_needs_compaction('compaction_step_idx'::regclass) AS nonowner_can_check; nonowner_can_check f (1 row) SELECT bm25_compact('compaction_step_idx'::regclass); ERROR: must be owner of index compaction_step_idx SELECT bm25_compact_step('compaction_step_idx'::regclass); ERROR: must be owner of index compaction_step_idx SELECT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid); ERROR: permission denied for function bm25_compact_step_if_current SELECT bm25_background_target_is_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid); ERROR: permission denied for function bm25_background_target_is_current RESET ROLE; DROP OWNED BY compaction_user CASCADE; DROP ROLE compaction_user; DROP TABLE compaction_background_target CASCADE; -- Local temporary indexes remain mutable in read-only transactions. SET pg_textsearch.segments_per_level = 64; CREATE TEMP TABLE compaction_temp_step (id serial, body text); CREATE TEMP TABLE compaction_temp_full (id serial, body text); CREATE INDEX compaction_temp_step_idx ON compaction_temp_step USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE INDEX compaction_temp_full_idx ON compaction_temp_full USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO compaction_temp_step (body) VALUES (format('temporary step document %s filler', n)); INSERT INTO compaction_temp_full (body) VALUES (format('temporary full document %s filler', n)); PERFORM bm25_spill_index('compaction_temp_step_idx'); PERFORM bm25_spill_index('compaction_temp_full_idx'); END LOOP; END $$; SET pg_textsearch.segments_per_level = 2; BEGIN READ ONLY; SELECT bm25_compact_step('compaction_temp_step_idx'::regclass) AS temp_step_ran; temp_step_ran t (1 row) SELECT bm25_compact('compaction_temp_full_idx'::regclass); bm25_compact (1 row) COMMIT; SELECT NOT bm25_needs_compaction('compaction_temp_step_idx'::regclass) AND NOT bm25_needs_compaction( 'compaction_temp_full_idx'::regclass) AS temp_indexes_converged; temp_indexes_converged t (1 row) -- A published pass is a physical change, so ROLLBACK does not undo it. -- A caller must not treat a step as transactional work. CREATE TABLE compaction_rollback (id serial, body text); CREATE INDEX compaction_rollback_idx ON compaction_rollback USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); SET pg_textsearch.segments_per_level = 64; DO $$ DECLARE n integer; BEGIN FOR n IN 1..4 LOOP INSERT INTO compaction_rollback (body) VALUES (format('rollback document %s filler', n)); PERFORM bm25_spill_index('compaction_rollback_idx'); END LOOP; END $$; SET pg_textsearch.segments_per_level = 2; CREATE TEMP TABLE compaction_rollback_before AS SELECT bm25_level_counts('compaction_rollback_idx'::regclass) AS counts; BEGIN; SELECT bm25_compact_step('compaction_rollback_idx'::regclass) AS rollback_step_ran; rollback_step_ran t (1 row) ROLLBACK; SELECT bm25_level_counts('compaction_rollback_idx'::regclass) <> before.counts AS rollback_does_not_undo_a_published_pass FROM compaction_rollback_before before; rollback_does_not_undo_a_published_pass t (1 row) DROP TABLE compaction_rollback CASCADE; -- A level can sit at the segment threshold with nothing to compact: -- every candidate group already exceeds max_segment_size, and an -- over-budget segment is an uncombinable singleton. This is not -- compaction debt, and bm25_needs_compaction() reports false so a -- scheduler looping on it does not spin. CREATE TABLE compaction_unreducible (id bigint PRIMARY KEY, body text); CREATE INDEX compaction_unreducible_idx ON compaction_unreducible USING bm25(body) WITH (text_config = 'simple'); SET pg_textsearch.max_segment_size = '1MB'; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.segments_per_level = 2; DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..1 LOOP INSERT INTO compaction_unreducible SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 32) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('compaction_unreducible_idx'); END LOOP; END $$; SELECT bm25_level_counts('compaction_unreducible_idx'::regclass) = ARRAY[2, 0, 0, 0, 0, 0, 0, 0] AS unreducible_level_is_at_threshold; unreducible_level_is_at_threshold t (1 row) SELECT NOT bm25_needs_compaction('compaction_unreducible_idx'::regclass) AS unreducible_level_reports_no_work; unreducible_level_reports_no_work t (1 row) SELECT bm25_compact_step('compaction_unreducible_idx'::regclass) AS unreducible_step_declines; unreducible_step_declines f (1 row) SELECT bm25_compact('compaction_unreducible_idx'::regclass); bm25_compact (1 row) SELECT bm25_level_counts('compaction_unreducible_idx'::regclass) = ARRAY[2, 0, 0, 0, 0, 0, 0, 0] AS unreducible_full_is_a_noop; unreducible_full_is_a_noop t (1 row) SELECT count(*) = 4000 AS unreducible_preserves_documents FROM ( SELECT 1 FROM compaction_unreducible ORDER BY body <@> to_bm25query('common', 'compaction_unreducible_idx') LIMIT 5000 ) ranked; unreducible_preserves_documents t (1 row) RESET pg_textsearch.max_segment_size; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP TABLE compaction_unreducible CASCADE; -- A pass must not copy segments it cannot combine. Spills prepend, so -- a level reads newest-first and its oldest, largest runs sit at the -- tail. Here two fresh small segments head a 2MB run that is already -- over max_segment_size: the pair merges, and the run is handed back -- instead of being rewritten to say the same thing in a new place. -- -- The handback is visible as a level count: a retained segment keeps -- its level, while a rewritten one is promoted. Without the trim this -- ends at [0, 2, ...] with the run copied to a fresh block, and the -- index grows by the full size of the run rather than by the pair. CREATE TABLE compaction_uncombinable_tail (id bigint PRIMARY KEY, body text); CREATE INDEX compaction_uncombinable_tail_idx ON compaction_uncombinable_tail USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.max_segment_size = '1MB'; -- Manual mode preserves the layout until the explicit pass. SET pg_textsearch.segments_per_level = 64; INSERT INTO compaction_uncombinable_tail SELECT gs, 'common ' || repeat(md5(gs::text), 32) FROM generate_series(1, 2000) gs; SELECT bm25_spill_index('compaction_uncombinable_tail_idx') > 0 AS tail_run_spilled; tail_run_spilled t (1 row) INSERT INTO compaction_uncombinable_tail SELECT 100000 + gs, 'common small ' || gs FROM generate_series(1, 20) gs; SELECT bm25_spill_index('compaction_uncombinable_tail_idx') > 0 AS head_pair_first_spilled; head_pair_first_spilled t (1 row) INSERT INTO compaction_uncombinable_tail SELECT 200000 + gs, 'common small ' || gs FROM generate_series(1, 20) gs; SELECT bm25_spill_index('compaction_uncombinable_tail_idx') > 0 AS head_pair_second_spilled; head_pair_second_spilled t (1 row) SET pg_textsearch.segments_per_level = 3; SELECT bm25_level_counts('compaction_uncombinable_tail_idx'::regclass) = ARRAY[3, 0, 0, 0, 0, 0, 0, 0] AS tail_level_is_at_threshold; tail_level_is_at_threshold t (1 row) CREATE TEMP TABLE compaction_tail_before AS SELECT pg_relation_size('compaction_uncombinable_tail_idx') AS bytes; SELECT bm25_compact('compaction_uncombinable_tail_idx'::regclass); bm25_compact (1 row) SELECT bm25_level_counts('compaction_uncombinable_tail_idx'::regclass) = ARRAY[1, 1, 0, 0, 0, 0, 0, 0] AS tail_run_retained_at_its_level; tail_run_retained_at_its_level t (1 row) SELECT pg_relation_size('compaction_uncombinable_tail_idx') - before.bytes < 1024 * 1024 AS tail_run_was_not_copied FROM compaction_tail_before before; tail_run_was_not_copied t (1 row) SELECT count(*) = 2040 AS tail_preserves_documents FROM ( SELECT 1 FROM compaction_uncombinable_tail ORDER BY body <@> to_bm25query( 'common', 'compaction_uncombinable_tail_idx') LIMIT 5000 ) ranked; tail_preserves_documents t (1 row) RESET pg_textsearch.max_segment_size; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP TABLE compaction_uncombinable_tail CASCADE; RESET pg_textsearch.segments_per_level; DROP TABLE compaction_partitioned CASCADE; DROP TABLE compaction_full CASCADE; DROP TABLE compaction_step CASCADE; SELECT NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AS dropped_step_target_rejected; dropped_step_target_rejected t (1 row) DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/compaction_error_injection.out000066400000000000000000000064021525762047700277030ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE TABLE compaction_unwind (id serial PRIMARY KEY, body text); CREATE INDEX compaction_unwind_idx ON compaction_unwind USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); SET pg_textsearch.segments_per_level = 2; -- An error raised anywhere in a compaction pass must unwind without -- losing documents or wedging the index: the pass is discarded, the -- published graph still answers queries, and a later pass succeeds. -- A buffer pin or index lock escaping the error path surfaces here as -- a WARNING in the test output. DO $$ DECLARE point text; n integer; docs integer; found integer; raised boolean; BEGIN FOREACH point IN ARRAY ARRAY[ 'pg-textsearch-compaction-after-select', 'pg-textsearch-compaction-alloc-output-data', 'pg-textsearch-compaction-alloc-page-index', 'pg-textsearch-before-compaction-publish', 'pg-textsearch-compaction-after-restamp', 'pg-textsearch-after-compaction-publish'] LOOP -- Accumulate level 0 debt without letting the spill's own -- inline compaction consume it. PERFORM set_config( 'pg_textsearch.segments_per_level', '64', true); FOR n IN 1..6 LOOP INSERT INTO compaction_unwind (body) SELECT format('unwind filler document %s %s', n, gs) FROM generate_series(1, 10) gs; PERFORM bm25_spill_index('compaction_unwind_idx'); END LOOP; PERFORM set_config( 'pg_textsearch.segments_per_level', '2', true); SELECT count(*) INTO docs FROM compaction_unwind; PERFORM injection_points_attach(point, 'error'); raised := false; BEGIN PERFORM bm25_compact_step( 'compaction_unwind_idx'::regclass); EXCEPTION WHEN others THEN raised := true; END; PERFORM injection_points_detach(point); IF NOT raised THEN RAISE EXCEPTION 'injected error at % did not fire', point; END IF; SELECT count(*) INTO found FROM ( SELECT 1 FROM compaction_unwind ORDER BY body <@> to_bm25query('filler', 'compaction_unwind_idx') ) ranked; IF found <> docs THEN RAISE EXCEPTION 'error at % lost documents: % of %', point, found, docs; END IF; PERFORM bm25_compact('compaction_unwind_idx'::regclass); SELECT count(*) INTO found FROM ( SELECT 1 FROM compaction_unwind ORDER BY body <@> to_bm25query('filler', 'compaction_unwind_idx') ) ranked; IF found <> docs THEN RAISE EXCEPTION 'compaction after % lost documents: % of %', point, found, docs; END IF; END LOOP; END $$; SELECT count(*) FROM compaction_unwind; count 360 (1 row) SELECT bm25_needs_compaction('compaction_unwind_idx'::regclass); bm25_needs_compaction f (1 row) RESET pg_textsearch.segments_per_level; DROP TABLE compaction_unwind CASCADE; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/compaction_injection.out000066400000000000000000000111161525762047700264700ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.segments_per_level = 64; CREATE TABLE compaction_terminal (id integer, body text); CREATE INDEX compaction_terminal_idx ON compaction_terminal USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO compaction_terminal SELECT batch * 100 + n, 'common document ' || n FROM generate_series(1, 100) n; PERFORM bm25_spill_index('compaction_terminal_idx'); END LOOP; END $$; -- Tiny outputs no longer climb to L7 naturally. Construct the legacy layout. SELECT pg_textsearch_test_move_level('compaction_terminal_idx', 0, 7); pg_textsearch_test_move_level (1 row) SELECT pg_textsearch_test_attach_segment_limit(2); pg_textsearch_test_attach_segment_limit (1 row) SET pg_textsearch.segments_per_level = 2; SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[0, 0, 0, 0, 0, 0, 0, 2] AND bm25_needs_compaction('compaction_terminal_idx') AS terminal_debt_constructed; terminal_debt_constructed t (1 row) SELECT bm25_compact_step('compaction_terminal_idx') AS terminal_debt_reduced; terminal_debt_reduced t (1 row) SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AND NOT bm25_needs_compaction('compaction_terminal_idx') AS terminal_output_placed_by_size; terminal_output_placed_by_size t (1 row) -- Compatible segments on different levels are debt below both thresholds. SELECT pg_textsearch_test_move_level('compaction_terminal_idx', 1, 7); pg_textsearch_test_move_level (1 row) SET pg_textsearch.segments_per_level = 64; INSERT INTO compaction_terminal SELECT 1000 + n, 'common document ' || n FROM generate_series(1, 200) n; SELECT bm25_spill_index('compaction_terminal_idx') > 0 AS lower_spilled; lower_spilled t (1 row) SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[1, 0, 0, 0, 0, 0, 0, 1] AND bm25_needs_compaction('compaction_terminal_idx') AS cross_level_debt_constructed; cross_level_debt_constructed t (1 row) SELECT bm25_compact_step('compaction_terminal_idx') AS cross_level_pass_ran; cross_level_pass_ran t (1 row) SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AND NOT bm25_compact_step('compaction_terminal_idx') AS cross_level_pass_converged; cross_level_pass_converged t (1 row) SELECT count(*) = 400 AS cross_level_merge_preserves_matches FROM (SELECT id FROM compaction_terminal ORDER BY body <@> to_bm25query('common', 'compaction_terminal_idx') LIMIT 1000) ranked; cross_level_merge_preserves_matches t (1 row) -- Force merge also accepts terminal-level inputs. SELECT pg_textsearch_test_move_level('compaction_terminal_idx', 1, 7); pg_textsearch_test_move_level (1 row) SELECT bm25_force_merge('compaction_terminal_idx'); bm25_force_merge (1 row) SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[1, 0, 0, 0, 0, 0, 0, 0] AS force_merge_reclassifies_terminal_segment; force_merge_reclassifies_terminal_segment t (1 row) SELECT injection_points_detach('pg-textsearch-segment-count-limit'); injection_points_detach (1 row) -- Background spills enqueue work without running the planner in the writer. CREATE TABLE compaction_signal (id integer, body text); CREATE INDEX compaction_signal_idx ON compaction_signal USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO compaction_signal SELECT n, 'common document ' || n FROM generate_series(1, 100) n; SELECT bm25_spill_index('compaction_signal_idx') > 0 AS initial_spill; initial_spill t (1 row) BEGIN; UPDATE pg_class SET reloptions = array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'compaction_signal_idx'::regclass; INSERT INTO compaction_signal SELECT n, 'common document ' || n FROM generate_series(101, 200) n; SELECT injection_points_attach( 'pg-textsearch-compaction-source-estimate', 'error'); injection_points_attach (1 row) SELECT bm25_spill_index('compaction_signal_idx') > 0 AS background_spill_defers_planning; background_spill_defers_planning t (1 row) ROLLBACK; SELECT injection_points_detach('pg-textsearch-compaction-source-estimate'); injection_points_detach (1 row) DROP TABLE compaction_signal; RESET pg_textsearch.segments_per_level; DROP TABLE compaction_terminal CASCADE; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/compaction_policy.out000066400000000000000000000176531525762047700260210ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_textsearch; \pset format unaligned SET client_min_messages = warning; SET pg_textsearch.segments_per_level = 64; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; -- Comparable spills are debt even below the per-level count threshold. CREATE TEMP TABLE policy_small (id integer, body text); CREATE INDEX policy_small_idx ON policy_small USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DO $$ BEGIN FOR batch IN 1..4 LOOP INSERT INTO policy_small SELECT batch * 100 + n, 'common document ' || n FROM generate_series(1, 100) n; PERFORM bm25_spill_index('policy_small_idx'); END LOOP; IF NOT bm25_needs_compaction('policy_small_idx') THEN RAISE EXCEPTION 'below-threshold small segments were not selected'; END IF; PERFORM bm25_compact('policy_small_idx'); IF (SELECT sum(n) FROM unnest( bm25_level_counts('policy_small_idx')) n) <> 1 OR bm25_needs_compaction('policy_small_idx') THEN RAISE EXCEPTION 'small-segment consolidation did not converge'; END IF; END $$; SELECT count(*) = 400 AS small_merge_preserves_matches FROM (SELECT id FROM policy_small ORDER BY body <@> to_bm25query('common', 'policy_small_idx') LIMIT 1000) ranked; small_merge_preserves_matches t (1 row) -- A tiny spill must not cause the much larger preceding output to rewrite. CREATE TEMP TABLE policy_unequal (id integer, body text); CREATE INDEX policy_unequal_idx ON policy_unequal USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO policy_unequal SELECT n, 'common ' || md5(n::text) FROM generate_series(1, 2000) n; SELECT bm25_spill_index('policy_unequal_idx') > 0 AS large_spilled; large_spilled t (1 row) INSERT INTO policy_unequal VALUES (2001, 'common tiny'); SELECT bm25_spill_index('policy_unequal_idx') > 0 AS tiny_spilled; tiny_spilled t (1 row) SELECT NOT bm25_needs_compaction('policy_unequal_idx') AND NOT bm25_compact_step('policy_unequal_idx') AS unequal_segments_are_not_repeatedly_rewritten; unequal_segments_are_not_repeatedly_rewritten t (1 row) -- A singleton becomes eligible at 50% dead, not just at zero survivors. CREATE TEMP TABLE policy_deleted (id integer, body text); INSERT INTO policy_deleted SELECT n, 'common document ' || n FROM generate_series(1, 100) n; CREATE INDEX policy_deleted_idx ON policy_deleted USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DELETE FROM policy_deleted WHERE id <= 49; VACUUM policy_deleted; SELECT NOT bm25_needs_compaction('policy_deleted_idx') AS minor_deletion_is_not_debt; minor_deletion_is_not_debt t (1 row) DELETE FROM policy_deleted WHERE id = 50; VACUUM policy_deleted; DO $$ BEGIN IF NOT bm25_needs_compaction('policy_deleted_idx') OR NOT bm25_compact_step('policy_deleted_idx') THEN RAISE EXCEPTION 'half-dead singleton was not rewritten'; END IF; IF bm25_needs_compaction('policy_deleted_idx') OR bm25_compact_step('policy_deleted_idx') THEN RAISE EXCEPTION 'clean singleton was rewritten again'; END IF; END $$; SELECT array_agg(id ORDER BY id) = ARRAY( SELECT generate_series(51, 100)) AS rewrite_preserves_survivors FROM (SELECT id FROM policy_deleted ORDER BY body <@> to_bm25query('common', 'policy_deleted_idx') LIMIT 1000) ranked; rewrite_preserves_survivors t (1 row) -- Inline maintenance gets an opportunity after VACUUM, without a new spill. CREATE TEMP TABLE policy_vacuum (id integer, body text); INSERT INTO policy_vacuum SELECT n, 'common document ' || n FROM generate_series(1, 100) n; CREATE INDEX policy_vacuum_idx ON policy_vacuum USING bm25(body) WITH (text_config = 'simple'); DELETE FROM policy_vacuum WHERE id <= 75; VACUUM policy_vacuum; SELECT bm25_pending_free_pages('policy_vacuum_idx') > 0 AND NOT bm25_needs_compaction('policy_vacuum_idx') AND NOT bm25_compact_step('policy_vacuum_idx') AS vacuum_runs_inline_policy; vacuum_runs_inline_policy t (1 row) SELECT count(*) = 25 AS vacuum_policy_preserves_survivors FROM (SELECT id FROM policy_vacuum ORDER BY body <@> to_bm25query('common', 'policy_vacuum_idx') LIMIT 1000) ranked; vacuum_policy_preserves_survivors t (1 row) DELETE FROM policy_deleted; VACUUM policy_deleted; SELECT (SELECT sum(n) FROM unnest( bm25_level_counts('policy_deleted_idx')) n) = 0 AS empty_segment_cleanup_preserved; empty_segment_cleanup_preserved t (1 row) -- Sparse cleanup can splice out an interior segment without merging its head. CREATE TEMP TABLE policy_interior (id integer, body text); CREATE INDEX policy_interior_idx ON policy_interior USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO policy_interior SELECT n, 'common document ' || n FROM generate_series(1, 100) n; SELECT bm25_spill_index('policy_interior_idx') > 0 AS older_spilled; older_spilled t (1 row) INSERT INTO policy_interior SELECT n, 'common document ' || n FROM generate_series(101, 200) n; SELECT bm25_spill_index('policy_interior_idx') > 0 AS newer_spilled; newer_spilled t (1 row) DELETE FROM policy_interior WHERE id <= 50; VACUUM policy_interior; CREATE TEMP TABLE policy_interior_before AS SELECT (regexp_match(summary, 'L0 Segment 1: block=([0-9]+),'))[1] AS clean_root, (regexp_match(summary, 'L0 Segment 2: block=([0-9]+),'))[1] AS sparse_root FROM (SELECT bm25_summarize_index('policy_interior_idx') AS summary) s; SELECT bm25_compact_step('policy_interior_idx') AS interior_rewrite_ran; interior_rewrite_ran t (1 row) SELECT bm25_level_counts('policy_interior_idx') = ARRAY[2, 0, 0, 0, 0, 0, 0, 0] AND bm25_summarize_index('policy_interior_idx') ~ E'total_docs: 150\n' AS interior_rewrite_does_not_copy_clean_head; interior_rewrite_does_not_copy_clean_head t (1 row) SELECT (regexp_match(summary, 'L0 Segment 1: block=([0-9]+),'))[1] = before.clean_root AND (regexp_match(summary, 'L0 Segment 2: block=([0-9]+),'))[1] <> before.sparse_root AS only_sparse_root_replaced FROM policy_interior_before before, (SELECT bm25_summarize_index('policy_interior_idx') AS summary) s; only_sparse_root_replaced t (1 row) SELECT count(*) = 150 AS interior_rewrite_preserves_matches FROM (SELECT id FROM policy_interior ORDER BY body <@> to_bm25query('common', 'policy_interior_idx') LIMIT 1000) ranked; interior_rewrite_preserves_matches t (1 row) DROP TABLE policy_small, policy_unequal, policy_deleted, policy_vacuum, policy_interior, policy_interior_before; -- The managed worker's physical-target entry point sees the same small debt. CREATE TABLE policy_background (id integer, body text); CREATE INDEX policy_background_idx ON policy_background USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO policy_background SELECT batch * 100 + n, 'common document ' || n FROM generate_series(1, 100) n; PERFORM bm25_spill_index('policy_background_idx'); END LOOP; END $$; UPDATE pg_class SET reloptions = array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'policy_background_idx'::regclass; SELECT bm25_compact_step_if_current( c.oid, d.oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace), pg_relation_filenode(c.oid), c.relowner) AS background_target_compacts_below_threshold FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'policy_background_idx'::regclass; background_target_compacts_below_threshold t (1 row) SELECT NOT bm25_needs_compaction('policy_background_idx') AS background_target_converged; background_target_converged t (1 row) DROP TABLE policy_background; RESET pg_textsearch.segments_per_level; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/compaction_request.out000066400000000000000000000133131525762047700261770ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_textsearch; \pset format unaligned SET client_min_messages = warning; SET pg_textsearch.segments_per_level = 2; SHOW pg_textsearch.background_compaction_schedule; pg_textsearch.background_compaction_schedule */5 * * * * (1 row) -- Legacy "off" remains accepted as an alias for manual mode. CREATE TABLE legacy_off_docs (id serial PRIMARY KEY, body text); CREATE INDEX legacy_off_docs_idx ON legacy_off_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); SELECT reloptions @> ARRAY['compaction=off'] FROM pg_class WHERE oid = 'legacy_off_docs_idx'::regclass; ?column? t (1 row) INSERT INTO legacy_off_docs (body) SELECT 'legacy off one ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('legacy_off_docs_idx') IS NOT NULL; ?column? t (1 row) INSERT INTO legacy_off_docs (body) SELECT 'legacy off two ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('legacy_off_docs_idx') IS NOT NULL; ?column? t (1 row) SELECT bm25_needs_compaction('legacy_off_docs_idx'::regclass) AS legacy_off_debt_remains; legacy_off_debt_remains t (1 row) -- The compaction policy and schedule are per-index options. CREATE TABLE relopt_docs (id serial PRIMARY KEY, body text); CREATE INDEX relopt_docs_idx ON relopt_docs USING bm25(body) WITH (text_config = 'english'); ALTER INDEX relopt_docs_idx SET (compaction = 'manual'); SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_class WHERE oid = 'relopt_docs_idx'::regclass; ?column? t (1 row) ALTER INDEX relopt_docs_idx SET (compaction_schedule = '17 * * * *'); SELECT reloptions @> ARRAY['compaction_schedule=17 * * * *'] FROM pg_class WHERE oid = 'relopt_docs_idx'::regclass; ?column? t (1 row) -- The post-ALTER activation hook preserves IF EXISTS no-op semantics. ALTER INDEX IF EXISTS missing_background_idx SET (compaction = 'background'); ALTER INDEX IF EXISTS missing_background_idx RESET (compaction_schedule); -- Managed background mode requires pg_durable, but temporary indexes are -- rejected before admission is attempted. CREATE INDEX CONCURRENTLY relopt_background_idx ON relopt_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); ERROR: background compaction requires pg_durable 0.2.8 or newer SELECT pg_catalog.to_regclass('relopt_background_idx') IS NULL AS rejected_cic_left_no_relation; rejected_cic_left_no_relation t (1 row) CREATE TEMP TABLE relopt_temp_docs (id integer, body text); CREATE INDEX relopt_temp_background_idx ON relopt_temp_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); ERROR: background compaction is not supported for temporary indexes CREATE INDEX CONCURRENTLY relopt_temp_background_cic_idx ON relopt_temp_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); ERROR: background compaction is not supported for temporary indexes -- Manual mode leaves spill-time compaction debt in place. CREATE TABLE manual_docs (id serial PRIMARY KEY, body text); CREATE INDEX manual_docs_idx ON manual_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO manual_docs (body) SELECT 'manual one ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('manual_docs_idx') IS NOT NULL; ?column? t (1 row) INSERT INTO manual_docs (body) SELECT 'manual two ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('manual_docs_idx') IS NOT NULL; ?column? t (1 row) SELECT bm25_needs_compaction('manual_docs_idx'::regclass) AS manual_debt_remains; manual_debt_remains t (1 row) -- The final serial build batch gets the same inline policy as -- intermediate batches. SET max_parallel_maintenance_workers = 0; SET maintenance_work_mem = '1MB'; CREATE TABLE final_build_docs ( id integer PRIMARY KEY, body text ); INSERT INTO final_build_docs SELECT i, 'token' || i || ' ' || repeat(md5(i::text) || ' ', 8) FROM generate_series(1, 10000) i; CREATE INDEX final_build_docs_idx ON final_build_docs USING bm25(body) WITH (text_config = 'simple'); SELECT NOT bm25_needs_compaction('final_build_docs_idx'::regclass) AS final_build_batch_compacted; final_build_batch_compacted t (1 row) -- CREATE INDEX honors manual mode, leaving its build batches uncompacted. CREATE TABLE manual_build_docs ( id integer PRIMARY KEY, body text ); INSERT INTO manual_build_docs SELECT i, 'token' || i || ' ' || repeat(md5(i::text) || ' ', 8) FROM generate_series(1, 10000) i; CREATE INDEX manual_build_docs_idx ON manual_build_docs USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); SELECT bm25_needs_compaction('manual_build_docs_idx'::regclass) AS manual_build_debt_remains; manual_build_debt_remains t (1 row) RESET maintenance_work_mem; RESET max_parallel_maintenance_workers; -- Inline mode compacts synchronously. CREATE TABLE inline_docs (id serial PRIMARY KEY, body text); CREATE INDEX inline_docs_idx ON inline_docs USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); INSERT INTO inline_docs (body) SELECT 'inline one ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('inline_docs_idx') IS NOT NULL; ?column? t (1 row) INSERT INTO inline_docs (body) SELECT 'inline two ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('inline_docs_idx') IS NOT NULL; ?column? t (1 row) SELECT NOT bm25_needs_compaction('inline_docs_idx'::regclass) AS inline_compacted; inline_compacted t (1 row) RESET pg_textsearch.segments_per_level; RESET client_min_messages; DROP TABLE inline_docs CASCADE; DROP TABLE manual_build_docs CASCADE; DROP TABLE final_build_docs CASCADE; DROP TABLE manual_docs CASCADE; DROP TABLE relopt_docs CASCADE; DROP TABLE legacy_off_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/compression.out000066400000000000000000000205631525762047700246410ustar00rootroot00000000000000-- Test compression functionality -- Tests that compression (now default) produces correct results CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Compression is now on by default SHOW pg_textsearch.compress_segments; pg_textsearch.compress_segments --------------------------------- on (1 row) -- Create test table CREATE TABLE compression_docs ( id serial PRIMARY KEY, content text ); -- Create BM25 index CREATE INDEX compression_idx ON compression_docs USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation compression_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert enough documents to create multiple blocks (block size is 128) -- We need at least 128+ docs with the same term to test block compression INSERT INTO compression_docs (content) SELECT 'the quick brown fox jumps over lazy dog number ' || i FROM generate_series(1, 200) AS i; -- Force segment creation by spilling memtable SELECT bm25_spill_index('compression_idx'); bm25_spill_index ------------------ 4 (1 row) -- Query should work correctly with compressed segments SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('fox', 'compression_idx')) sub; count ------- 200 (1 row) -- Test with multiple query terms SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('quick brown', 'compression_idx')) sub; count ------- 200 (1 row) -- Test that scores are reasonable (not corrupted by compression) -- Scores should be negative (BM25 returns negative for ORDER BY ASC) SELECT id, content <@> to_bm25query('fox', 'compression_idx') AS score FROM compression_docs ORDER BY score LIMIT 5; id | score ----+------------------------ 1 | -0.0024906613398343325 2 | -0.0024906613398343325 3 | -0.0024906613398343325 4 | -0.0024906613398343325 5 | -0.0024906613398343325 (5 rows) -- Add more documents and test again INSERT INTO compression_docs (content) SELECT 'different content with unique words like zephyr ' || i FROM generate_series(1, 100) AS i; -- Spill again to create a second segment SELECT bm25_spill_index('compression_idx'); bm25_spill_index ------------------ 9 (1 row) -- Query across both segments SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('fox', 'compression_idx')) sub; count ------- 200 (1 row) -- Query the new content SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('zephyr', 'compression_idx')) sub; count ------- 100 (1 row) -- Test BMW query with compressed segments SELECT id FROM compression_docs ORDER BY content <@> to_bm25query('fox', 'compression_idx') LIMIT 5; id ---- 1 2 3 4 5 (5 rows) -- Clean up DROP TABLE compression_docs; -- Test that compression can be disabled SET pg_textsearch.compress_segments = off; SHOW pg_textsearch.compress_segments; pg_textsearch.compress_segments --------------------------------- off (1 row) -- Create another table with compression off CREATE TABLE uncompressed_docs ( id serial PRIMARY KEY, content text ); CREATE INDEX uncompressed_idx ON uncompressed_docs USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation uncompressed_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO uncompressed_docs (content) SELECT 'testing uncompressed segments ' || i FROM generate_series(1, 150) AS i; SELECT bm25_spill_index('uncompressed_idx'); bm25_spill_index ------------------ 3 (1 row) -- Should work the same SELECT count(*) FROM (SELECT 1 FROM uncompressed_docs ORDER BY content <@> to_bm25query('testing', 'uncompressed_idx')) sub; count ------- 150 (1 row) -- Clean up DROP TABLE uncompressed_docs; -- ============================================================================= -- Test mixed compression: toggle GUC after index has segments -- ============================================================================= -- Reset to default (compression on) SET pg_textsearch.compress_segments = on; -- Create table and index CREATE TABLE mixed_docs ( id serial PRIMARY KEY, content text ); CREATE INDEX mixed_idx ON mixed_docs USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation mixed_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert first batch with compression ON INSERT INTO mixed_docs (content) SELECT 'alpha beta gamma document ' || i FROM generate_series(1, 150) AS i; SELECT bm25_spill_index('mixed_idx'); bm25_spill_index ------------------ 3 (1 row) -- Verify compressed segment works SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('alpha', 'mixed_idx')) sub; count ------- 150 (1 row) -- Now turn compression OFF and add more data SET pg_textsearch.compress_segments = off; INSERT INTO mixed_docs (content) SELECT 'alpha delta epsilon document ' || i FROM generate_series(151, 300) AS i; -- This creates an uncompressed segment SELECT bm25_spill_index('mixed_idx'); bm25_spill_index ------------------ 8 (1 row) -- Query should work across mixed compressed/uncompressed segments SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('alpha', 'mixed_idx')) sub; count ------- 300 (1 row) -- Query term only in compressed segment SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('beta', 'mixed_idx')) sub; count ------- 150 (1 row) -- Query term only in uncompressed segment SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('delta', 'mixed_idx')) sub; count ------- 150 (1 row) -- Turn compression back on and add a third segment SET pg_textsearch.compress_segments = on; INSERT INTO mixed_docs (content) SELECT 'alpha zeta theta document ' || i FROM generate_series(301, 450) AS i; SELECT bm25_spill_index('mixed_idx'); bm25_spill_index ------------------ 13 (1 row) -- Now we have: compressed segment, uncompressed segment, compressed segment -- Query should still work SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('alpha', 'mixed_idx')) sub; count ------- 450 (1 row) -- Clean up DROP TABLE mixed_docs; -- ============================================================================= -- Test: start uncompressed, then enable compression -- ============================================================================= SET pg_textsearch.compress_segments = off; CREATE TABLE mixed_docs2 ( id serial PRIMARY KEY, content text ); CREATE INDEX mixed_idx2 ON mixed_docs2 USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation mixed_idx2 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- First batch uncompressed INSERT INTO mixed_docs2 (content) SELECT 'apple banana cherry ' || i FROM generate_series(1, 150) AS i; SELECT bm25_spill_index('mixed_idx2'); bm25_spill_index ------------------ 3 (1 row) -- Enable compression SET pg_textsearch.compress_segments = on; -- Second batch compressed INSERT INTO mixed_docs2 (content) SELECT 'apple date elderberry ' || i FROM generate_series(151, 300) AS i; SELECT bm25_spill_index('mixed_idx2'); bm25_spill_index ------------------ 8 (1 row) -- Query across uncompressed and compressed segments SELECT count(*) FROM (SELECT 1 FROM mixed_docs2 ORDER BY content <@> to_bm25query('apple', 'mixed_idx2')) sub; count ------- 300 (1 row) SELECT count(*) FROM (SELECT 1 FROM mixed_docs2 ORDER BY content <@> to_bm25query('banana', 'mixed_idx2')) sub; count ------- 150 (1 row) SELECT count(*) FROM (SELECT 1 FROM mixed_docs2 ORDER BY content <@> to_bm25query('date', 'mixed_idx2')) sub; count ------- 150 (1 row) -- Clean up DROP TABLE mixed_docs2; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/concurrent_build.out000066400000000000000000000067311525762047700256420ustar00rootroot00000000000000-- Test CREATE INDEX CONCURRENTLY with bm25 indexes -- -- CIC requires proper ambulkdelete callback implementation to report all -- indexed TIDs during validation phase. This ensures the validate_index -- function can determine which tuples are already indexed and avoid -- duplicating entries. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -------------------------------------------------------------------------------- -- Test 1: Basic CREATE INDEX CONCURRENTLY -------------------------------------------------------------------------------- CREATE TABLE cic_basic ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO cic_basic (content) VALUES ('the quick brown fox jumps over the lazy dog'), ('postgresql is a powerful database system'), ('full text search with bm25 ranking'); -- Create index concurrently - should complete without error CREATE INDEX CONCURRENTLY cic_basic_idx ON cic_basic USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation cic_basic_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=5.00 -- Verify index is valid (indisvalid = true) SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_basic_idx'::regclass; indisvalid ------------ t (1 row) -- Verify no duplicate entries (this was a bug symptom before the fix) SELECT COUNT(*) = 1 AS no_duplicates FROM ( SELECT 1 FROM cic_basic ORDER BY content <@> to_bm25query('database', 'cic_basic_idx') ) sub; no_duplicates --------------- t (1 row) -- Verify index is used in query plan EXPLAIN (COSTS OFF) SELECT id FROM cic_basic ORDER BY content <@> to_bm25query('database', 'cic_basic_idx') LIMIT 3; QUERY PLAN --------------------------------------------------------------------- Limit -> Index Scan using cic_basic_idx on cic_basic Order By: (content <@> 'cic_basic_idx:database'::bm25query) (3 rows) -------------------------------------------------------------------------------- -- Test 2: DROP INDEX CONCURRENTLY -------------------------------------------------------------------------------- DROP INDEX CONCURRENTLY cic_basic_idx; -- Verify index is gone SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'cic_basic_idx'; count ------- 0 (1 row) -------------------------------------------------------------------------------- -- Test 3: Recreate with regular CREATE INDEX for comparison -------------------------------------------------------------------------------- CREATE INDEX cic_basic_idx ON cic_basic USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation cic_basic_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=5.00 -- Verify index is valid SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_basic_idx'::regclass; indisvalid ------------ t (1 row) -- Query should work and return expected results SELECT id FROM cic_basic ORDER BY content <@> to_bm25query('database', 'cic_basic_idx'); id ---- 2 (1 row) -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- DROP TABLE cic_basic CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/coverage.out000066400000000000000000000321441525762047700240710ustar00rootroot00000000000000-- Additional coverage tests for untested code paths -- Tests debug functions, segment dump, and text<@>text operator CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================================= -- Test 1: Basic debug functions with memtable-only data -- ============================================================================= -- Create a simple table and index CREATE TABLE coverage_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO coverage_docs (content) VALUES ('hello world test document'), ('database search query optimization'), ('full text search engine postgresql'); CREATE INDEX coverage_idx ON coverage_docs USING bm25(content) WITH (text_config='english', compaction='manual'); NOTICE: BM25 index build started for relation coverage_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=4.33 -- Test bm25_summarize_index with memtable data SELECT bm25_summarize_index('coverage_idx') IS NOT NULL AS summarize_memtable; summarize_memtable -------------------- t (1 row) -- Test bm25_dump_index with memtable data SELECT length(bm25_dump_index('coverage_idx')) > 0 AS dump_memtable; dump_memtable --------------- t (1 row) -- ============================================================================= -- Test 2: Debug functions with segment data (exercises segment dump code) -- ============================================================================= -- Force segment spill to have data on disk -- Note: With build mode, data is already spilled during CREATE INDEX, -- so memtable is empty here and returns NULL SELECT bm25_spill_index('coverage_idx'); bm25_spill_index ------------------ (1 row) -- Test bm25_summarize_index with segment data -- This exercises tp_summarize_index_to_output with segments SELECT bm25_summarize_index('coverage_idx') ~ E'L0 Segment 1:.*\n Total: 1 segments' AS summarize_segment; summarize_segment ------------------- t (1 row) -- Test bm25_dump_index with segment data -- This exercises tp_dump_segment_to_output, read_dict_entry, read_term_at_index SELECT length(bm25_dump_index('coverage_idx')) > 0 AS dump_segment; dump_segment -------------- t (1 row) -- ============================================================================= -- Test 3: bm25query equality (tpquery_eq) -- ============================================================================= -- Test equality of identical queries SELECT to_bm25query('hello', 'coverage_idx') = to_bm25query('hello', 'coverage_idx') AS same_query_eq; same_query_eq --------------- t (1 row) -- Test inequality of different queries SELECT to_bm25query('hello', 'coverage_idx') = to_bm25query('world', 'coverage_idx') AS diff_query_eq; diff_query_eq --------------- f (1 row) -- ============================================================================= -- Test 4: text <@> text operator (bm25_text_text_score) -- This is the implicit form without explicit bm25query -- ============================================================================= -- Insert more data to create variation INSERT INTO coverage_docs (content) VALUES ('additional text for testing'), ('more content with different words'); -- Test text <@> text implicit scoring -- This uses find_first_child_index to locate the bm25 index SET enable_seqscan = off; -- The implicit form should find coverage_idx automatically SELECT content, content <@> 'hello'::text AS score FROM coverage_docs ORDER BY content <@> 'hello'::text LIMIT 3; content | score ---------------------------+--------------------- hello world test document | -1.3570750951766968 (1 row) SET enable_seqscan = on; -- ============================================================================= -- Test 5: Multiple segments (exercises segment iteration in dump) -- ============================================================================= -- Add more data and spill again to create multiple segments INSERT INTO coverage_docs (content) SELECT 'document number ' || i || ' with varying content for coverage' FROM generate_series(1, 100) AS i; SELECT bm25_spill_index('coverage_idx'); bm25_spill_index ------------------ 5 (1 row) -- Both debug readers must enumerate every root recorded by the metapage. -- Count the segment headers rather than merely matching one: a LIKE test -- passes with a single segment and would not catch a dump that stops -- after the first root. SELECT bm25_summarize_index('coverage_idx') ~ E'Total: 2 segments' AND array_length( string_to_array(bm25_dump_index('coverage_idx'), '========== Segment at block'), 1) - 1 = 2 AS dump_multi_segment; dump_multi_segment -------------------- t (1 row) -- ============================================================================= -- Test 6: Page visualization - moved behind DEBUG_DUMP_INDEX compile flag -- ============================================================================= -- ============================================================================= -- Test 7: Score logging (exercises am/scan.c log_scores path) -- ============================================================================= SET pg_textsearch.log_scores = true; SELECT content <@> to_bm25query('hello', 'coverage_idx') AS score FROM coverage_docs ORDER BY content <@> to_bm25query('hello', 'coverage_idx') LIMIT 1; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-4.9028 score -------------------- -4.902772426605225 (1 row) SET pg_textsearch.log_scores = false; -- ============================================================================= -- Test 8: BMW stats logging (exercises score.c bmw_stats path) -- ============================================================================= SET pg_textsearch.log_bmw_stats = true; SET enable_seqscan = off; -- Single-term query SELECT count(*) FROM ( SELECT 1 FROM coverage_docs ORDER BY content <@> to_bm25query('hello', 'coverage_idx') ) sub; count ------- 1 (1 row) -- Multi-term query SELECT count(*) FROM ( SELECT 1 FROM coverage_docs ORDER BY content <@> to_bm25query('hello world', 'coverage_idx') ) sub; count ------- 1 (1 row) SET enable_seqscan = on; SET pg_textsearch.log_bmw_stats = false; -- ============================================================================= -- Test 9: AM handler property callbacks (exercises handler.c tp_property) -- ============================================================================= -- Test pg_indexam_has_property for our access method SELECT pg_indexam_has_property(am.oid, 'can_order') AS can_order, pg_indexam_has_property(am.oid, 'can_unique') AS can_unique FROM pg_am am WHERE am.amname = 'bm25'; can_order | can_unique -----------+------------ f | f (1 row) -- Test index-level properties SELECT pg_index_has_property('coverage_idx'::regclass, 'clusterable') AS clusterable, pg_index_has_property('coverage_idx'::regclass, 'index_scan') AS index_scan; clusterable | index_scan -------------+------------ f | t (1 row) -- Test column-level property (distance_orderable triggers tp_property) SELECT pg_index_column_has_property('coverage_idx'::regclass, 1, 'distance_orderable') AS dist_orderable; dist_orderable ---------------- t (1 row) -- ============================================================================= -- Test 10: AM validate error path (exercises opclass resolution) -- ============================================================================= -- Try creating a bm25 index on integer column (should fail validation) CREATE TABLE validate_test (id serial, num integer); CREATE INDEX validate_test_idx ON validate_test USING bm25(num) WITH (text_config='english'); ERROR: data type integer has no default operator class for access method "bm25" HINT: You must specify an operator class for the index or define a default operator class for the data type. DROP TABLE validate_test; -- ============================================================================= -- Test 11: File-writing dump - moved behind DEBUG_DUMP_INDEX compile flag -- ============================================================================= -- ============================================================================= -- Test 12: Segment BMW seek (exercises segment/scan.c seek path) -- Uses ORDER BY ... LIMIT to trigger top-k BMW with segment data -- ============================================================================= -- We already have segment data from Test 5; add more for multi-block INSERT INTO coverage_docs (content) SELECT 'seek test document with term alpha ' || i FROM generate_series(1, 200) AS i; SELECT bm25_spill_index('coverage_idx'); bm25_spill_index ------------------ 10 (1 row) -- ORDER BY score LIMIT triggers BMW seek path on segments -- Wrap in outer query with id tiebreaker for deterministic ordering -- (adding secondary sort key to inner query would prevent index use) SELECT id FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('alpha', 'coverage_idx') LIMIT 5 ) sub ORDER BY id; id ----- 106 107 108 109 110 (5 rows) -- Multi-term BMW seek with segment data SELECT id FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('seek alpha', 'coverage_idx') LIMIT 3 ) sub ORDER BY id; id ----- 106 107 108 (3 rows) -- ============================================================================= -- Test 13: tpquery_in with index name prefix (exercises query.c parsing) -- ============================================================================= -- Cast string with index:query format to bm25query type SELECT 'coverage_idx:hello world'::bm25query IS NOT NULL AS tpquery_in_with_index; tpquery_in_with_index ----------------------- t (1 row) -- ============================================================================= -- Test 14: BMW stats logging with segment ORDER BY LIMIT queries -- ============================================================================= SET pg_textsearch.log_bmw_stats = true; -- Single-term with ORDER BY LIMIT (BMW seek on segments) SELECT count(*) FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('alpha', 'coverage_idx') LIMIT 10 ) sub; count ------- 10 (1 row) -- Multi-term with ORDER BY LIMIT SELECT count(*) FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('seek alpha', 'coverage_idx') LIMIT 10 ) sub; count ------- 10 (1 row) SET pg_textsearch.log_bmw_stats = false; -- ============================================================================= -- Test 15: Plain text ORDER BY (exercises scan.c sk_subtype == TEXTOID) -- ============================================================================= SET enable_seqscan = off; -- The implicit text <@> text form in ORDER BY SELECT id FROM ( SELECT id FROM coverage_docs ORDER BY content <@> 'alpha'::text LIMIT 3 ) sub ORDER BY id; id ----- 106 107 108 (3 rows) SET enable_seqscan = on; -- ============================================================================= -- Test 16: GROUP BY / HAVING with BM25 operator -- ============================================================================= -- Use BM25 operator in HAVING clause SELECT left(content, 20) AS prefix, count(*) AS cnt FROM ( SELECT content FROM coverage_docs ORDER BY content <@> to_bm25query('document', 'coverage_idx') ) sub GROUP BY left(content, 20) HAVING count(*) > 1 ORDER BY cnt DESC LIMIT 3; prefix | cnt ----------------------+----- seek test document w | 200 (1 row) -- ============================================================================= -- Test 17: Build-mode spill (exercises state.c tp_clear_memtable build path) -- ============================================================================= -- Create table with enough data to trigger mid-build spill CREATE TABLE build_spill_test (id serial PRIMARY KEY, content text); INSERT INTO build_spill_test (content) SELECT 'buildspill unique term' || i || ' extra words for document ' || i FROM generate_series(1, 500) AS i; -- Set very low spill threshold so CREATE INDEX spills during build SET pg_textsearch.memtable_pages_threshold = 1; CREATE INDEX build_spill_idx ON build_spill_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation build_spill_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=7.00 -- Reset threshold RESET pg_textsearch.memtable_pages_threshold; -- Verify index works after build-mode spill SELECT count(*) FROM ( SELECT 1 FROM build_spill_test ORDER BY content <@> to_bm25query('buildspill', 'build_spill_idx') ) sub; count ------- 500 (1 row) DROP TABLE build_spill_test; -- ============================================================================= -- Cleanup -- ============================================================================= DROP TABLE coverage_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/deletion.out000066400000000000000000000064651525762047700241100ustar00rootroot00000000000000-- Test handling of row deletions in indexed tables -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table CREATE TABLE deletion_test ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert some documents INSERT INTO deletion_test (content) VALUES ('first document with some content'), ('second document with different content'), ('third document with test keyword'), ('fourth document without the keyword'), ('fifth document with test again'); -- Create index CREATE INDEX deletion_idx ON deletion_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation deletion_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.60 -- Force index scan (small table would otherwise prefer seq scan) SET enable_seqscan = off; -- Initial search - should find documents with "test" SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; id | content_preview ----+---------------------------------- 5 | fifth document with test again 3 | third document with test keyword (2 rows) -- Delete one of the documents that contains "test" DELETE FROM deletion_test WHERE id = 3; -- Search again - deleted document should not appear -- (Note: corpus statistics won't be updated, but shouldn't crash) SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; id | content_preview ----+-------------------------------- 5 | fifth document with test again (1 row) -- Delete a document that doesn't contain "test" DELETE FROM deletion_test WHERE id = 4; -- Search again - should still work SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; id | content_preview ----+-------------------------------- 5 | fifth document with test again (1 row) -- Delete all remaining documents with "test" DELETE FROM deletion_test WHERE content LIKE '%test%'; -- Search again - should return no results (but not crash) SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; id | content_preview ----+----------------- (0 rows) -- Insert new documents after deletions INSERT INTO deletion_test (content) VALUES ('new document with test keyword'), ('another new document'); -- Search again - should find the new document SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; id | content_preview ----+-------------------------------- 6 | new document with test keyword (1 row) -- Test with empty table after deleting everything DELETE FROM deletion_test; -- Search in empty table - should return no results SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; id | content_preview ----+----------------- (0 rows) -- Clean up DROP TABLE deletion_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/dropped.out000066400000000000000000000145641525762047700237410ustar00rootroot00000000000000-- Test behavior when using a dropped index name in queries -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table CREATE TABLE dropped_idx_test ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO dropped_idx_test (content) VALUES ('first test document'), ('second test document'), ('third test document'), ('fourth test document'), ('fifth test document'); -- Create index CREATE INDEX dropped_idx ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 -- Query should work with index (using explicit index reference) SELECT COUNT(*) AS with_index FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'dropped_idx') ) sub; with_index ------------ 5 (1 row) -- Drop the index DROP INDEX dropped_idx; -- Query should ERROR when index doesn't exist SELECT COUNT(*) AS after_drop FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'dropped_idx') ) sub; ERROR: index "dropped_idx" does not exist -- Also test with a completely non-existent index name SELECT COUNT(*) AS nonexistent FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'totally_fake_index') ) sub; ERROR: index "totally_fake_index" does not exist -- Test registry slot cleanup (issue #83) -- Create multiple indexes, drop them, then create new ones -- This verifies that registry slots are properly freed on drop -- Create 5 indexes CREATE INDEX dropped_idx_1 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_1 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_2 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_2 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_3 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_3 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_4 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_4 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_5 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_5 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 -- Drop all 5 DROP INDEX dropped_idx_1; DROP INDEX dropped_idx_2; DROP INDEX dropped_idx_3; DROP INDEX dropped_idx_4; DROP INDEX dropped_idx_5; -- Create 5 more indexes (should succeed if slots were freed) CREATE INDEX dropped_idx_6 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_6 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_7 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_7 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_8 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_8 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_9 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_9 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 CREATE INDEX dropped_idx_10 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_10 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 -- Test that the new indexes work SELECT COUNT(*) AS recycled_slots_work FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'dropped_idx_6') ) sub; recycled_slots_work --------------------- 5 (1 row) -- Also test CASCADE drop (table drop should clean up index slots) CREATE TABLE cascade_test (id SERIAL, content TEXT); INSERT INTO cascade_test (content) VALUES ('cascade test doc'); CREATE INDEX cascade_idx ON cascade_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation cascade_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=3.00 -- Drop table should cascade to drop index and free its registry slot DROP TABLE cascade_test CASCADE; -- Create another index to verify the cascade-dropped slot was freed CREATE INDEX dropped_idx_11 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation dropped_idx_11 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.00 -- Clean up DROP TABLE dropped_idx_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/empty.out000066400000000000000000000034141525762047700234320ustar00rootroot00000000000000-- Test handling of empty and whitespace-only documents -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table with various empty/whitespace content CREATE TABLE empty_docs ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert various empty/whitespace documents INSERT INTO empty_docs (content) VALUES (''), -- completely empty (' '), -- only spaces (' '), -- only tabs (' '), -- only newlines (' '), -- mixed whitespace (NULL); -- NULL content -- Create index on content CREATE INDEX empty_docs_idx ON empty_docs USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation empty_docs_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=0.00 -- Force index scan (small table would otherwise prefer seq scan) SET enable_seqscan = off; -- Try searching - should return no results without warnings SELECT id, content FROM empty_docs ORDER BY content <@> to_bm25query('test', 'empty_docs_idx') LIMIT 5; id | content ----+--------- (0 rows) -- Add a document with actual content INSERT INTO empty_docs (content) VALUES ('test document with content'); -- Search again - should find only the document with content SELECT id, substring(content, 1, 30) as content_preview FROM empty_docs ORDER BY content <@> to_bm25query('test', 'empty_docs_idx') LIMIT 5; id | content_preview ----+---------------------------- 7 | test document with content (1 row) -- Clean up DROP TABLE empty_docs; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/explicit_index.out000066400000000000000000000124631525762047700253100ustar00rootroot00000000000000-- Test explicit index name in to_bm25query -- Validates fixes for GitHub issues #183 and #194 CREATE EXTENSION IF NOT EXISTS pg_textsearch; ----------------------------------------------------------------------- -- Issue #194: to_bm25query with explicit index ignores index column -- Using an index on a different column should error ----------------------------------------------------------------------- -- Create table with TWO text columns CREATE TABLE explicit_index_test ( id SERIAL PRIMARY KEY, title TEXT, content TEXT ); -- Index on 'content' column only CREATE INDEX content_idx ON explicit_index_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test data INSERT INTO explicit_index_test (title, content) VALUES ('rust c java', 'postgres mysql'), ('python ruby', 'redis mongodb'); -- This should ERROR: using content_idx to search 'title' column -- The index is on 'content' but we're searching 'title' SELECT * FROM explicit_index_test ORDER BY title <@> to_bm25query('postgres', 'content_idx'); ERROR: index "content_idx" is not on column "title" DETAIL: The explicitly specified index is not built on the column being searched. HINT: Use an index that is built on column "title" of table "explicit_index_test", or omit the index name to use automatic index resolution. -- Positive test: using correct column should work SELECT id, content <@> to_bm25query('postgres', 'content_idx') AS score FROM explicit_index_test ORDER BY score LIMIT 10; id | score ----+--------------------- 1 | -0.6931471824645996 (1 row) -- This should ERROR: left operand is a constant, not a column SELECT 'hello' <@> to_bm25query('postgres', 'content_idx') AS score; ERROR: left operand of <@> must reference a table column HINT: Use column_name <@> bm25query, not a constant. -- Clean up DROP TABLE explicit_index_test; ----------------------------------------------------------------------- -- Issue #183: to_bm25query with explicit index name does not force -- planner to use that index ----------------------------------------------------------------------- -- Create table CREATE TABLE multi_index_test ( id SERIAL PRIMARY KEY, category_id INT, content TEXT ); -- Insert test data - enough rows to make planner consider index choices INSERT INTO multi_index_test (category_id, content) SELECT (i % 10), 'Les ressources humaines gèrent les employés et leurs contrats de travail' FROM generate_series(1, 1000) i; -- Create two BM25 indexes with different text configurations on same column CREATE INDEX content_idx_french ON multi_index_test USING bm25 (content) WITH (text_config = 'french'); NOTICE: BM25 index build started for relation content_idx_french NOTICE: Using text search configuration: french NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1000 documents, avg_length=9.00 CREATE INDEX content_idx_simple ON multi_index_test USING bm25 (content) WITH (text_config = 'simple'); NOTICE: BM25 index build started for relation content_idx_simple NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1000 documents, avg_length=11.00 -- Analyze for statistics ANALYZE multi_index_test; -- Query explicitly requesting the French index -- The planner should use content_idx_french as specified EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('ressources', 'content_idx_french') AS score FROM multi_index_test WHERE category_id = 1 ORDER BY score LIMIT 10; QUERY PLAN ---------------------------------------------------------------------------- Limit -> Index Scan using content_idx_french on multi_index_test Order By: (content <@> 'content_idx_french:ressources'::bm25query) Filter: (category_id = 1) (4 rows) -- Positive test: without an explicit index, implicit resolution still -- works. All matching rows share the same text, so the BM25 scores are -- deterministic even though the tied tuple ids (and which tied index is -- scanned) are not. SET client_min_messages = error; \pset format unaligned SELECT content <@> 'ressources' AS score FROM multi_index_test WHERE category_id = 1 ORDER BY score LIMIT 3; score -0.0004996252828277647 -0.0004996252828277647 -0.0004996252828277647 (3 rows) \pset format aligned RESET client_min_messages; -- Positive test: explicit index matching the scanned index works -- Drop the simple index so french is the only choice DROP INDEX content_idx_simple; EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('ressources', 'content_idx_french') AS score FROM multi_index_test WHERE category_id = 1 ORDER BY score LIMIT 10; QUERY PLAN ---------------------------------------------------------------------------- Limit -> Index Scan using content_idx_french on multi_index_test Order By: (content <@> 'content_idx_french:ressources'::bm25query) Filter: (category_id = 1) (4 rows) -- Clean up DROP TABLE multi_index_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/expression_index.out000066400000000000000000000313751525762047700256710ustar00rootroot00000000000000-- Expression index tests for pg_textsearch CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================ -- JSONB expression index -- ============================================================ CREATE TABLE expr_jsonb ( id SERIAL PRIMARY KEY, data JSONB ); INSERT INTO expr_jsonb (data) VALUES ('{"content": "postgresql database management system"}'), ('{"content": "machine learning algorithms"}'), ('{"content": "full text search and retrieval"}'); CREATE INDEX expr_jsonb_idx ON expr_jsonb USING bm25 ((data->>'content')) WITH (text_config='english', compaction='manual'); NOTICE: BM25 index build started for relation expr_jsonb_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.67 SET enable_seqscan = off; -- Query with explicit index naming SELECT id, ROUND(((data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx') LIMIT 5; id | score ----+--------- 1 | -0.9457 (1 row) -- ============================================================ -- Multi-column concatenation expression index -- ============================================================ CREATE TABLE expr_concat ( id SERIAL PRIMARY KEY, title TEXT, body TEXT ); INSERT INTO expr_concat (title, body) VALUES ('Database Systems', 'An introduction to relational databases'), ('Machine Learning', 'Algorithms for pattern recognition'), ('Search Engines', 'Full text search and information retrieval'); CREATE INDEX expr_concat_idx ON expr_concat USING bm25 ((coalesce(title, '') || ' ' || coalesce(body, ''))) WITH (text_config='english'); NOTICE: BM25 index build started for relation expr_concat_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=5.67 SELECT id, ROUND(((coalesce(title, '') || ' ' || coalesce(body, '')) <@> to_bm25query('database', 'expr_concat_idx'))::numeric, 4) AS score FROM expr_concat ORDER BY (coalesce(title, '') || ' ' || coalesce(body, '')) <@> to_bm25query('database', 'expr_concat_idx') LIMIT 5; id | score ----+--------- 1 | -1.3948 (1 row) -- ============================================================ -- Simple transformation expression index -- ============================================================ CREATE TABLE expr_lower ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO expr_lower (content) VALUES ('PostgreSQL Database Management'), ('Machine Learning Algorithms'), ('Full Text Search and Retrieval'); CREATE INDEX expr_lower_idx ON expr_lower USING bm25 ((lower(content))) WITH (text_config='simple'); NOTICE: BM25 index build started for relation expr_lower_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.67 SELECT id, ROUND(((lower(content)) <@> to_bm25query('database', 'expr_lower_idx'))::numeric, 4) AS score FROM expr_lower ORDER BY (lower(content)) <@> to_bm25query('database', 'expr_lower_idx') LIMIT 5; id | score ----+--------- 1 | -1.0596 (1 row) -- ============================================================ -- Implicit resolution (no explicit index name) -- ============================================================ SELECT id, ROUND(((data->>'content') <@> to_bm25query('database'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database') LIMIT 5; id | score ----+--------- 1 | -0.9457 (1 row) -- ============================================================ -- Verify index scan is used -- ============================================================ EXPLAIN (COSTS OFF) SELECT id FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx') LIMIT 5; QUERY PLAN ----------------------------------------------------------------------------------------- Limit -> Index Scan using expr_jsonb_idx on expr_jsonb Order By: ((data ->> 'content'::text) <@> 'expr_jsonb_idx:database'::bm25query) (3 rows) -- ============================================================ -- NULL handling: row with missing JSONB key -- ============================================================ INSERT INTO expr_jsonb (data) VALUES ('{"other_field": "no content key"}'), (NULL); -- These should not affect results SELECT id, ROUND(((data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx') LIMIT 5; id | score ----+--------- 1 | -0.9457 (1 row) -- ============================================================ -- Delete a document and verify index still works -- ============================================================ DELETE FROM expr_jsonb WHERE id = 1; SELECT id, ROUND(((data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx') LIMIT 5; id | score ----+--------- 3 | -0.9457 (1 row) -- ============================================================ -- VACUUM: delete rows and vacuum to exercise rebuild -- ============================================================ DELETE FROM expr_jsonb WHERE id <= 2; VACUUM expr_jsonb; -- Verify index still works after vacuum SELECT id, ROUND(((data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx') LIMIT 5; id | score ----+--------- 3 | -0.9457 (1 row) -- ============================================================ -- Partitioned table with expression index -- ============================================================ CREATE TABLE expr_part ( id SERIAL, data JSONB, created_at DATE NOT NULL ) PARTITION BY RANGE (created_at); CREATE TABLE expr_part_2024 PARTITION OF expr_part FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); CREATE TABLE expr_part_2025 PARTITION OF expr_part FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); INSERT INTO expr_part (data, created_at) VALUES ('{"text": "database management systems"}', '2024-06-15'), ('{"text": "machine learning algorithms"}', '2024-09-20'), ('{"text": "search engine optimization"}', '2025-03-10'); CREATE INDEX expr_part_idx ON expr_part USING bm25 ((data->>'text')) WITH (text_config='english'); NOTICE: BM25 index build started for relation expr_part_2024_expr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents across 2 partitions, avg_length=3.00 SELECT id, ROUND(((data->>'text') <@> to_bm25query('database', 'expr_part_idx'))::numeric, 4) AS score FROM expr_part ORDER BY (data->>'text') <@> to_bm25query('database', 'expr_part_idx') LIMIT 5; id | score ----+--------- 1 | -0.6931 (1 row) DROP TABLE expr_part CASCADE; -- ============================================================ -- Multiple expression indexes trigger warning -- ============================================================ -- Suppress build NOTICEs: the reported doc count is non-deterministic -- here (recently-dead tuples from the earlier delete/vacuum may be -- indexed) and irrelevant to this multi-index test. SET client_min_messages = warning; CREATE INDEX expr_jsonb_idx2 ON expr_jsonb USING bm25 ((data->>'content')) WITH (text_config='simple'); RESET client_min_messages; -- Implicit resolution warns about multiple matching indexes. SELECT id, ROUND(((data->>'content') <@> to_bm25query('database'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database') LIMIT 3; WARNING: multiple BM25 indexes match the expression HINT: Use explicit to_bm25query('query', 'index_name') to specify which index. WARNING: multiple BM25 indexes match the expression HINT: Use explicit to_bm25query('query', 'index_name') to specify which index. id | score ----+------- (0 rows) -- Explicit naming with multiple expression indexes. SELECT id, ROUND(((data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx2'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx2') LIMIT 3; id | score ----+------- (0 rows) DROP INDEX expr_jsonb_idx2; -- ============================================================ -- Implicit text <@> text with expression index -- ============================================================ SELECT id, ROUND(((data->>'content') <@> 'database')::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> 'database' LIMIT 3; id | score ----+------- (0 rows) -- ============================================================ -- Non-NULL INSERT/UPDATE values must use the expression's type, -- not heap attribute zero. Run these under ASan as well. -- ============================================================ INSERT INTO expr_lower (content) VALUES ('Insertedneedle'); SELECT id FROM expr_lower ORDER BY lower(content) <@> to_bm25query('insertedneedle', 'expr_lower_idx') LIMIT 5; id ---- 4 (1 row) UPDATE expr_lower SET content = 'Updatedneedle' WHERE id = 4; SELECT id FROM expr_lower ORDER BY lower(content) <@> to_bm25query('updatedneedle', 'expr_lower_idx') LIMIT 5; id ---- 4 (1 row) -- Array-valued expressions need flattening in both build and insert. CREATE TABLE expr_array (id int, title text, body text); INSERT INTO expr_array VALUES (1, 'buildneedle', 'common'); CREATE INDEX expr_array_idx ON expr_array USING bm25 ((ARRAY[title, body])) WITH (text_config='simple'); NOTICE: BM25 index build started for relation expr_array_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('buildneedle', 'expr_array_idx') LIMIT 5; id ---- 1 (1 row) INSERT INTO expr_array VALUES (2, 'insertneedle', 'common'); SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('insertneedle', 'expr_array_idx') LIMIT 5; id ---- 2 (1 row) UPDATE expr_array SET body = 'common updateneedle' WHERE id = 1; SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('updateneedle', 'expr_array_idx') LIMIT 5; id ---- 1 (1 row) -- Ignore recently-dead rows in the reported REINDEX build count. SET client_min_messages = warning; REINDEX INDEX expr_array_idx; RESET client_min_messages; SELECT array_agg(id ORDER BY id) AS rebuilt_ids FROM ( SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('common', 'expr_array_idx') LIMIT 5 ) hits; rebuilt_ids ------------- {1,2} (1 row) DROP TABLE expr_array; -- ============================================================ -- Error cases -- ============================================================ \set ON_ERROR_STOP off -- Non-text expression type CREATE INDEX expr_bad_idx ON expr_lower USING bm25 ((length(content))) WITH (text_config='simple'); ERROR: data type integer has no default operator class for access method "bm25" HINT: You must specify an operator class for the index or define a default operator class for the data type. -- Constant left operand with bm25query SELECT 'hello' <@> to_bm25query('world', 'expr_jsonb_idx'); ERROR: left operand of <@> must reference a table column HINT: Use column_name <@> bm25query, not a constant. -- Constant left operand with text SELECT 'hello' <@> 'world'; ERROR: left operand of <@> must reference a table column HINT: Use column_name <@> 'query text', not a constant. \set ON_ERROR_STOP on -- ============================================================ -- Cleanup -- ============================================================ DROP TABLE expr_jsonb CASCADE; DROP TABLE expr_concat CASCADE; DROP TABLE expr_lower CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/filtered_seed.out000066400000000000000000001040361525762047700250740ustar00rootroot00000000000000-- Test: selectivity-seeded top-K for filtered BM25 search -- -- A filtered top-k query -- WHERE ORDER BY LIMIT k -- -- is planned as a BM25 top-k index scan with applied as a -- Filter above it. pg_textsearch.filtered_seed seeds the scan's -- internal top-K from the planner's estimated filter selectivity so a -- single scoring pass usually surfaces enough matching rows, avoiding -- the executor's backoff re-drives. The seed only changes scan depth, -- never which rows win, so results must be IDENTICAL to the un-seeded -- plan. -- -- This test asserts correctness, not latency: -- * parity - seeded result == un-seeded result (same code path, -- so immune to score ties) -- * oracle - the filtered index scan returns exactly the set of -- documents that match the filter AND contain a query -- term, checked against an independent plain-SQL regex -- (k large enough that the full match set is returned, -- so immune to ties and to score values) -- * no-op - a query with no filter is unaffected by seeding -- * robustness - the result is independent of filtered_seed_margin -- * GUC - defaults and range enforcement SET log_duration = off; SET client_min_messages = WARNING; -- suppress index-build NOTICE chatter CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = false; ------------------------------------------------------------------------ -- Setup: 1000 docs, a 50-bucket facet column, controlled vocabulary. -- 'common' in every doc; 'alpha' in even ids; 'beta' every 3rd; -- 'gamma' every 7th; a unique 'doc' token per row. -- facet_id = id % 50 => 20 docs per bucket (2% selectivity each). ------------------------------------------------------------------------ CREATE TABLE fs_docs ( id int PRIMARY KEY, facet_id int, body text ); INSERT INTO fs_docs SELECT g, g % 50, concat_ws(' ', 'common', CASE WHEN g % 2 = 0 THEN 'alpha' END, CASE WHEN g % 3 = 0 THEN 'beta' END, CASE WHEN g % 7 = 0 THEN 'gamma' END, 'doc' || g) FROM generate_series(1, 1000) g; CREATE INDEX fs_docs_idx ON fs_docs USING bm25(body) WITH (text_config='english'); -- No index on facet_id: the facet is applied as a Filter above the -- BM25 index scan, which is exactly the path filtered_seed optimizes. -- (With a facet index the planner could instead scan the facet and -- sort by score, bypassing the seed; that plan is equally correct but -- not what this test exercises.) ANALYZE fs_docs; ------------------------------------------------------------------------ -- Parity: seeded (default on) vs un-seeded top-k are identical. -- Compared as sets (array_agg ORDER BY id); same index-scan code path -- on both sides, so any score ties break identically. ------------------------------------------------------------------------ -- Confirm the plan under test: a BM25 index scan with the facet applied -- as a Filter -- the path filtered_seed seeds. If this ever becomes a -- facet-index scan + sort, the checks below stop exercising the seed. EXPLAIN (COSTS OFF) SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10; QUERY PLAN ------------------------------------------------------------- Limit -> Index Scan using fs_docs_idx on fs_docs Order By: (body <@> 'fs_docs_idx:alpha'::bm25query) Filter: (facet_id = 6) (4 rows) CREATE FUNCTION fs_check(qry text, pred text, k int) RETURNS text LANGUAGE plpgsql AS $$ DECLARE q text; seeded int[]; unseeded int[]; BEGIN IF pred IS NULL THEN q := format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (SELECT id FROM fs_docs ' ' ORDER BY body <@> to_bm25query(%L, ''fs_docs_idx'') ' ' LIMIT %s) s', qry, k); ELSE q := format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (SELECT id FROM fs_docs WHERE %s ' ' ORDER BY body <@> to_bm25query(%L, ''fs_docs_idx'') ' ' LIMIT %s) s', pred, qry, k); END IF; SET LOCAL pg_textsearch.filtered_seed = on; EXECUTE q INTO seeded; SET LOCAL pg_textsearch.filtered_seed = off; EXECUTE q INTO unseeded; IF seeded IS NOT DISTINCT FROM unseeded THEN RETURN format('PASS (%s rows)', coalesce(array_length(seeded, 1), 0)); END IF; RETURN format('FAIL seeded=%s unseeded=%s', seeded, unseeded); END; $$; -- Independent oracle: the filtered index scan must return exactly the -- documents matching the filter AND containing the query term. k is -- large enough (>= facet bucket size) that the whole match set is -- returned, so this is immune to ranking, ties, and score values. CREATE FUNCTION fs_oracle_check(qry text, term text, pred text, k int) RETURNS text LANGUAGE plpgsql AS $$ DECLARE idx int[]; ora int[]; BEGIN EXECUTE format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (SELECT id FROM fs_docs WHERE %s ' ' ORDER BY body <@> to_bm25query(%L, ''fs_docs_idx'') ' ' LIMIT %s) s', pred, qry, k) INTO idx; EXECUTE format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM fs_docs WHERE (%s) AND body ~ %L', pred, '\y' || term || '\y') INTO ora; IF idx IS NOT DISTINCT FROM ora THEN RETURN format('PASS (%s rows)', coalesce(array_length(idx, 1), 0)); END IF; RETURN format('FAIL index=%s oracle=%s', idx, ora); END; $$; -- Realistic filtered top-k (k < facet size, partial matches, ties). SELECT n, qry, pred, fs_check(qry, pred, k) AS result FROM (VALUES (1, 'alpha', 'facet_id = 6', 10), (2, 'beta', 'facet_id < 5', 10), (3, 'common', 'facet_id = 13', 5), (4, 'alpha beta', 'facet_id < 10', 8), (5, 'gamma', 'facet_id IN (0, 7, 14)', 6), (6, 'gamma', 'facet_id >= 45', 7) ) v(n, qry, pred, k) ORDER BY n; n | qry | pred | result ---+------------+------------------------+---------------- 1 | alpha | facet_id = 6 | PASS (10 rows) 2 | beta | facet_id < 5 | PASS (10 rows) 3 | common | facet_id = 13 | PASS (5 rows) 4 | alpha beta | facet_id < 10 | PASS (8 rows) 5 | gamma | facet_id IN (0, 7, 14) | PASS (6 rows) 6 | gamma | facet_id >= 45 | PASS (7 rows) (6 rows) -- Cross-check the returned set against a plain-SQL regex oracle. SELECT n, qry, pred, fs_oracle_check(qry, term, pred, k) AS result FROM (VALUES (1, 'alpha', 'alpha', 'facet_id = 6', 1000), (2, 'gamma', 'gamma', 'facet_id = 9', 1000), (3, 'beta', 'beta', 'facet_id = 2', 1000), (4, 'common', 'common', 'facet_id = 40', 1000) ) v(n, qry, term, pred, k) ORDER BY n; n | qry | pred | result ---+--------+---------------+---------------- 1 | alpha | facet_id = 6 | PASS (20 rows) 2 | gamma | facet_id = 9 | PASS (3 rows) 3 | beta | facet_id = 2 | PASS (6 rows) 4 | common | facet_id = 40 | PASS (20 rows) (4 rows) ------------------------------------------------------------------------ -- No-op: a query with no filter is unaffected by seeding. ------------------------------------------------------------------------ SELECT n, qry, fs_check(qry, NULL, k) AS result FROM (VALUES (1, 'alpha', 10), (2, 'common', 5) ) v(n, qry, k) ORDER BY n; n | qry | result ---+--------+---------------- 1 | alpha | PASS (10 rows) 2 | common | PASS (5 rows) (2 rows) ------------------------------------------------------------------------ -- Robustness: the result is independent of the margin (margin only -- affects how deep the seeded pass scores, never the final top-k). ------------------------------------------------------------------------ SET pg_textsearch.filtered_seed_margin = 1.0; SELECT 'margin=1.0' AS margin, fs_check('alpha', 'facet_id = 6', 10) AS result; margin | result ------------+---------------- margin=1.0 | PASS (10 rows) (1 row) SET pg_textsearch.filtered_seed_margin = 1000.0; SELECT 'margin=1000' AS margin, fs_check('alpha', 'facet_id = 6', 10) AS result; margin | result -------------+---------------- margin=1000 | PASS (10 rows) (1 row) RESET pg_textsearch.filtered_seed_margin; ------------------------------------------------------------------------ -- GUC contract: defaults and range enforcement. ------------------------------------------------------------------------ SHOW pg_textsearch.filtered_seed; pg_textsearch.filtered_seed ----------------------------- on (1 row) SHOW pg_textsearch.filtered_seed_margin; pg_textsearch.filtered_seed_margin ------------------------------------ 3 (1 row) -- Out of range (min 1.0, max 1000.0) -- both should error. SET pg_textsearch.filtered_seed_margin = 0.5; ERROR: 0.5 is outside the valid range for parameter "pg_textsearch.filtered_seed_margin" (1 .. 1000) SET pg_textsearch.filtered_seed_margin = 2000; ERROR: 2000 is outside the valid range for parameter "pg_textsearch.filtered_seed_margin" (1 .. 1000) ------------------------------------------------------------------------ -- Scan identity (issue #435): the seed is bound per index scan, so -- several BM25 scans of one index in one statement each get the seed -- derived from their own Filter and their own Limit. Results are -- identical either way -- Filter + Limit + backoff still decide the -- top-k -- so these are parity checks; the depth assertions that -- actually prove per-scan seeding are in the section after this one. ------------------------------------------------------------------------ -- Parity for an arbitrary query shape: run it with seeding on and off -- and compare the results as a multiset. q must yield a column named -- id. CREATE FUNCTION fs_check_q(q text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE wrapped text; seeded int[]; unseeded int[]; BEGIN wrapped := format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (%s) s', q); SET LOCAL pg_textsearch.filtered_seed = on; EXECUTE wrapped INTO seeded; SET LOCAL pg_textsearch.filtered_seed = off; EXECUTE wrapped INTO unseeded; IF seeded IS NOT DISTINCT FROM unseeded THEN RETURN format('PASS (%s rows)', coalesce(array_length(seeded, 1), 0)); END IF; RETURN format('FAIL seeded=%s unseeded=%s', seeded, unseeded); END; $$; SELECT n, what, fs_check_q(q) AS result FROM (VALUES (1, 'UNION ALL, two facets, same LIMIT', $q$ (SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 12 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) $q$), (2, 'UNION ALL, two different LIMITs', $q$ (SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id < 10 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50) $q$), (3, 'self-join, two facets', $q$ SELECT a.id AS id FROM (SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 5) a CROSS JOIN (SELECT id FROM fs_docs WHERE facet_id = 12 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 5) b WHERE b.id > a.id $q$), (4, 'CTE referenced twice', $q$ WITH t AS ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) SELECT id FROM t UNION ALL SELECT id FROM t $q$), (5, 'CTE NOT MATERIALIZED referenced twice', $q$ WITH t AS NOT MATERIALIZED ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) SELECT id FROM t UNION ALL SELECT id FROM t $q$), (6, 'unfiltered UNION ALL, two LIMITs', $q$ (SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50) $q$), (7, 'LIMIT with OFFSET', $q$ SELECT id FROM fs_docs WHERE facet_id < 10 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10 OFFSET 20 $q$), (8, 'Sort above the BM25 subquery', $q$ SELECT id FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx')) s ORDER BY id LIMIT 10 $q$), (9, 'BM25 scan in a correlated subquery', $q$ SELECT f.facet_id AS id FROM (SELECT DISTINCT facet_id FROM fs_docs WHERE facet_id < 4) f WHERE EXISTS ( SELECT 1 FROM fs_docs d WHERE d.facet_id = f.facet_id ORDER BY d.body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 3) $q$) ) v(n, what, q) ORDER BY n; n | what | result ---+---------------------------------------+---------------- 1 | UNION ALL, two facets, same LIMIT | PASS (20 rows) 2 | UNION ALL, two different LIMITs | PASS (60 rows) 3 | self-join, two facets | PASS (13 rows) 4 | CTE referenced twice | PASS (20 rows) 5 | CTE NOT MATERIALIZED referenced twice | PASS (20 rows) 6 | unfiltered UNION ALL, two LIMITs | PASS (60 rows) 7 | LIMIT with OFFSET | PASS (10 rows) 8 | Sort above the BM25 subquery | PASS (10 rows) 9 | BM25 scan in a correlated subquery | PASS (4 rows) (9 rows) -- Each arm of a two-arm UNION ALL must return what that arm returns -- standalone. Before #435 the two scans shared one per-index limit -- slot, so one arm could run at the other's depth (or at the default). CREATE FUNCTION fs_arms_check(q1 text, q2 text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE agg text := 'coalesce(array_agg(id ORDER BY id), ARRAY[]::int[])'; solo1 int[]; solo2 int[]; both1 int[] := ARRAY[]::int[]; both2 int[] := ARRAY[]::int[]; r record; BEGIN EXECUTE format('SELECT %s FROM (%s) s', agg, q1) INTO solo1; EXECUTE format('SELECT %s FROM (%s) s', agg, q2) INTO solo2; -- One statement, two scans of fs_docs_idx. The GROUP BY sits -- above the UNION ALL so neither arm can be pruned away. FOR r IN EXECUTE format( 'SELECT arm, %s AS ids FROM (' ' SELECT 1 AS arm, id FROM (%s) a ' ' UNION ALL ' ' SELECT 2 AS arm, id FROM (%s) b) u ' 'GROUP BY arm', agg, q1, q2) LOOP IF r.arm = 1 THEN both1 := r.ids; ELSE both2 := r.ids; END IF; END LOOP; IF both1 IS NOT DISTINCT FROM solo1 AND both2 IS NOT DISTINCT FROM solo2 THEN RETURN format('PASS (%s + %s rows)', coalesce(array_length(solo1, 1), 0), coalesce(array_length(solo2, 1), 0)); END IF; RETURN format('FAIL arm1 both=%s solo=%s / arm2 both=%s solo=%s', both1, solo1, both2, solo2); END; $$; SELECT n, what, fs_arms_check(q1, q2) AS result FROM (VALUES (1, 'two facets, same LIMIT', $q$SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$, $q$SELECT id FROM fs_docs WHERE facet_id = 12 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$), (2, 'two facets, two LIMITs', $q$SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$, $q$SELECT id FROM fs_docs WHERE facet_id < 10 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50$q$), (3, 'no filter, two LIMITs', $q$SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$, $q$SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50$q$) ) v(n, what, q1, q2) ORDER BY n; n | what | result ---+------------------------+--------------------- 1 | two facets, same LIMIT | PASS (10 + 10 rows) 2 | two facets, two LIMITs | PASS (10 + 50 rows) 3 | no filter, two LIMITs | PASS (10 + 50 rows) (3 rows) -- Cached plans: possible custom-to-generic plan changes and changes -- to the margin between executions must preserve results, whether the -- scan receives a query hint or falls back to normal backoff. PREPARE fs_union(int, int) AS SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) FROM ( (SELECT id FROM fs_docs WHERE facet_id = $1 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = $2 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10)) s; EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) SET pg_textsearch.filtered_seed_margin = 50.0; EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) RESET pg_textsearch.filtered_seed_margin; EXECUTE fs_union(6, 12); coalesce ---------------------------------------------------------------------------------- {62,106,206,212,256,262,356,362,412,506,512,556,562,656,662,706,712,806,856,862} (1 row) DEALLOCATE fs_union; -- An unresolved LIMIT $1 cannot be carried in a planner query hint. -- Both parameter types therefore use normal backoff on a generic plan. PREPARE fs_lim(bigint) AS SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT $1) s; EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) EXECUTE fs_lim(10); coalesce ------------------------------------------- {106,206,256,356,506,556,656,706,806,856} (1 row) DEALLOCATE fs_lim; -- Agg above a limited BM25 subquery. SELECT count(*) AS agg_rows FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) s; agg_rows ---------- 10 (1 row) ------------------------------------------------------------------------ -- Depth: the acceptance test for issue #435. -- -- Scan depth is invisible in results, so the parity checks above pass -- with or without the fix. bm25_debug_scoring_passes counts scoring -- passes: a scan seeded for its own filter costs exactly one, and each -- executor backoff re-drive adds another. -- -- Both arms below filter down to fewer than k rows, so the executor -- always exhausts the batch and backs off -- the pass count is then a -- function of the seed alone, not of score ties. With a correct -- per-scan seed each arm costs one pass; when one arm steals the -- other's seed and the loser falls back to pg_textsearch.default_limit, -- the loser pays a chain of doublings instead. ------------------------------------------------------------------------ CREATE FUNCTION fs_passes(q text) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE n bigint; BEGIN PERFORM bm25_debug_scoring_passes(true); EXECUTE q; SELECT bm25_debug_scoring_passes() INTO n; RETURN n; END; $$; -- A small default_limit lengthens the backoff chain an unseeded arm -- pays, so the two cases are far apart rather than adjacent. SET pg_textsearch.default_limit = 100; -- Confirm the plan under test: two BM25 index scans of one index, each -- under its own Limit and with its own Filter. Counted rather than -- printed, so subquery aliases and node order cannot make this brittle. CREATE FUNCTION fs_plan_shape(q text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE r record; scans int := 0; limits int := 0; filters int := 0; BEGIN FOR r IN EXECUTE 'EXPLAIN (COSTS OFF) ' || q LOOP IF r."QUERY PLAN" LIKE '%Index Scan using fs_docs_idx%' THEN scans := scans + 1; ELSIF r."QUERY PLAN" LIKE '%Limit%' THEN limits := limits + 1; ELSIF r."QUERY PLAN" LIKE '%Filter:%' THEN filters := filters + 1; END IF; END LOOP; RETURN format('%s bm25 scans, %s limits, %s filters', scans, limits, filters); END; $$; SELECT fs_plan_shape($q$ (SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 13 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) $q$) AS plan_shape; plan_shape ----------------------------------- 2 bm25 scans, 2 limits, 2 filters (1 row) -- Control: one scan, one Limit, one Filter -- one pass. This holds -- with or without the fix; it calibrates the counter. SELECT fs_passes($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 $q$) AS single_scan_passes; single_scan_passes -------------------- 1 (1 row) -- The issue #435 example: two scans of one index, each with its own -- Filter. Must be one pass per arm. SELECT fs_passes($q$ (SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 13 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) $q$) AS union_all_passes; union_all_passes ------------------ 2 (1 row) -- Three arms, to show the count tracks the number of scans rather -- than happening to be small. SELECT fs_passes($q$ (SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 13 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 21 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) $q$) AS three_arm_passes; three_arm_passes ------------------ 3 (1 row) -- The planner walker finds the Limit inside a correlated SubPlan. -- Every rescan reads the hint from its query value, restoring the seed -- on each of the three evaluations. SELECT fs_passes($q$ SELECT sum(c) FROM ( SELECT (SELECT count(*) FROM ( SELECT id FROM fs_docs WHERE facet_id = v.fid AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) x) AS c FROM (VALUES (6), (13), (21)) v(fid)) y $q$) AS correlated_subplan_passes; correlated_subplan_passes --------------------------- 3 (1 row) -- Materialized CTEs are reached through their InitPlan. Referencing -- the CTE twice still executes its single seeded scan only once. SELECT fs_passes($q$ WITH t AS MATERIALIZED ( SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) SELECT id FROM t UNION ALL SELECT id FROM t $q$) AS materialized_cte_passes; materialized_cte_passes ------------------------- 1 (1 row) -- An uncorrelated scalar subquery exercises the other InitPlan case. SELECT fs_passes($q$ SELECT (SELECT count(*) FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) s) $q$) AS initplan_passes; initplan_passes ----------------- 1 (1 row) -- Only direct Limit -> IndexScan pairs are seeded. LockRows is an -- intentional boundary, even though it preserves ordering. With -- fewer than 10 matches, the unseeded scan exhausts all 1000 candidates -- through backoff (100, 200, 400, 800, 1600): five scoring passes. EXPLAIN (COSTS OFF) SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 FOR UPDATE; QUERY PLAN -------------------------------------------------------------------- Limit -> LockRows -> Index Scan using fs_docs_idx on fs_docs Order By: (body <@> 'fs_docs_idx:common'::bm25query) Filter: ((facet_id = 6) AND ((id % 7) = 3)) (5 rows) SELECT fs_passes($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 FOR UPDATE $q$) AS lockrows_fallback_passes; lockrows_fallback_passes -------------------------- 5 (1 row) SELECT fs_check_q($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 FOR UPDATE $q$) AS lockrows_fallback_parity; lockrows_fallback_parity -------------------------- PASS (3 rows) (1 row) -- Typed literals resolved to an index must retain variable-length metadata -- when the planner appends a hint and PostgreSQL copies the query value. SELECT fs_passes($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> 'common'::bm25query LIMIT 10 $q$) AS typed_literal_passes; typed_literal_passes ---------------------- 1 (1 row) -- Planning a saved query must not depend on the last parsed statement. -- Force generic plans rather than relying on the custom-plan cost heuristic. SET plan_cache_mode = force_generic_plan; PREPARE fs_const_hint AS SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> 'common'::bm25query LIMIT 10; SELECT 1 AS unrelated_statement; unrelated_statement --------------------- 1 (1 row) SELECT fs_passes('EXECUTE fs_const_hint') AS prepared_constant_passes; prepared_constant_passes -------------------------- 1 (1 row) SELECT fs_passes('EXECUTE fs_const_hint') AS cached_constant_passes; cached_constant_passes ------------------------ 1 (1 row) DISCARD PLANS; SELECT 1 AS unrelated_statement; unrelated_statement --------------------- 1 (1 row) SELECT fs_passes('EXECUTE fs_const_hint') AS replanned_constant_passes; replanned_constant_passes --------------------------- 1 (1 row) DEALLOCATE fs_const_hint; -- A runtime query operand cannot carry a hint. The cached plan must use -- default-limit backoff and read the new query value on each execution. PREPARE fs_query(bm25query) AS SELECT array_agg(id ORDER BY id) FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> $1 LIMIT 10) s; SELECT fs_plan_shape( 'EXECUTE fs_query(to_bm25query(''common'', ''fs_docs_idx''))' ) AS runtime_query_shape; runtime_query_shape ----------------------------------- 1 bm25 scans, 1 limits, 1 filters (1 row) SELECT fs_passes( 'EXECUTE fs_query(to_bm25query(''common'', ''fs_docs_idx''))' ) AS runtime_query_passes; runtime_query_passes ---------------------- 5 (1 row) EXECUTE fs_query(to_bm25query('common', 'fs_docs_idx')); array_agg --------------- {206,556,906} (1 row) EXECUTE fs_query(to_bm25query('beta', 'fs_docs_idx')); array_agg ----------- {906} (1 row) DEALLOCATE fs_query; RESET plan_cache_mode; -- A generic plan's LIMIT is a PARAM_EXTERN and cannot be folded into the -- planner-carried query hint, so it uses normal backoff. -- The parser coerces a LIMIT to bigint, while an int parameter arrives -- wrapped in an int8() coercion. Both cases intentionally fall back. PREPARE fs_dp_big(bigint) AS SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT $1; PREPARE fs_dp_int(int) AS SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT $1; -- Runs stmt n times first, so the plan has flipped from custom to -- generic before the measured execution. CREATE FUNCTION fs_generic_passes(stmt text, n int) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE p bigint; BEGIN FOR i IN 1..n LOOP EXECUTE stmt; END LOOP; PERFORM bm25_debug_scoring_passes(true); EXECUTE stmt; SELECT bm25_debug_scoring_passes() INTO p; RETURN p; END; $$; SELECT fs_generic_passes('EXECUTE fs_dp_big(10)', 6) AS bigint_param_passes; bigint_param_passes --------------------- 5 (1 row) SELECT fs_generic_passes('EXECUTE fs_dp_int(10)', 6) AS int_param_passes; int_param_passes ------------------ 5 (1 row) DEALLOCATE fs_dp_big; DEALLOCATE fs_dp_int; RESET pg_textsearch.default_limit; ------------------------------------------------------------------------ -- Saturating LIMIT: offset + count at or above INT_MAX yields no k. -- -- The seed becomes so->limit, which sizes a palloc of k -- ItemPointerDatas, so a k of INT_MAX asks for 12 GB and the query -- dies with "invalid memory alloc request size". LIMIT 2147483647 is -- the usual generated-SQL spelling of "no limit", so these must fall -- back to the default rather than bind a k that cannot be served. ------------------------------------------------------------------------ -- 20 docs per facet bucket; each query must return the whole bucket. SELECT count(*) AS int_max_limit FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 2147483647) s; int_max_limit --------------- 20 (1 row) SELECT count(*) AS count_over_int_max FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 9999999999) s; count_over_int_max -------------------- 20 (1 row) -- k is offset + count, so a large OFFSET saturates it just as a large -- count does. Nothing survives the offset; the point is that it runs. SELECT count(*) AS int_max_offset FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 OFFSET 3000000000) s; int_max_offset ---------------- 0 (1 row) -- A k that binds normally, so the cases above are shown to be about -- saturation and not about large limits in general. SELECT count(*) AS large_bound_limit FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 1000000) s; large_bound_limit ------------------- 20 (1 row) ------------------------------------------------------------------------ -- Cleanup ------------------------------------------------------------------------ DROP FUNCTION fs_check(text, text, int); DROP FUNCTION fs_oracle_check(text, text, text, int); DROP FUNCTION fs_check_q(text); DROP FUNCTION fs_arms_check(text, text); DROP FUNCTION fs_plan_shape(text); DROP FUNCTION fs_passes(text); DROP FUNCTION fs_generic_passes(text, int); DROP TABLE fs_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/force_merge.out000066400000000000000000000603701525762047700245550ustar00rootroot00000000000000-- Test case: force_merge -- Tests bm25_force_merge() bounded best-effort compaction. -- -- This test exercises: -- 1. Multiple segments across levels -- 2. Force merge into size-bounded segments -- 3. Queries return correct results after merge -- 4. Inserts work normally after force merge CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Use 2 segments per level so we get multi-level segments quickly SET pg_textsearch.segments_per_level = 2; CREATE TABLE force_merge_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX force_merge_idx ON force_merge_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation force_merge_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Empty indexes are already a valid zero-segment force-merge result. DO $$ DECLARE summary text; BEGIN PERFORM bm25_force_merge('force_merge_idx'); summary := bm25_summarize_index('force_merge_idx'); IF summary !~ E'Segments:\n \\(none\\)' THEN RAISE EXCEPTION 'empty force merge created a segment: %', summary; END IF; END $$; -------------------------------------------------------------------------------- -- Build up segments across multiple levels -------------------------------------------------------------------------------- -- Batch 1 -> spill to L0 INSERT INTO force_merge_test (content) VALUES ('hello world database'), ('goodbye cruel world'), ('hello goodbye friend'), ('world peace harmony'); SELECT bm25_spill_index('force_merge_idx') IS NOT NULL AS spill1; spill1 -------- t (1 row) -- Batch 2 -> spill to L0, triggers L0->L1 merge INSERT INTO force_merge_test (content) VALUES ('database indexing query'), ('search engine optimization'), ('database world news'), ('goodbye database friend'); SELECT bm25_spill_index('force_merge_idx') IS NOT NULL AS spill2; spill2 -------- t (1 row) -- Batch 3 -> new L0 segment (so we have L0 + L1) INSERT INTO force_merge_test (content) VALUES ('hello search results'), ('optimization database tuning'); SELECT bm25_spill_index('force_merge_idx') IS NOT NULL AS spill3; spill3 -------- t (1 row) -- Verify counts before force merge SELECT COUNT(*) AS hello_before FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('hello', 'force_merge_idx') LIMIT 100 ) t; hello_before -------------- 3 (1 row) SELECT COUNT(*) AS database_before FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('database', 'force_merge_idx') LIMIT 100 ) t; database_before ----------------- 5 (1 row) -------------------------------------------------------------------------------- -- Force merge all segments into one -------------------------------------------------------------------------------- SELECT bm25_force_merge('force_merge_idx'); bm25_force_merge ------------------ (1 row) DO $$ DECLARE summary text := bm25_summarize_index('force_merge_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 THEN RAISE EXCEPTION 'force merge left multiple segments: %', summary; END IF; END $$; -- Verify same counts after force merge SELECT COUNT(*) AS hello_after FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('hello', 'force_merge_idx') LIMIT 100 ) t; hello_after ------------- 3 (1 row) SELECT COUNT(*) AS database_after FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('database', 'force_merge_idx') LIMIT 100 ) t; database_after ---------------- 5 (1 row) -- Validate BM25 scoring is correct after force merge SELECT validate_bm25_scoring('force_merge_test', 'content', 'force_merge_idx', 'hello', 'english', 1.2, 0.75) AS hello_scores_valid; hello_scores_valid -------------------- t (1 row) SELECT validate_bm25_scoring('force_merge_test', 'content', 'force_merge_idx', 'database', 'english', 1.2, 0.75) AS database_scores_valid; database_scores_valid ----------------------- t (1 row) -------------------------------------------------------------------------------- -- Verify inserts work after force merge -------------------------------------------------------------------------------- INSERT INTO force_merge_test (content) VALUES ('hello after merge'), ('new database entry'); SELECT COUNT(*) AS hello_with_new FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('hello', 'force_merge_idx') LIMIT 100 ) t; hello_with_new ---------------- 4 (1 row) SELECT validate_bm25_scoring('force_merge_test', 'content', 'force_merge_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid_after_insert; hello_valid_after_insert -------------------------- t (1 row) DROP TABLE force_merge_test CASCADE; -------------------------------------------------------------------------------- -- A force-merge spill must not run ordinary threshold compaction first. -- -- Build two L0 segments below the initial threshold, then lower the threshold -- while a memtable remains pending. Force merge must spill the memtable -- without running threshold compaction before its own merge plan. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 3; CREATE TABLE force_spill_cascade (id serial PRIMARY KEY, content text); CREATE INDEX force_spill_cascade_idx ON force_spill_cascade USING bm25(content) WITH (text_config='english', compaction='manual'); NOTICE: BM25 index build started for relation force_spill_cascade_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO force_spill_cascade (content) VALUES (format('spill cascade document %s filler', n)); PERFORM bm25_spill_index('force_spill_cascade_idx'); END LOOP; INSERT INTO force_spill_cascade (content) VALUES ('spill cascade pending memtable document filler'); END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_spill_cascade_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 2 OR summary !~ 'L0 Segment 2:' OR summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'force spill cascade layout was not constructed: %', summary; END IF; END $$; SET pg_textsearch.segments_per_level = 2; ALTER INDEX force_spill_cascade_idx SET (compaction = 'inline'); DO $$ BEGIN PERFORM bm25_force_merge('force_spill_cascade_idx'); END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_spill_cascade_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'force spill cascade did not reach one segment: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_spill_cascade ORDER BY content <@> to_bm25query('filler', 'force_spill_cascade_idx') ) ranked) <> 3 THEN RAISE EXCEPTION 'force spill cascade lost documents'; END IF; END $$; DROP TABLE force_spill_cascade CASCADE; -------------------------------------------------------------------------------- -- Force merge is bounded best-effort compaction, not forceMerge(1). -- -- Four layouts: a single L1 segment, repeated merges that stay in L1, -- a mixed L0+L1 pair, and an L1 segment with a pending memtable. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 2; CREATE TABLE force_tier_single (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_single_idx ON force_tier_single USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation force_tier_single_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE TABLE force_tier_deep (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_deep_idx ON force_tier_deep USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation force_tier_deep_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE TABLE force_tier_mixed (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_mixed_idx ON force_tier_mixed USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation force_tier_mixed_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE TABLE force_tier_memtable (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_memtable_idx ON force_tier_memtable USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation force_tier_memtable_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ DECLARE n integer; BEGIN FOR n IN 1..256 LOOP INSERT INTO force_tier_deep (content) VALUES (format('multiple terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_deep_idx'); IF n <= 128 THEN INSERT INTO force_tier_single (content) VALUES (format('single terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_single_idx'); INSERT INTO force_tier_mixed (content) VALUES (format('mixed terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_mixed_idx'); INSERT INTO force_tier_memtable (content) VALUES (format('memtable terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_memtable_idx'); END IF; END LOOP; -- Runtime inline policy performs one bounded pass per spill. Explicitly -- drain the remaining cascade to construct the size-tiered fixtures. PERFORM bm25_compact('force_tier_deep_idx'::regclass); PERFORM bm25_compact('force_tier_single_idx'::regclass); PERFORM bm25_compact('force_tier_mixed_idx'::regclass); PERFORM bm25_compact('force_tier_memtable_idx'::regclass); ALTER INDEX force_tier_mixed_idx SET (compaction = 'manual'); INSERT INTO force_tier_mixed (content) VALUES ('mixed lower filler'); PERFORM bm25_spill_index('force_tier_mixed_idx'); INSERT INTO force_tier_memtable (content) VALUES ('memtable pending filler'); END $$; DO $$ DECLARE single_summary text := bm25_summarize_index('force_tier_single_idx'); deep_summary text := bm25_summarize_index('force_tier_deep_idx'); mixed_summary text := bm25_summarize_index('force_tier_mixed_idx'); memtable_summary text := bm25_summarize_index('force_tier_memtable_idx'); BEGIN IF regexp_count(single_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR single_summary !~ 'L1 Segment 1:' THEN RAISE EXCEPTION 'single L1 layout was not constructed: %', single_summary; END IF; IF regexp_count(deep_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR deep_summary !~ 'L1 Segment 1:' THEN RAISE EXCEPTION 'deep cascade did not settle at one L1 segment: %', deep_summary; END IF; IF regexp_count(mixed_summary, 'L[0-7] Segment [0-9]+:') <> 2 OR mixed_summary !~ 'L0 Segment 1:' OR mixed_summary !~ 'L1 Segment 1:' THEN RAISE EXCEPTION 'mixed L1 layout was not constructed: %', mixed_summary; END IF; IF regexp_count(memtable_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR memtable_summary !~ 'L1 Segment 1:' OR memtable_summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'memtable L1 layout was not constructed: %', memtable_summary; END IF; END $$; DO $$ DECLARE single_before text := bm25_summarize_index('force_tier_single_idx'); deep_before text := bm25_summarize_index('force_tier_deep_idx'); mixed_before text := bm25_summarize_index('force_tier_mixed_idx'); memtable_before text := bm25_summarize_index('force_tier_memtable_idx'); single_summary text; deep_summary text; mixed_summary text; memtable_summary text; BEGIN PERFORM bm25_force_merge('force_tier_single_idx'); PERFORM bm25_force_merge('force_tier_deep_idx'); PERFORM bm25_force_merge('force_tier_mixed_idx'); PERFORM bm25_force_merge('force_tier_memtable_idx'); single_summary := bm25_summarize_index('force_tier_single_idx'); deep_summary := bm25_summarize_index('force_tier_deep_idx'); mixed_summary := bm25_summarize_index('force_tier_mixed_idx'); memtable_summary := bm25_summarize_index('force_tier_memtable_idx'); IF regexp_count(single_summary, 'L[0-7] Segment [0-9]+:') > regexp_count(single_before, 'L[0-7] Segment [0-9]+:') THEN RAISE EXCEPTION 'force merge increased segment count: before %, after %', single_before, single_summary; END IF; IF regexp_count(deep_summary, 'L[0-7] Segment [0-9]+:') > regexp_count(deep_before, 'L[0-7] Segment [0-9]+:') THEN RAISE EXCEPTION 'force merge increased segment count: before %, after %', deep_before, deep_summary; END IF; -- The mixed fixture is the multi-segment case, so force merge must -- actually combine its L0 and L1 segments. Asserting only that the -- count did not increase would also accept a no-op. IF regexp_count(mixed_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR mixed_summary !~ 'L0 Segment 1:' THEN RAISE EXCEPTION 'force merge did not combine the mixed layout: before %, after %', mixed_before, mixed_summary; END IF; IF regexp_count(memtable_summary, 'L[0-7] Segment [0-9]+:') > regexp_count(memtable_before, 'L[0-7] Segment [0-9]+:') THEN RAISE EXCEPTION 'force merge increased segment count: before %, after %', memtable_before, memtable_summary; END IF; IF memtable_summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'force merge did not clear the memtable: %', memtable_summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_tier_single ORDER BY content <@> to_bm25query('filler', 'force_tier_single_idx') ) ranked) <> 128 OR (SELECT count(*) FROM ( SELECT 1 FROM force_tier_deep ORDER BY content <@> to_bm25query('filler', 'force_tier_deep_idx') ) ranked) <> 256 OR (SELECT count(*) FROM ( SELECT 1 FROM force_tier_mixed ORDER BY content <@> to_bm25query('filler', 'force_tier_mixed_idx') ) ranked) <> 129 OR (SELECT count(*) FROM ( SELECT 1 FROM force_tier_memtable ORDER BY content <@> to_bm25query('filler', 'force_tier_memtable_idx') ) ranked) <> 129 THEN RAISE EXCEPTION 'size-tiered force merge lost documents'; END IF; END $$; -- A termless memtable contributes document accounting but no prospective -- segment, so it can be cleared without rewriting a compacted singleton. DO $$ BEGIN IF bm25_test_chain_source( 'force_tier_single_idx', 'one_doc_no_terms') <> 'OK' THEN RAISE EXCEPTION 'could not construct termless singleton memtable'; END IF; END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_tier_single_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ 'L0 Segment 1:' OR summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'termless singleton layout was not constructed: %', summary; END IF; PERFORM bm25_force_merge('force_tier_single_idx'); summary := bm25_summarize_index('force_tier_single_idx'); IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ 'L0 Segment 1:' OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'termless force merge changed segment layout: %', summary; END IF; IF bm25_dump_index('force_tier_single_idx')::text !~ E'total_docs: 129\n total_len: 519\n' THEN RAISE EXCEPTION 'termless force merge lost document accounting'; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_tier_single ORDER BY content <@> to_bm25query('filler', 'force_tier_single_idx') ) ranked) <> 128 THEN RAISE EXCEPTION 'termless force merge changed query results'; END IF; END $$; DROP TABLE force_tier_single CASCADE; DROP TABLE force_tier_deep CASCADE; DROP TABLE force_tier_mixed CASCADE; DROP TABLE force_tier_memtable CASCADE; -------------------------------------------------------------------------------- -- Termless accounting survives a non-no-op force compaction. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 64; CREATE TABLE force_termless_multi ( id bigint PRIMARY KEY, content text ); CREATE INDEX force_termless_multi_idx ON force_termless_multi USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation force_termless_multi_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO force_termless_multi VALUES (batch, 'accounted'); PERFORM bm25_spill_index('force_termless_multi_idx'); END LOOP; IF bm25_test_chain_source( 'force_termless_multi_idx', 'one_doc_no_terms') <> 'OK' THEN RAISE EXCEPTION 'could not append termless accounting record'; END IF; END $$; SELECT bm25_force_merge('force_termless_multi_idx'); bm25_force_merge ------------------ (1 row) DO $$ DECLARE dump text := bm25_dump_index('force_termless_multi_idx')::text; summary text := bm25_summarize_index('force_termless_multi_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'termless multi-source force merge did not compact: %', summary; END IF; IF dump !~ E'total_docs: 3\n total_len: 9\n' THEN RAISE EXCEPTION 'termless multi-source accounting was lost: %', dump; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_termless_multi ORDER BY content <@> to_bm25query('accounted', 'force_termless_multi_idx') ) ranked) <> 2 THEN RAISE EXCEPTION 'termless multi-source compaction lost documents'; END IF; END $$; DROP TABLE force_termless_multi CASCADE; -------------------------------------------------------------------------------- -- Skip-entry estimates are per term. Two low-frequency vocabularies fit the -- budget only if skip blocks are incorrectly rounded after global summation. -------------------------------------------------------------------------------- SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE force_low_frequency ( id bigint PRIMARY KEY, content text ); CREATE INDEX force_low_frequency_idx ON force_low_frequency USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation force_low_frequency_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO force_low_frequency SELECT batch, 'lowfrequency ' || string_agg(format('lf%sx%s', batch, term), ' ' ORDER BY term) FROM generate_series(1, 9000) term; PERFORM bm25_spill_index('force_low_frequency_idx'); END LOOP; END $$; SELECT bm25_force_merge('force_low_frequency_idx'); bm25_force_merge ------------------ (1 row) DO $$ DECLARE summary text := bm25_summarize_index('force_low_frequency_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 2 THEN RAISE EXCEPTION 'low-frequency sources were merged over budget: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_low_frequency ORDER BY content <@> to_bm25query('lowfrequency', 'force_low_frequency_idx') ) ranked) <> 2 THEN RAISE EXCEPTION 'low-frequency force merge lost documents'; END IF; END $$; RESET pg_textsearch.max_segment_size; DROP TABLE force_low_frequency CASCADE; -------------------------------------------------------------------------------- -- A bounded force merge produces maximal batches without exceeding the -- configured segment-size budget. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 64; SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE force_bounded (id bigint PRIMARY KEY, content text); CREATE INDEX force_bounded_idx ON force_bounded USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation force_bounded_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..3 LOOP INSERT INTO force_bounded SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 4) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('force_bounded_idx'); END LOOP; END $$; SELECT bm25_force_merge('force_bounded_idx'); bm25_force_merge ------------------ (1 row) DO $$ DECLARE first_summary text := bm25_summarize_index('force_bounded_idx'); second_summary text; output_pages bigint; parsed_outputs integer := 0; BEGIN IF regexp_count(first_summary, 'L[0-7] Segment [0-9]+:') <> 2 THEN RAISE EXCEPTION 'bounded force merge did not produce two segments: %', first_summary; END IF; FOR output_pages IN SELECT captures[1]::bigint FROM regexp_matches( first_summary, 'pages=([0-9]+), size=', 'g') AS m(captures) LOOP parsed_outputs := parsed_outputs + 1; IF (output_pages + 1) * current_setting('block_size')::bigint > 1024 * 1024 THEN RAISE EXCEPTION 'bounded output exceeds 1 MiB: %', first_summary; END IF; END LOOP; IF parsed_outputs <> 2 THEN RAISE EXCEPTION 'could not parse bounded outputs: %', first_summary; END IF; PERFORM bm25_force_merge('force_bounded_idx'); second_summary := bm25_summarize_index('force_bounded_idx'); IF second_summary <> first_summary THEN RAISE EXCEPTION 'bounded force merge was not idempotent: before %, after %', first_summary, second_summary; END IF; END $$; SELECT count(*) = 8000 AS bounded_docs_preserved FROM ( SELECT 1 FROM force_bounded ORDER BY content <@> to_bm25query('common', 'force_bounded_idx') LIMIT 8000 ) ranked; bounded_docs_preserved ------------------------ t (1 row) RESET pg_textsearch.max_segment_size; DROP TABLE force_bounded CASCADE; -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- RESET pg_textsearch.segments_per_level; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/force_merge_injection.out000066400000000000000000000046671525762047700266260ustar00rootroot00000000000000SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE force_lowered_capacity ( id bigint PRIMARY KEY, content text ); CREATE INDEX force_lowered_capacity_idx ON force_lowered_capacity USING bm25(content) WITH (text_config='simple'); DO $$ BEGIN FOR batch IN 1..9 LOOP INSERT INTO force_lowered_capacity SELECT batch, 'capacitytoken ' || string_agg(format('cap%sx%s', batch, term), ' ' ORDER BY term) FROM generate_series(1, 12000) term; PERFORM bm25_spill_index('force_lowered_capacity_idx'); END LOOP; END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_lowered_capacity_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 9 OR summary !~ 'L0 Segment 9:' THEN RAISE EXCEPTION 'lowered-capacity layout was not constructed: %', summary; END IF; END $$; SELECT pg_textsearch_test_attach_segment_limit(1); pg_textsearch_test_attach_segment_limit ----------------------------------------- (1 row) DO $$ BEGIN PERFORM bm25_force_merge('force_lowered_capacity_idx'); END $$; INSERT INTO force_lowered_capacity VALUES (10, 'capacitytoken pending spill'); DO $$ BEGIN PERFORM bm25_force_merge('force_lowered_capacity_idx'); END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_lowered_capacity_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 9 OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'lowered-capacity force merge changed topology: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_lowered_capacity ORDER BY content <@> to_bm25query('capacitytoken', 'force_lowered_capacity_idx') ) ranked) <> 10 THEN RAISE EXCEPTION 'lowered-capacity force merge lost documents'; END IF; END $$; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); injection_points_detach ------------------------- (1 row) RESET pg_textsearch.max_segment_size; DROP TABLE force_lowered_capacity CASCADE; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/implicit.out000066400000000000000000000305621525762047700241120ustar00rootroot00000000000000-- Test implicit index resolution via planner hook -- Tests that to_bm25query() without an index name automatically finds the BM25 index CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -- Create test table with BM25 index CREATE TABLE implicit_docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX implicit_docs_idx ON implicit_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation implicit_docs_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents INSERT INTO implicit_docs (content) VALUES ('hello world example'), ('database system design'), ('hello database world'); -- Test 1: Explicit index name works (baseline) -- Note: ORDER BY must use the raw expression, not the ROUND() alias, for index scan SELECT id, content, ROUND((content <@> to_bm25query('hello', 'implicit_docs_idx'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('hello', 'implicit_docs_idx'), id LIMIT 10; id | content | score ----+----------------------+--------- 1 | hello world example | -0.4700 3 | hello database world | -0.4700 (2 rows) -- Test 2: Implicit index resolution (main feature being tested) SELECT id, content, ROUND((content <@> to_bm25query('hello'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('hello'), id LIMIT 10; id | content | score ----+----------------------+--------- 1 | hello world example | -0.4700 3 | hello database world | -0.4700 (2 rows) -- Test 3: Implicit resolution works with different query terms SELECT id, content, ROUND((content <@> to_bm25query('database'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('database'), id LIMIT 10; id | content | score ----+------------------------+--------- 2 | database system design | -0.4700 3 | hello database world | -0.4700 (2 rows) -- Test 4: Works with ORDER BY only (no WHERE clause) SELECT id, content, ROUND((content <@> to_bm25query('database'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('database'), id LIMIT 3; id | content | score ----+------------------------+--------- 2 | database system design | -0.4700 3 | hello database world | -0.4700 (2 rows) -- Test 5: Multiple BM25 indexes on different columns - should pick correct one CREATE TABLE multi_col_docs ( id SERIAL PRIMARY KEY, title TEXT, body TEXT ); CREATE INDEX multi_col_title_idx ON multi_col_docs USING bm25(title) WITH (text_config='english'); NOTICE: BM25 index build started for relation multi_col_title_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE INDEX multi_col_body_idx ON multi_col_docs USING bm25(body) WITH (text_config='english'); NOTICE: BM25 index build started for relation multi_col_body_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO multi_col_docs (title, body) VALUES ('hello title', 'goodbye body content'), ('goodbye title', 'hello body content'), ('neutral title', 'neutral body content'); -- Query on title column should use title index SELECT id, title, ROUND((title <@> to_bm25query('hello'))::numeric, 4) as score FROM multi_col_docs ORDER BY title <@> to_bm25query('hello'), id LIMIT 10; id | title | score ----+-------------+--------- 1 | hello title | -0.9808 (1 row) -- Query on body column should use body index SELECT id, body, ROUND((body <@> to_bm25query('hello'))::numeric, 4) as score FROM multi_col_docs ORDER BY body <@> to_bm25query('hello'), id LIMIT 10; id | body | score ----+--------------------+--------- 2 | hello body content | -0.9808 (1 row) -- Test 6: Schema-qualified table still works with implicit resolution CREATE SCHEMA implicit_schema; CREATE TABLE implicit_schema.schema_docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX schema_docs_idx ON implicit_schema.schema_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation schema_docs_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO implicit_schema.schema_docs (content) VALUES ('hello from schema'), ('goodbye from schema'); -- Implicit resolution should work even with schema-qualified tables SELECT id, content, ROUND((content <@> to_bm25query('hello'))::numeric, 4) as score FROM implicit_schema.schema_docs ORDER BY content <@> to_bm25query('hello'), id LIMIT 10; id | content | score ----+-------------------+--------- 1 | hello from schema | -0.6931 (1 row) -- Test 7: text <@> text implicit syntax for standalone scoring -- The planner hook transforms text <@> text to text <@> bm25query for non-ORDER BY -- expressions, enabling implicit index resolution without to_bm25query(). SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM implicit_docs ORDER BY content <@> 'hello', id LIMIT 10; id | content | score ----+----------------------+--------- 1 | hello world example | -0.4700 3 | hello database world | -0.4700 (2 rows) -- Test 8: text <@> text works in ORDER BY with implicit index SELECT id, content FROM implicit_docs ORDER BY content <@> 'database' LIMIT 3; id | content ----+------------------------ 2 | database system design 3 | hello database world (2 rows) -- Test 9: Combining implicit text <@> text scoring with ORDER BY SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM implicit_docs ORDER BY content <@> 'hello', id LIMIT 3; id | content | score ----+----------------------+--------- 1 | hello world example | -0.4700 3 | hello database world | -0.4700 (2 rows) -- Test 10: Prepared statements require explicit index name -- Implicit resolution doesn't work because to_bm25query($1) can't be -- folded to a constant at plan time when the parameter isn't known. PREPARE explicit_search(text) AS SELECT id, content, ROUND((content <@> to_bm25query($1, 'implicit_docs_idx'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx'), id LIMIT 10; EXECUTE explicit_search('hello'); id | content | score ----+----------------------+--------- 1 | hello world example | -0.4700 3 | hello database world | -0.4700 (2 rows) EXECUTE explicit_search('database'); id | content | score ----+------------------------+--------- 2 | database system design | -0.4700 3 | hello database world | -0.4700 (2 rows) DEALLOCATE explicit_search; -- The score rewrite must use the query being planned, not parse-hook state -- left over from PREPARE. The DO statement itself clears that old state. PREPARE prepared_score(text) AS SELECT id FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx') LIMIT 10; -- The planner-time BM25 check must descend into a FROM subquery. The outer -- SELECT has no BM25 operator of its own. PREPARE prepared_subquery_score(text) AS SELECT id FROM ( SELECT id FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx') LIMIT 2 ) AS ranked; -- An inlined CTE also has no BM25 operator at the outer query level. PREPARE prepared_cte_score(text) AS WITH ranked AS ( SELECT id FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx') LIMIT 2 ) SELECT id FROM ranked; DO $$ DECLARE plan json; mode text; statement text; BEGIN FOREACH mode IN ARRAY ARRAY['force_generic_plan', 'force_custom_plan'] LOOP PERFORM set_config('plan_cache_mode', mode, true); FOREACH statement IN ARRAY ARRAY[ 'prepared_score', 'prepared_subquery_score', 'prepared_cte_score' ] LOOP EXECUTE format( 'EXPLAIN (FORMAT JSON, COSTS OFF, VERBOSE) EXECUTE %I(%L)', statement, 'hello') INTO plan; IF plan::text NOT LIKE '%bm25_get_current_score()%' THEN RAISE EXCEPTION '% (%) did not reuse the index score: %', statement, mode, plan; END IF; END LOOP; END LOOP; END $$; DEALLOCATE prepared_score; DEALLOCATE prepared_subquery_score; DEALLOCATE prepared_cte_score; -- Nested scoring must not force its explicit index onto the outer scan. CREATE TABLE planner_scope_docs (id int PRIMARY KEY, content text); INSERT INTO planner_scope_docs VALUES (1, 'running'), (2, 'runs'), (3, 'run'); CREATE INDEX planner_scope_simple ON planner_scope_docs USING bm25(content) WITH (text_config = 'simple') WHERE id = 1; NOTICE: BM25 index build started for relation planner_scope_simple NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=1.00 CREATE INDEX planner_scope_english ON planner_scope_docs USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation planner_scope_english NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=1.00 DO $$ DECLARE baseline_ids int[]; nested_ids int[]; BEGIN SELECT array_agg(id ORDER BY id) INTO baseline_ids FROM ( SELECT id FROM planner_scope_docs ORDER BY content <@> 'running' LIMIT 3 ) AS ranked; IF baseline_ids IS DISTINCT FROM ARRAY[1, 2, 3] THEN RAISE EXCEPTION 'unexpected outer scan results: %', baseline_ids; END IF; SELECT array_agg(id ORDER BY id) INTO nested_ids FROM ( SELECT id FROM planner_scope_docs WHERE ( SELECT content <@> to_bm25query('running', 'planner_scope_simple') FROM planner_scope_docs WHERE id = 1 ) IS NOT NULL ORDER BY content <@> 'running' LIMIT 3 ) AS ranked; IF nested_ids IS DISTINCT FROM baseline_ids THEN RAISE EXCEPTION 'nested index changed outer results: % vs %', nested_ids, baseline_ids; END IF; END $$; DROP TABLE planner_scope_docs; -- Test 11: Implicit text <@> text in DELETE subquery (issue #213) -- The implicit form should work inside subqueries of DML statements DELETE FROM implicit_docs WHERE id = ( SELECT id FROM implicit_docs ORDER BY content <@> 'database' LIMIT 1 ); -- Verify the top-scoring document for 'database' was deleted SELECT id, content FROM implicit_docs ORDER BY content <@> 'database' LIMIT 3; id | content ----+---------------------- 3 | hello database world (1 row) -- Re-insert the deleted row for subsequent tests INSERT INTO implicit_docs (content) VALUES ('database system design'); -- Test 12: Implicit text <@> text in UPDATE subquery UPDATE implicit_docs SET content = 'updated via subquery' WHERE id = ( SELECT id FROM implicit_docs ORDER BY content <@> 'hello' LIMIT 1 ); -- Verify the update happened to the top-scoring 'hello' document SELECT id, content FROM implicit_docs ORDER BY id; id | content ----+------------------------ 1 | updated via subquery 3 | hello database world 4 | database system design (3 rows) -- Test 13: Implicit form in UPDATE SET subquery UPDATE implicit_docs SET content = ( SELECT content FROM implicit_docs ORDER BY content <@> 'database' LIMIT 1 ) WHERE id = (SELECT max(id) FROM implicit_docs); SELECT id, content FROM implicit_docs ORDER BY id; id | content ----+---------------------- 1 | updated via subquery 3 | hello database world 4 | hello database world (3 rows) -- Test 14: Nested subquery (exercises recursive SubLink handling) SELECT id, content FROM implicit_docs WHERE id IN ( SELECT id FROM implicit_docs WHERE id = ( SELECT id FROM implicit_docs ORDER BY content <@> 'database' LIMIT 1 ) ); id | content ----+---------------------- 3 | hello database world (1 row) -- Cleanup DROP TABLE implicit_schema.schema_docs CASCADE; DROP SCHEMA implicit_schema CASCADE; DROP TABLE multi_col_docs CASCADE; DROP TABLE implicit_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/index.out000066400000000000000000000052721525762047700234070ustar00rootroot00000000000000-- Test pg_textsearch index access method functionality -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Setup test table CREATE TABLE test_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO test_docs (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz'), ('hello world example'), ('postgresql full text search'); -- Test index creation with different text_config options CREATE INDEX docs_english_idx ON test_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_english_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.20 CREATE INDEX docs_simple_idx ON test_docs USING bm25(content) WITH (text_config='simple', k1=1.5, b=0.8); NOTICE: BM25 index build started for relation docs_simple_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.50, b=0.80 NOTICE: BM25 index build completed: 5 documents, avg_length=4.00 -- Verify indexes were created \d+ test_docs Table "public.test_docs" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------------------------------------+----------+--------------+------------- id | integer | | not null | nextval('test_docs_id_seq'::regclass) | plain | | content | text | | | | extended | | Indexes: "test_docs_pkey" PRIMARY KEY, btree (id) "docs_english_idx" bm25 (content) WITH (text_config=english) "docs_simple_idx" bm25 (content) WITH (text_config=simple, k1='1.5', b='0.8') Not-null constraints: "test_docs_id_not_null" NOT NULL "id" -- Test basic index operations INSERT INTO test_docs (content) VALUES ('new document for testing'); -- Test that index options are correctly stored and retrievable SELECT i.relname as index_name, CASE WHEN i.reloptions IS NOT NULL THEN 'has_options' ELSE 'no_options' END as options_status FROM pg_class i WHERE i.relname IN ('docs_english_idx', 'docs_simple_idx') ORDER BY i.relname; index_name | options_status ------------------+---------------- docs_english_idx | has_options docs_simple_idx | has_options (2 rows) -- Cleanup DROP INDEX docs_english_idx; DROP INDEX docs_simple_idx; DROP TABLE test_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/index_1.out000066400000000000000000000051721525762047700236260ustar00rootroot00000000000000-- Test pg_textsearch index access method functionality -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Setup test table CREATE TABLE test_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO test_docs (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz'), ('hello world example'), ('postgresql full text search'); -- Test index creation with different text_config options CREATE INDEX docs_english_idx ON test_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_english_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.20 CREATE INDEX docs_simple_idx ON test_docs USING bm25(content) WITH (text_config='simple', k1=1.5, b=0.8); NOTICE: BM25 index build started for relation docs_simple_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.50, b=0.80 NOTICE: BM25 index build completed: 5 documents, avg_length=4.00 -- Verify indexes were created \d+ test_docs Table "public.test_docs" Column | Type | Collation | Nullable | Default | Storage | Stats target | Description ---------+---------+-----------+----------+---------------------------------------+----------+--------------+------------- id | integer | | not null | nextval('test_docs_id_seq'::regclass) | plain | | content | text | | | | extended | | Indexes: "test_docs_pkey" PRIMARY KEY, btree (id) "docs_english_idx" bm25 (content) WITH (text_config=english) "docs_simple_idx" bm25 (content) WITH (text_config=simple, k1='1.5', b='0.8') -- Test basic index operations INSERT INTO test_docs (content) VALUES ('new document for testing'); -- Test that index options are correctly stored and retrievable SELECT i.relname as index_name, CASE WHEN i.reloptions IS NOT NULL THEN 'has_options' ELSE 'no_options' END as options_status FROM pg_class i WHERE i.relname IN ('docs_english_idx', 'docs_simple_idx') ORDER BY i.relname; index_name | options_status ------------------+---------------- docs_english_idx | has_options docs_simple_idx | has_options (2 rows) -- Cleanup DROP INDEX docs_english_idx; DROP INDEX docs_simple_idx; DROP TABLE test_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/inheritance.out000066400000000000000000000307061525762047700245710ustar00rootroot00000000000000-- Test BM25 index behavior with table inheritance -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================================= -- Test 1: PostgreSQL native table inheritance -- BM25 indexes on parent tables only index direct rows (ONLY semantics), -- not inherited rows from child tables -- ============================================================================= CREATE TABLE parent_docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE child_docs1 ( category TEXT DEFAULT 'category1' ) INHERITS (parent_docs); CREATE TABLE child_docs2 ( category TEXT DEFAULT 'category2' ) INHERITS (parent_docs); -- Insert data into child tables INSERT INTO child_docs1 (content) VALUES ('The quick brown fox jumps over the lazy dog'), ('PostgreSQL inheritance allows tables to share structure'); INSERT INTO child_docs2 (content) VALUES ('Database normalization reduces redundancy'), ('Indexes help improve query performance'); -- Verify parent sees inherited data SELECT COUNT(*) as parent_count FROM parent_docs; parent_count -------------- 4 (1 row) SELECT COUNT(*) as parent_only_count FROM ONLY parent_docs; parent_only_count ------------------- 0 (1 row) -- BM25 index on parent only indexes direct rows (0 documents) \set VERBOSITY terse CREATE INDEX parent_bm25_idx ON parent_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parent_bm25_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY default -- BM25 index on child table should work CREATE INDEX child1_bm25_idx ON child_docs1 USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation child1_bm25_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=6.00 -- Verify child index was created successfully with correct document count SELECT bm25_dump_index('child1_bm25_idx')::text ~ 'total_docs: 2' as has_two_docs; has_two_docs -------------- t (1 row) -- BM25 search on the child table returns its top hit. SELECT content FROM child_docs1 ORDER BY content <@> to_bm25query('postgresql inheritance', 'child1_bm25_idx') LIMIT 1; content --------------------------------------------------------- PostgreSQL inheritance allows tables to share structure (1 row) -- Cleanup inheritance test DROP TABLE child_docs1 CASCADE; DROP TABLE child_docs2 CASCADE; DROP TABLE parent_docs CASCADE; -- ============================================================================= -- Test 2: Regular tables (should work normally) -- ============================================================================= CREATE TABLE regular_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO regular_docs (content) VALUES ('Regular table without inheritance'), ('Should work perfectly with BM25'); -- BM25 index on regular table should work CREATE INDEX regular_bm25_idx ON regular_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation regular_bm25_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=3.50 -- Verify regular index works SELECT bm25_dump_index('regular_bm25_idx')::text ~ 'total_docs: 2' as has_two_docs; has_two_docs -------------- t (1 row) -- Cleanup DROP TABLE regular_docs CASCADE; -- ============================================================================= -- Test 3: Inheritance scoring via find_first_child_bm25_index -- When querying a parent index that has 0 docs, the scoring code -- falls back to the first child's BM25 index for actual scoring. -- ============================================================================= CREATE TABLE inh_parent ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE inh_child ( category TEXT DEFAULT 'cat1' ) INHERITS (inh_parent); -- Insert data only into child INSERT INTO inh_child (content) VALUES ('the quick brown fox jumps over the lazy dog'), ('postgresql full text search engine'), ('database query optimization techniques'), ('information retrieval and ranking'); -- Create BM25 index on parent (will have 0 docs since data is in child) \set VERBOSITY terse CREATE INDEX inh_parent_bm25 ON inh_parent USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation inh_parent_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY default -- Create BM25 index on child (will have the actual data) CREATE INDEX inh_child_bm25 ON inh_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation inh_child_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 4 documents, avg_length=4.50 -- Query via parent index — should fall back to child index. -- ORDER BY makes this an index scan, which resolves the parent in the -- planner, so it does not reach find_first_child_bm25_index. SELECT content, content <@> to_bm25query('fox', 'inh_parent_bm25') AS score FROM inh_parent ORDER BY content <@> to_bm25query('fox', 'inh_parent_bm25') LIMIT 3; content | score ---------------------------------------------+--------------------- the quick brown fox jumps over the lazy dog | -1.0594960451126099 (1 row) -- Standalone scoring against the storage-less parent. Deliberately no -- ORDER BY: that is what forces the standalone path (the documented -- exception for tests whose target *is* standalone scoring), which is -- the only caller of find_first_child_bm25_index. Scoring every row -- exercises the parent -> first-child hop, which swaps the relation, -- the index state, the segment snapshot and the memtable source -- together; non-matching rows score 0. -- The plan assertion guards against this silently becoming an index -- scan, which is how the query above stopped covering this path. EXPLAIN (COSTS OFF) SELECT content, content <@> to_bm25query('fox', 'inh_parent_bm25') FROM inh_parent; QUERY PLAN ------------------------------------------------- Result -> Append -> Seq Scan on inh_parent inh_parent_1 -> Seq Scan on inh_child inh_parent_2 (4 rows) SELECT content, round((content <@> to_bm25query('fox', 'inh_parent_bm25'))::numeric, 4) AS score FROM inh_parent ORDER BY content; content | score ---------------------------------------------+--------- database query optimization techniques | 0.0000 information retrieval and ranking | 0.0000 postgresql full text search engine | 0.0000 the quick brown fox jumps over the lazy dog | -1.0595 (4 rows) -- Cleanup DROP TABLE inh_child CASCADE; DROP TABLE inh_parent CASCADE; -- ============================================================================= -- Test 4: Inheritance with dropped columns (attnum drift) -- When a parent table has dropped columns, child tables created after -- the drop get different physical attnum values for the same logical -- column. BM25 index matching must handle this by comparing column -- names rather than raw attnums. (GitHub issue #288) -- ============================================================================= -- 4a: Index scan path (tests indexes_match_by_attribute in scan.c) CREATE TABLE drift_parent ( id SERIAL PRIMARY KEY, extra_col TEXT, content TEXT ); -- Insert data into parent before dropping column INSERT INTO drift_parent (extra_col, content) VALUES ('filler', 'the quick brown fox jumps over the lazy dog'), ('filler', 'postgresql full text search engine'); -- Drop a column to create an attnum gap in the parent ALTER TABLE drift_parent DROP COLUMN extra_col; -- Create child AFTER the column drop. The child will NOT inherit the -- dropped-column slot, so "content" gets a different attnum than in -- the parent. CREATE TABLE drift_child () INHERITS (drift_parent); INSERT INTO drift_child (content) VALUES ('memory store retrieval and caching layer'), ('database query optimization techniques'); -- Create BM25 indexes on both parent and child \set VERBOSITY terse CREATE INDEX drift_parent_bm25 ON drift_parent USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation drift_parent_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=5.50 \set VERBOSITY default CREATE INDEX drift_child_bm25 ON drift_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation drift_child_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=4.50 -- Verify attnum drift exists: parent content attnum != child content attnum SELECT (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_parent_bm25'::regclass) AS parent_attnum, (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_child_bm25'::regclass) AS child_attnum, (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_parent_bm25'::regclass) <> (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_child_bm25'::regclass) AS attnums_differ; parent_attnum | child_attnum | attnums_differ ---------------+--------------+---------------- 3 | 2 | t (1 row) -- Index scan through parent should work despite attnum mismatch. -- Before the fix, this failed with "tpquery index mismatch". SET enable_seqscan = off; SELECT id, content <@> 'fox' AS score FROM drift_parent ORDER BY content <@> 'fox' LIMIT 3; id | score ----+--------------------- 1 | -0.6682932376861572 (1 row) RESET enable_seqscan; -- Cleanup DROP TABLE drift_child CASCADE; DROP TABLE drift_parent CASCADE; -- 4b: Scoring fallback path (tests find_first_child_bm25_index in -- query.c). Parent index has 0 docs so the scorer falls back to -- the first child index. With attnum drift that lookup used to fail. CREATE TABLE drift2_parent ( id SERIAL PRIMARY KEY, extra_col TEXT, content TEXT ); ALTER TABLE drift2_parent DROP COLUMN extra_col; CREATE TABLE drift2_child () INHERITS (drift2_parent); INSERT INTO drift2_child (content) VALUES ('the quick brown fox jumps over the lazy dog'), ('postgresql full text search engine'), ('database query optimization techniques'), ('information retrieval and ranking'); \set VERBOSITY terse CREATE INDEX drift2_parent_bm25 ON drift2_parent USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation drift2_parent_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY default CREATE INDEX drift2_child_bm25 ON drift2_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation drift2_child_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 4 documents, avg_length=4.50 -- Verify attnum drift SELECT (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift2_parent_bm25'::regclass) <> (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift2_child_bm25'::regclass) AS attnums_differ; attnums_differ ---------------- t (1 row) -- Query via parent index — should fall back to child index for scoring SELECT content, content <@> to_bm25query('fox', 'drift2_parent_bm25') AS score FROM drift2_parent ORDER BY content <@> to_bm25query('fox', 'drift2_parent_bm25') LIMIT 3; content | score ---------------------------------------------+--------------------- the quick brown fox jumps over the lazy dog | -1.0594960451126099 (1 row) -- Cleanup DROP TABLE drift2_child CASCADE; DROP TABLE drift2_parent CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/large_documents.out000066400000000000000000000040571525762047700254530ustar00rootroot00000000000000-- Test indexing of documents whose unique-token volume would otherwise -- exceed Postgres's tsvector 1 MB lexeme dictionary cap. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET client_min_messages = WARNING; -- Sanity: bare to_tsvector still rejects oversized inputs. DO $$ DECLARE big_doc text; BEGIN SELECT string_agg('w' || gs::text, ' ') INTO big_doc FROM generate_series(1, 250000) gs; PERFORM to_tsvector('english', big_doc); RAISE EXCEPTION 'to_tsvector unexpectedly succeeded'; EXCEPTION WHEN program_limit_exceeded THEN RAISE NOTICE 'to_tsvector rejected oversized input as expected'; END $$; -- CREATE INDEX over a >2 MB many-unique-token document must succeed. CREATE TABLE large_docs (id int, content text); INSERT INTO large_docs VALUES (1, 'short doc with hello world'); INSERT INTO large_docs (id, content) SELECT 2, string_agg('w' || gs::text, ' ') FROM generate_series(1, 250000) gs; INSERT INTO large_docs VALUES (3, 'another short doc mentioning hello'); SELECT id, length(content) > 1048576 AS oversized FROM large_docs ORDER BY id; id | oversized ----+----------- 1 | f 2 | t 3 | f (3 rows) CREATE INDEX large_idx ON large_docs USING bm25 (content text_bm25_ops) WITH (text_config = 'english'); -- Index scan: a token from the large doc must be retrievable. SELECT id FROM large_docs ORDER BY content <@> to_bm25query('w12345', 'large_idx') LIMIT 1; id ---- 2 (1 row) -- Index scan over a token that is in two short docs (sanity). SELECT id FROM large_docs ORDER BY content <@> to_bm25query('hello', 'large_idx') LIMIT 5; id ---- 1 3 2 (3 rows) -- INSERT (aminsert) of a >2 MB many-unique-token row. INSERT INTO large_docs (id, content) SELECT 4, string_agg('z' || gs::text, ' ') FROM generate_series(1, 250000) gs; SELECT id FROM large_docs ORDER BY content <@> to_bm25query('z67890', 'large_idx') LIMIT 1; id ---- 4 (1 row) -- VACUUM after deleting the large rows must succeed. DELETE FROM large_docs WHERE id IN (2, 4); VACUUM large_docs; DROP TABLE large_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/limits.out000066400000000000000000000320571525762047700236020ustar00rootroot00000000000000-- Test file for LIMIT detection and optimization in Tapir -- Disable duration logging to avoid timing differences in sanitizer tests SET log_duration = off; -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Create test table with sufficient data for meaningful LIMIT testing CREATE TABLE limit_test ( id SERIAL PRIMARY KEY, title TEXT, content TEXT ); -- Insert test data with varied content for BM25 scoring INSERT INTO limit_test (title, content) VALUES ('Database Systems', 'postgresql database management with advanced indexing and query optimization'), ('Search Algorithms', 'full text search algorithms with relevance scoring and ranking mechanisms'), ('Information Retrieval', 'information retrieval systems using vector space models and term frequency'), ('Machine Learning', 'machine learning algorithms for data analysis and pattern recognition'), ('Data Mining', 'data mining techniques for knowledge discovery in large databases'), ('Text Processing', 'natural language processing and text analysis with stemming algorithms'), ('Database Performance', 'database optimization strategies for improving query performance'), ('Search Engines', 'search engine design with ranking algorithms and user experience'), ('Big Data Analytics', 'big data processing platforms for massive dataset analysis'), ('Distributed Systems', 'distributed computing and scalability in cloud environments'), ('Database Architecture', 'database system architecture with storage and indexing strategies'), ('Query Optimization', 'query optimization techniques for efficient database operations'), ('Text Analytics', 'text mining and document classification using statistical methods'), ('Information Systems', 'information management systems with data organization principles'), ('Database Theory', 'theoretical foundations of database systems and relational algebra'), ('Search Technology', 'search technology innovations with semantic understanding capabilities'), ('Data Structures', 'efficient data structures for database storage and retrieval operations'), ('Algorithm Design', 'algorithm design principles for optimal computational performance'), ('System Performance', 'system performance tuning and resource optimization strategies'), ('Database Security', 'database security measures and access control mechanisms'); -- Create pg_textsearch index CREATE INDEX limit_test_idx ON limit_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation limit_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 20 documents, avg_length=6.60 -- Test 1: Basic LIMIT functionality -- Should detect and optimize for LIMIT 5 EXPLAIN (COSTS OFF) SELECT title, content, content <@> 'database' as score FROM limit_test ORDER BY content <@> 'database' LIMIT 5; QUERY PLAN ---------------------------------------------------------------------- Limit -> Index Scan using limit_test_idx on limit_test Order By: (content <@> 'limit_test_idx:database'::bm25query) (3 rows) SELECT title, content, ROUND((content <@> 'database')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'database' LIMIT 5; NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,11), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,12), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,15), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,20), BM25_score=-0.9394 title | content | score -----------------------+--------------------------------------------------------------------+--------- Database Performance | database optimization strategies for improving query performance | -0.9394 Database Architecture | database system architecture with storage and indexing strategies | -0.9394 Query Optimization | query optimization techniques for efficient database operations | -0.9394 Database Theory | theoretical foundations of database systems and relational algebra | -0.9394 Database Security | database security measures and access control mechanisms | -0.9394 (5 rows) -- Test 2: Different LIMIT values -- Test LIMIT 1 (should be highly optimized) SELECT title, ROUND((content <@> 'search')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'search' LIMIT 1; NOTICE: BM25 index scan: tid=(0,16), BM25_score=-1.8610 title | score -------------------+--------- Search Technology | -1.8610 (1 row) -- Test LIMIT 3 SELECT title, ROUND((content <@> 'optimization')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'optimization' LIMIT 3; NOTICE: BM25 index scan: tid=(0,7), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,12), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.3915 title | score ----------------------+--------- Database Performance | -1.3915 Query Optimization | -1.3915 Algorithm Design | -1.3915 (3 rows) -- Test LIMIT 10 SELECT title, ROUND((content <@> 'algorithm')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'algorithm' LIMIT 10; NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.2328 title | score -------------------+--------- Algorithm Design | -1.3915 Machine Learning | -1.3074 Text Processing | -1.3074 Search Engines | -1.3074 Search Algorithms | -1.2328 (5 rows) -- Test 3: LIMIT with WHERE clause (should prevent pushdown for safety) -- This should NOT use LIMIT pushdown due to additional WHERE clause EXPLAIN (COSTS OFF) SELECT title, ROUND((content <@> 'database system')::numeric, 4) as score FROM limit_test WHERE id > 5 ORDER BY content <@> 'database system' LIMIT 7; QUERY PLAN ----------------------------------------------------------------------------- Limit -> Index Scan using limit_test_idx on limit_test Order By: (content <@> 'limit_test_idx:database system'::bm25query) Filter: (id > 5) (4 rows) SELECT title, ROUND((content <@> 'database system')::numeric, 4) as score FROM limit_test WHERE id > 5 ORDER BY content <@> 'database system' LIMIT 7; NOTICE: BM25 index scan: tid=(0,11), BM25_score=-2.3309 NOTICE: BM25 index scan: tid=(0,15), BM25_score=-2.3309 NOTICE: BM25 index scan: tid=(0,14), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,19), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-1.1663 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,12), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,20), BM25_score=-0.9394 title | score -----------------------+--------- Database Architecture | -2.3309 Database Theory | -2.3309 Information Systems | -1.3915 System Performance | -1.3915 Database Performance | -0.9394 Query Optimization | -0.9394 Database Security | -0.9394 (7 rows) -- Test 4: No LIMIT (should use default behavior) -- Note: Query plan varies by PG version - not testing EXPLAIN here -- Test 5: LIMIT with OFFSET SELECT title, ROUND((content <@> 'performance')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'performance' LIMIT 5 OFFSET 2; NOTICE: BM25 index scan: tid=(0,7), BM25_score=-1.8610 NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.8610 NOTICE: BM25 index scan: tid=(0,19), BM25_score=-1.8610 title | score --------------------+--------- System Performance | -1.8610 (1 row) -- Test 6: Very small LIMIT (edge case) SELECT title, ROUND((content <@> 'text')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'text' LIMIT 1; NOTICE: BM25 index scan: tid=(0,6), BM25_score=-1.7484 title | score -----------------+--------- Text Processing | -1.7484 (1 row) -- Test 7: Large LIMIT (should still use index optimization) SELECT COUNT(*) > 0 as has_results FROM ( SELECT title, content <@> to_bm25query('xyzabc123', 'limit_test_idx') as score FROM limit_test -- Test for non-matching terms (should return no results) ORDER BY content <@> to_bm25query('xyzabc123', 'limit_test_idx') LIMIT 1000 ) subq; has_results ------------- f (1 row) -- Test 8: LIMIT in subquery (requires explicit index name) SELECT * FROM ( SELECT title, ROUND((content <@> to_bm25query('mining', 'limit_test_idx'))::numeric, 4) as score FROM limit_test ORDER BY content <@> to_bm25query('mining', 'limit_test_idx') LIMIT 3 ) sub; NOTICE: BM25 index scan: tid=(0,5), BM25_score=-2.0767 NOTICE: BM25 index scan: tid=(0,13), BM25_score=-2.0767 title | score ----------------+--------- Data Mining | -2.0767 Text Analytics | -2.0767 (2 rows) -- Test 9: Multiple queries with different LIMIT values to test limit storage/cleanup SELECT 'Query 1' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('database', 'limit_test_idx') LIMIT 2 ) q1; NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,11), BM25_score=-0.9394 query_name | results ------------+--------- Query 1 | 2 (1 row) SELECT 'Query 2' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('search', 'limit_test_idx') LIMIT 8 ) q2; NOTICE: BM25 index scan: tid=(0,16), BM25_score=-1.8610 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.7484 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.6487 query_name | results ------------+--------- Query 2 | 3 (1 row) SELECT 'Query 3' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('algorithm', 'limit_test_idx') LIMIT 4 ) q3; NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.3074 query_name | results ------------+--------- Query 3 | 4 (1 row) -- Test 10: LIMIT pushdown safety verification -- These tests verify that LIMIT pushdown is only used when safe -- Safe case: Simple ORDER BY with pg_textsearch score, no WHERE clause -- This SHOULD allow LIMIT pushdown EXPLAIN (COSTS OFF) SELECT title, content <@> 'simple' as score FROM limit_test ORDER BY content <@> 'simple' LIMIT 3; QUERY PLAN -------------------------------------------------------------------- Limit -> Index Scan using limit_test_idx on limit_test Order By: (content <@> 'limit_test_idx:simple'::bm25query) (3 rows) -- Case: Multiple ORDER BY clauses -- Tapir requires exactly one ORDER BY, so not using the index here is correct behavior -- Query plan varies by PG version (Sort vs Incremental Sort) - not testing EXPLAIN -- Unsafe case: Complex WHERE clause with additional conditions -- This should NOT allow LIMIT pushdown due to WHERE clause -- Query plan varies by PG version - not testing EXPLAIN -- Test 11: Basic LIMIT functionality verification -- Test with safe pushdown case SELECT COUNT(*) as pushdown_safe_count FROM ( SELECT title, content <@> to_bm25query('pushdown_safe', 'limit_test_idx') as score FROM limit_test ORDER BY content <@> to_bm25query('pushdown_safe', 'limit_test_idx') LIMIT 2 ) subq; pushdown_safe_count --------------------- 0 (1 row) -- Test with unsafe pushdown case (has WHERE clause) SELECT COUNT(*) as pushdown_unsafe_count FROM ( SELECT title, content <@> to_bm25query('pushdown_unsafe', 'limit_test_idx') as score FROM limit_test WHERE id % 2 = 0 ORDER BY content <@> to_bm25query('pushdown_unsafe', 'limit_test_idx') LIMIT 2 ) subq; pushdown_unsafe_count ----------------------- 0 (1 row) -- Test 12: Edge cases for LIMIT pushdown -- Very large LIMIT (should still use index optimization efficiently) SELECT COUNT(*) > 0 as large_limit_has_results FROM ( SELECT title, content <@> to_bm25query('qwertyuiop999', 'limit_test_idx') as score FROM limit_test -- Test with non-matching nonsense term ORDER BY content <@> to_bm25query('qwertyuiop999', 'limit_test_idx') LIMIT 50000 -- Much larger than our dataset ) subq; large_limit_has_results ------------------------- f (1 row) -- LIMIT 0 edge case SELECT title, content <@> to_bm25query('zero_limit', 'limit_test_idx') as score FROM limit_test ORDER BY content <@> to_bm25query('zero_limit', 'limit_test_idx') LIMIT 0; title | score -------+------- (0 rows) -- Cleanup DROP TABLE limit_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/limits_1.out000066400000000000000000000317411525762047700240210ustar00rootroot00000000000000-- Test file for LIMIT detection and optimization in Tapir -- Disable duration logging to avoid timing differences in sanitizer tests SET log_duration = off; -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Create test table with sufficient data for meaningful LIMIT testing CREATE TABLE limit_test ( id SERIAL PRIMARY KEY, title TEXT, content TEXT ); -- Insert test data with varied content for BM25 scoring INSERT INTO limit_test (title, content) VALUES ('Database Systems', 'postgresql database management with advanced indexing and query optimization'), ('Search Algorithms', 'full text search algorithms with relevance scoring and ranking mechanisms'), ('Information Retrieval', 'information retrieval systems using vector space models and term frequency'), ('Machine Learning', 'machine learning algorithms for data analysis and pattern recognition'), ('Data Mining', 'data mining techniques for knowledge discovery in large databases'), ('Text Processing', 'natural language processing and text analysis with stemming algorithms'), ('Database Performance', 'database optimization strategies for improving query performance'), ('Search Engines', 'search engine design with ranking algorithms and user experience'), ('Big Data Analytics', 'big data processing platforms for massive dataset analysis'), ('Distributed Systems', 'distributed computing and scalability in cloud environments'), ('Database Architecture', 'database system architecture with storage and indexing strategies'), ('Query Optimization', 'query optimization techniques for efficient database operations'), ('Text Analytics', 'text mining and document classification using statistical methods'), ('Information Systems', 'information management systems with data organization principles'), ('Database Theory', 'theoretical foundations of database systems and relational algebra'), ('Search Technology', 'search technology innovations with semantic understanding capabilities'), ('Data Structures', 'efficient data structures for database storage and retrieval operations'), ('Algorithm Design', 'algorithm design principles for optimal computational performance'), ('System Performance', 'system performance tuning and resource optimization strategies'), ('Database Security', 'database security measures and access control mechanisms'); -- Create pg_textsearch index CREATE INDEX limit_test_idx ON limit_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation limit_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 20 documents, avg_length=6.60 -- Test 1: Basic LIMIT functionality -- Should detect and optimize for LIMIT 5 EXPLAIN (COSTS OFF) SELECT title, content, content <@> 'database' as score FROM limit_test ORDER BY content <@> 'database' LIMIT 5; QUERY PLAN ---------------------------------------------------------------------- Limit -> Index Scan using limit_test_idx on limit_test Order By: (content <@> 'limit_test_idx:database'::bm25query) (3 rows) SELECT title, content, ROUND((content <@> 'database')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'database' LIMIT 5; NOTICE: BM25 index scan: tid=(0,11), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,20), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,15), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,12), BM25_score=-0.9394 title | content | score -----------------------+--------------------------------------------------------------------+--------- Database Architecture | database system architecture with storage and indexing strategies | -0.9394 Database Performance | database optimization strategies for improving query performance | -0.9394 Database Security | database security measures and access control mechanisms | -0.9394 Database Theory | theoretical foundations of database systems and relational algebra | -0.9394 Query Optimization | query optimization techniques for efficient database operations | -0.9394 (5 rows) -- Test 2: Different LIMIT values -- Test LIMIT 1 (should be highly optimized) SELECT title, ROUND((content <@> 'search')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'search' LIMIT 1; NOTICE: BM25 index scan: tid=(0,16), BM25_score=-1.8610 title | score -------------------+--------- Search Technology | -1.8610 (1 row) -- Test LIMIT 3 SELECT title, ROUND((content <@> 'optimization')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'optimization' LIMIT 3; NOTICE: BM25 index scan: tid=(0,12), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,19), BM25_score=-1.3915 title | score --------------------+--------- Query Optimization | -1.3915 Algorithm Design | -1.3915 System Performance | -1.3915 (3 rows) -- Test LIMIT 10 SELECT title, ROUND((content <@> 'algorithm')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'algorithm' LIMIT 10; NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.2328 title | score -------------------+--------- Algorithm Design | -1.3915 Search Engines | -1.3074 Machine Learning | -1.3074 Text Processing | -1.3074 Search Algorithms | -1.2328 (5 rows) -- Test 3: LIMIT with WHERE clause (should prevent pushdown for safety) -- This should NOT use LIMIT pushdown due to additional WHERE clause EXPLAIN (COSTS OFF) SELECT title, ROUND((content <@> 'database system')::numeric, 4) as score FROM limit_test WHERE id > 5 ORDER BY content <@> 'database system' LIMIT 7; QUERY PLAN ----------------------------------------------------------------------------- Limit -> Index Scan using limit_test_idx on limit_test Order By: (content <@> 'limit_test_idx:database system'::bm25query) Filter: (id > 5) (4 rows) SELECT title, ROUND((content <@> 'database system')::numeric, 4) as score FROM limit_test WHERE id > 5 ORDER BY content <@> 'database system' LIMIT 7; NOTICE: BM25 index scan: tid=(0,11), BM25_score=-2.3309 NOTICE: BM25 index scan: tid=(0,15), BM25_score=-2.3309 NOTICE: BM25 index scan: tid=(0,19), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,14), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-1.1663 NOTICE: BM25 index scan: tid=(0,20), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,12), BM25_score=-0.9394 title | score -----------------------+--------- Database Architecture | -2.3309 Database Theory | -2.3309 System Performance | -1.3915 Information Systems | -1.3915 Database Security | -0.9394 Query Optimization | -0.9394 (6 rows) -- Test 4: No LIMIT (should use default behavior) -- Note: Query plan varies by PG version - not testing EXPLAIN here -- Test 5: LIMIT with OFFSET SELECT title, ROUND((content <@> 'performance')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'performance' LIMIT 5 OFFSET 2; NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.8610 NOTICE: BM25 index scan: tid=(0,19), BM25_score=-1.8610 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-1.8610 title | score ----------------------+--------- Database Performance | -1.8610 (1 row) -- Test 6: Very small LIMIT (edge case) SELECT title, ROUND((content <@> 'text')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'text' LIMIT 1; NOTICE: BM25 index scan: tid=(0,13), BM25_score=-1.7484 title | score ----------------+--------- Text Analytics | -1.7484 (1 row) -- Test 7: Large LIMIT (should still use index optimization) SELECT COUNT(*) > 0 as has_results FROM ( SELECT title, content <@> to_bm25query('xyzabc123', 'limit_test_idx') as score FROM limit_test -- Test for non-matching terms (should return no results) ORDER BY content <@> to_bm25query('xyzabc123', 'limit_test_idx') LIMIT 1000 ) subq; has_results ------------- f (1 row) -- Test 8: LIMIT in subquery (requires explicit index name) SELECT * FROM ( SELECT title, ROUND((content <@> to_bm25query('mining', 'limit_test_idx'))::numeric, 4) as score FROM limit_test ORDER BY content <@> to_bm25query('mining', 'limit_test_idx') LIMIT 3 ) sub WHERE score < 0; NOTICE: BM25 index scan: tid=(0,13), BM25_score=-2.0767 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-2.0767 title | score ----------------+--------- Text Analytics | -2.0767 Data Mining | -2.0767 (2 rows) -- Test 9: Multiple queries with different LIMIT values to test limit storage/cleanup SELECT 'Query 1' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('database', 'limit_test_idx') LIMIT 2 ) q1; NOTICE: BM25 index scan: tid=(0,11), BM25_score=-0.9394 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.9394 query_name | results ------------+--------- Query 1 | 2 (1 row) SELECT 'Query 2' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('search', 'limit_test_idx') LIMIT 8 ) q2; NOTICE: BM25 index scan: tid=(0,16), BM25_score=-1.8610 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.7484 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.6487 query_name | results ------------+--------- Query 2 | 3 (1 row) SELECT 'Query 3' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('algorithm', 'limit_test_idx') LIMIT 4 ) q3; NOTICE: BM25 index scan: tid=(0,18), BM25_score=-1.3915 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-1.3074 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-1.3074 query_name | results ------------+--------- Query 3 | 4 (1 row) -- Test 10: LIMIT pushdown safety verification -- These tests verify that LIMIT pushdown is only used when safe -- Safe case: Simple ORDER BY with pg_textsearch score, no WHERE clause -- This SHOULD allow LIMIT pushdown EXPLAIN (COSTS OFF) SELECT title, content <@> 'simple' as score FROM limit_test ORDER BY content <@> 'simple' LIMIT 3; QUERY PLAN -------------------------------------------------------------------- Limit -> Index Scan using limit_test_idx on limit_test Order By: (content <@> 'limit_test_idx:simple'::bm25query) (3 rows) -- Case: Multiple ORDER BY clauses -- Tapir requires exactly one ORDER BY, so not using the index here is correct behavior -- Query plan varies by PG version (Sort vs Incremental Sort) - not testing EXPLAIN -- Unsafe case: Complex WHERE clause with additional conditions -- This should NOT allow LIMIT pushdown due to WHERE clause -- Query plan varies by PG version - not testing EXPLAIN -- Test 11: Basic LIMIT functionality verification -- Test with safe pushdown case SELECT COUNT(*) as pushdown_safe_count FROM ( SELECT title, content <@> to_bm25query('pushdown_safe', 'limit_test_idx') as score FROM limit_test ORDER BY content <@> to_bm25query('pushdown_safe', 'limit_test_idx') LIMIT 2 ) subq; pushdown_safe_count --------------------- 0 (1 row) -- Test with unsafe pushdown case (has WHERE clause) SELECT COUNT(*) as pushdown_unsafe_count FROM ( SELECT title, content <@> to_bm25query('pushdown_unsafe', 'limit_test_idx') as score FROM limit_test WHERE id % 2 = 0 ORDER BY content <@> to_bm25query('pushdown_unsafe', 'limit_test_idx') LIMIT 2 ) subq; pushdown_unsafe_count ----------------------- 0 (1 row) -- Test 12: Edge cases for LIMIT pushdown -- Very large LIMIT (should still use index optimization efficiently) SELECT COUNT(*) > 0 as large_limit_has_results FROM ( SELECT title, content <@> to_bm25query('qwertyuiop999', 'limit_test_idx') as score FROM limit_test -- Test with non-matching nonsense term ORDER BY content <@> to_bm25query('qwertyuiop999', 'limit_test_idx') LIMIT 50000 -- Much larger than our dataset ) subq; large_limit_has_results ------------------------- f (1 row) -- LIMIT 0 edge case SELECT title, content <@> to_bm25query('zero_limit', 'limit_test_idx') as score FROM limit_test ORDER BY content <@> to_bm25query('zero_limit', 'limit_test_idx') LIMIT 0; title | score -------+------- (0 rows) -- Cleanup DROP TABLE limit_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/lock.out000066400000000000000000000042021525762047700232200ustar00rootroot00000000000000-- Test lock upgrade from shared to exclusive within a single transaction -- This exercises the tp_acquire_index_lock upgrade path CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table and index CREATE TABLE lock_upgrade_test ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO lock_upgrade_test (content) VALUES ('database systems and concurrent access'), ('full text search indexing'), ('query optimization techniques'); CREATE INDEX lock_upgrade_idx ON lock_upgrade_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation lock_upgrade_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.67 -- Force index scan (small tables default to seq scan) SET enable_seqscan = off; SET enable_bitmapscan = off; -- Test: Single transaction that first reads (shared lock) then writes (exclusive lock) -- This should trigger the lock upgrade path in tp_acquire_index_lock BEGIN; -- First operation: SELECT acquires shared lock via index scan SELECT id, content FROM lock_upgrade_test ORDER BY content <@> 'database' LIMIT 5; id | content ----+---------------------------------------- 1 | database systems and concurrent access (1 row) -- Second operation: INSERT in same transaction should upgrade to exclusive lock INSERT INTO lock_upgrade_test (content) VALUES ('new document about database concurrency'); -- Verify the insert worked SELECT COUNT(*) FROM lock_upgrade_test; count ------- 4 (1 row) COMMIT; -- Verify final state SELECT COUNT(*) FROM lock_upgrade_test; count ------- 4 (1 row) -- Test that subsequent operations work normally SELECT id, content FROM lock_upgrade_test ORDER BY content <@> 'database concurrency' LIMIT 5; id | content ----+----------------------------------------- 1 | database systems and concurrent access 4 | new document about database concurrency (2 rows) -- Clean up DROP TABLE lock_upgrade_test CASCADE; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/manyterms.out000066400000000000000000000054611525762047700243170ustar00rootroot00000000000000-- Test hash table automatic resizing with many terms -- This test creates enough unique terms to trigger hash table resize -- and verifies the system continues to work correctly CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Create test table CREATE TABLE manyterms_test (id serial, content text); -- Create index to initialize hash table with small initial size CREATE INDEX manyterms_idx ON manyterms_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation manyterms_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Add enough unique terms to exceed initial hash table capacity -- With 64 buckets at 0.75 load factor, >48 unique terms should trigger automatic resize INSERT INTO manyterms_test (content) VALUES ('alpha'), ('beta'), ('gamma'), ('delta'), ('epsilon'), ('zeta'), ('eta'), ('theta'), ('iota'), ('kappa'), ('lambda'), ('mu'), ('nu'), ('xi'), ('omicron'), ('pi'), ('rho'), ('sigma'), ('tau'), ('upsilon'), ('phi'), ('chi'), ('psi'), ('omega'), ('one'), ('two'), ('three'), ('four'), ('five'), ('six'), ('seven'), ('eight'), ('nine'), ('ten'), ('eleven'), ('twelve'), ('thirteen'), ('fourteen'), ('fifteen'), ('sixteen'), ('seventeen'), ('eighteen'), ('nineteen'), ('twenty'), ('twentyone'), ('twentytwo'), ('twentythree'), ('twentyfour'), ('twentyfive'), ('twentysix'), ('twentyseven'), ('twentyeight'), ('twentynine'), ('thirty'), ('thirtyone'), ('thirtytwo'), ('thirtythree'), ('thirtyfour'), ('thirtyfive'), ('thirtysix'), ('thirtyseven'), ('thirtyeight'), ('thirtynine'), ('forty'), ('fortyone'), ('fortytwo'), ('fortythree'), ('fortyfour'), ('fortyfive'), ('fortysix'), ('fortyseven'), ('fortyeight'), ('fortynine'), ('fifty'), ('fiftyone'), ('fiftytwo'), ('fiftythree'), ('fiftyfour'), ('fiftyfive'), ('fiftysix'); -- Verify we can count all documents successfully (no errors from resize) SELECT COUNT(*) as total_docs FROM manyterms_test; total_docs ------------ 80 (1 row) -- Test searching for specific terms works correctly after resize SELECT id, content FROM manyterms_test WHERE content = 'alpha' ORDER BY id LIMIT 1; id | content ----+--------- 1 | alpha (1 row) SELECT id, content FROM manyterms_test WHERE content = 'omega' ORDER BY id LIMIT 1; id | content ----+--------- 24 | omega (1 row) -- Test BM25 scoring still works after hash table resize SELECT id, content, ROUND((content <@> 'fifty')::numeric, 4) as score FROM manyterms_test WHERE content = 'fifty' ORDER BY score LIMIT 1; id | content | score ----+---------+--------- 74 | fifty | -3.9890 (1 row) -- Clean up DROP INDEX manyterms_idx; DROP TABLE manyterms_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/memory.out000066400000000000000000000037771525762047700236200ustar00rootroot00000000000000-- Test bulk insert with auto-spill for pg_textsearch indexes -- Create extension if not exists CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table DROP TABLE IF EXISTS memory_test CASCADE; NOTICE: table "memory_test" does not exist, skipping CREATE TABLE memory_test (id serial PRIMARY KEY, content text); -- Suppress auto-spill NOTICE messages during test to avoid non-deterministic output -- (The exact DSA memory values vary between runs) SET client_min_messages = WARNING; -- Create an index CREATE INDEX idx_memory_test ON memory_test USING bm25(content) WITH (text_config='english'); -- Insert some documents INSERT INTO memory_test (content) VALUES ('This is a test document with some content'); INSERT INTO memory_test (content) VALUES ('Another document with different words'); -- Insert many documents with lots of unique terms to exercise auto-spill. -- With 1000 docs * 100 unique terms = 100K terms per transaction, -- this should trigger the bulk_load_threshold (default 100K). DO $$ DECLARE i integer; doc text; BEGIN FOR i IN 1..1000 LOOP -- Generate document with unique terms to increase memory usage doc := 'Document number ' || i || ' with unique terms: '; -- Add lots of unique words per document FOR j IN 1..100 LOOP doc := doc || ' term_' || i || '_' || j; END LOOP; INSERT INTO memory_test (content) VALUES (doc); END LOOP; END $$; -- Re-enable notices for verification output RESET client_min_messages; -- Verify that all documents were inserted and index works SELECT COUNT(*) AS total_docs FROM memory_test; total_docs ------------ 1002 (1 row) -- Search should find documents SELECT id, left(content, 50) AS content_preview FROM memory_test ORDER BY content <@> 'test' LIMIT 5; id | content_preview ----+------------------------------------------- 1 | This is a test document with some content (1 row) -- Clean up DROP TABLE memory_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/memtable_append.out000066400000000000000000000142661525762047700254200ustar00rootroot00000000000000-- Unit-level coverage for the on-disk write path -- tp_memtable_append() (memtable v2, issue #374), driven through -- the bm25_test_memtable_append() scaffold function, plus the -- chain-inspection SRF bm25_memtable_chain(). -- -- These tests use an ordinary pg_textsearch index purely as a -- host for the memtable pages; the assertions are about the -- chain layout, not end-to-end query semantics (which the rest -- of the regression suite covers). CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE memtable_append_t (id int, body text); CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_append_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Empty index: head and tail are InvalidBlockNumber, chain is empty. SELECT bm25_test_memtable_append('memtable_append_idx', 'empty_index'); bm25_test_memtable_append --------------------------- OK (1 row) SELECT count(*) AS chain_pages FROM bm25_memtable_chain('memtable_append_idx'); chain_pages ------------- 0 (1 row) -- One append: head == tail, chain length 1, page has 1 record. SELECT bm25_test_memtable_append('memtable_append_idx', 'single_append'); bm25_test_memtable_append --------------------------- OK (1 row) SELECT n_records, next_block IS NULL AS is_tail FROM bm25_memtable_chain('memtable_append_idx'); n_records | is_tail -----------+--------- 1 | t (1 row) -- Drop and recreate so each scenario starts with an empty chain. DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_append_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Many small appends all fit on the first page. SELECT bm25_test_memtable_append('memtable_append_idx', 'fits_on_one_page'); bm25_test_memtable_append --------------------------- OK (1 row) SELECT count(*) AS chain_pages, sum(n_records)::int AS total_records FROM bm25_memtable_chain('memtable_append_idx'); chain_pages | total_records -------------+--------------- 1 | 10 (1 row) DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_append_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Larger records force the chain to extend past one page. SELECT bm25_test_memtable_append('memtable_append_idx', 'extends_chain'); bm25_test_memtable_append --------------------------- OK (1 row) SELECT (count(*) >= 2) AS chain_extended, sum(n_records) > 0 AS has_records FROM bm25_memtable_chain('memtable_append_idx'); chain_extended | has_records ----------------+------------- t | t (1 row) DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_append_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- ctid, doc_length, and vector_bytes all round-trip exactly. SELECT bm25_test_memtable_append('memtable_append_idx', 'roundtrip'); bm25_test_memtable_append --------------------------- OK (1 row) DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_append_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Oversized record goes through the multi-page (fragment) path: -- a fragment head page followed by N continuation pages and a -- fresh tail page, with a small bookend record on either side -- to verify the head record stays on the original page and the -- new tail can still accept subsequent appends. SELECT bm25_test_memtable_append('memtable_append_idx', 'multi_page_fragment'); bm25_test_memtable_append --------------------------- OK (1 row) -- Page shape: -- small_record_page (n_records=1, flags=0) -- fragment_head (n_records=1, FRAGMENT flag set on the record; -- page flags=0) -- continuation*N (n_records=0, CONTINUATION flag set on the page, -- page flags=0x02) -- new_tail (n_records=1, flags=0) SELECT n_records FROM bm25_memtable_chain('memtable_append_idx'); n_records ----------- 1 1 0 0 0 1 (6 rows) -- Assert page-level flag bits. The continuation pages must carry -- TP_MEMTABLE_PAGE_FLAG_CONTINUATION (0x02); every other page must -- have flags = 0. Iterated record_count of fragment continuation -- pages is 0 because the fragment head holds the record's metadata -- and continuations carry only opaque tail bytes. SELECT sum(((flags & 2) <> 0)::int)::int AS continuation_pages, sum(CASE WHEN (flags & 2) <> 0 THEN 0 ELSE 1 END)::int AS regular_pages, bool_and((flags & 2) <> 0 OR flags = 0) AS only_known_flags FROM bm25_memtable_chain('memtable_append_idx'); continuation_pages | regular_pages | only_known_flags --------------------+---------------+------------------ 3 | 3 | t (1 row) SELECT bool_and( CASE WHEN (flags & 2) <> 0 THEN n_records = 0 ELSE n_records >= 0 END ) AS continuation_pages_have_zero_records FROM bm25_memtable_chain('memtable_append_idx'); continuation_pages_have_zero_records -------------------------------------- t (1 row) -- Clean up DROP EXTENSION pg_textsearch CASCADE; NOTICE: drop cascades to index memtable_append_idx timescale-pg_textsearch-b63fe58/test/expected/memtable_page.out000066400000000000000000000037741525762047700250670ustar00rootroot00000000000000-- Unit-level coverage for the new on-disk memtable page format. -- Exercises tp_memtable_page_{init,append,iterate,can_fit,is_valid} -- via the bm25_test_memtable_page() scaffold function. Each case -- returns 'OK' on success or 'FAIL: ' otherwise. -- -- Memtable v2 page format (see plan / issue #374). CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- An uninitialized (zero-filled) page must be rejected. SELECT bm25_test_memtable_page('reject_uninitialized'); bm25_test_memtable_page ------------------------- OK (1 row) -- After init, the page is valid, has 0 records, an invalid next_block, -- and free_offset pointing past the header. SELECT bm25_test_memtable_page('init_empty'); bm25_test_memtable_page ------------------------- OK (1 row) -- Iteration over an empty page yields no records. SELECT bm25_test_memtable_page('iterate_empty'); bm25_test_memtable_page ------------------------- OK (1 row) -- A single appended record round-trips its ctid, doc_length, and -- vector bytes verbatim. SELECT bm25_test_memtable_page('one_record_roundtrip'); bm25_test_memtable_page ------------------------- OK (1 row) -- Many records of varying sizes appended sequentially are all -- recoverable in insertion order. SELECT bm25_test_memtable_page('many_records_roundtrip'); bm25_test_memtable_page ------------------------- OK (1 row) -- can_fit() returns false when the requested record would not fit. SELECT bm25_test_memtable_page('can_fit_boundary'); bm25_test_memtable_page ------------------------- OK (1 row) -- Appending records until the page is full leaves the page in a -- consistent state (n_records matches what was actually appended, -- and iteration walks them all). SELECT bm25_test_memtable_page('fill_until_full'); bm25_test_memtable_page ------------------------- OK (1 row) -- next_block setter / getter round-trip. SELECT bm25_test_memtable_page('next_block_roundtrip'); bm25_test_memtable_page ------------------------- OK (1 row) -- Clean up DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/memtable_reclaim.out000066400000000000000000000152621525762047700255620ustar00rootroot00000000000000-- Regression: Step 2 FSM reclaim of DEAD memtable orphans. -- After spill + VACUUM, tp_reclaim_dead_memtable_pages returns blocks -- to the index FSM; tp_memtable_alloc_page should reuse them when the -- chain grows again (main fork should not grow by ~one block per dead -- page if reuse works). CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Temporary relations have a session-local reclaim horizon, unaffected -- by other backends or standby feedback. CREATE TEMP TABLE memtable_reclaim_t (id serial PRIMARY KEY, body text); CREATE INDEX memtable_reclaim_idx ON memtable_reclaim_t USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_reclaim_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SELECT count(*)::int AS dead_pages_before_spill FROM bm25_memtable_dead_pages('memtable_reclaim_idx'); dead_pages_before_spill ------------------------- 0 (1 row) INSERT INTO memtable_reclaim_t (body) SELECT 'reclaim ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; CREATE TEMP TABLE spill_stats AS SELECT count(*)::int AS live_pages FROM bm25_memtable_chain('memtable_reclaim_idx'); SELECT live_pages > 1 AS chain_multi_page FROM spill_stats; chain_multi_page ------------------ t (1 row) SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS spill_ok; spill_ok ---------- t (1 row) SELECT count(*)::int AS live_chain_after_spill FROM bm25_memtable_chain('memtable_reclaim_idx'); live_chain_after_spill ------------------------ 0 (1 row) CREATE TEMP TABLE reclaim_dead AS SELECT count(*)::int AS dead_count FROM bm25_memtable_dead_pages('memtable_reclaim_idx'); SELECT dead_count > 0 AS spilled_dead_pages FROM reclaim_dead; spilled_dead_pages -------------------- t (1 row) SELECT r.dead_count = s.live_pages AS dead_matches_spilled_chain FROM reclaim_dead r, spill_stats s; dead_matches_spilled_chain ---------------------------- t (1 row) CREATE TEMP TABLE reclaim_sizes AS SELECT pg_relation_size('memtable_reclaim_idx'::regclass, 'main')::bigint AS sz_after_spill; -- Horizon: spill xact is committed (autocommit per statement). VACUUM ANALYZE memtable_reclaim_t; -- Idempotent second pass (RecordFreeIndexPage on same blocks is safe). VACUUM ANALYZE memtable_reclaim_t; SELECT count(*) >= 1 AS search_after_vacuum FROM ( SELECT 1 FROM memtable_reclaim_t ORDER BY body <@> to_bm25query('reclaim', 'memtable_reclaim_idx') LIMIT 10 ) sub; search_after_vacuum --------------------- t (1 row) -- Rebuild chain without auto-spill so pages come from FSM vs extend. SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO memtable_reclaim_t (body) SELECT 'reclaim2 ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; SELECT count(*)::int > 0 AS chain_rebuilt FROM bm25_memtable_chain('memtable_reclaim_idx'); chain_rebuilt --------------- t (1 row) -- VACUUM freed the DEAD pages and this rebuild reused them, so the main -- fork does not grow and no DEAD pages remain. SELECT (pg_relation_size('memtable_reclaim_idx'::regclass, 'main') - (SELECT sz_after_spill FROM reclaim_sizes)) = 0 AND (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_reclaim_idx')) = 0 AS single_cycle_growth_valid; single_cycle_growth_valid --------------------------- t (1 row) -- Multi-cycle: five spill→VACUUM cycles (VACUUM cannot run inside DO). -- With per-cycle FSM reuse, cumulative growth stays bounded by the live -- memtable work per cycle; without reuse it would grow ~N×K pages. CREATE TEMP TABLE multi_cycle_bounds ( sz_start bigint, max_live_pages int, num_cycles int ); INSERT INTO multi_cycle_bounds (sz_start, max_live_pages, num_cycles) SELECT pg_relation_size('memtable_reclaim_idx'::regclass, 'main')::bigint, 0, 5; -- Cycle 1 INSERT INTO memtable_reclaim_t (body) SELECT 'mc1 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_1; mc_spill_1 ------------ t (1 row) VACUUM ANALYZE memtable_reclaim_t; -- Cycle 2 INSERT INTO memtable_reclaim_t (body) SELECT 'mc2 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_2; mc_spill_2 ------------ t (1 row) VACUUM ANALYZE memtable_reclaim_t; -- Cycle 3 INSERT INTO memtable_reclaim_t (body) SELECT 'mc3 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_3; mc_spill_3 ------------ t (1 row) VACUUM ANALYZE memtable_reclaim_t; -- Cycle 4 INSERT INTO memtable_reclaim_t (body) SELECT 'mc4 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_4; mc_spill_4 ------------ t (1 row) VACUUM ANALYZE memtable_reclaim_t; -- Cycle 5 INSERT INTO memtable_reclaim_t (body) SELECT 'mc5 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_5; mc_spill_5 ------------ t (1 row) VACUUM ANALYZE memtable_reclaim_t; SELECT max_live_pages > 1 AS multi_cycle_chain_multi_page FROM multi_cycle_bounds; multi_cycle_chain_multi_page ------------------------------ t (1 row) -- Cumulative growth must stay within one live chain per cycle -- (num_cycles * max_live_pages). SELECT (pg_relation_size('memtable_reclaim_idx'::regclass, 'main') - (SELECT sz_start FROM multi_cycle_bounds)) <= (SELECT num_cycles * max_live_pages FROM multi_cycle_bounds) * current_setting('block_size')::bigint AS multi_cycle_growth_valid; multi_cycle_growth_valid -------------------------- t (1 row) DROP TABLE memtable_reclaim_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/memtable_spill.out000066400000000000000000000112371525762047700252670ustar00rootroot00000000000000-- Tests the on-disk memtable chain lifecycle introduced by the -- page-backed memtable rewrite (issue #374): -- -- * inserts append to chain pages, -- * manual bm25_spill_index() drains the chain into an L0 -- segment, -- * memtable_pages_threshold triggers an auto-spill at -- transaction end, -- * queries return correct results across spills. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE memtable_spill_test (id serial PRIMARY KEY, body text); CREATE INDEX memtable_spill_idx ON memtable_spill_test USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_spill_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Test 1: a freshly built empty index has no chain pages -- (metapage.memtable_head_blkno = InvalidBlockNumber). SELECT count(*) AS chain_pages_at_create FROM bm25_memtable_chain('memtable_spill_idx'); chain_pages_at_create ----------------------- 0 (1 row) -- Test 2: inserts populate the chain. Use enough distinct rows -- that more than one chain page is required (records are -- ~40-80 bytes after packing, plus per-doc TpVector overhead). INSERT INTO memtable_spill_test (body) SELECT 'alpha doc ' || i || ' ' || repeat('filler ', 4) FROM generate_series(1, 200) i; SELECT count(*) > 1 AS chain_grew_past_one_page FROM bm25_memtable_chain('memtable_spill_idx'); chain_grew_past_one_page -------------------------- t (1 row) -- Test 3: queries see memtable contents directly (no segments yet). SELECT count(*) AS alpha_hits_before_spill FROM ( SELECT 1 FROM memtable_spill_test ORDER BY body <@> to_bm25query('alpha', 'memtable_spill_idx') LIMIT 1000 ) sub; alpha_hits_before_spill ------------------------- 200 (1 row) -- Test 4: manual spill flushes the chain into an L0 segment and -- resets metapage.memtable_head_blkno/tail_blkno to Invalid. SELECT bm25_spill_index('memtable_spill_idx') IS NOT NULL AS produced_segment; produced_segment ------------------ t (1 row) SELECT count(*) AS chain_pages_after_spill FROM bm25_memtable_chain('memtable_spill_idx'); chain_pages_after_spill ------------------------- 0 (1 row) -- Test 5: queries still return the same answers (data is now in -- the segment, not the memtable). SELECT count(*) AS alpha_hits_after_spill FROM ( SELECT 1 FROM memtable_spill_test ORDER BY body <@> to_bm25query('alpha', 'memtable_spill_idx') LIMIT 1000 ) sub; alpha_hits_after_spill ------------------------ 200 (1 row) -- Test 6: spilling an empty chain is a no-op and returns NULL. SELECT bm25_spill_index('memtable_spill_idx') AS empty_chain_spill; empty_chain_spill ------------------- (1 row) -- Test 7: subsequent inserts build a fresh chain. Run them in -- a transaction with the auto-spill threshold disabled so the -- chain remains observable at COMMIT. SET pg_textsearch.memtable_pages_threshold = 0; BEGIN; INSERT INTO memtable_spill_test (body) SELECT 'beta ' || i FROM generate_series(1, 30) i; SELECT count(*) > 0 AS chain_rebuilt FROM bm25_memtable_chain('memtable_spill_idx'); chain_rebuilt --------------- t (1 row) COMMIT; -- After commit with threshold = 0 the chain still has the new -- entries; verify they survive across the COMMIT boundary. SELECT count(*) > 0 AS chain_persists_after_commit FROM bm25_memtable_chain('memtable_spill_idx'); chain_persists_after_commit ----------------------------- t (1 row) -- Test 8: memtable_pages_threshold (in chain pages) drains the -- chain at COMMIT. Spill once to start from empty, then insert -- past the threshold and verify the chain ends empty. SELECT bm25_spill_index('memtable_spill_idx') IS NOT NULL OR true AS reset; reset ------- t (1 row) SET pg_textsearch.memtable_pages_threshold = 1; INSERT INTO memtable_spill_test (body) SELECT 'gamma ' || i || ' delta ' || (i+1) || ' epsilon ' || (i+2) FROM generate_series(1, 100) i; SELECT count(*) AS chain_pages_after_autospill FROM bm25_memtable_chain('memtable_spill_idx'); chain_pages_after_autospill ----------------------------- 0 (1 row) RESET pg_textsearch.memtable_pages_threshold; -- Test 9: query the (now-spilled) data to confirm end-to-end -- correctness across the auto-spill path. SELECT count(*) > 0 AS gamma_queryable FROM ( SELECT 1 FROM memtable_spill_test ORDER BY body <@> to_bm25query('gamma', 'memtable_spill_idx') LIMIT 1000 ) sub; gamma_queryable ----------------- t (1 row) DROP TABLE memtable_spill_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/memtable_spill_dead.out000066400000000000000000000123531525762047700262440ustar00rootroot00000000000000-- Regression tests for spill step 4: tp_memtable_mark_chain_dead -- stamps every page of the spilled chain (including fragment -- continuation pages) with TP_MEMTABLE_PAGE_FLAG_DEAD before -- tp_spill_finalize unlinks the chain from the metapage. -- Orphans are visible via bm25_memtable_dead_pages(). CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Test 1: heap inserts, manual spill, dead orphan count matches -- the live chain page count that existed immediately before spill. CREATE TABLE memtable_spill_dead_t (id serial PRIMARY KEY, body text); CREATE INDEX memtable_spill_dead_idx ON memtable_spill_dead_t USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_spill_dead_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SELECT count(*) AS dead_pages_before_spill FROM bm25_memtable_dead_pages('memtable_spill_dead_idx'); dead_pages_before_spill ------------------------- 0 (1 row) INSERT INTO memtable_spill_dead_t (body) SELECT 'spill dead ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; CREATE TEMP TABLE spill_dead_live_pages AS SELECT count(*)::int AS n FROM bm25_memtable_chain('memtable_spill_dead_idx'); SELECT n > 1 AS chain_multi_page FROM spill_dead_live_pages; chain_multi_page ------------------ t (1 row) SELECT bm25_spill_index('memtable_spill_dead_idx') IS NOT NULL AS spill_ok; spill_ok ---------- t (1 row) SELECT count(*) AS live_chain_pages_after_spill FROM bm25_memtable_chain('memtable_spill_dead_idx'); live_chain_pages_after_spill ------------------------------ 0 (1 row) SELECT (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_spill_dead_idx')) = (SELECT n FROM spill_dead_live_pages) AS dead_matches_spilled_chain; dead_matches_spilled_chain ---------------------------- t (1 row) SELECT bool_and((flags & 1) <> 0) AS all_dead_flag_set, bool_and(dead_fxid > 0) AS all_dead_fxid_set FROM bm25_memtable_dead_pages('memtable_spill_dead_idx'); all_dead_flag_set | all_dead_fxid_set -------------------+------------------- t | t (1 row) -- Second spill on an empty chain must not add more dead pages. DO $$ BEGIN PERFORM bm25_spill_index('memtable_spill_dead_idx'); END $$; SELECT (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_spill_dead_idx')) = (SELECT n FROM spill_dead_live_pages) AS dead_count_unchanged_after_empty_respill; dead_count_unchanged_after_empty_respill ------------------------------------------ t (1 row) DROP TABLE memtable_spill_dead_t; -- Test 2: large INSERT forces fragment pages; spill stamps them DEAD. CREATE TABLE memtable_spill_dead_frag_t (id int, body text); CREATE INDEX memtable_spill_dead_frag_idx ON memtable_spill_dead_frag_t USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation memtable_spill_dead_frag_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO memtable_spill_dead_frag_t (body) SELECT string_agg('tok' || gs::text || ' ', '') FROM generate_series(1, 5000) gs; CREATE TEMP TABLE frag_spill_live_pages AS SELECT count(*)::int AS n FROM bm25_memtable_chain('memtable_spill_dead_frag_idx'); SELECT n > 2 AS frag_chain_has_continuations FROM frag_spill_live_pages; frag_chain_has_continuations ------------------------------ t (1 row) DO $$ BEGIN PERFORM bm25_spill_index('memtable_spill_dead_frag_idx'); END $$; SELECT count(*) AS live_pages_after_frag_spill FROM bm25_memtable_chain('memtable_spill_dead_frag_idx'); live_pages_after_frag_spill ----------------------------- 0 (1 row) SELECT (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx')) = (SELECT n FROM frag_spill_live_pages) AS frag_dead_matches_spilled_chain; frag_dead_matches_spilled_chain --------------------------------- t (1 row) SELECT bool_and((flags & 1) <> 0) AS frag_all_dead_flag, sum(((flags & 2) <> 0)::int)::int AS frag_dead_continuation_pages, sum(CASE WHEN (flags & 2) <> 0 THEN 0 ELSE 1 END)::int AS frag_dead_regular_pages FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx'); frag_all_dead_flag | frag_dead_continuation_pages | frag_dead_regular_pages --------------------+------------------------------+------------------------- t | 5 | 2 (1 row) SELECT sum(((flags & 2) <> 0)::int)::int > 0 AS frag_has_dead_continuation_pages FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx'); frag_has_dead_continuation_pages ---------------------------------- t (1 row) SELECT bool_and( CASE WHEN (flags & 2) <> 0 THEN n_records = 0 ELSE n_records >= 0 END ) AS frag_dead_continuation_zero_records FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx'); frag_dead_continuation_zero_records ------------------------------------- t (1 row) DROP TABLE memtable_spill_dead_frag_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/memtable_stale_tail_injection.out000066400000000000000000000106271525762047700303310ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION dblink; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE tail_retry (id int, body text); CREATE INDEX tail_retry_idx ON tail_retry USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO tail_retry VALUES (0, 'retry seed'); SELECT dblink_connect('retry_writer', format( 'host=%s port=%s dbname=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database())); dblink_connect OK (1 row) SELECT dblink_exec('retry_writer', 'SET pg_textsearch.memtable_pages_threshold = 0'); dblink_exec SET (1 row) SELECT dblink_exec('retry_writer', 'SET pg_textsearch.bulk_load_threshold = 0'); dblink_exec SET (1 row) SELECT * FROM dblink('retry_writer', 'SELECT injection_points_set_local()') AS t(result text); result (1 row) -- Force both append paths to encounter a legitimately advanced tail. DO $$ DECLARE point text := 'pg-textsearch-memtable-tail-read'; writer_pid int; terms int; old_tail bigint; new_tail bigint; inserted_id int; deadline timestamptz; BEGIN SELECT pid INTO writer_pid FROM dblink('retry_writer', 'SELECT pg_backend_pid()') AS t(pid int); FOREACH terms IN ARRAY ARRAY[60, 1000] LOOP SELECT blkno INTO STRICT old_tail FROM bm25_memtable_chain('tail_retry_idx') WHERE next_block IS NULL; PERFORM result FROM dblink('retry_writer', format( 'SELECT injection_points_attach(%L, %L)', point, 'wait') ) AS t(result text); PERFORM dblink_send_query('retry_writer', format($q$ INSERT INTO tail_retry SELECT %s, 'retry ' || string_agg(md5(j::text), ' ') FROM generate_series(1, %s) j RETURNING id $q$, terms, terms)); deadline := clock_timestamp() + interval '30 seconds'; LOOP PERFORM pg_stat_clear_snapshot(); EXIT WHEN EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = writer_pid AND wait_event_type = 'InjectionPoint' AND wait_event = point ); IF dblink_is_busy('retry_writer') = 0 THEN RAISE EXCEPTION 'writer finished without the tail-read pause'; END IF; IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'timed out waiting for the tail-read pause'; END IF; PERFORM pg_sleep(0.01); END LOOP; -- An oversized append always publishes a new tail. INSERT INTO tail_retry SELECT -terms, 'retry ' || string_agg(md5(j::text), ' ') FROM generate_series(1, 1000) j; SELECT blkno INTO STRICT new_tail FROM bm25_memtable_chain('tail_retry_idx') WHERE next_block IS NULL; IF new_tail = old_tail OR NOT EXISTS ( SELECT 1 FROM bm25_memtable_chain('tail_retry_idx') WHERE blkno = old_tail AND next_block IS NOT NULL ) THEN RAISE EXCEPTION 'concurrent append did not advance the old tail'; END IF; -- Detach before waking so the metapage reread cannot pause again. PERFORM injection_points_detach(point); PERFORM injection_points_wakeup(point); deadline := clock_timestamp() + interval '30 seconds'; WHILE dblink_is_busy('retry_writer') <> 0 LOOP IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'writer did not retry the advanced tail'; END IF; PERFORM pg_sleep(0.01); END LOOP; SELECT id INTO STRICT inserted_id FROM dblink_get_result('retry_writer') AS t(id int); IF inserted_id <> terms THEN RAISE EXCEPTION 'retry inserted the wrong document'; END IF; PERFORM id FROM dblink_get_result('retry_writer') AS t(id int); END LOOP; END $$; SELECT dblink_disconnect('retry_writer'); dblink_disconnect OK (1 row) SELECT array_agg(id ORDER BY id) FROM ( SELECT id FROM tail_retry ORDER BY body <@> to_bm25query('retry', 'tail_retry_idx') LIMIT 10 ) ranked; array_agg {-1000,-60,0,60,1000} (1 row) DROP TABLE tail_retry; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP EXTENSION dblink; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/merge.out000066400000000000000000000311231525762047700233710ustar00rootroot00000000000000-- Test case: merge -- Tests segment merge functionality and queries across multiple segment levels. -- -- This test exercises: -- 1. Multiple L0 segments -- 2. L0 -> L1 merge triggered by segments_per_level = 2 -- 3. Queries across multiple segment levels (L0, L1) CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Enable merge: 2 segments per level triggers L0->L1 merge on 2nd spill SET pg_textsearch.segments_per_level = 2; -- Create table and index CREATE TABLE merge_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX merge_test_idx ON merge_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation merge_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -------------------------------------------------------------------------------- -- Phase 1: First batch - will become first L0 segment after spill -------------------------------------------------------------------------------- INSERT INTO merge_test (content) VALUES ('hello world database'); INSERT INTO merge_test (content) VALUES ('goodbye cruel world'); INSERT INTO merge_test (content) VALUES ('hello goodbye friend'); INSERT INTO merge_test (content) VALUES ('world peace harmony'); -- Verify data is queryable in memtable SELECT 'Phase 1: memtable only' AS phase; phase ------------------------ Phase 1: memtable only (1 row) SELECT COUNT(*) AS count_before_spill FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; count_before_spill -------------------- 2 (1 row) -- First spill creates segment 1 in L0 SELECT bm25_spill_index('merge_test_idx') IS NOT NULL AS first_spill; first_spill ------------- t (1 row) -- Verify data is still queryable from L0 segment SELECT 'Phase 1b: after first spill (1 segment in L0)' AS phase; phase ----------------------------------------------- Phase 1b: after first spill (1 segment in L0) (1 row) SELECT COUNT(*) AS hello_count_after_first_spill FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; hello_count_after_first_spill ------------------------------- 2 (1 row) -------------------------------------------------------------------------------- -- Phase 2: Second batch - triggers L0->L1 merge (segments_per_level=2) -------------------------------------------------------------------------------- INSERT INTO merge_test (content) VALUES ('database indexing query'); INSERT INTO merge_test (content) VALUES ('search engine optimization'); INSERT INTO merge_test (content) VALUES ('database world news'); INSERT INTO merge_test (content) VALUES ('goodbye database friend'); -- Second spill creates segment 2 in L0, which triggers merge to L1 -- After merge: L0 is empty (cleared), L1 has merged segment -- Note: Run with client_min_messages=debug1 to see merge DEBUG messages SELECT bm25_spill_index('merge_test_idx') IS NOT NULL AS second_spill; second_spill -------------- t (1 row) -- After merge: merged segment in L1, L0 is empty SELECT 'Phase 2: after second spill + merge (L0 empty, 1 segment in L1)' AS phase; phase ----------------------------------------------------------------- Phase 2: after second spill + merge (L0 empty, 1 segment in L1) (1 row) -- Should find 2 documents with 'hello' (both in L1 merged segment) SELECT COUNT(*) AS hello_count_after_merge FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; hello_count_after_merge ------------------------- 2 (1 row) -- Should find 4 documents with 'database' (all in L1 merged segment) SELECT COUNT(*) AS database_count_after_merge FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('database', 'merge_test_idx') LIMIT 100 ) t; database_count_after_merge ---------------------------- 4 (1 row) -- Validate BM25 scoring is correct across L1 data SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid_after_merge; hello_valid_after_merge ------------------------- t (1 row) SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'database', 'english', 1.2, 0.75) AS database_valid_after_merge; database_valid_after_merge ---------------------------- t (1 row) SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid_after_merge; world_valid_after_merge ------------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 3: Post-merge inserts (new memtable data + 1 L1 segment) -------------------------------------------------------------------------------- INSERT INTO merge_test (content) VALUES ('hello new insertion'); INSERT INTO merge_test (content) VALUES ('database transaction log'); SELECT 'Phase 3: post-merge inserts (memtable + 1 L1 segment)' AS phase; phase ------------------------------------------------------- Phase 3: post-merge inserts (memtable + 1 L1 segment) (1 row) -- Should find 3 documents with 'hello' (2 in L1, 1 in memtable) SELECT COUNT(*) AS hello_count_with_new_inserts FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; hello_count_with_new_inserts ------------------------------ 3 (1 row) -- Should find 5 documents with 'database' (4 in L1, 1 in memtable) SELECT COUNT(*) AS database_count_with_new_inserts FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('database', 'merge_test_idx') LIMIT 100 ) t; database_count_with_new_inserts --------------------------------- 5 (1 row) -- Validate BM25 scoring with mixed sources (memtable + L1 segment) SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid_mixed; hello_valid_mixed ------------------- t (1 row) SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'database', 'english', 1.2, 0.75) AS database_valid_mixed; database_valid_mixed ---------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 4: Verify total document count -------------------------------------------------------------------------------- SELECT 'Phase 4: final verification' AS phase; phase ----------------------------- Phase 4: final verification (1 row) SELECT COUNT(*) AS total_documents FROM merge_test; total_documents ----------------- 10 (1 row) -- Show final scores for reference SELECT id, content, ROUND((content <@> to_bm25query('database', 'merge_test_idx'))::numeric, 4) AS database_score FROM merge_test ORDER BY content <@> to_bm25query('database', 'merge_test_idx'), id; id | content | database_score ----+--------------------------+---------------- 1 | hello world database | -0.6931 5 | database indexing query | -0.6931 7 | database world news | -0.6931 8 | goodbye database friend | -0.6931 10 | database transaction log | -0.6931 (5 rows) -------------------------------------------------------------------------------- -- Phase 7: Bound ordinary compaction outputs -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 4; SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE merge_bounded (id bigint PRIMARY KEY, content text); CREATE INDEX merge_bounded_idx ON merge_bounded USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation merge_bounded_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..3 LOOP INSERT INTO merge_bounded SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 4) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('merge_bounded_idx'); END LOOP; END $$; DO $$ DECLARE summary text := bm25_summarize_index('merge_bounded_idx'); BEGIN IF regexp_count(summary, 'L1 Segment [0-9]+:') <> 2 OR summary ~ 'L0 Segment' THEN RAISE EXCEPTION 'ordinary bounded compaction layout is wrong: %', summary; END IF; END $$; SELECT count(*) = 8000 AS bounded_merge_docs_preserved FROM ( SELECT 1 FROM merge_bounded ORDER BY content <@> to_bm25query('common', 'merge_bounded_idx') LIMIT 8000 ) ranked; bounded_merge_docs_preserved ------------------------------ t (1 row) RESET pg_textsearch.max_segment_size; RESET pg_textsearch.segments_per_level; DROP TABLE merge_bounded CASCADE; -- A level with no legal count reduction must not starve a higher -- level that has one. Explicit spills only, so the layout below is -- deterministic. SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.max_segment_size = '1MB'; SET pg_textsearch.segments_per_level = 2; CREATE TABLE merge_starve (id bigint PRIMARY KEY, content text); CREATE INDEX merge_starve_idx ON merge_starve USING bm25(content) WITH (text_config='simple', compaction='manual'); NOTICE: BM25 index build started for relation merge_starve_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DO $$ DECLARE batch integer; BEGIN -- Eight pairable segments compact into a four-segment L1 run. FOR batch IN 0..7 LOOP INSERT INTO merge_starve SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 4) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('merge_starve_idx'); IF batch % 2 = 1 AND NOT bm25_compact_step('merge_starve_idx') THEN RAISE EXCEPTION 'failed to construct an L1 segment'; END IF; END LOOP; PERFORM set_config('pg_textsearch.segments_per_level', '8', false); -- Each of these alone exceeds the budget, so L0 becomes an -- uncombinable run while staying below the level threshold. FOR batch IN 8..11 LOOP INSERT INTO merge_starve SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 32) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('merge_starve_idx'); END LOOP; END $$; -- Widening the budget makes L1's run legally mergeable (4 x 0.8MB fits) -- while L0's 2.1MB segments still cannot pair. Lowering the threshold -- triggers both levels at once. SET pg_textsearch.max_segment_size = '4MB'; SET pg_textsearch.segments_per_level = 4; INSERT INTO merge_starve SELECT 24000 + gs, 'common ' || repeat(md5((24000 + gs)::text), 32) FROM generate_series(1, 2000) gs; SELECT bm25_spill_index('merge_starve_idx') > 0 AS starve_spilled; starve_spilled ---------------- t (1 row) DO $$ DECLARE summary text := bm25_summarize_index('merge_starve_idx'); BEGIN IF NOT bm25_compact_step('merge_starve_idx') THEN RAISE EXCEPTION 'higher-level compaction did not run'; END IF; summary := bm25_summarize_index('merge_starve_idx'); IF regexp_count(summary, 'L1 Segment [0-9]+:') <> 1 THEN RAISE EXCEPTION 'uncombinable L0 starved a compactable L1: %', summary; END IF; END $$; SELECT count(*) = 26000 AS starve_docs_preserved FROM ( SELECT 1 FROM merge_starve ORDER BY content <@> to_bm25query('common', 'merge_starve_idx') LIMIT 26000 ) ranked; starve_docs_preserved ----------------------- t (1 row) RESET pg_textsearch.max_segment_size; RESET pg_textsearch.segments_per_level; RESET pg_textsearch.bulk_load_threshold; RESET pg_textsearch.memtable_pages_threshold; DROP TABLE merge_starve CASCADE; -- Cleanup DROP TABLE merge_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/merge_injection.out000066400000000000000000000120051525762047700254310ustar00rootroot00000000000000SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.segments_per_level = 3; SELECT pg_textsearch_test_attach_segment_limit(2); pg_textsearch_test_attach_segment_limit ----------------------------------------- (1 row) CREATE TABLE merge_capacity (id serial PRIMARY KEY, content text); CREATE INDEX merge_capacity_idx ON merge_capacity USING bm25(content) WITH (text_config='english', compaction='manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO merge_capacity (content) VALUES (format('capacity document %s filler', n)); PERFORM bm25_spill_index('merge_capacity_idx'); END LOOP; END $$; INSERT INTO merge_capacity (content) VALUES ('capacity pending document filler'); DO $$ BEGIN BEGIN PERFORM bm25_spill_index('merge_capacity_idx'); RAISE EXCEPTION 'expected segment count limit error'; EXCEPTION WHEN program_limit_exceeded THEN IF SQLERRM <> 'bm25 segment count limit reached at level 0' THEN RAISE; END IF; END; END $$; DO $$ DECLARE summary text := bm25_summarize_index('merge_capacity_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 2 OR summary !~ 'L0 Segment 2:' OR summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'segment capacity failure mutated the index: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM merge_capacity ORDER BY content <@> to_bm25query('capacity', 'merge_capacity_idx') ) ranked) <> 3 THEN RAISE EXCEPTION 'segment capacity failure lost documents'; END IF; END $$; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); injection_points_detach ------------------------- (1 row) DROP TABLE merge_capacity CASCADE; CREATE TABLE merge_nested_full (id serial PRIMARY KEY, content text); CREATE INDEX merge_nested_full_idx ON merge_nested_full USING bm25(content) WITH (text_config='english', compaction='manual'); -- Construct two full destination levels without relying on tiny outputs -- being promoted repeatedly. DO $$ DECLARE n integer; BEGIN FOR level IN REVERSE 2..1 LOOP FOR n IN 1..2 LOOP INSERT INTO merge_nested_full (content) SELECT format('nested full document %s filler', gs) FROM generate_series(1, 6) gs; PERFORM bm25_spill_index('merge_nested_full_idx'); END LOOP; PERFORM pg_textsearch_test_move_level( 'merge_nested_full_idx', 0, level); END LOOP; END $$; DO $$ DECLARE summary text := bm25_summarize_index('merge_nested_full_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 4 OR summary !~ 'L1 Segment 2:' OR summary !~ 'L2 Segment 2:' OR summary ~ 'L0 Segment 1:' THEN RAISE EXCEPTION 'nested capacity layout was not constructed: %', summary; END IF; END $$; SELECT pg_textsearch_test_attach_segment_limit(2); pg_textsearch_test_attach_segment_limit ----------------------------------------- (1 row) SET pg_textsearch.segments_per_level = 2; DO $$ DECLARE n integer; BEGIN FOR n IN 25..26 LOOP INSERT INTO merge_nested_full (content) VALUES (format('nested full document %s filler', n)); PERFORM bm25_spill_index('merge_nested_full_idx'); END LOOP; END $$; SELECT bm25_compact_step('merge_nested_full_idx'); bm25_compact_step ------------------- t (1 row) DO $$ DECLARE summary text := bm25_summarize_index('merge_nested_full_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 4 OR summary !~ 'L1 Segment 2:' OR summary !~ 'L2 Segment 2:' OR summary ~ 'L0 Segment 1:' THEN RAISE EXCEPTION 'nested capacity blockers were not drained: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM merge_nested_full ORDER BY content <@> to_bm25query('nested', 'merge_nested_full_idx') ) ranked) <> 26 THEN RAISE EXCEPTION 'nested capacity compaction lost documents'; END IF; END $$; SELECT bm25_compact('merge_nested_full_idx'); bm25_compact -------------- (1 row) SELECT bm25_level_counts('merge_nested_full_idx') = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AND NOT bm25_needs_compaction('merge_nested_full_idx') AS capacity_recourse_converges_without_promotion; capacity_recourse_converges_without_promotion ----------------------------------------------- t (1 row) SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); injection_points_detach ------------------------- (1 row) RESET pg_textsearch.segments_per_level; DROP TABLE merge_nested_full CASCADE; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/mixed.out000066400000000000000000000373501525762047700234100ustar00rootroot00000000000000-- Test concurrent operations on pg_textsearch indexes -- This test verifies that concurrent access to shared memory structures is safe -- and that operations like inserts, searches, and index building work correctly CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Clean up from any previous tests DROP TABLE IF EXISTS concurrent_test_docs CASCADE; NOTICE: table "concurrent_test_docs" does not exist, skipping DROP TABLE IF EXISTS concurrent_test_docs2 CASCADE; NOTICE: table "concurrent_test_docs2" does not exist, skipping -- Create test tables for concurrent operations CREATE TABLE concurrent_test_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL, category VARCHAR(50), created_at TIMESTAMP DEFAULT NOW() ); CREATE TABLE concurrent_test_docs2 ( id SERIAL PRIMARY KEY, content TEXT NOT NULL, priority INTEGER DEFAULT 1 ); -- Test 1: Sequential inserts to verify basic functionality \echo 'Test 1: Sequential baseline test' Test 1: Sequential baseline test INSERT INTO concurrent_test_docs (content, category) VALUES ('database query optimization techniques', 'tech'), ('concurrent programming patterns', 'tech'), ('shared memory management systems', 'tech'), ('lock-free data structures', 'tech'), ('database indexing strategies', 'tech'); -- Create index after initial data CREATE INDEX concurrent_idx1 ON concurrent_test_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation concurrent_idx1 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.80 -- Verify basic search works (only matching documents via index) SELECT id, content, ROUND((content <@> to_bm25query('database concurrent', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY content <@> to_bm25query('database concurrent', 'concurrent_idx1') LIMIT 10; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.5169 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-0.9580 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.8570 id | content | score ----+----------------------------------------+--------- 2 | concurrent programming patterns | -1.5169 5 | database indexing strategies | -0.9580 1 | database query optimization techniques | -0.8570 (3 rows) -- Test 2: Simulate concurrent inserts with same terms \echo 'Test 2: Concurrent inserts with overlapping terms' Test 2: Concurrent inserts with overlapping terms -- Insert documents that will compete for same posting lists INSERT INTO concurrent_test_docs (content, category) VALUES ('database systems require concurrent access control', 'tech'), ('concurrent database operations need careful synchronization', 'tech'), ('database concurrent processing improves performance', 'tech'); -- Search to verify posting lists updated correctly SELECT id, content, ROUND((content <@> to_bm25query('database concurrent', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 5; id | content | score ----+-------------------------------------------------------------+--------- 8 | database concurrent processing improves performance | -1.1341 6 | database systems require concurrent access control | -1.0433 7 | concurrent database operations need careful synchronization | -1.0433 2 | concurrent programming patterns | -0.8026 5 | database indexing strategies | -0.5702 (5 rows) -- Test 3: Multiple index creation (simulates concurrent index builds) \echo 'Test 3: Multiple index creation' Test 3: Multiple index creation -- Create second table and index to test multiple indexes INSERT INTO concurrent_test_docs2 (content, priority) VALUES ('high priority database tasks', 3), ('concurrent processing systems', 2), ('shared memory data structures', 1), ('database performance optimization', 3), ('concurrent system design patterns', 2); CREATE INDEX concurrent_idx2 ON concurrent_test_docs2 USING bm25(content) WITH (text_config='english', k1=1.5, b=0.8); NOTICE: BM25 index build started for relation concurrent_idx2 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.50, b=0.80 NOTICE: BM25 index build completed: 5 documents, avg_length=3.60 -- Verify both indexes work independently SELECT 'Table 1' as source, id, content FROM concurrent_test_docs UNION ALL SELECT 'Table 2' as source, id, content FROM concurrent_test_docs2 ORDER BY source, id; source | id | content ---------+----+------------------------------------------------------------- Table 1 | 1 | database query optimization techniques Table 1 | 2 | concurrent programming patterns Table 1 | 3 | shared memory management systems Table 1 | 4 | lock-free data structures Table 1 | 5 | database indexing strategies Table 1 | 6 | database systems require concurrent access control Table 1 | 7 | concurrent database operations need careful synchronization Table 1 | 8 | database concurrent processing improves performance Table 2 | 1 | high priority database tasks Table 2 | 2 | concurrent processing systems Table 2 | 3 | shared memory data structures Table 2 | 4 | database performance optimization Table 2 | 5 | concurrent system design patterns (13 rows) -- Test 4: Stress test with many inserts \echo 'Test 4: Bulk insert stress test' Test 4: Bulk insert stress test -- Generate series of documents with overlapping terms INSERT INTO concurrent_test_docs (content, category) SELECT 'test document number ' || i || ' contains database and concurrent terms', 'stress' FROM generate_series(1, 50) i; -- Verify search still works with larger dataset SELECT COUNT(*) as total_matches FROM concurrent_test_docs; total_matches --------------- 58 (1 row) -- Test top results still make sense SELECT id, substring(content, 1, 50) || '...' as content_preview, ROUND((content <@> to_bm25query('database concurrent', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 8; id | content_preview | score ----+-------------------------------------------------------+--------- 8 | database concurrent processing improves performanc... | -0.1628 7 | concurrent database operations need careful synchr... | -0.1531 6 | database systems require concurrent access control... | -0.1531 10 | test document number 2 contains database and concu... | -0.1369 9 | test document number 1 contains database and concu... | -0.1369 11 | test document number 3 contains database and concu... | -0.1369 12 | test document number 4 contains database and concu... | -0.1369 13 | test document number 5 contains database and concu... | -0.1369 (8 rows) -- Test 5: Mixed operations (insert while searching) \echo 'Test 5: Mixed insert and search operations' Test 5: Mixed insert and search operations -- This simulates a session inserting while another searches BEGIN; INSERT INTO concurrent_test_docs (content, category) VALUES ('new concurrent database research paper', 'research'); -- Search within same transaction SELECT id, content, ROUND((content <@> to_bm25query('research database', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 3; id | content | score ----+----------------------------------------+--------- 59 | new concurrent database research paper | -4.3363 5 | database indexing strategies | -0.0796 1 | database query optimization techniques | -0.0742 (3 rows) COMMIT; -- Verify the insert is visible after commit SELECT id, content, ROUND((content <@> to_bm25query('research database', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 3; id | content | score ----+----------------------------------------+--------- 59 | new concurrent database research paper | -4.3363 5 | database indexing strategies | -0.0796 1 | database query optimization techniques | -0.0742 (3 rows) -- Test 6: Index integrity under updates \echo 'Test 6: Update operations' Test 6: Update operations -- Update some documents to test posting list maintenance UPDATE concurrent_test_docs SET content = 'updated database system with enhanced concurrent features' WHERE id IN (1, 2); -- Verify search finds updated documents SELECT * FROM ( SELECT id, content, ROUND((content <@> to_bm25query('enhanced database', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY content <@> to_bm25query('enhanced database', 'concurrent_idx1') LIMIT 10 ) sub ORDER BY score, id; NOTICE: BM25 index scan: tid=(0,60), BM25_score=-3.5477 NOTICE: BM25 index scan: tid=(0,61), BM25_score=-3.5477 NOTICE: BM25 index scan: tid=(0,5), BM25_score=-0.0768 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.0716 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-0.0671 NOTICE: BM25 index scan: tid=(0,59), BM25_score=-0.0671 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-0.0631 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.0631 NOTICE: BM25 index scan: tid=(0,9), BM25_score=-0.0563 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-0.0563 NOTICE: BM25 index scan: tid=(0,11), BM25_score=-0.0563 id | content | score ----+---------------------------------------------------------------+--------- 1 | updated database system with enhanced concurrent features | -3.5477 2 | updated database system with enhanced concurrent features | -3.5477 5 | database indexing strategies | -0.0768 8 | database concurrent processing improves performance | -0.0671 59 | new concurrent database research paper | -0.0671 6 | database systems require concurrent access control | -0.0631 7 | concurrent database operations need careful synchronization | -0.0631 9 | test document number 1 contains database and concurrent terms | -0.0563 10 | test document number 2 contains database and concurrent terms | -0.0563 11 | test document number 3 contains database and concurrent terms | -0.0563 (10 rows) -- Test 7: Delete operations \echo 'Test 7: Delete operations' Test 7: Delete operations -- Count before delete SELECT COUNT(*) as count_before_delete FROM concurrent_test_docs; count_before_delete --------------------- 59 (1 row) -- Delete some stress test documents DELETE FROM concurrent_test_docs WHERE content LIKE 'test document number%' AND id % 5 = 0; -- Count after delete to verify cleanup SELECT COUNT(*) as count_after_delete FROM concurrent_test_docs; count_after_delete -------------------- 49 (1 row) -- Test 8: Verify string interning consistency \echo 'Test 8: String interning consistency' Test 8: String interning consistency -- Insert documents with exact same terms to test string interning INSERT INTO concurrent_test_docs (content, category) VALUES ('exact same terms for testing consistency', 'test'), ('exact same terms for testing consistency', 'test'), ('testing consistency with exact same terms', 'test'); -- Search should find all variants SELECT id, content, ROUND((content <@> to_bm25query('exact same terms', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY content <@> to_bm25query('exact same terms', 'concurrent_idx1'), id LIMIT 10; id | content | score ----+---------------------------------------------------------------+--------- 60 | exact same terms for testing consistency | -3.8184 61 | exact same terms for testing consistency | -3.8184 62 | testing consistency with exact same terms | -3.8184 9 | test document number 1 contains database and concurrent terms | -0.1870 11 | test document number 3 contains database and concurrent terms | -0.1870 12 | test document number 4 contains database and concurrent terms | -0.1870 13 | test document number 5 contains database and concurrent terms | -0.1870 14 | test document number 6 contains database and concurrent terms | -0.1870 16 | test document number 8 contains database and concurrent terms | -0.1870 17 | test document number 9 contains database and concurrent terms | -0.1870 (10 rows) -- Test 9: Multiple indexes on same table \echo 'Test 9: Multiple indexes on same table' Test 9: Multiple indexes on same table -- Create a test table for multiple indexes CREATE TABLE multi_idx_test ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); -- Insert initial data INSERT INTO multi_idx_test (content) VALUES ('hello world from the test'), ('the quick brown fox jumps'), ('hello from another document'); -- Create English index first (only BM25 index, planner must use it) CREATE INDEX multi_idx_english ON multi_idx_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation multi_idx_english NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.33 -- Insert more data after index creation INSERT INTO multi_idx_test (content) VALUES ('world of databases and indexes'), ('hello database world'); -- Query English index while it's the only BM25 index SET pg_textsearch.log_scores = false; SELECT * FROM ( SELECT id, content, ROUND((content <@> to_bm25query('hello world', 'multi_idx_english'))::numeric, 4) as english_score FROM multi_idx_test ORDER BY content <@> to_bm25query('hello world', 'multi_idx_english') LIMIT 10 ) sub ORDER BY english_score, id; id | content | english_score ----+--------------------------------+--------------- 1 | hello world from the test | -1.1063 5 | hello database world | -1.1063 3 | hello from another document | -0.5531 4 | world of databases and indexes | -0.5531 (4 rows) -- Now create Simple index too CREATE INDEX multi_idx_simple ON multi_idx_test USING bm25(content) WITH (text_config='simple', k1=1.5, b=0.8); NOTICE: BM25 index build started for relation multi_idx_simple NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.50, b=0.80 NOTICE: BM25 index build completed: 5 documents, avg_length=4.40 -- Verify both indexes exist and function independently SELECT i.indexrelid::regclass as index_name, pg_size_pretty(pg_relation_size(i.indexrelid)) as index_size FROM pg_index i JOIN pg_class c ON c.oid = i.indrelid WHERE c.relname = 'multi_idx_test' ORDER BY index_name; index_name | index_size ---------------------+------------ multi_idx_test_pkey | 16 kB multi_idx_english | 32 kB multi_idx_simple | 24 kB (3 rows) -- Clean up test table DROP TABLE multi_idx_test CASCADE; -- Final verification \echo 'Final verification: Index statistics' Final verification: Index statistics SELECT 'concurrent_test_docs' as table_name, COUNT(*) as total_docs, COUNT(DISTINCT content) as unique_docs FROM concurrent_test_docs UNION ALL SELECT 'concurrent_test_docs2' as table_name, COUNT(*) as total_docs, COUNT(DISTINCT content) as unique_docs FROM concurrent_test_docs2; table_name | total_docs | unique_docs -----------------------+------------+------------- concurrent_test_docs | 52 | 50 concurrent_test_docs2 | 5 | 5 (2 rows) -- Clean up DROP TABLE concurrent_test_docs CASCADE; DROP TABLE concurrent_test_docs2 CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/parallel_bmw.out000066400000000000000000000070231525762047700247350ustar00rootroot00000000000000-- Test case: parallel_bmw -- Validates that parallel index build computes correct block_max_norm -- (MIN fieldnorm) for BMW skip entries, matching the serial build path. -- -- The bug: build_parallel.c computed MAX fieldnorm instead of MIN, -- causing BMW to underestimate block score upper bounds and incorrectly -- skip blocks containing high-scoring short documents. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; SET min_parallel_table_scan_size = 0; SET maintenance_work_mem = '256MB'; -------------------------------------------------------------------------------- -- Setup: 3-tier document design to expose block_max_norm bug -------------------------------------------------------------------------------- -- The posting list for "target" is ordered by CTID (insertion order): -- -- Rows 1-5000: MEDIUM docs (50 words with "target") -> set threshold -- Rows 5001-7000: SHORT + LONG docs (alternating, both with "target") -- Rows 7001-150000: FILLER docs (no "target", for parallel build sizing) -- -- BMW processes blocks sequentially. Medium-doc blocks are processed first, -- establishing a threshold. Then mixed short+long blocks follow: -- -- Correct (MIN fieldnorm): upper bound from short doc -> above threshold -- Buggy (MAX fieldnorm): upper bound from long doc -> below threshold -- -- With the bug, BMW skips the mixed blocks entirely, missing the short -- docs that should rank highest in top-k results. SET max_parallel_maintenance_workers = 2; CREATE TABLE parallel_bmw_test ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_bmw_test (content) SELECT CASE -- Medium docs: "target" + 49 filler words = 50 total WHEN i <= 5000 THEN 'target ' || repeat('alpha ', 49) -- Short docs (odd): just "target" = 1 word WHEN i <= 7000 AND i % 2 = 1 THEN 'target' -- Long docs (even): "target" + 199 filler words = 200 total WHEN i <= 7000 THEN 'target ' || repeat('beta ', 199) -- Filler: no "target" term ELSE 'gamma delta' END FROM generate_series(1, 150000) AS i; ANALYZE parallel_bmw_test; CREATE INDEX parallel_bmw_test_idx ON parallel_bmw_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_bmw_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 150000 documents, avg_length=4.91 -------------------------------------------------------------------------------- -- Test 1: BMW top-k must find short docs despite them being in later blocks -------------------------------------------------------------------------------- -- Short docs (1 word) score highest due to BM25 length normalization. -- With the bug, their blocks are skipped because the upper bound is -- computed from the long docs (200 words) sharing those blocks, and that -- upper bound falls below the threshold set by medium docs (50 words). SELECT COUNT(*) AS short_docs_in_top10 FROM ( SELECT content FROM parallel_bmw_test ORDER BY content <@> to_bm25query('target', 'parallel_bmw_test_idx') LIMIT 10 ) sub WHERE content = 'target'; short_docs_in_top10 --------------------- 10 (1 row) -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- DROP TABLE parallel_bmw_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/parallel_build.out000066400000000000000000000755671525762047700252710ustar00rootroot00000000000000-- Test case: parallel_build -- Tests parallel index build with various worker counts and corner cases CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -- Lower threshold so our test tables qualify for parallel build SET min_parallel_table_scan_size = 0; -- Sufficient memory for parallel workers (32MB per worker + leader) SET maintenance_work_mem = '256MB'; -------------------------------------------------------------------------------- -- Test 1: Serial build (0 workers) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 0; CREATE TABLE parallel_test_serial ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_serial (content) SELECT 'Document ' || i || ' contains words like apple banana cherry' FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_serial; -- Should NOT show "parallel index build" notice (serial build) CREATE INDEX parallel_test_serial_idx ON parallel_test_serial USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_serial_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100000 documents, avg_length=8.00 -- Verify queries work SELECT COUNT(*) AS apple_count FROM (SELECT 1 FROM parallel_test_serial ORDER BY content <@> to_bm25query('apple', 'parallel_test_serial_idx')) sub; apple_count ------------- 100000 (1 row) -------------------------------------------------------------------------------- -- Test 2: Single worker (edge case - minimum parallelism) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 1; CREATE TABLE parallel_test_1worker ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_1worker (content) SELECT 'Document ' || i || ' with database query optimization terms' FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_1worker; -- Should show "parallel index build: launched 1 of 1 requested workers" CREATE INDEX parallel_test_1worker_idx ON parallel_test_1worker USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_1worker_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 1 of 1 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=6.00 SELECT COUNT(*) AS database_count FROM (SELECT 1 FROM parallel_test_1worker ORDER BY content <@> to_bm25query('database', 'parallel_test_1worker_idx')) sub; database_count ---------------- 100000 (1 row) REINDEX INDEX parallel_test_1worker_idx; NOTICE: BM25 index build started for relation parallel_test_1worker_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 1 of 1 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=6.00 -- Parallel workers must flatten array-valued index expressions too. CREATE INDEX parallel_test_array_expr_idx ON parallel_test_1worker USING bm25 ((ARRAY[content, 'arrayneedle'])) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_array_expr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 1 of 1 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=7.00 SELECT count(*) AS array_expression_hits FROM ( SELECT id FROM parallel_test_1worker ORDER BY ARRAY[content, 'arrayneedle'] <@> to_bm25query('arrayneedle', 'parallel_test_array_expr_idx') LIMIT 5 ) hits; array_expression_hits ----------------------- 5 (1 row) DROP INDEX parallel_test_array_expr_idx; -------------------------------------------------------------------------------- -- Test 3: Two workers (common case) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; CREATE TABLE parallel_test_2workers ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_2workers (content) SELECT 'Document number ' || i || ' contains words like ' || CASE i % 5 WHEN 0 THEN 'apple banana cherry' WHEN 1 THEN 'database query optimization' WHEN 2 THEN 'machine learning algorithm' WHEN 3 THEN 'distributed systems design' ELSE 'network protocol security' END || ' and additional text to make it longer ' || CASE i % 3 WHEN 0 THEN 'with some extra content here' WHEN 1 THEN 'including more varied terms' ELSE 'plus different phrases altogether' END FROM generate_series(1, 150000) AS i; ANALYZE parallel_test_2workers; -- Should show "parallel index build: launched 2 of 2 requested workers" CREATE INDEX parallel_test_2workers_idx ON parallel_test_2workers USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_2workers_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 150000 documents, avg_length=16.00 -- Verify all document groups are searchable SELECT COUNT(*) AS apple_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('apple', 'parallel_test_2workers_idx')) sub; apple_count ------------- 30000 (1 row) SELECT COUNT(*) AS database_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('database', 'parallel_test_2workers_idx')) sub; database_count ---------------- 30000 (1 row) SELECT COUNT(*) AS machine_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('machine', 'parallel_test_2workers_idx')) sub; machine_count --------------- 30000 (1 row) SELECT COUNT(*) AS distributed_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('distributed', 'parallel_test_2workers_idx')) sub; distributed_count ------------------- 30000 (1 row) SELECT COUNT(*) AS network_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('network', 'parallel_test_2workers_idx')) sub; network_count --------------- 30000 (1 row) -- Verify top-k ordering (limit to deterministic results) SELECT id, ROUND((content <@> to_bm25query('machine learning', 'parallel_test_2workers_idx'))::numeric, 4) AS score FROM parallel_test_2workers WHERE id <= 100 ORDER BY content <@> to_bm25query('machine learning', 'parallel_test_2workers_idx'), id LIMIT 5; id | score ----+--------- 12 | -3.3033 27 | -3.3033 42 | -3.3033 57 | -3.3033 72 | -3.3033 (5 rows) -------------------------------------------------------------------------------- -- Test 4: Four workers (higher parallelism) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 4; CREATE TABLE parallel_test_4workers ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_4workers (content) SELECT 'Document ' || i || ' with search engine indexing text ' || CASE i % 7 WHEN 0 THEN 'quick brown fox' WHEN 1 THEN 'lazy sleeping dog' WHEN 2 THEN 'postgres full text' WHEN 3 THEN 'inverted index structure' WHEN 4 THEN 'bm25 ranking algorithm' WHEN 5 THEN 'term frequency idf' ELSE 'document relevance score' END FROM generate_series(1, 200000) AS i; ANALYZE parallel_test_4workers; -- Should show "parallel index build: launched N of 4 requested workers" -- (may launch fewer if system limits) CREATE INDEX parallel_test_4workers_idx ON parallel_test_4workers USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_4workers_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 4 of 4 requested workers NOTICE: BM25 index build completed: 200000 documents, avg_length=9.00 SELECT COUNT(*) AS fox_count FROM (SELECT 1 FROM parallel_test_4workers ORDER BY content <@> to_bm25query('fox', 'parallel_test_4workers_idx')) sub; fox_count ----------- 28571 (1 row) SELECT COUNT(*) AS postgres_count FROM (SELECT 1 FROM parallel_test_4workers ORDER BY content <@> to_bm25query('postgres', 'parallel_test_4workers_idx')) sub; postgres_count ---------------- 28572 (1 row) -------------------------------------------------------------------------------- -- Test 5: Corner case - very short documents (single term) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; CREATE TABLE parallel_test_short ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_short (content) SELECT CASE i % 10 WHEN 0 THEN 'alpha' WHEN 1 THEN 'beta' WHEN 2 THEN 'gamma' WHEN 3 THEN 'delta' WHEN 4 THEN 'epsilon' WHEN 5 THEN 'zeta' WHEN 6 THEN 'eta' WHEN 7 THEN 'theta' WHEN 8 THEN 'iota' ELSE 'kappa' END FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_short; CREATE INDEX parallel_test_short_idx ON parallel_test_short USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_short_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=1.00 -- Each term should appear in 10% of documents SELECT COUNT(*) AS alpha_count FROM (SELECT 1 FROM parallel_test_short ORDER BY content <@> to_bm25query('alpha', 'parallel_test_short_idx')) sub; alpha_count ------------- 10000 (1 row) SELECT COUNT(*) AS beta_count FROM (SELECT 1 FROM parallel_test_short ORDER BY content <@> to_bm25query('beta', 'parallel_test_short_idx')) sub; beta_count ------------ 10000 (1 row) -------------------------------------------------------------------------------- -- Test 6: Corner case - documents with many duplicate terms -------------------------------------------------------------------------------- CREATE TABLE parallel_test_dupes ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_dupes (content) SELECT repeat('repeat ', (i % 50) + 1) || 'unique' || i FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_dupes; CREATE INDEX parallel_test_dupes_idx ON parallel_test_dupes USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_dupes_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=26.50 -- "repeat" appears in all documents with varying term frequency SELECT COUNT(*) AS repeat_count FROM (SELECT 1 FROM parallel_test_dupes ORDER BY content <@> to_bm25query('repeat', 'parallel_test_dupes_idx')) sub; repeat_count -------------- 100000 (1 row) -- Each "unique" term appears exactly once SELECT COUNT(*) AS unique_count FROM (SELECT 1 FROM parallel_test_dupes ORDER BY content <@> to_bm25query('unique1', 'parallel_test_dupes_idx')) sub; unique_count -------------- 1 (1 row) -------------------------------------------------------------------------------- -- Test 7: Corner case - NULL content (should be skipped) -------------------------------------------------------------------------------- CREATE TABLE parallel_test_nulls ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_nulls (content) SELECT CASE WHEN i % 10 = 0 THEN NULL ELSE 'document ' || i || ' has content' END FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_nulls; CREATE INDEX parallel_test_nulls_idx ON parallel_test_nulls USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_nulls_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 90000 documents, avg_length=3.00 -- Only non-NULL documents should be indexed (90%) SELECT COUNT(*) AS document_count FROM (SELECT 1 FROM parallel_test_nulls ORDER BY content <@> to_bm25query('document', 'parallel_test_nulls_idx')) sub; document_count ---------------- 90000 (1 row) -------------------------------------------------------------------------------- -- Test 8: Post-build operations work correctly -------------------------------------------------------------------------------- -- Use the 2-worker table for post-build tests -- Additional inserts after parallel build INSERT INTO parallel_test_2workers (content) VALUES ('completely new unique document inserted after parallel build'); -- Search for newly inserted content SELECT COUNT(*) AS new_doc_found FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('completely', 'parallel_test_2workers_idx')) sub; new_doc_found --------------- 1 (1 row) -- VACUUM should work VACUUM parallel_test_2workers; -- Queries still work after VACUUM SELECT COUNT(*) AS apple_after_vacuum FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('apple', 'parallel_test_2workers_idx')) sub; apple_after_vacuum -------------------- 30000 (1 row) -- Index stats still available SELECT bm25_summarize_index('parallel_test_2workers_idx') IS NOT NULL AS has_summary; has_summary ------------- t (1 row) -------------------------------------------------------------------------------- -- Test 9: Custom BM25 parameters with parallel build -------------------------------------------------------------------------------- CREATE TABLE parallel_test_custom ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_custom (content) SELECT 'Testing custom parameters with document ' || i || ' and some text' FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_custom; -- Use non-default k1 and b parameters CREATE INDEX parallel_test_custom_idx ON parallel_test_custom USING bm25(content) WITH (text_config='english', k1='1.5', b='0.5'); NOTICE: BM25 index build started for relation parallel_test_custom_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.50, b=0.50 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=6.00 SELECT COUNT(*) AS custom_count FROM (SELECT 1 FROM parallel_test_custom ORDER BY content <@> to_bm25query('custom', 'parallel_test_custom_idx')) sub; custom_count -------------- 100000 (1 row) -------------------------------------------------------------------------------- -- Test 10: Below threshold - should use serial even with workers configured -------------------------------------------------------------------------------- -- Table with fewer than 100K rows should use serial build CREATE TABLE parallel_test_below_threshold ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_below_threshold (content) SELECT 'Small table document ' || i FROM generate_series(1, 50000) AS i; ANALYZE parallel_test_below_threshold; -- Should NOT show "parallel index build" (below 100K threshold) CREATE INDEX parallel_test_below_idx ON parallel_test_below_threshold USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_below_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50000 documents, avg_length=4.00 SELECT COUNT(*) AS small_count FROM (SELECT 1 FROM parallel_test_below_threshold ORDER BY content <@> to_bm25query('small', 'parallel_test_below_idx')) sub; small_count ------------- 50000 (1 row) -------------------------------------------------------------------------------- -- Test 11: Parallel build with documents that produce no lexemes -------------------------------------------------------------------------------- CREATE TABLE parallel_test_empty ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO parallel_test_empty (content) SELECT CASE WHEN i % 2 = 0 THEN '' ELSE 'the' END FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_empty; CREATE INDEX parallel_test_empty_idx ON parallel_test_empty USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_empty_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=0.00 SELECT COUNT(*) AS empty_count FROM parallel_test_empty WHERE content @@ to_tsquery('english', '!missing'); empty_count ------------- 100000 (1 row) -------------------------------------------------------------------------------- -- Test 12: Zero-lexeme documents respect the serial build memory budget -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 0; SET maintenance_work_mem = '1MB'; CREATE TABLE serial_test_empty ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO serial_test_empty (content) SELECT '' FROM generate_series(1, 140000); CREATE INDEX serial_test_empty_idx ON serial_test_empty USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation serial_test_empty_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 140000 documents, avg_length=0.00 SELECT bm25_summarize_index('serial_test_empty_idx') ~ 'Total: 2 segments,.*0 terms, 140000 docs' AS document_arrays_flushed; document_arrays_flushed ------------------------- t (1 row) -------------------------------------------------------------------------------- -- Test 13: Low parallel budgets are divided across actual workers -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; SET maintenance_work_mem = '64MB'; INSERT INTO parallel_test_2workers (content) VALUES (repeat('large ', 20000)), (''); ANALYZE parallel_test_2workers; CREATE INDEX parallel_test_low_budget_idx ON parallel_test_2workers USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation parallel_test_low_budget_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 1 of 1 requested workers NOTICE: BM25 index build completed: 150003 documents, avg_length=16.00 SELECT COUNT(*) AS low_budget_documents FROM ( SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query( 'database', 'parallel_test_low_budget_idx') ) ranked; low_budget_documents ---------------------- 30000 (1 row) SELECT id AS parallel_large_document_id FROM parallel_test_2workers ORDER BY content <@> to_bm25query( 'large', 'parallel_test_low_budget_idx'), id LIMIT 1; parallel_large_document_id ---------------------------- 150002 (1 row) SELECT bm25_summarize_index('parallel_test_low_budget_idx') ~ E'total_docs: 150003\n' AS parallel_tokenless_row_counted; parallel_tokenless_row_counted -------------------------------- t (1 row) -------------------------------------------------------------------------------- -- Test 14: Serial and parallel streaming builds produce identical results -------------------------------------------------------------------------------- SET maintenance_work_mem = '256MB'; CREATE TABLE parallel_stream_test AS SELECT i AS id, CASE WHEN i % 100 = 0 THEN '' ELSE 'anchor shared t' || i::text END AS content FROM generate_series(1, 100000) i; ALTER TABLE parallel_stream_test SET (parallel_workers=2); ANALYZE parallel_stream_test; CREATE TYPE parallel_stream_result AS ( id integer, score double precision ); SET max_parallel_maintenance_workers=0; CREATE INDEX parallel_stream_serial ON parallel_stream_test USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation parallel_stream_serial NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100000 documents, avg_length=2.97 CREATE TEMP TABLE parallel_stream_expected AS SELECT ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'anchor shared', 'parallel_stream_serial') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'anchor shared', 'parallel_stream_serial'), id LIMIT 100000 ) ranked ) AS shared_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 't99999', 'parallel_stream_serial') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 't99999', 'parallel_stream_serial'), id LIMIT 100000 ) ranked ) AS unique_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'missing', 'parallel_stream_serial') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'missing', 'parallel_stream_serial'), id LIMIT 100000 ) ranked ) AS missing_results; SELECT cardinality(shared_results) AS shared_count, ARRAY(SELECT (result).id FROM unnest(unique_results) result) AS unique_ids, cardinality(missing_results) AS missing_count FROM parallel_stream_expected; shared_count | unique_ids | missing_count --------------+------------+--------------- 99000 | {99999} | 0 (1 row) SELECT bm25_summarize_index('parallel_stream_serial') ~ E'total_docs: 100000\n total_len: 297000\n' AS serial_corpus_totals; serial_corpus_totals ---------------------- t (1 row) DROP INDEX parallel_stream_serial; SET max_parallel_maintenance_workers=2; CREATE INDEX parallel_stream_parallel ON parallel_stream_test USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation parallel_stream_parallel NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=2.97 WITH actual AS ( SELECT ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS shared_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 't99999', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 't99999', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS unique_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'missing', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'missing', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS missing_results ) SELECT actual.shared_results = expected.shared_results AS shared_matches, actual.unique_results = expected.unique_results AS unique_matches, actual.missing_results = expected.missing_results AS missing_matches FROM actual, parallel_stream_expected expected; shared_matches | unique_matches | missing_matches ----------------+----------------+----------------- t | t | t (1 row) SELECT bm25_summarize_index('parallel_stream_parallel') ~ E'total_docs: 100000\n total_len: 297000\n' AS parallel_corpus_totals; parallel_corpus_totals ------------------------ t (1 row) DROP INDEX parallel_stream_parallel; SET pg_textsearch.compress_segments=off; CREATE INDEX parallel_stream_parallel ON parallel_stream_test USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation parallel_stream_parallel NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=2.97 WITH actual AS ( SELECT ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS shared_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 't99999', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 't99999', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS unique_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'missing', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'missing', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS missing_results ) SELECT actual.shared_results = expected.shared_results AS shared_matches, actual.unique_results = expected.unique_results AS unique_matches, actual.missing_results = expected.missing_results AS missing_matches FROM actual, parallel_stream_expected expected; shared_matches | unique_matches | missing_matches ----------------+----------------+----------------- t | t | t (1 row) SELECT bm25_summarize_index('parallel_stream_parallel') ~ E'total_docs: 100000\n total_len: 297000\n' AS uncompressed_corpus_totals; uncompressed_corpus_totals ---------------------------- t (1 row) RESET pg_textsearch.compress_segments; -------------------------------------------------------------------------------- -- Test 15: Small worker budgets can produce more than 64 segments per worker -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; SET maintenance_work_mem = '4MB'; CREATE TABLE parallel_many_segments AS SELECT i AS id, 'anchor ' || ( SELECT string_agg('t' || lpad(((i - 1) * 10 + j)::text, 9, '0'), ' ' ORDER BY j) FROM generate_series(1, 10) j ) AS content FROM generate_series(1, 100000) i; ALTER TABLE parallel_many_segments SET (parallel_workers=2); ANALYZE parallel_many_segments; CREATE INDEX parallel_many_segments_idx ON parallel_many_segments USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation parallel_many_segments_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=11.00 REINDEX INDEX parallel_many_segments_idx; NOTICE: BM25 index build started for relation parallel_many_segments_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: parallel index build: launched 2 of 2 requested workers NOTICE: BM25 index build completed: 100000 documents, avg_length=11.00 SELECT count(*) AS all_worker_segments_indexed FROM ( SELECT id FROM parallel_many_segments ORDER BY content <@> to_bm25query('anchor', 'parallel_many_segments_idx') LIMIT 100000 ) ranked; all_worker_segments_indexed ----------------------------- 100000 (1 row) SELECT id AS last_worker_document FROM parallel_many_segments ORDER BY content <@> to_bm25query('t001000000', 'parallel_many_segments_idx') LIMIT 10; last_worker_document ---------------------- 100000 (1 row) -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- DROP TABLE parallel_test_serial CASCADE; DROP TABLE parallel_test_1worker CASCADE; DROP TABLE parallel_test_2workers CASCADE; DROP TABLE parallel_test_4workers CASCADE; DROP TABLE parallel_test_short CASCADE; DROP TABLE parallel_test_dupes CASCADE; DROP TABLE parallel_test_nulls CASCADE; DROP TABLE parallel_test_custom CASCADE; DROP TABLE parallel_test_below_threshold CASCADE; DROP TABLE parallel_test_empty CASCADE; DROP TABLE serial_test_empty CASCADE; DROP TABLE parallel_stream_test CASCADE; DROP TABLE parallel_many_segments CASCADE; DROP TABLE parallel_stream_expected; DROP TYPE parallel_stream_result; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/partial_index.out000066400000000000000000000235441525762047700251250ustar00rootroot00000000000000-- Partial index tests for pg_textsearch CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================ -- Basic partial index -- ============================================================ CREATE TABLE partial_docs ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO partial_docs (content, category) VALUES ('postgresql database management', 'tech'), ('machine learning algorithms', 'tech'), ('cooking recipes for beginners', 'lifestyle'), ('full text search and retrieval', 'tech'), ('gardening tips and tricks', 'lifestyle'); CREATE INDEX partial_tech_idx ON partial_docs USING bm25 (content) WITH (text_config='english', compaction='manual') WHERE category = 'tech'; NOTICE: BM25 index build started for relation partial_tech_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.33 SET enable_seqscan = off; -- Explicit query against partial index SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; id | content | score ----+--------------------------------+--------- 1 | postgresql database management | -1.0227 (1 row) -- Lifestyle rows should not appear SELECT id, content, ROUND((content <@> to_bm25query('cooking', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('cooking', 'partial_tech_idx') LIMIT 5; id | content | score ----+---------+------- (0 rows) -- ============================================================ -- Multi-language use case -- ============================================================ CREATE TABLE multilang ( id SERIAL PRIMARY KEY, content TEXT, lang CHAR(2) NOT NULL ); INSERT INTO multilang (content, lang) VALUES ('the quick brown fox jumps over the lazy dog', 'en'), ('postgresql database management system', 'en'), ('full text searching and information retrieval', 'en'), ('simple words without stemming needed', 'xx'), ('another simple document for testing', 'xx'); CREATE INDEX multilang_en_idx ON multilang USING bm25 (content) WITH (text_config='english') WHERE lang = 'en'; NOTICE: BM25 index build started for relation multilang_en_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=5.00 CREATE INDEX multilang_xx_idx ON multilang USING bm25 (content) WITH (text_config='simple') WHERE lang = 'xx'; NOTICE: BM25 index build started for relation multilang_xx_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=5.00 -- English stemming: "databases" matches "database" SELECT id, content, ROUND((content <@> to_bm25query('databases', 'multilang_en_idx'))::numeric, 4) AS score FROM multilang WHERE lang = 'en' ORDER BY content <@> to_bm25query('databases', 'multilang_en_idx') LIMIT 5; id | content | score ----+---------------------------------------+--------- 2 | postgresql database management system | -1.0682 (1 row) -- Simple: no stemming SELECT id, content, ROUND((content <@> to_bm25query('simple', 'multilang_xx_idx'))::numeric, 4) AS score FROM multilang WHERE lang = 'xx' ORDER BY content <@> to_bm25query('simple', 'multilang_xx_idx') LIMIT 5; id | content | score ----+--------------------------------------+--------- 4 | simple words without stemming needed | -0.1823 5 | another simple document for testing | -0.1823 (2 rows) -- ============================================================ -- Expression + partial combined -- ============================================================ CREATE TABLE expr_partial ( id SERIAL PRIMARY KEY, data JSONB ); INSERT INTO expr_partial (data) VALUES ('{"text": "postgresql database management", "type": "article"}'), ('{"text": "machine learning algorithms", "type": "article"}'), ('{"text": "random user comment", "type": "comment"}'), ('{"text": "full text search retrieval", "type": "article"}'); CREATE INDEX expr_partial_idx ON expr_partial USING bm25 ((data->>'text')) WITH (text_config='english') WHERE (data->>'type') = 'article'; NOTICE: BM25 index build started for relation expr_partial_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.33 SELECT id, ROUND(((data->>'text') <@> to_bm25query('database', 'expr_partial_idx'))::numeric, 4) AS score FROM expr_partial WHERE (data->>'type') = 'article' ORDER BY (data->>'text') <@> to_bm25query('database', 'expr_partial_idx') LIMIT 5; id | score ----+--------- 1 | -1.0227 (1 row) -- ============================================================ -- Insert into partial index -- predicate filtering -- ============================================================ -- Insert a tech row that matches the predicate and query INSERT INTO partial_docs (content, category) VALUES ('advanced database optimization techniques', 'tech'); -- New tech row should appear in results SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; id | content | score ----+-------------------------------------------+--------- 1 | postgresql database management | -0.7362 6 | advanced database optimization techniques | -0.6549 (2 rows) -- Insert a row that does NOT match the predicate INSERT INTO partial_docs (content, category) VALUES ('database administration basics', 'lifestyle'); -- It should not appear in partial index results SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; id | content | score ----+-------------------------------------------+--------- 1 | postgresql database management | -0.7362 6 | advanced database optimization techniques | -0.6549 (2 rows) -- ============================================================ -- VACUUM on partial index -- ============================================================ DELETE FROM partial_docs WHERE id = 1; VACUUM partial_docs; -- Verify partial index still works after vacuum SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; id | content | score ----+-------------------------------------------+--------- 6 | advanced database optimization techniques | -0.6549 (1 row) -- ============================================================ -- Verify partial index scan is used -- ============================================================ EXPLAIN (COSTS OFF) SELECT id FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; QUERY PLAN ------------------------------------------------------------------------ Limit -> Index Scan using partial_tech_idx on partial_docs Order By: (content <@> 'partial_tech_idx:database'::bm25query) (3 rows) -- ============================================================ -- Implicit resolution skips partial indexes -- ============================================================ -- Table with ONLY a partial index (no non-partial BM25 index) CREATE TABLE partial_only ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO partial_only (content, category) VALUES ('database management', 'tech'), ('cooking recipes', 'lifestyle'); CREATE INDEX partial_only_idx ON partial_only USING bm25 (content) WITH (text_config='english') WHERE category = 'tech'; NOTICE: BM25 index build started for relation partial_only_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- Implicit resolution should ERROR because only partial -- indexes exist on this column. \set ON_ERROR_STOP off SELECT id, ROUND((content <@> 'database')::numeric, 4) AS score FROM partial_only ORDER BY content <@> 'database' LIMIT 5; ERROR: no non-partial BM25 index found for this column (1 partial index(es) skipped) HINT: Use to_bm25query('query', 'index_name') to specify which partial index to use. \set ON_ERROR_STOP on DROP TABLE partial_only CASCADE; -- ============================================================ -- Cleanup -- ============================================================ DROP TABLE partial_docs CASCADE; DROP TABLE multilang CASCADE; DROP TABLE expr_partial CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/partitioned.out000066400000000000000000000412141525762047700246160ustar00rootroot00000000000000-- Test BM25 index behavior with partitioned tables -- -- IMPORTANT: BM25 indexes on partitioned tables use PARTITION-LOCAL STATISTICS. -- Each partition maintains its own: -- - Document count (total_docs) -- - Average document length (avg_doc_len) -- - Per-term document frequencies for IDF calculation -- -- This means: -- - Queries targeting a single partition compute accurate BM25 scores using -- that partition's statistics -- - Queries spanning multiple partitions return scores computed independently -- per partition, which may not be directly comparable across partitions -- -- This test validates that per-partition scores are computed correctly using -- partition-local statistics. -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Load validation functions \set ECHO none SET enable_seqscan = off; -- ============================================================================= -- Test 1: Basic partitioned table with BM25 index -- When creating an index on a partitioned table, PostgreSQL automatically -- creates indexes on all existing partitions. The parent partitioned table -- itself doesn't call our index build functions, only the partitions do. -- ============================================================================= CREATE TABLE partitioned_docs ( id SERIAL, content TEXT, created_at TIMESTAMP ) PARTITION BY RANGE (created_at); -- Create a partition first CREATE TABLE partition_2024 PARTITION OF partitioned_docs FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); -- Insert data into partition INSERT INTO partition_2024 (content, created_at) VALUES ('Data in the partition', '2024-06-15'); -- Creating index on partitioned table will automatically create it on partitions -- This creates an index with auto-generated name like partition_2024_content_idx CREATE INDEX partitioned_bm25_idx ON partitioned_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation partition_2024_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- Verify the auto-created partition index works (using the generated name) SELECT indexrelid::regclass as index_name FROM pg_index WHERE indrelid = 'partition_2024'::regclass AND indexrelid::regclass::text LIKE '%content%'; index_name ---------------------------- partition_2024_content_idx (1 row) -- Create another partition after index exists CREATE TABLE partition_2025 PARTITION OF partitioned_docs FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); NOTICE: BM25 index build started for relation partition_2025_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert data into new partition INSERT INTO partition_2025 (content, created_at) VALUES ('Data in the 2025 partition', '2025-06-15'); -- Verify the auto-created index on new partition SELECT COUNT(*) as num_partition_indexes FROM pg_index WHERE indrelid IN ('partition_2024'::regclass, 'partition_2025'::regclass); num_partition_indexes ----------------------- 2 (1 row) -- ============================================================================= -- Test 2: Implicit index resolution with <@> operator on partitioned table -- The planner hook should resolve partitioned_bm25_idx, and the executor -- should map it to the correct partition index at scan time -- ============================================================================= -- Query partition_2024 - should use partition_2024_content_idx EXPLAIN (COSTS OFF) SELECT content FROM partitioned_docs WHERE created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit -> Index Scan using partition_2024_content_idx on partition_2024 partitioned_docs Order By: (content <@> 'partitioned_bm25_idx:data'::bm25query) Filter: ((created_at >= 'Mon Jan 01 00:00:00 2024'::timestamp without time zone) AND (created_at < 'Wed Jan 01 00:00:00 2025'::timestamp without time zone)) (4 rows) -- Actually execute to verify the query works SELECT content FROM partitioned_docs WHERE created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; content ----------------------- Data in the partition (1 row) -- Query partition_2025 - should use partition_2025_content_idx SELECT content FROM partitioned_docs WHERE created_at >= '2025-01-01' AND created_at < '2026-01-01' ORDER BY content <@> 'partition' LIMIT 1; content ---------------------------- Data in the 2025 partition (1 row) -- Cleanup first test DROP TABLE partitioned_docs CASCADE; -- ============================================================================= -- Test 3: Sub-partitioned table (multi-level partition hierarchy) -- Tests that is_partition_of() correctly walks up the inheritance chain -- ============================================================================= CREATE TABLE subpart_docs ( id SERIAL, content TEXT, region TEXT, created_at TIMESTAMP ) PARTITION BY LIST (region); -- Create sub-partitioned partition CREATE TABLE subpart_docs_us PARTITION OF subpart_docs FOR VALUES IN ('us') PARTITION BY RANGE (created_at); -- Create leaf partition CREATE TABLE subpart_docs_us_2024 PARTITION OF subpart_docs_us FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); -- Create another region partition (not sub-partitioned) CREATE TABLE subpart_docs_eu PARTITION OF subpart_docs FOR VALUES IN ('eu'); -- Insert test data INSERT INTO subpart_docs_us_2024 (content, region, created_at) VALUES ('US data from 2024', 'us', '2024-06-15'); INSERT INTO subpart_docs_eu (content, region, created_at) VALUES ('EU data', 'eu', '2024-06-15'); -- Create index on top-level partitioned table CREATE INDEX subpart_bm25_idx ON subpart_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation subpart_docs_eu_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents across 2 partitions, avg_length=2.50 -- Verify indexes were created at all levels SELECT c.relname, c.relkind FROM pg_class c JOIN pg_index i ON c.oid = i.indexrelid WHERE i.indrelid IN ( 'subpart_docs'::regclass, 'subpart_docs_us'::regclass, 'subpart_docs_us_2024'::regclass, 'subpart_docs_eu'::regclass ) ORDER BY c.relname; relname | relkind ----------------------------------+--------- subpart_bm25_idx | I subpart_docs_eu_content_idx | i subpart_docs_us_2024_content_idx | i subpart_docs_us_content_idx | I (4 rows) -- Query that hits leaf partition (3-level: top -> sub -> leaf) -- This tests is_partition_of() walking up multiple levels SELECT content FROM subpart_docs WHERE region = 'us' AND created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; content ------------------- US data from 2024 (1 row) -- Query that hits non-sub-partitioned partition (2-level: top -> leaf) SELECT content FROM subpart_docs WHERE region = 'eu' ORDER BY content <@> 'data' LIMIT 1; content --------- EU data (1 row) -- Cleanup DROP TABLE subpart_docs CASCADE; -- ============================================================================= -- Test 4: Partition-local BM25 score validation -- -- This test validates that BM25 scores are computed correctly using -- PARTITION-LOCAL statistics, not global statistics across all partitions. -- -- We create two partitions with different corpus characteristics: -- - partition_small: 3 documents -- - partition_large: 10 documents -- -- The same term will have different IDF values in each partition because -- IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) depends on partition-local N and df. -- -- We validate scores by querying each partition's table directly and comparing -- against pure SQL BM25 computation using that partition's statistics. -- ============================================================================= CREATE TABLE partitioned_scoring ( id SERIAL, content TEXT, partition_key INT ) PARTITION BY LIST (partition_key); -- Create small partition (3 docs) CREATE TABLE partition_small PARTITION OF partitioned_scoring FOR VALUES IN (1); -- Create large partition (10 docs) CREATE TABLE partition_large PARTITION OF partitioned_scoring FOR VALUES IN (2); -- Insert documents into small partition INSERT INTO partition_small (content, partition_key) VALUES ('database search query', 1), ('full text search engine', 1), ('postgresql database system', 1); -- Insert documents into large partition (with different term distribution) INSERT INTO partition_large (content, partition_key) VALUES ('database management system', 2), ('search engine optimization', 2), ('web application development', 2), ('cloud computing services', 2), ('machine learning algorithms', 2), ('data analysis tools', 2), ('software engineering practices', 2), ('network security protocols', 2), ('mobile app development', 2), ('database query optimization', 2); -- Create index on partitioned table CREATE INDEX scoring_bm25_idx ON partitioned_scoring USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation partition_small_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 13 documents across 2 partitions, avg_length=3.08 -- Get the auto-generated partition index names SELECT i.indrelid::regclass as partition_table, i.indexrelid::regclass as partition_index FROM pg_index i JOIN pg_class c ON i.indexrelid = c.oid WHERE i.indrelid IN ('partition_small'::regclass, 'partition_large'::regclass) ORDER BY i.indrelid::regclass::text; partition_table | partition_index -----------------+----------------------------- partition_large | partition_large_content_idx partition_small | partition_small_content_idx (2 rows) -- Validate scoring on SMALL partition (3 docs) -- The validation function computes expected BM25 using partition-local statistics SELECT validate_bm25_scoring( 'partition_small', 'content', 'partition_small_content_idx', 'database', 'english', 1.2, 0.75 ) as small_partition_database_valid; small_partition_database_valid -------------------------------- t (1 row) SELECT validate_bm25_scoring( 'partition_small', 'content', 'partition_small_content_idx', 'search', 'english', 1.2, 0.75 ) as small_partition_search_valid; small_partition_search_valid ------------------------------ t (1 row) -- Validate scoring on LARGE partition (10 docs) SELECT validate_bm25_scoring( 'partition_large', 'content', 'partition_large_content_idx', 'database', 'english', 1.2, 0.75 ) as large_partition_database_valid; large_partition_database_valid -------------------------------- t (1 row) SELECT validate_bm25_scoring( 'partition_large', 'content', 'partition_large_content_idx', 'development', 'english', 1.2, 0.75 ) as large_partition_development_valid; large_partition_development_valid ----------------------------------- t (1 row) -- Show that the same query produces DIFFERENT IDF values in each partition -- because N (total docs) differs: 3 vs 10 -- For 'database': df=2 in small (2/3 docs), df=2 in large (2/10 docs) -- Small IDF = ln(1 + (3 - 2 + 0.5) / (2 + 0.5)) = ln(1 + 1.5/2.5) = ln(1.6) ≈ 0.47 -- Large IDF = ln(1 + (10 - 2 + 0.5) / (2 + 0.5)) = ln(1 + 8.5/2.5) = ln(4.4) ≈ 1.48 -- -- This demonstrates partition-local statistics in action: -- Show actual scores from small partition (only matching documents) SELECT 'small' as partition, content, ROUND((content <@> to_bm25query('database', 'partition_small_content_idx'))::numeric, 4) as score FROM partition_small ORDER BY content <@> to_bm25query('database', 'partition_small_content_idx') LIMIT 10; partition | content | score -----------+----------------------------+--------- small | database search query | -0.4901 small | postgresql database system | -0.4901 (2 rows) -- Show actual scores from large partition (only matching documents) SELECT 'large' as partition, content, ROUND((content <@> to_bm25query('database', 'partition_large_content_idx'))::numeric, 4) as score FROM partition_large ORDER BY content <@> to_bm25query('database', 'partition_large_content_idx') LIMIT 10; partition | content | score -----------+-----------------------------+--------- large | database management system | -1.4816 large | database query optimization | -1.4816 (2 rows) -- Validate index scan vs standalone scoring consistency per partition SELECT validate_index_vs_standalone( 'partition_small', 'content', 'partition_small_content_idx', 'database' ) as small_partition_modes_match; small_partition_modes_match ----------------------------- t (1 row) SELECT validate_index_vs_standalone( 'partition_large', 'content', 'partition_large_content_idx', 'database' ) as large_partition_modes_match; large_partition_modes_match ----------------------------- t (1 row) -- ============================================================================= -- Test 5: Score expression replacement via parent table (MergeAppend) -- -- This tests that the planner hook correctly replaces score expressions when -- querying through the parent partitioned table. The query plan uses MergeAppend -- to combine results from partition index scans. The planner hook must: -- 1. Detect BM25 index scans inside MergeAppend children -- 2. Replace score expressions in SELECT with stub functions that retrieve -- cached scores from the index scan -- -- Without proper handling, the SELECT score would use standalone scoring -- (which looks up the parent table's index, not partition indexes), resulting -- in incorrect or zero scores. -- ============================================================================= -- Query across all partitions via parent table with score in both SELECT and ORDER BY -- The planner should use MergeAppend with partition index scans EXPLAIN (COSTS OFF) SELECT content, -(content <@> to_bm25query('database', 'scoring_bm25_idx')) as score FROM partitioned_scoring ORDER BY content <@> to_bm25query('database', 'scoring_bm25_idx') LIMIT 5; QUERY PLAN --------------------------------------------------------------------------------------------------------- Limit -> Result -> Merge Append Sort Key: ((partitioned_scoring.content <@> 'scoring_bm25_idx:database'::bm25query)) -> Index Scan using partition_small_content_idx on partition_small partitioned_scoring_1 Order By: (content <@> 'scoring_bm25_idx:database'::bm25query) -> Index Scan using partition_large_content_idx on partition_large partitioned_scoring_2 Order By: (content <@> 'scoring_bm25_idx:database'::bm25query) (8 rows) -- Verify scores are non-zero when querying via parent table -- If the planner hook fails to replace the score expression, we'd get 0 scores SELECT content, ROUND(-(content <@> to_bm25query('database', 'scoring_bm25_idx'))::numeric, 4) as score FROM partitioned_scoring ORDER BY content <@> to_bm25query('database', 'scoring_bm25_idx') LIMIT 5; content | score -----------------------------+-------- database search query | 0.4901 postgresql database system | 0.4901 database management system | 0.4901 database query optimization | 0.4901 (4 rows) -- Verify the returned scores are all non-zero (would fail if planner hook broken) SELECT COUNT(*) as nonzero_scores FROM ( SELECT content, -(content <@> to_bm25query('database', 'scoring_bm25_idx')) as score FROM partitioned_scoring ORDER BY content <@> to_bm25query('database', 'scoring_bm25_idx') LIMIT 5 ) t; nonzero_scores ---------------- 4 (1 row) -- Cleanup SET enable_seqscan = on; DROP TABLE partitioned_scoring CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/partitioned_1.out000066400000000000000000000334211525762047700250370ustar00rootroot00000000000000-- Test BM25 index behavior with partitioned tables -- -- IMPORTANT: BM25 indexes on partitioned tables use PARTITION-LOCAL STATISTICS. -- Each partition maintains its own: -- - Document count (total_docs) -- - Average document length (avg_doc_len) -- - Per-term document frequencies for IDF calculation -- -- This means: -- - Queries targeting a single partition compute accurate BM25 scores using -- that partition's statistics -- - Queries spanning multiple partitions return scores computed independently -- per partition, which may not be directly comparable across partitions -- -- This test validates that per-partition scores are computed correctly using -- partition-local statistics. -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Load validation functions \set ECHO none SET enable_seqscan = off; -- ============================================================================= -- Test 1: Basic partitioned table with BM25 index -- When creating an index on a partitioned table, PostgreSQL automatically -- creates indexes on all existing partitions. The parent partitioned table -- itself doesn't call our index build functions, only the partitions do. -- ============================================================================= CREATE TABLE partitioned_docs ( id SERIAL, content TEXT, created_at TIMESTAMP ) PARTITION BY RANGE (created_at); -- Create a partition first CREATE TABLE partition_2024 PARTITION OF partitioned_docs FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); -- Insert data into partition INSERT INTO partition_2024 (content, created_at) VALUES ('Data in the partition', '2024-06-15'); -- Creating index on partitioned table will automatically create it on partitions -- This creates an index with auto-generated name like partition_2024_content_idx CREATE INDEX partitioned_bm25_idx ON partitioned_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation partition_2024_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- Verify the auto-created partition index works (using the generated name) SELECT indexrelid::regclass as index_name FROM pg_index WHERE indrelid = 'partition_2024'::regclass AND indexrelid::regclass::text LIKE '%content%'; index_name ---------------------------- partition_2024_content_idx (1 row) -- Create another partition after index exists CREATE TABLE partition_2025 PARTITION OF partitioned_docs FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); NOTICE: BM25 index build started for relation partition_2025_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert data into new partition INSERT INTO partition_2025 (content, created_at) VALUES ('Data in the 2025 partition', '2025-06-15'); -- Verify the auto-created index on new partition SELECT COUNT(*) as num_partition_indexes FROM pg_index WHERE indrelid IN ('partition_2024'::regclass, 'partition_2025'::regclass); num_partition_indexes ----------------------- 2 (1 row) -- ============================================================================= -- Test 2: Implicit index resolution with <@> operator on partitioned table -- The planner hook should resolve partitioned_bm25_idx, and the executor -- should map it to the correct partition index at scan time -- ============================================================================= -- Query partition_2024 - should use partition_2024_content_idx EXPLAIN (COSTS OFF) SELECT content FROM partitioned_docs WHERE created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; QUERY PLAN ---------------------------------------------------------------------------------------------------------------------------------------------------------------------- Limit -> Index Scan using partition_2024_content_idx on partition_2024 partitioned_docs Order By: (content <@> 'partitioned_bm25_idx:data'::bm25query) Filter: ((created_at >= 'Mon Jan 01 00:00:00 2024'::timestamp without time zone) AND (created_at < 'Wed Jan 01 00:00:00 2025'::timestamp without time zone)) (4 rows) -- Actually execute to verify the query works SELECT content FROM partitioned_docs WHERE created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; content ----------------------- Data in the partition (1 row) -- Query partition_2025 - should use partition_2025_content_idx SELECT content FROM partitioned_docs WHERE created_at >= '2025-01-01' AND created_at < '2026-01-01' ORDER BY content <@> 'partition' LIMIT 1; content ---------------------------- Data in the 2025 partition (1 row) -- Cleanup first test DROP TABLE partitioned_docs CASCADE; -- ============================================================================= -- Test 3: Sub-partitioned table (multi-level partition hierarchy) -- Tests that is_partition_of() correctly walks up the inheritance chain -- ============================================================================= CREATE TABLE subpart_docs ( id SERIAL, content TEXT, region TEXT, created_at TIMESTAMP ) PARTITION BY LIST (region); -- Create sub-partitioned partition CREATE TABLE subpart_docs_us PARTITION OF subpart_docs FOR VALUES IN ('us') PARTITION BY RANGE (created_at); -- Create leaf partition CREATE TABLE subpart_docs_us_2024 PARTITION OF subpart_docs_us FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); -- Create another region partition (not sub-partitioned) CREATE TABLE subpart_docs_eu PARTITION OF subpart_docs FOR VALUES IN ('eu'); -- Insert test data INSERT INTO subpart_docs_us_2024 (content, region, created_at) VALUES ('US data from 2024', 'us', '2024-06-15'); INSERT INTO subpart_docs_eu (content, region, created_at) VALUES ('EU data', 'eu', '2024-06-15'); -- Create index on top-level partitioned table CREATE INDEX subpart_bm25_idx ON subpart_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation subpart_docs_eu_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents across 2 partitions, avg_length=2.50 -- Verify indexes were created at all levels SELECT c.relname, c.relkind FROM pg_class c JOIN pg_index i ON c.oid = i.indexrelid WHERE i.indrelid IN ( 'subpart_docs'::regclass, 'subpart_docs_us'::regclass, 'subpart_docs_us_2024'::regclass, 'subpart_docs_eu'::regclass ) ORDER BY c.relname; relname | relkind ----------------------------------+--------- subpart_bm25_idx | I subpart_docs_eu_content_idx | i subpart_docs_us_2024_content_idx | i subpart_docs_us_content_idx | I (4 rows) -- Query that hits leaf partition (3-level: top -> sub -> leaf) -- This tests is_partition_of() walking up multiple levels SELECT content FROM subpart_docs WHERE region = 'us' AND created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; content ------------------- US data from 2024 (1 row) -- Query that hits non-sub-partitioned partition (2-level: top -> leaf) SELECT content FROM subpart_docs WHERE region = 'eu' ORDER BY content <@> 'data' LIMIT 1; content --------- EU data (1 row) -- Cleanup DROP TABLE subpart_docs CASCADE; -- ============================================================================= -- Test 4: Partition-local BM25 score validation -- -- This test validates that BM25 scores are computed correctly using -- PARTITION-LOCAL statistics, not global statistics across all partitions. -- -- We create two partitions with different corpus characteristics: -- - partition_small: 3 documents -- - partition_large: 10 documents -- -- The same term will have different IDF values in each partition because -- IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) depends on partition-local N and df. -- -- We validate scores by querying each partition's table directly and comparing -- against pure SQL BM25 computation using that partition's statistics. -- ============================================================================= CREATE TABLE partitioned_scoring ( id SERIAL, content TEXT, partition_key INT ) PARTITION BY LIST (partition_key); -- Create small partition (3 docs) CREATE TABLE partition_small PARTITION OF partitioned_scoring FOR VALUES IN (1); -- Create large partition (10 docs) CREATE TABLE partition_large PARTITION OF partitioned_scoring FOR VALUES IN (2); -- Insert documents into small partition INSERT INTO partition_small (content, partition_key) VALUES ('database search query', 1), ('full text search engine', 1), ('postgresql database system', 1); -- Insert documents into large partition (with different term distribution) INSERT INTO partition_large (content, partition_key) VALUES ('database management system', 2), ('search engine optimization', 2), ('web application development', 2), ('cloud computing services', 2), ('machine learning algorithms', 2), ('data analysis tools', 2), ('software engineering practices', 2), ('network security protocols', 2), ('mobile app development', 2), ('database query optimization', 2); -- Create index on partitioned table CREATE INDEX scoring_bm25_idx ON partitioned_scoring USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation partition_small_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 13 documents across 2 partitions, avg_length=3.08 -- Get the auto-generated partition index names SELECT i.indrelid::regclass as partition_table, i.indexrelid::regclass as partition_index FROM pg_index i JOIN pg_class c ON i.indexrelid = c.oid WHERE i.indrelid IN ('partition_small'::regclass, 'partition_large'::regclass) ORDER BY i.indrelid::regclass::text; partition_table | partition_index -----------------+----------------------------- partition_large | partition_large_content_idx partition_small | partition_small_content_idx (2 rows) -- Validate scoring on SMALL partition (3 docs) -- The validation function computes expected BM25 using partition-local statistics SELECT validate_bm25_scoring( 'partition_small', 'content', 'partition_small_content_idx', 'database', 'english', 1.2, 0.75 ) as small_partition_database_valid; small_partition_database_valid -------------------------------- t (1 row) SELECT validate_bm25_scoring( 'partition_small', 'content', 'partition_small_content_idx', 'search', 'english', 1.2, 0.75 ) as small_partition_search_valid; small_partition_search_valid ------------------------------ t (1 row) -- Validate scoring on LARGE partition (10 docs) SELECT validate_bm25_scoring( 'partition_large', 'content', 'partition_large_content_idx', 'database', 'english', 1.2, 0.75 ) as large_partition_database_valid; large_partition_database_valid -------------------------------- t (1 row) SELECT validate_bm25_scoring( 'partition_large', 'content', 'partition_large_content_idx', 'development', 'english', 1.2, 0.75 ) as large_partition_development_valid; large_partition_development_valid ----------------------------------- t (1 row) -- Show that the same query produces DIFFERENT IDF values in each partition -- because N (total docs) differs: 3 vs 10 -- For 'database': df=2 in small (2/3 docs), df=2 in large (2/10 docs) -- Small IDF = ln(1 + (3 - 2 + 0.5) / (2 + 0.5)) = ln(1 + 1.5/2.5) = ln(1.6) ≈ 0.47 -- Large IDF = ln(1 + (10 - 2 + 0.5) / (2 + 0.5)) = ln(1 + 8.5/2.5) = ln(4.4) ≈ 1.48 -- -- This demonstrates partition-local statistics in action: -- Show actual scores from small partition (only matching documents) SELECT 'small' as partition, content, ROUND((content <@> to_bm25query('database', 'partition_small_content_idx'))::numeric, 4) as score FROM partition_small ORDER BY content <@> to_bm25query('database', 'partition_small_content_idx') LIMIT 10; partition | content | score -----------+----------------------------+--------- small | database search query | -0.4901 small | postgresql database system | -0.4901 (2 rows) -- Show actual scores from large partition (only matching documents) SELECT 'large' as partition, content, ROUND((content <@> to_bm25query('database', 'partition_large_content_idx'))::numeric, 4) as score FROM partition_large ORDER BY content <@> to_bm25query('database', 'partition_large_content_idx') LIMIT 10; partition | content | score -----------+-----------------------------+--------- large | database management system | -1.4816 large | database query optimization | -1.4816 (2 rows) -- Validate index scan vs standalone scoring consistency per partition SELECT validate_index_vs_standalone( 'partition_small', 'content', 'partition_small_content_idx', 'database' ) as small_partition_modes_match; small_partition_modes_match ----------------------------- t (1 row) SELECT validate_index_vs_standalone( 'partition_large', 'content', 'partition_large_content_idx', 'database' ) as large_partition_modes_match; large_partition_modes_match ----------------------------- t (1 row) -- Cleanup SET enable_seqscan = on; DROP TABLE partitioned_scoring CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/partitioned_many.out000066400000000000000000000037411525762047700256450ustar00rootroot00000000000000-- Test BM25 index creation on partitioned table with many partitions -- This tests for the "too many LWLocks taken" error that occurs when -- creating indexes on tables with many partitions containing data. -- -- The bug triggers during document processing when per-index locks accumulate. -- With 200 partitions and data, the old code would exceed MAX_SIMUL_LWLOCKS. CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create a partitioned table with 200 partitions -- (200 is enough to trigger the bug - old limit was ~142 partitions) CREATE TABLE docs_many_parts ( id serial, content text, category int ) PARTITION BY RANGE (category); -- Create 200 range partitions DO $$ DECLARE i int; BEGIN FOR i IN 0..199 LOOP EXECUTE format( 'CREATE TABLE docs_many_parts_%s PARTITION OF docs_many_parts FOR VALUES FROM (%s) TO (%s)', i, i, i + 1 ); END LOOP; END $$; -- Insert one row per partition - data is required to trigger the bug -- (the lock is acquired during document processing, not index creation) INSERT INTO docs_many_parts (content, category) SELECT 'test document ' || i, i FROM generate_series(0, 199) AS i; -- This should not fail with "too many LWLocks taken" -- Previously failed at around partition 142 when documents were processed CREATE INDEX docs_many_parts_idx ON docs_many_parts USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_many_parts_0_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 200 documents across 200 partitions, avg_length=3.00 -- Verify all 200 partition indexes were created SELECT COUNT(*) AS partition_index_count FROM pg_indexes WHERE indexname LIKE 'docs_many_parts_%_content_idx'; partition_index_count ----------------------- 200 (1 row) -- Clean up DROP TABLE docs_many_parts CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/pgstats.out000066400000000000000000000042221525762047700237570ustar00rootroot00000000000000-- Test that BM25 index scans report statistics to pg_stat_user_indexes CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create a test table with BM25 index CREATE TABLE test_pgstats (id serial PRIMARY KEY, content text); CREATE INDEX test_pgstats_idx ON test_pgstats USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation test_pgstats_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO test_pgstats (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz judge my vow'); -- Reset statistics for our index SELECT pg_stat_reset_single_table_counters('test_pgstats_idx'::regclass); pg_stat_reset_single_table_counters ------------------------------------- (1 row) -- Run a query that uses the BM25 index scan SELECT content FROM test_pgstats ORDER BY content <@> to_bm25query('fox', 'test_pgstats_idx') LIMIT 5; content --------------------- the quick brown fox (1 row) -- Check that idx_scan was incremented -- (pg_stat counters are updated asynchronously, so force a stats flush) SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) SELECT idx_scan > 0 AS has_scan_stats, idx_tup_read > 0 AS has_tup_read_stats FROM pg_stat_user_indexes WHERE indexrelname = 'test_pgstats_idx'; has_scan_stats | has_tup_read_stats ----------------+-------------------- t | t (1 row) -- Run a second query and verify the count increases SELECT content FROM test_pgstats ORDER BY content <@> to_bm25query('dog', 'test_pgstats_idx') LIMIT 5; content -------------------------- jumped over the lazy dog (1 row) SELECT pg_stat_force_next_flush(); pg_stat_force_next_flush -------------------------- (1 row) SELECT idx_scan > 1 AS multiple_scans FROM pg_stat_user_indexes WHERE indexrelname = 'test_pgstats_idx'; multiple_scans ---------------- t (1 row) -- Cleanup DROP TABLE test_pgstats; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/preload.out000066400000000000000000000036501525762047700237240ustar00rootroot00000000000000-- Preloaded utility hooks must work without CREATE EXTENSION in this database. SELECT EXISTS (SELECT FROM pg_am WHERE amname = 'bm25') AS bm25_installed; bm25_installed ---------------- f (1 row) CREATE SCHEMA preload_test; CREATE TABLE preload_test.docs (id integer PRIMARY KEY, content text); INSERT INTO preload_test.docs VALUES (1, 'alpha'), (2, 'bravo'); ALTER TABLE preload_test.docs CLUSTER ON docs_pkey; REINDEX DATABASE :"DBNAME"; REINDEX SCHEMA preload_test; REINDEX TABLE preload_test.docs; REINDEX INDEX preload_test.docs_pkey; VACUUM FULL; VACUUM (FULL, SKIP_LOCKED); CLUSTER; -- Ownership changes use the same database-wide BM25 discovery. CREATE ROLE preload_owner; ALTER TABLE preload_test.docs OWNER TO preload_owner; REASSIGN OWNED BY preload_owner TO CURRENT_USER; DROP ROLE preload_owner; SELECT * FROM preload_test.docs ORDER BY id; id | content ----+--------- 1 | alpha 2 | bravo (2 rows) -- Creating and dropping the extension in this backend must not leave stale -- access-method state in the preloaded hooks. SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE INDEX docs_bm25 ON preload_test.docs USING bm25 (content) WITH (text_config = 'english'); REINDEX DATABASE :"DBNAME"; REINDEX SCHEMA preload_test; VACUUM FULL; CLUSTER; SET enable_seqscan = off; SELECT id FROM preload_test.docs ORDER BY content <@> to_bm25query('alpha', 'preload_test.docs_bm25') LIMIT 1; id ---- 1 (1 row) RESET enable_seqscan; DROP INDEX preload_test.docs_bm25; DROP EXTENSION pg_textsearch; RESET client_min_messages; SELECT EXISTS (SELECT FROM pg_am WHERE amname = 'bm25') AS bm25_installed; bm25_installed ---------------- f (1 row) REINDEX DATABASE :"DBNAME"; REINDEX SCHEMA preload_test; VACUUM FULL; VACUUM (FULL, SKIP_LOCKED); CLUSTER; SELECT * FROM preload_test.docs ORDER BY id; id | content ----+--------- 1 | alpha 2 | bravo (2 rows) DROP TABLE preload_test.docs; DROP SCHEMA preload_test; timescale-pg_textsearch-b63fe58/test/expected/queries.out000066400000000000000000000310551525762047700237530ustar00rootroot00000000000000-- This test demonstrates top-k pg_textsearch query patterns for efficient text search -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; -- Setup test data with realistic documents CREATE TABLE articles ( id SERIAL PRIMARY KEY, title TEXT, content TEXT, category TEXT ); INSERT INTO articles (title, content, category) VALUES ('Introduction to Databases', 'postgresql database management system with advanced indexing capabilities for fast query processing', 'technology'), ('Machine Learning Fundamentals', 'machine learning algorithms and artificial intelligence techniques for data analysis and prediction', 'technology'), ('Text Search Algorithms', 'full text search with tapir ranking and relevance scoring for information retrieval systems', 'technology'), ('Information Retrieval', 'information retrieval and search engines using vector space models and term frequency analysis', 'technology'), ('Natural Language Processing', 'natural language processing techniques including parsing stemming and semantic analysis', 'technology'), ('Database Optimization', 'database indexing and query optimization strategies for improving performance in large datasets', 'technology'), ('Document Classification', 'text mining and document classification using machine learning and statistical methods', 'technology'), ('Search Engine Design', 'search relevance and ranking algorithms with focus on user experience and result quality', 'technology'), ('Distributed Computing', 'distributed systems and scalability challenges in modern cloud computing environments', 'technology'), ('Data Mining Techniques', 'data mining algorithms for pattern discovery and knowledge extraction from large databases', 'technology'), ('Web Search Evolution', 'evolution of web search from simple keyword matching to semantic understanding', 'history'), ('Big Data Analytics', 'big data analytics platforms and tools for processing massive datasets efficiently', 'technology'); -- Create pg_textsearch index with text_config CREATE INDEX articles_tapir_idx ON articles USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation articles_tapir_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 12 documents, avg_length=9.17 -- Disable sequential scans to force index usage SET enable_seqscan = off; -- Test 1: Basic text similarity search with LIMIT (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('database', 'articles_tapir_idx') as score FROM articles ORDER BY content <@> 'database' LIMIT 5; QUERY PLAN -------------------------------------------------------------------------- Limit -> Index Scan using articles_tapir_idx on articles Order By: (content <@> 'articles_tapir_idx:database'::bm25query) (3 rows) SELECT title, content, ROUND((content <@> to_bm25query('database', 'articles_tapir_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> 'database' LIMIT 5; NOTICE: BM25 index scan: tid=(0,6), BM25_score=-1.3220 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-1.3220 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-1.2651 title | content | score ---------------------------+-----------------------------------------------------------------------------------------------------+--------- Database Optimization | database indexing and query optimization strategies for improving performance in large datasets | -1.3220 Data Mining Techniques | data mining algorithms for pattern discovery and knowledge extraction from large databases | -1.3220 Introduction to Databases | postgresql database management system with advanced indexing capabilities for fast query processing | -1.2651 (3 rows) -- Test 2: Multi-term search with ranking (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('machine learning', 'articles_tapir_idx') as score FROM articles ORDER BY content <@> 'machine learning' LIMIT 3; QUERY PLAN ---------------------------------------------------------------------------------- Limit -> Index Scan using articles_tapir_idx on articles Order By: (content <@> 'articles_tapir_idx:machine learning'::bm25query) (3 rows) SELECT title, content, ROUND((content <@> to_bm25query('machine learning', 'articles_tapir_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> 'machine learning' LIMIT 3; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-3.3220 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-3.3220 title | content | score -------------------------------+-----------------------------------------------------------------------------------------------------+--------- Machine Learning Fundamentals | machine learning algorithms and artificial intelligence techniques for data analysis and prediction | -3.3220 Document Classification | text mining and document classification using machine learning and statistical methods | -3.3220 (2 rows) -- Test 3: Category-filtered search (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('search algorithms', 'articles_tapir_idx') as score FROM articles WHERE category = 'technology' ORDER BY content <@> 'search algorithms' LIMIT 10; QUERY PLAN ----------------------------------------------------------------------------------- Limit -> Index Scan using articles_tapir_idx on articles Order By: (content <@> 'articles_tapir_idx:search algorithms'::bm25query) Filter: (category = 'technology'::text) (4 rows) SELECT title, content, ROUND((content <@> to_bm25query('search algorithms', 'articles_tapir_idx'))::numeric, 4) as score FROM articles WHERE category = 'technology' ORDER BY content <@> 'search algorithms' LIMIT 10; NOTICE: BM25 index scan: tid=(0,8), BM25_score=-2.3908 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.3220 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-1.3220 NOTICE: BM25 index scan: tid=(0,11), BM25_score=-1.1191 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-1.0228 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.9806 title | content | score -------------------------------+-----------------------------------------------------------------------------------------------------+--------- Search Engine Design | search relevance and ranking algorithms with focus on user experience and result quality | -2.3908 Machine Learning Fundamentals | machine learning algorithms and artificial intelligence techniques for data analysis and prediction | -1.3220 Data Mining Techniques | data mining algorithms for pattern discovery and knowledge extraction from large databases | -1.3220 Text Search Algorithms | full text search with tapir ranking and relevance scoring for information retrieval systems | -1.0228 Information Retrieval | information retrieval and search engines using vector space models and term frequency analysis | -0.9806 (5 rows) -- Test 4: Find similar articles to a specific one (standalone scoring with explicit corpus) -- Note: This query uses explicit index name because of dynamic query text SELECT a2.title, a2.content, ROUND((a2.content <@> to_bm25query(a1.content, 'articles_tapir_idx'))::numeric, 4) as score FROM articles a1, articles a2 WHERE a1.id = 3 -- "Text Search Algorithms" article AND a2.id != a1.id ORDER BY a2.content <@> to_bm25query(a1.content, 'articles_tapir_idx') LIMIT 5; title | content | score ---------------------------+-----------------------------------------------------------------------------------------------------+--------- Search Engine Design | search relevance and ranking algorithms with focus on user experience and result quality | -4.3908 Information Retrieval | information retrieval and search engines using vector space models and term frequency analysis | -4.0286 Document Classification | text mining and document classification using machine learning and statistical methods | -1.6610 Distributed Computing | distributed systems and scalability challenges in modern cloud computing environments | -1.3843 Introduction to Databases | postgresql database management system with advanced indexing capabilities for fast query processing | -1.2651 (5 rows) -- Test 5: Top results with scoring (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('database optimization', 'articles_tapir_idx') as score FROM articles ORDER BY content <@> 'database optimization' LIMIT 10; QUERY PLAN --------------------------------------------------------------------------------------- Limit -> Index Scan using articles_tapir_idx on articles Order By: (content <@> 'articles_tapir_idx:database optimization'::bm25query) (3 rows) SELECT title, content, ROUND((content <@> to_bm25query('database optimization', 'articles_tapir_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> 'database optimization' LIMIT 10; NOTICE: BM25 index scan: tid=(0,6), BM25_score=-3.4977 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-1.3220 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-1.2651 title | content | score ---------------------------+-----------------------------------------------------------------------------------------------------+--------- Database Optimization | database indexing and query optimization strategies for improving performance in large datasets | -3.4977 Data Mining Techniques | data mining algorithms for pattern discovery and knowledge extraction from large databases | -1.3220 Introduction to Databases | postgresql database management system with advanced indexing capabilities for fast query processing | -1.2651 (3 rows) -- Test 6: Batch search with different queries -- Note: Uses explicit index name because query text comes from a subquery -- Note: Query plan varies by PG version so EXPLAIN is skipped -- Test scoring consistency for multi-term query (implicit index resolution) -- Note: standalone_results uses explicit index name since it has a WHERE clause \echo 'Testing ORDER BY vs standalone scoring for multi-term queries' Testing ORDER BY vs standalone scoring for multi-term queries WITH order_by_results AS ( SELECT id, title, ROUND((content <@> to_bm25query('machine learning algorithm', 'articles_tapir_idx'))::numeric, 6) as order_by_score FROM articles ORDER BY content <@> 'machine learning algorithm' LIMIT 2 ), standalone_results AS ( SELECT id, title, ROUND((content <@> to_bm25query('machine learning algorithm', 'articles_tapir_idx'))::numeric, 6) as standalone_score FROM articles WHERE id IN (SELECT id FROM order_by_results) ) SELECT o.id, o.title, o.order_by_score, s.standalone_score, CASE WHEN abs(o.order_by_score - s.standalone_score) < 0.000001 THEN '✓ CONSISTENT' ELSE '✗ INCONSISTENT' END as consistency_check FROM order_by_results o JOIN standalone_results s ON o.id = s.id ORDER BY o.id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-4.6440 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-3.3220 id | title | order_by_score | standalone_score | consistency_check ----+-------------------------------+----------------+------------------+------------------- 2 | Machine Learning Fundamentals | -4.644046 | -4.644046 | ✓ CONSISTENT 7 | Document Classification | -3.322026 | -3.322026 | ✓ CONSISTENT (2 rows) -- Cleanup DROP TABLE articles CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/query_hint_roundtrip.out000066400000000000000000000031471525762047700265740ustar00rootroot00000000000000-- Uses the existing test module installed in injection-enabled CI builds. -- This case itself does not require injection points. SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION pg_textsearch_test; CREATE TABLE hint_copy_docs (id int, body text); INSERT INTO hint_copy_docs SELECT g, 'common' FROM generate_series(1, 1000) g; CREATE INDEX hint_copy_idx ON hint_copy_docs USING bm25(body) WITH (text_config='english'); ANALYZE hint_copy_docs; SET enable_seqscan = off; -- Validate zeroed padding, full datum copying and plan serialization. -- Use both implicit index resolution and an explicitly named index. SELECT pg_textsearch_test_query_hint_roundtrip($q$ SELECT id FROM hint_copy_docs WHERE id < 100 ORDER BY body <@> 'common'::bm25query LIMIT 10 OFFSET 2 $q$, 12) AS typed_query_roundtrip; typed_query_roundtrip ----------------------- t (1 row) SELECT pg_textsearch_test_query_hint_roundtrip($q$ SELECT id FROM hint_copy_docs WHERE id < 100 ORDER BY body <@> to_bm25query('common', 'hint_copy_idx') LIMIT 20 $q$, 20) AS explicit_query_roundtrip; explicit_query_roundtrip -------------------------- t (1 row) -- Exercise a large query value, where redundant datum copies cost more. SELECT pg_textsearch_test_query_hint_roundtrip(format($q$ SELECT id FROM hint_copy_docs WHERE id < 100 ORDER BY body <@> %L::bm25query LIMIT 10 $q$, repeat('common ', 10000)), 10) AS large_query_roundtrip; large_query_roundtrip ----------------------- t (1 row) RESET enable_seqscan; DROP TABLE hint_copy_docs; DROP EXTENSION pg_textsearch_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/quoted_identifiers.out000066400000000000000000000121131525762047700261560ustar00rootroot00000000000000-- Test case: quoted_identifiers -- Tests that BM25 indexes work correctly with quoted (case-sensitive) -- identifiers for both index names and schema names. -- Regression test for https://github.com/timescale/pg_textsearch/issues/285 CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Test 1: Uppercase index name (the reported bug) CREATE TABLE qi_test1 ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX "IX_qi_test1_content" ON qi_test1 USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation IX_qi_test1_content NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO qi_test1 (content) VALUES ('lorem ipsum dolor sit amet'); INSERT INTO qi_test1 (content) VALUES ('quick brown fox jumps'); INSERT INTO qi_test1 (content) VALUES ('lorem ipsum again'); SELECT id, content FROM qi_test1 ORDER BY id; id | content ----+---------------------------- 1 | lorem ipsum dolor sit amet 2 | quick brown fox jumps 3 | lorem ipsum again (3 rows) -- Test 2: Query with uppercase index name SET enable_seqscan = off; SELECT id, content, ROUND((content <@> to_bm25query('lorem', 'IX_qi_test1_content'))::numeric, 4) AS score FROM qi_test1 ORDER BY content <@> to_bm25query('lorem', 'IX_qi_test1_content'), id; id | content | score ----+----------------------------+--------- 3 | lorem ipsum again | -0.5774 1 | lorem ipsum dolor sit amet | -0.4091 (2 rows) -- Test 3: Uppercase schema name with lowercase index CREATE SCHEMA "MySchema"; CREATE TABLE "MySchema".docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX idx_myschema_docs ON "MySchema".docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation idx_myschema_docs NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO "MySchema".docs (content) VALUES ('hello world'); INSERT INTO "MySchema".docs (content) VALUES ('goodbye world'); SELECT id, content FROM "MySchema".docs ORDER BY id; id | content ----+--------------- 1 | hello world 2 | goodbye world (2 rows) -- Query using schema-qualified index SELECT id, content, ROUND((content <@> to_bm25query('hello', '"MySchema".idx_myschema_docs'))::numeric, 4) AS score FROM "MySchema".docs ORDER BY content <@> to_bm25query('hello', '"MySchema".idx_myschema_docs'), id; id | content | score ----+-------------+--------- 1 | hello world | -0.6931 (1 row) -- Test 4: Both uppercase schema and uppercase index CREATE TABLE "MySchema".docs2 ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX "IX_MyDocs" ON "MySchema".docs2 USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation IX_MyDocs NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO "MySchema".docs2 (content) VALUES ('alpha beta gamma'); INSERT INTO "MySchema".docs2 (content) VALUES ('delta epsilon'); SELECT id, content FROM "MySchema".docs2 ORDER BY id; id | content ----+------------------ 1 | alpha beta gamma 2 | delta epsilon (2 rows) -- Query using both-uppercase schema-qualified index SELECT id, content, ROUND((content <@> to_bm25query('alpha', '"MySchema"."IX_MyDocs"'))::numeric, 4) AS score FROM "MySchema".docs2 ORDER BY content <@> to_bm25query('alpha', '"MySchema"."IX_MyDocs"'), id; id | content | score ----+------------------+--------- 1 | alpha beta gamma | -0.6407 (1 row) -- Test 5: UPDATE with uppercase index name UPDATE qi_test1 SET content = 'updated lorem ipsum' WHERE id = 1; SELECT id, content FROM qi_test1 ORDER BY id; id | content ----+----------------------- 1 | updated lorem ipsum 2 | quick brown fox jumps 3 | lorem ipsum again (3 rows) -- Test 6: DELETE with uppercase index name DELETE FROM qi_test1 WHERE id = 2; SELECT id, content FROM qi_test1 ORDER BY id; id | content ----+--------------------- 1 | updated lorem ipsum 3 | lorem ipsum again (2 rows) -- Test 7: VACUUM with uppercase index name VACUUM qi_test1; SELECT id, content FROM qi_test1 ORDER BY id; id | content ----+--------------------- 1 | updated lorem ipsum 3 | lorem ipsum again (2 rows) -- Test 8: bm25_dump_index with schema-qualified uppercase index SELECT bm25_dump_index('"MySchema"."IX_MyDocs"')::text ~ 'Tapir Index Debug' AS dump_works; dump_works ------------ t (1 row) -- Test 9: bm25_summarize_index with schema-qualified uppercase index SELECT bm25_summarize_index('"MySchema"."IX_MyDocs"')::text ~ 'total_docs: 2' AS summarize_works; summarize_works ----------------- t (1 row) -- Cleanup DROP TABLE qi_test1 CASCADE; DROP TABLE "MySchema".docs CASCADE; DROP TABLE "MySchema".docs2 CASCADE; DROP SCHEMA "MySchema" CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/rescan.out000066400000000000000000000316251525762047700235540ustar00rootroot00000000000000-- Test: iterative index scan (rescan with increasing limit) -- -- Demonstrates and tests the interaction between LIMIT, WHERE clause -- post-filtering, and the index's internal result limit. -- -- The core problem: the index computes top-k results in one shot. -- When a WHERE clause filters out rows, fewer than LIMIT rows may be -- returned even though qualifying rows exist in the table. -- -- The fix: when the executor exhausts the index's result set without -- satisfying LIMIT, the index rescans with a larger internal limit -- (exponential backoff: 2x, 4x, ...) until enough rows are found. SET log_duration = off; CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET client_min_messages = NOTICE; SET enable_seqscan = false; ------------------------------------------------------------------------ -- Setup: table with two categories, deliberately skewed BM25 scores ------------------------------------------------------------------------ CREATE TABLE rescan_test ( id SERIAL PRIMARY KEY, category TEXT, content TEXT ); -- 30 'common' rows: high BM25 score for 'database' (term appears 3x, -- short document) INSERT INTO rescan_test (category, content) SELECT 'common', 'database database database management optimization' FROM generate_series(1, 30); -- 20 'rare' rows: lower BM25 score for 'database' (term appears 1x, -- longer document, diluted by other terms) INSERT INTO rescan_test (category, content) SELECT 'rare', 'the modern application framework provides a database ' 'connection pooling layer for enterprise deployments' FROM generate_series(1, 20); CREATE INDEX rescan_idx ON rescan_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rescan_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=7.00 ------------------------------------------------------------------------ -- Baseline: without WHERE, LIMIT works correctly ------------------------------------------------------------------------ -- Should return exactly 5 rows SELECT COUNT(*) AS without_where FROM ( SELECT id FROM rescan_test ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 5 ) q; without_where --------------- 5 (1 row) ------------------------------------------------------------------------ -- Test 1: LIMIT + WHERE — complete miss -- -- LIMIT 5 is pushed down to the index (no indexclauses to prevent it). -- The index returns top-5 results, all 'common' rows (higher score). -- WHERE category='rare' filters out all 5 — returns 0 rows. -- -- After fix: index rescans with larger limit until 5 'rare' rows found. ------------------------------------------------------------------------ SELECT COUNT(*) AS with_where_complete_miss FROM ( SELECT id, category FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 5 ) q; with_where_complete_miss -------------------------- 5 (1 row) ------------------------------------------------------------------------ -- Test 2: default_limit + WHERE — same problem without explicit LIMIT -- -- When no LIMIT clause, the default_limit (set low here) controls -- how many results the index computes. WHERE can still starve results. ------------------------------------------------------------------------ SET pg_textsearch.default_limit = 10; SELECT COUNT(*) AS default_limit_miss FROM ( SELECT id, category FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') ) q; default_limit_miss -------------------- 20 (1 row) -- Reset to normal SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 3: partial miss — some results pass, but fewer than LIMIT -- -- Mix categories so that some top-k results pass the filter. ------------------------------------------------------------------------ INSERT INTO rescan_test (category, content) VALUES ('mixed', 'database systems and database optimization techniques'), ('mixed', 'database management in enterprise systems'); -- Refresh index knowledge REINDEX INDEX rescan_idx; NOTICE: BM25 index build started for relation rescan_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 52 documents, avg_length=6.90 -- Ask for 10 'mixed' rows but only 2 exist. With a small internal -- limit, the index may not even find those 2 among the top-k results. SET pg_textsearch.default_limit = 5; SELECT COUNT(*) AS partial_miss FROM ( SELECT id, category FROM rescan_test WHERE category = 'mixed' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 10 ) q; partial_miss -------------- 2 (1 row) SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 4: verify result ordering after rescan -- -- Even when the index rescans with a larger limit, results must still -- be returned in correct BM25 score order. ------------------------------------------------------------------------ -- All 20 'rare' rows should be returned, in score order (they all -- have the same score, so order by id is acceptable as tiebreaker) SELECT id, category, ROUND((content <@> to_bm25query('database', 'rescan_idx'))::numeric, 4) AS score FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 5; id | category | score ----+----------+--------- 31 | rare | -0.0080 32 | rare | -0.0080 33 | rare | -0.0080 34 | rare | -0.0080 35 | rare | -0.0080 (5 rows) ------------------------------------------------------------------------ -- Test 5: large table with aggressive filtering -- -- Scale up to make the problem more realistic. With 500 total rows -- and only 10 matching the filter, the index needs multiple rescans. ------------------------------------------------------------------------ CREATE TABLE rescan_large ( id SERIAL PRIMARY KEY, tag INT, content TEXT ); -- 490 rows with tag=0, high score for 'server' INSERT INTO rescan_large (tag, content) SELECT 0, 'server cluster server configuration server deployment' FROM generate_series(1, 490); -- 10 rows with tag=1, lower score for 'server' INSERT INTO rescan_large (tag, content) SELECT 1, 'the application connects to a server through a ' 'load balanced proxy for high availability' FROM generate_series(1, 10); CREATE INDEX rescan_large_idx ON rescan_large USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rescan_large_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 500 documents, avg_length=6.04 -- Ask for 8 rows with tag=1. Only 10 exist out of 500 total. -- The index must scan deep enough to find them. SELECT COUNT(*) AS large_table_filter FROM ( SELECT id FROM rescan_large WHERE tag = 1 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 8 ) q; large_table_filter -------------------- 8 (1 row) ------------------------------------------------------------------------ -- Test 6: WHERE on indexed column's table, not a separate column -- -- Even when the WHERE references the same table, if it's not an -- index qual, the same problem applies. ------------------------------------------------------------------------ SELECT COUNT(*) AS where_on_id FROM ( SELECT id FROM rescan_large WHERE id > 495 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 3 ) q; where_on_id ------------- 3 (1 row) ------------------------------------------------------------------------ -- Test 7: LIMIT + OFFSET + WHERE -- -- OFFSET makes this harder: we need to skip rows AND satisfy LIMIT, -- so the index needs even more candidates. ------------------------------------------------------------------------ SELECT COUNT(*) AS limit_offset_where FROM ( SELECT id FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 3 OFFSET 2 ) q; limit_offset_where -------------------- 3 (1 row) ------------------------------------------------------------------------ -- Test 8: multiple backoffs required -- -- With default_limit=5, finding 3 rows with tag=1 among 500 rows -- requires the limit to grow: 5 -> 10 -> 20 -> 40 -> 80 -> 160 -> -- 320 -> 640. That's 7 doublings. The NOTICE trace messages from -- the rescan code confirm each backoff step. ------------------------------------------------------------------------ SET pg_textsearch.default_limit = 5; SELECT COUNT(*) AS multi_backoff FROM ( SELECT id FROM rescan_large WHERE tag = 1 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 3 ) q; multi_backoff --------------- 3 (1 row) SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 9: backoff terminates when all matching rows exhausted -- -- Ask for 20 rows with tag=1, but only 10 exist. The index should -- keep doubling until it has scanned all 500 docs, then stop. ------------------------------------------------------------------------ SET pg_textsearch.default_limit = 5; SELECT COUNT(*) AS exhausted_backoff FROM ( SELECT id FROM rescan_large WHERE tag = 1 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 20 ) q; exhausted_backoff ------------------- 10 (1 row) SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 10: Postgres-level rescan via correlated LATERAL join -- -- Forces the executor to call the index's rescan function after -- results have already been returned. The correlation (category = -- v.x) prevents Postgres from materializing the subquery, so each -- outer row triggers a true index rescan that must clean up the -- result arrays from the previous iteration. ------------------------------------------------------------------------ SELECT v.x, t.id, t.category FROM (VALUES ('common'), ('rare')) AS v(x), LATERAL ( SELECT id, category FROM rescan_test WHERE category = v.x ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 2 ) t ORDER BY v.x, t.id; x | id | category --------+----+---------- common | 1 | common common | 2 | common rare | 31 | rare rare | 32 | rare (4 rows) -- Filtered and invisible candidates must not impose a scan-wide limit. CREATE TABLE rescan_deep ( id int, tag int, content text ) WITH (autovacuum_enabled = false); INSERT INTO rescan_deep SELECT g, 0, 'alpha beta' FROM generate_series(1, 100050) g; INSERT INTO rescan_deep SELECT 100050 + g, 1, 'alpha gamma gamma gamma gamma gamma gamma gamma' FROM generate_series(1, 5) g; CREATE INDEX rescan_deep_idx ON rescan_deep USING bm25(content) WITH (text_config = 'simple'); NOTICE: BM25 index build started for relation rescan_deep_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100055 documents, avg_length=2.00 ANALYZE rescan_deep; SELECT array_agg(id ORDER BY id) AS deep_filtered FROM ( SELECT id FROM rescan_deep WHERE tag = 1 ORDER BY content <@> to_bm25query('alpha', 'rescan_deep_idx') LIMIT 10 ) s; deep_filtered -------------------------------------- {100051,100052,100053,100054,100055} (1 row) DELETE FROM rescan_deep WHERE tag = 0; SELECT array_agg(id ORDER BY id) AS deep_deleted FROM ( SELECT id FROM rescan_deep ORDER BY content <@> to_bm25query('alpha', 'rescan_deep_idx') LIMIT 10 ) s; deep_deleted -------------------------------------- {100051,100052,100053,100054,100055} (1 row) DROP TABLE rescan_deep; -- Incremental Sort must see a whole tie group, even beyond 100,000 rows. CREATE TABLE rescan_ties (id int, content text); INSERT INTO rescan_ties SELECT g, 'alpha beta' FROM generate_series(100055, 1, -1) g; CREATE INDEX rescan_ties_idx ON rescan_ties USING bm25(content) WITH (text_config = 'simple'); NOTICE: BM25 index build started for relation rescan_ties_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100055 documents, avg_length=2.00 SELECT array_agg(id ORDER BY id) AS deep_ties FROM ( SELECT id, content <@> to_bm25query('alpha', 'rescan_ties_idx') FROM rescan_ties ORDER BY content <@> to_bm25query('alpha', 'rescan_ties_idx'), id LIMIT 10 ) s; deep_ties ------------------------ {1,2,3,4,5,6,7,8,9,10} (1 row) DROP TABLE rescan_ties; DROP TABLE rescan_test CASCADE; DROP TABLE rescan_large CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/rls.out000066400000000000000000000713251525762047700231020ustar00rootroot00000000000000-- BM25/RLS combinations are allowed by default, but administrators can -- disable creation, rebuild, or later RLS enablement. CREATE EXTENSION pg_textsearch; SHOW pg_textsearch.allow_rls; pg_textsearch.allow_rls ------------------------- on (1 row) -- RLS policies filter BM25 index results when the combination is allowed. CREATE ROLE rls_visibility_reader; CREATE TABLE rls_visibility (id integer, content text); INSERT INTO rls_visibility VALUES (1, 'target target target target target target target target'), (2, 'target filler filler filler filler filler'); ALTER TABLE rls_visibility ENABLE ROW LEVEL SECURITY; CREATE POLICY rls_visibility_policy ON rls_visibility TO rls_visibility_reader USING (id = 2); CREATE INDEX rls_visibility_idx ON rls_visibility USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_visibility_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=7.00 GRANT SELECT ON rls_visibility TO rls_visibility_reader; CREATE FUNCTION pg_temp.first_plan_child(query text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE plan json; BEGIN EXECUTE 'EXPLAIN (FORMAT JSON, COSTS OFF) ' || query INTO plan; RETURN plan->0->'Plan'->'Plans'->0->>'Node Type'; END $$; SET enable_seqscan = off; \pset format unaligned SELECT id AS unrestricted_top FROM rls_visibility ORDER BY content <@> to_bm25query('target', 'rls_visibility_idx') LIMIT 1; unrestricted_top 1 (1 row) SET ROLE rls_visibility_reader; SELECT pg_temp.first_plan_child($query$ SELECT id FROM rls_visibility ORDER BY content <@> to_bm25query('target', 'rls_visibility_idx') LIMIT 1 $query$) = 'Index Scan' AS rls_uses_bm25_index; rls_uses_bm25_index t (1 row) SELECT id AS rls_visible_top FROM rls_visibility ORDER BY content <@> to_bm25query('target', 'rls_visibility_idx') LIMIT 1; rls_visible_top 2 (1 row) RESET ROLE; \pset format aligned RESET enable_seqscan; DROP TABLE rls_visibility; DROP ROLE rls_visibility_reader; CREATE TABLE rls_existing (id integer, content text); INSERT INTO rls_existing VALUES (1, 'known secret term'); ALTER TABLE rls_existing ENABLE ROW LEVEL SECURITY; CREATE INDEX rls_existing_idx ON rls_existing USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_existing_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=3.00 SET pg_textsearch.allow_rls = off; ALTER TABLE IF EXISTS rls_missing ENABLE ROW LEVEL SECURITY; NOTICE: relation "rls_missing" does not exist, skipping CREATE INDEX IF NOT EXISTS rls_existing_idx ON rls_existing USING bm25(content) WITH (text_config='english'); NOTICE: relation "rls_existing_idx" already exists, skipping \pset format unaligned SELECT indisvalid AS existing_index_valid FROM pg_index WHERE indexrelid = 'rls_existing_idx'::regclass; existing_index_valid t (1 row) \pset format aligned -- Existing combinations remain usable when the GUC changes. SELECT content FROM rls_existing ORDER BY content <@> to_bm25query('known', 'rls_existing_idx') LIMIT 1; content ------------------- known secret term (1 row) INSERT INTO rls_existing VALUES (2, 'another known term'); \set VERBOSITY terse REINDEX INDEX rls_existing_idx; ERROR: BM25 indexes are not allowed on row-level security relation "rls_existing" when "pg_textsearch.allow_rls" is off \set VERBOSITY default CREATE TABLE rls_before_index (id integer, content text); ALTER TABLE rls_before_index ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse BEGIN; CREATE INDEX CONCURRENTLY rls_concurrent_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ERROR: CREATE INDEX CONCURRENTLY cannot run inside a transaction block ROLLBACK; CREATE INDEX CONCURRENTLY rls_concurrent_top_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_before_index" when "pg_textsearch.allow_rls" is off \pset format unaligned SELECT to_regclass('rls_concurrent_top_idx') IS NULL AS no_index_catalog_entry; no_index_catalog_entry t (1 row) \pset format aligned CREATE INDEX CONCURRENTLY IF NOT EXISTS rls_concurrent_ifne_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_before_index" when "pg_textsearch.allow_rls" is off \pset format unaligned SELECT to_regclass('rls_concurrent_ifne_idx') IS NULL AS no_if_not_exists_catalog_entry; no_if_not_exists_catalog_entry t (1 row) \pset format aligned CREATE INDEX rls_before_index_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_before_index" when "pg_textsearch.allow_rls" is off \set VERBOSITY default CREATE TABLE index_before_rls (id integer, content text); CREATE INDEX index_before_rls_idx ON index_before_rls USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation index_before_rls_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY terse ALTER VIEW index_before_rls ENABLE ROW LEVEL SECURITY; ERROR: "index_before_rls" is not a view ALTER TABLE index_before_rls DROP COLUMN missing, ENABLE ROW LEVEL SECURITY; ERROR: column "missing" of relation "index_before_rls" does not exist \pset format unaligned SELECT relrowsecurity FROM pg_class WHERE oid = 'index_before_rls'::regclass; relrowsecurity f (1 row) ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY, DISABLE ROW LEVEL SECURITY; SELECT relrowsecurity FROM pg_class WHERE oid = 'index_before_rls'::regclass; relrowsecurity f (1 row) ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY; ERROR: cannot enable row-level security on relation "index_before_rls" because it has a BM25 index while "pg_textsearch.allow_rls" is off SELECT relrowsecurity FROM pg_class WHERE oid = 'index_before_rls'::regclass; relrowsecurity f (1 row) \pset format aligned \set VERBOSITY default -- A child index is protected by RLS enabled on an inheritance ancestor. CREATE TABLE rls_parent (id integer, content text); CREATE TABLE rls_child () INHERITS (rls_parent); ALTER TABLE rls_parent ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_child_idx ON rls_child USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_parent" when "pg_textsearch.allow_rls" is off \set VERBOSITY default -- A direct partition-child index is protected by RLS on its parent. CREATE TABLE rls_direct_partitioned (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE rls_direct_partition PARTITION OF rls_direct_partitioned FOR VALUES FROM (0) TO (10); ALTER TABLE rls_direct_partitioned ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_direct_partition_idx ON rls_direct_partition USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_direct_partitioned" when "pg_textsearch.allow_rls" is off \set VERBOSITY default -- A multi-level partition descendant is protected by RLS at the top. CREATE TABLE rls_partition_top (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE rls_partition_mid PARTITION OF rls_partition_top FOR VALUES FROM (0) TO (100) PARTITION BY RANGE (id); CREATE TABLE rls_partition_leaf PARTITION OF rls_partition_mid FOR VALUES FROM (0) TO (10); ALTER TABLE rls_partition_top ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_partition_leaf_idx ON rls_partition_leaf USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_partition_top" when "pg_textsearch.allow_rls" is off \set VERBOSITY default -- A multiple-inheritance child is protected if either parent has RLS. CREATE TABLE rls_multi_parent_a (id integer, content text); CREATE TABLE rls_multi_parent_b (extra text); CREATE TABLE rls_multi_child () INHERITS (rls_multi_parent_a, rls_multi_parent_b); ALTER TABLE rls_multi_parent_b ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_multi_child_idx ON rls_multi_child USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_multi_parent_b" when "pg_textsearch.allow_rls" is off \set VERBOSITY default -- An existing BM25-indexed table cannot be attached below an RLS ancestor. CREATE TABLE rls_attach_parent (id integer, content text); ALTER TABLE rls_attach_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_attach_child (id integer, content text); CREATE INDEX rls_attach_child_idx ON rls_attach_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_attach_child_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY terse ALTER TABLE rls_attach_child INHERIT rls_attach_parent; ERROR: BM25 indexes are not allowed on row-level security relation "rls_attach_parent" when "pg_textsearch.allow_rls" is off \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_attach_child'::regclass AND inhparent = 'rls_attach_parent'::regclass ) AS inherit_not_attached; inherit_not_attached t (1 row) \pset format aligned \set VERBOSITY default -- An existing BM25-indexed table cannot become an RLS parent's partition. CREATE TABLE rls_attach_partitioned (id integer, content text) PARTITION BY RANGE (id); ALTER TABLE rls_attach_partitioned ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_attach_partition (id integer, content text); CREATE INDEX rls_attach_partition_idx ON rls_attach_partition USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_attach_partition_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY terse ALTER TABLE rls_attach_partitioned ATTACH PARTITION rls_attach_partition FOR VALUES FROM (0) TO (10); ERROR: BM25 indexes are not allowed on row-level security relation "rls_attach_partitioned" when "pg_textsearch.allow_rls" is off \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_attach_partition'::regclass AND inhparent = 'rls_attach_partitioned'::regclass ) AS partition_not_attached; partition_not_attached t (1 row) \pset format aligned \set VERBOSITY default -- Successful checks do not retain locks on traversed relations. CREATE TABLE rls_lock_root (id integer, content text); CREATE TABLE rls_lock_parent () INHERITS (rls_lock_root); CREATE TABLE rls_lock_child () INHERITS (rls_lock_parent); BEGIN; CREATE INDEX rls_lock_child_idx ON rls_lock_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_lock_child_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \pset format unaligned SELECT count(*) = 0 AS no_ancestor_locks_held FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'relation' AND relation IN ('rls_lock_root'::regclass, 'rls_lock_parent'::regclass) AND mode = 'AccessShareLock' AND granted; no_ancestor_locks_held t (1 row) \pset format aligned ROLLBACK; CREATE TABLE rls_lock_enable_root (id integer, content text); CREATE TABLE rls_lock_enable_child () INHERITS (rls_lock_enable_root); BEGIN; ALTER TABLE rls_lock_enable_root ENABLE ROW LEVEL SECURITY; \pset format unaligned SELECT count(*) = 0 AS no_descendant_locks_held FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'relation' AND relation = 'rls_lock_enable_child'::regclass AND mode = 'AccessShareLock' AND granted; no_descendant_locks_held t (1 row) \pset format aligned ROLLBACK; -- Enabling RLS on a parent is blocked by a BM25 index on a descendant. CREATE TABLE index_parent (id integer, content text); CREATE TABLE index_child () INHERITS (index_parent); CREATE INDEX index_child_idx ON index_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation index_child_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY terse ALTER TABLE index_parent ENABLE ROW LEVEL SECURITY; ERROR: cannot enable row-level security on relation "index_parent" because descendant relation "index_child" has a BM25 index while "pg_textsearch.allow_rls" is off \set VERBOSITY default -- A partition build is protected by RLS enabled on the partitioned parent. CREATE TABLE rls_partitioned (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE rls_partition PARTITION OF rls_partitioned FOR VALUES FROM (0) TO (10); ALTER TABLE rls_partitioned ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_partitioned_idx ON rls_partitioned USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_partitioned" when "pg_textsearch.allow_rls" is off \set VERBOSITY default -- Enabling RLS on a partitioned parent is blocked by descendant BM25 indexes. CREATE TABLE index_partitioned (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE index_partition PARTITION OF index_partitioned FOR VALUES FROM (0) TO (10); CREATE INDEX index_partition_idx ON index_partition USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation index_partition_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 \set VERBOSITY terse ALTER TABLE index_partitioned ENABLE ROW LEVEL SECURITY; ERROR: cannot enable row-level security on relation "index_partitioned" because descendant relation "index_partition" has a BM25 index while "pg_textsearch.allow_rls" is off \set VERBOSITY default -- Event-trigger DDL must be attributed to the command that actually creates -- or alters each catalog object, not to a stale pre-trigger name lookup. CREATE FUNCTION rls_ddl_race_start() RETURNS event_trigger AS $$ DECLARE action text := current_setting('rls_test.action', true); BEGIN IF action IS NULL OR action IN ('', 'running', 'done', 'drop_enable', 'drop_inherit', 'drop_attach') THEN RETURN; END IF; PERFORM set_config('rls_test.action', 'running', true); IF action = 'swap_enable' THEN EXECUTE 'ALTER TABLE rls_evt_enable_target ' 'RENAME TO rls_evt_enable_swap'; EXECUTE 'ALTER TABLE rls_evt_enable_guarded ' 'RENAME TO rls_evt_enable_target'; EXECUTE 'ALTER TABLE rls_evt_enable_swap ' 'RENAME TO rls_evt_enable_guarded'; PERFORM set_config('rls_test.action', 'restore_enable', true); ELSIF action = 'swap_inherit' THEN EXECUTE 'ALTER TABLE rls_evt_inherit_target ' 'RENAME TO rls_evt_inherit_swap'; EXECUTE 'ALTER TABLE rls_evt_inherit_guarded ' 'RENAME TO rls_evt_inherit_target'; EXECUTE 'ALTER TABLE rls_evt_inherit_swap ' 'RENAME TO rls_evt_inherit_guarded'; PERFORM set_config('rls_test.action', 'restore_inherit', true); ELSIF action = 'swap_attach' THEN EXECUTE 'ALTER TABLE rls_evt_attach_target ' 'RENAME TO rls_evt_attach_swap'; EXECUTE 'ALTER TABLE rls_evt_attach_guarded ' 'RENAME TO rls_evt_attach_target'; EXECUTE 'ALTER TABLE rls_evt_attach_swap ' 'RENAME TO rls_evt_attach_guarded'; PERFORM set_config('rls_test.action', 'restore_attach', true); ELSIF action = 'drop_ifne_collision' THEN EXECUTE 'DROP INDEX rls_evt_ifne_idx'; PERFORM set_config('rls_test.action', 'done', true); ELSIF action = 'drop_extension' THEN PERFORM set_config('client_min_messages', 'warning', true); EXECUTE 'DROP EXTENSION pg_textsearch CASCADE'; PERFORM set_config('rls_test.action', 'done', true); END IF; END $$ LANGUAGE plpgsql; CREATE FUNCTION rls_ddl_race_end() RETURNS event_trigger AS $$ DECLARE action text := current_setting('rls_test.action', true); BEGIN IF action IS NULL OR action NOT IN ('restore_enable', 'restore_inherit', 'restore_attach', 'drop_enable', 'drop_inherit', 'drop_attach') THEN RETURN; END IF; PERFORM set_config('rls_test.action', 'running', true); IF action = 'restore_enable' THEN EXECUTE 'ALTER TABLE rls_evt_enable_target ' 'RENAME TO rls_evt_enable_swap'; EXECUTE 'ALTER TABLE rls_evt_enable_guarded ' 'RENAME TO rls_evt_enable_target'; EXECUTE 'ALTER TABLE rls_evt_enable_swap ' 'RENAME TO rls_evt_enable_guarded'; ELSIF action = 'restore_inherit' THEN EXECUTE 'ALTER TABLE rls_evt_inherit_target ' 'RENAME TO rls_evt_inherit_swap'; EXECUTE 'ALTER TABLE rls_evt_inherit_guarded ' 'RENAME TO rls_evt_inherit_target'; EXECUTE 'ALTER TABLE rls_evt_inherit_swap ' 'RENAME TO rls_evt_inherit_guarded'; ELSIF action = 'restore_attach' THEN EXECUTE 'ALTER TABLE rls_evt_attach_target ' 'RENAME TO rls_evt_attach_swap'; EXECUTE 'ALTER TABLE rls_evt_attach_guarded ' 'RENAME TO rls_evt_attach_target'; EXECUTE 'ALTER TABLE rls_evt_attach_swap ' 'RENAME TO rls_evt_attach_guarded'; ELSIF action = 'drop_enable' THEN EXECUTE 'DROP TABLE rls_evt_drop_enable_target CASCADE'; ELSIF action = 'drop_inherit' THEN EXECUTE 'DROP TABLE rls_evt_drop_inherit_child CASCADE'; ELSE EXECUTE 'DROP TABLE rls_evt_drop_attach_child CASCADE'; END IF; PERFORM set_config('rls_test.action', 'done', true); END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER rls_ddl_race_start_trigger ON ddl_command_start WHEN TAG IN ('ALTER TABLE', 'CREATE INDEX') EXECUTE FUNCTION rls_ddl_race_start(); CREATE EVENT TRIGGER rls_ddl_race_end_trigger ON ddl_command_end WHEN TAG IN ('ALTER TABLE') EXECUTE FUNCTION rls_ddl_race_end(); CREATE TABLE rls_evt_enable_target (id integer, content text); CREATE TABLE rls_evt_enable_guarded (id integer, content text); CREATE INDEX rls_evt_enable_idx ON rls_evt_enable_guarded USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_enable_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET rls_test.action = 'swap_enable'; \set VERBOSITY terse ALTER TABLE rls_evt_enable_target ENABLE ROW LEVEL SECURITY; ERROR: cannot enable row-level security on relation "rls_evt_enable_guarded" because it has a BM25 index while "pg_textsearch.allow_rls" is off \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT relrowsecurity AS swapped_enable_rolled_back FROM pg_class WHERE oid = 'rls_evt_enable_guarded'::regclass; swapped_enable_rolled_back f (1 row) \pset format aligned CREATE TABLE rls_evt_inherit_parent (id integer, content text); ALTER TABLE rls_evt_inherit_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_inherit_target (id integer, content text); CREATE TABLE rls_evt_inherit_guarded (id integer, content text); CREATE INDEX rls_evt_inherit_idx ON rls_evt_inherit_guarded USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_inherit_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET rls_test.action = 'swap_inherit'; \set VERBOSITY terse ALTER TABLE rls_evt_inherit_target INHERIT rls_evt_inherit_parent; ERROR: BM25 indexes are not allowed on row-level security relation "rls_evt_inherit_parent" when "pg_textsearch.allow_rls" is off \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_evt_inherit_guarded'::regclass AND inhparent = 'rls_evt_inherit_parent'::regclass ) AS swapped_inherit_rolled_back; swapped_inherit_rolled_back t (1 row) \pset format aligned CREATE TABLE rls_evt_attach_parent (id integer, content text) PARTITION BY RANGE (id); ALTER TABLE rls_evt_attach_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_attach_target (id integer, content text); CREATE TABLE rls_evt_attach_guarded (id integer, content text); CREATE INDEX rls_evt_attach_idx ON rls_evt_attach_guarded USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_attach_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET rls_test.action = 'swap_attach'; \set VERBOSITY terse ALTER TABLE rls_evt_attach_parent ATTACH PARTITION rls_evt_attach_target FOR VALUES FROM (0) TO (10); ERROR: BM25 indexes are not allowed on row-level security relation "rls_evt_attach_parent" when "pg_textsearch.allow_rls" is off \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_evt_attach_guarded'::regclass AND inhparent = 'rls_evt_attach_parent'::regclass ) AS swapped_attach_rolled_back; swapped_attach_rolled_back t (1 row) \pset format aligned CREATE TABLE rls_evt_drop_enable_target (id integer, content text); CREATE INDEX rls_evt_drop_enable_idx ON rls_evt_drop_enable_target USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_drop_enable_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET rls_test.action = 'drop_enable'; ALTER TABLE rls_evt_drop_enable_target ENABLE ROW LEVEL SECURITY; RESET rls_test.action; \pset format unaligned SELECT to_regclass('rls_evt_drop_enable_target') IS NULL AS dropped_enable_target_absent; dropped_enable_target_absent t (1 row) \pset format aligned CREATE TABLE rls_evt_drop_inherit_parent (id integer, content text); ALTER TABLE rls_evt_drop_inherit_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_drop_inherit_child (id integer, content text); CREATE INDEX rls_evt_drop_inherit_idx ON rls_evt_drop_inherit_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_drop_inherit_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET rls_test.action = 'drop_inherit'; ALTER TABLE rls_evt_drop_inherit_child INHERIT rls_evt_drop_inherit_parent; RESET rls_test.action; \pset format unaligned SELECT to_regclass('rls_evt_drop_inherit_child') IS NULL AS dropped_inherit_child_absent; dropped_inherit_child_absent t (1 row) \pset format aligned CREATE TABLE rls_evt_drop_attach_parent (id integer, content text) PARTITION BY RANGE (id); ALTER TABLE rls_evt_drop_attach_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_drop_attach_child (id integer, content text); CREATE INDEX rls_evt_drop_attach_idx ON rls_evt_drop_attach_child USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_drop_attach_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET rls_test.action = 'drop_attach'; ALTER TABLE rls_evt_drop_attach_parent ATTACH PARTITION rls_evt_drop_attach_child FOR VALUES FROM (0) TO (10); RESET rls_test.action; \pset format unaligned SELECT to_regclass('rls_evt_drop_attach_child') IS NULL AS dropped_attach_child_absent; dropped_attach_child_absent t (1 row) \pset format aligned CREATE TABLE rls_evt_ifne_heap (id integer, content text); ALTER TABLE rls_evt_ifne_heap ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_ifne_collision (id integer); CREATE INDEX rls_evt_ifne_idx ON rls_evt_ifne_collision(id); SET rls_test.action = 'drop_ifne_collision'; \set VERBOSITY terse CREATE INDEX CONCURRENTLY IF NOT EXISTS rls_evt_ifne_idx ON rls_evt_ifne_heap USING bm25(content) WITH (text_config='english'); ERROR: BM25 indexes are not allowed on row-level security relation "rls_evt_ifne_heap" when "pg_textsearch.allow_rls" is off \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT EXISTS ( SELECT 1 FROM pg_index WHERE indexrelid = 'rls_evt_ifne_idx'::regclass AND indrelid = 'rls_evt_ifne_collision'::regclass AND indisvalid ) AS ifne_collision_restored, NOT EXISTS ( SELECT 1 FROM pg_index WHERE indrelid = 'rls_evt_ifne_heap'::regclass ) AS no_forbidden_ifne_index; ifne_collision_restored|no_forbidden_ifne_index t|t (1 row) \pset format aligned CREATE TABLE rls_evt_progress_heap (id integer, content text); CREATE INDEX rls_evt_progress_idx ON rls_evt_progress_heap USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation rls_evt_progress_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 CREATE TABLE rls_evt_missing_am_heap (id integer, content text); SET rls_test.action = 'drop_extension'; \set VERBOSITY terse CREATE INDEX rls_evt_missing_am_idx ON rls_evt_missing_am_heap USING bm25(content) WITH (text_config='english'); ERROR: access method "bm25" does not exist \set VERBOSITY default RESET rls_test.action; REINDEX INDEX rls_evt_progress_idx; NOTICE: BM25 index build started for relation rls_evt_progress_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DROP EVENT TRIGGER rls_ddl_race_start_trigger; DROP EVENT TRIGGER rls_ddl_race_end_trigger; DROP FUNCTION rls_ddl_race_start(); DROP FUNCTION rls_ddl_race_end(); RESET pg_textsearch.allow_rls; ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY; DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rls_guc_user') THEN EXECUTE 'REASSIGN OWNED BY rls_guc_user TO CURRENT_USER'; EXECUTE 'DROP OWNED BY rls_guc_user CASCADE'; DROP ROLE rls_guc_user; END IF; END $$; CREATE ROLE rls_guc_user; SET pg_textsearch.allow_rls = off; SET ROLE rls_guc_user; \set VERBOSITY terse CREATE INDEX rls_unauthorized_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ERROR: must be owner of table rls_before_index ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY; ERROR: must be owner of table index_before_rls SET pg_textsearch.allow_rls = off; ERROR: permission denied to set parameter "pg_textsearch.allow_rls" \set VERBOSITY default RESET ROLE; DROP OWNED BY rls_guc_user; DROP ROLE rls_guc_user; DROP TABLE rls_existing, rls_before_index, index_before_rls CASCADE; DROP TABLE rls_child, rls_parent, index_child, index_parent CASCADE; DROP TABLE rls_partitioned, index_partitioned CASCADE; DROP TABLE rls_direct_partitioned, rls_partition_top CASCADE; DROP TABLE rls_multi_child, rls_multi_parent_a, rls_multi_parent_b CASCADE; DROP TABLE rls_attach_child, rls_attach_parent CASCADE; DROP TABLE rls_attach_partitioned, rls_attach_partition CASCADE; DROP TABLE rls_lock_child, rls_lock_parent, rls_lock_root CASCADE; DROP TABLE rls_evt_enable_target, rls_evt_enable_guarded CASCADE; DROP TABLE rls_evt_inherit_target, rls_evt_inherit_guarded, rls_evt_inherit_parent CASCADE; DROP TABLE rls_evt_attach_target, rls_evt_attach_guarded, rls_evt_attach_parent CASCADE; DROP TABLE rls_evt_drop_inherit_parent, rls_evt_drop_attach_parent CASCADE; DROP TABLE rls_evt_ifne_heap, rls_evt_ifne_collision CASCADE; DROP TABLE rls_evt_progress_heap, rls_evt_missing_am_heap CASCADE; CREATE TABLE rls_without_extension (id integer); SET pg_textsearch.allow_rls = off; DROP EXTENSION pg_textsearch; ALTER TABLE rls_without_extension ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_missing_am_idx ON rls_without_extension USING bm25(id); ERROR: access method "bm25" does not exist \set VERBOSITY default DROP TABLE rls_without_extension; timescale-pg_textsearch-b63fe58/test/expected/schema.out000066400000000000000000000165411525762047700235410ustar00rootroot00000000000000-- Test case: schema -- Tests index operations with schema-qualified tables CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- Create a custom schema CREATE SCHEMA docs; -- Create table in the custom schema CREATE TABLE docs.articles ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO docs.articles (content) VALUES ('hello world'); INSERT INTO docs.articles (content) VALUES ('goodbye cruel world'); INSERT INTO docs.articles (content) VALUES ('hello cruel goodbye'); -- Create index on schema-qualified table CREATE INDEX articles_idx ON docs.articles USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation articles_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=2.67 -- Test 1: Unqualified index name (not in search path - should fail) SELECT id, content, ROUND((content <@> to_bm25query('hello', 'articles_idx'))::numeric, 4) as score FROM docs.articles ORDER BY content <@> to_bm25query('hello', 'articles_idx'), id; ERROR: index "articles_idx" does not exist -- Test 2: Schema-qualified index name works correctly SELECT id, content, ROUND((content <@> to_bm25query('hello', 'docs.articles_idx'))::numeric, 4) as score FROM docs.articles ORDER BY content <@> to_bm25query('hello', 'docs.articles_idx'), id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.5235 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.4471 id | content | score ----+---------------------+--------- 1 | hello world | -0.5235 3 | hello cruel goodbye | -0.4471 (2 rows) -- Test 3: With search_path set, unqualified names work SET search_path = docs, public; SELECT id, content, ROUND((content <@> to_bm25query('hello', 'articles_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> to_bm25query('hello', 'articles_idx'), id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.5235 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.4471 id | content | score ----+---------------------+--------- 1 | hello world | -0.5235 3 | hello cruel goodbye | -0.4471 (2 rows) -- Reset search_path SET search_path = public; -- Test 4: Table in public schema works with unqualified names CREATE TABLE public_articles ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO public_articles (content) VALUES ('hello world'); INSERT INTO public_articles (content) VALUES ('goodbye cruel world'); CREATE INDEX public_articles_idx ON public_articles USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation public_articles_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=2.50 -- Unqualified index name works for public schema tables SELECT id, content, ROUND((content <@> to_bm25query('hello', 'public_articles_idx'))::numeric, 4) as score FROM public_articles ORDER BY content <@> to_bm25query('hello', 'public_articles_idx'), id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.7549 id | content | score ----+-------------+--------- 1 | hello world | -0.7549 (1 row) -- Test 5: INSERT operations with schema-qualified tables and implicit index -- names (reproduces the reported bug) CREATE SCHEMA bar; CREATE TABLE bar.baz(content TEXT NOT NULL); -- Create index with implicit name (PostgreSQL will name it baz_content_idx) CREATE INDEX ON bar.baz USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation baz_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- INSERTs work correctly with schema-qualified tables INSERT INTO bar.baz VALUES('really'); INSERT INTO bar.baz VALUES('testing schemas'); INSERT INTO bar.baz VALUES('another document'); -- Verify data was inserted correctly SELECT * FROM bar.baz ORDER BY content; content ------------------ another document really testing schemas (3 rows) -- Test search functionality SELECT content, ROUND((content <@> to_bm25query('testing', 'bar.baz_content_idx'))::numeric, 4) as score FROM bar.baz ORDER BY content <@> to_bm25query('testing', 'bar.baz_content_idx'), content; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.9066 content | score -----------------+--------- testing schemas | -0.9066 (1 row) -- Test BM25 edge case: With only 2 documents where a term appears in exactly -- 50% of them (1 out of 2), the IDF becomes log(1) = 0, making the entire -- BM25 score 0 regardless of term frequency. This is mathematically correct -- behavior, not a bug. CREATE SCHEMA edge_case; CREATE TABLE edge_case.demo(content TEXT NOT NULL); CREATE INDEX ON edge_case.demo USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation demo_content_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert exactly 2 documents, one with "testing" INSERT INTO edge_case.demo VALUES('really'); INSERT INTO edge_case.demo VALUES('testing schemas'); -- With 2 docs and "testing" in 1 (50%), IDF = log((2-1+0.5)/(1+0.5)) = log(1) = 0 SELECT content, ROUND((content <@> to_bm25query('testing', 'edge_case.demo_content_idx'))::numeric, 4) as score FROM edge_case.demo ORDER BY content <@> to_bm25query('testing', 'edge_case.demo_content_idx'), content; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6100 content | score -----------------+--------- testing schemas | -0.6100 (1 row) DROP TABLE edge_case.demo CASCADE; DROP SCHEMA edge_case CASCADE; -- Test 6: bm25_dump_index with schema-qualified indexes -- Test that the debug function properly handles schema resolution -- Test debug with public schema index (should work) SELECT bm25_dump_index('public_articles_idx')::text ~ 'Tapir Index Debug: public_articles_idx' AS debug_public_works; debug_public_works -------------------- t (1 row) -- Test debug with schema-qualified index name (should work) SELECT bm25_dump_index('bar.baz_content_idx')::text ~ 'Tapir Index Debug: bar.baz_content_idx' AS debug_schema_qualified; debug_schema_qualified ------------------------ t (1 row) -- Test debug with unqualified name for schema index (should fail when not in search path) SELECT bm25_dump_index('baz_content_idx')::text ~ 'ERROR: Index ''baz_content_idx'' not found' AS debug_unqualified_fails; debug_unqualified_fails ------------------------- t (1 row) -- Test debug with search_path set SET search_path = bar, public; SELECT bm25_dump_index('baz_content_idx')::text ~ 'Tapir Index Debug: baz_content_idx' AS debug_with_search_path; debug_with_search_path ------------------------ t (1 row) -- Reset search_path SET search_path = public; -- Test debug with non-existent index SELECT bm25_dump_index('nonexistent_idx')::text ~ 'ERROR: Index ''nonexistent_idx'' not found' AS debug_nonexistent; debug_nonexistent ------------------- t (1 row) -- Cleanup DROP TABLE bar.baz CASCADE; DROP SCHEMA bar CASCADE; DROP TABLE docs.articles CASCADE; DROP TABLE public_articles CASCADE; DROP SCHEMA docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring1.out000066400000000000000000000114221525762047700240170ustar00rootroot00000000000000-- Test case: scoring1 -- Generated BM25 test with 2 documents and 2 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring1_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring1_bulk (content) VALUES ('hello world'); INSERT INTO scoring1_bulk (content) VALUES ('goodbye cruel world'); -- Create index after data insertion (bulk build) CREATE INDEX scoring1_bulk_idx ON scoring1_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring1_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=2.50 -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring1_bulk ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.7549 id | content | score ----+-------------+--------- 1 | hello world | -0.7549 (1 row) -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_valid; hello_valid ------------- t (1 row) -- Validate index scan vs standalone scoring match for 'hello' SELECT validate_index_vs_standalone('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'hello') as hello_modes_match; hello_modes_match ------------------- t (1 row) -- Bulk mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring1_bulk ORDER BY content <@> 'cruel', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6407 id | content | score ----+---------------------+--------- 2 | goodbye cruel world | -0.6407 (1 row) -- Validate BM25 scoring for 'cruel' SELECT validate_bm25_scoring('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'cruel', 'english', 1.2, 0.75) as cruel_valid; cruel_valid ------------- t (1 row) -- Validate index scan vs standalone scoring match for 'cruel' SELECT validate_index_vs_standalone('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'cruel') as cruel_modes_match; cruel_modes_match ------------------- t (1 row) -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring1_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring1_incr_idx ON scoring1_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring1_incr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents incrementally INSERT INTO scoring1_incr (content) VALUES ('hello world'); INSERT INTO scoring1_incr (content) VALUES ('goodbye cruel world'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring1_incr ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.7549 id | content | score ----+-------------+--------- 1 | hello world | -0.7549 (1 row) -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring1_incr', 'content', 'scoring1_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; hello_incr_valid ------------------ t (1 row) -- Validate index scan vs standalone scoring match for 'hello' (incremental) SELECT validate_index_vs_standalone('scoring1_incr', 'content', 'scoring1_incr_idx', 'hello') as hello_incr_modes_match; hello_incr_modes_match ------------------------ t (1 row) -- Incremental mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring1_incr ORDER BY content <@> 'cruel', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6407 id | content | score ----+---------------------+--------- 2 | goodbye cruel world | -0.6407 (1 row) -- Validate BM25 scoring for 'cruel' (incremental) SELECT validate_bm25_scoring('scoring1_incr', 'content', 'scoring1_incr_idx', 'cruel', 'english', 1.2, 0.75) as cruel_incr_valid; cruel_incr_valid ------------------ t (1 row) -- Validate index scan vs standalone scoring match for 'cruel' (incremental) SELECT validate_index_vs_standalone('scoring1_incr', 'content', 'scoring1_incr_idx', 'cruel') as cruel_incr_modes_match; cruel_incr_modes_match ------------------------ t (1 row) -- Cleanup DROP TABLE scoring1_bulk CASCADE; DROP TABLE scoring1_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring2.out000066400000000000000000000164101525762047700240220ustar00rootroot00000000000000-- Test case: scoring2 -- Generated BM25 test with 5 documents and 4 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring2_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring2_bulk (content) VALUES ('hello world'); INSERT INTO scoring2_bulk (content) VALUES ('goodbye world'); INSERT INTO scoring2_bulk (content) VALUES ('hello goodbye'); INSERT INTO scoring2_bulk (content) VALUES ('world domination'); INSERT INTO scoring2_bulk (content) VALUES ('hello'); -- Create index after data insertion (bulk build) CREATE INDEX scoring2_bulk_idx ON scoring2_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring2_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=1.80 -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,5), BM25_score=-0.6588 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.5156 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.5156 id | content | score ----+---------------+--------- 5 | hello | -0.6588 1 | hello world | -0.5156 3 | hello goodbye | -0.5156 (3 rows) -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_bulk_valid; hello_bulk_valid ------------------ t (1 row) -- Bulk mode query 2: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'world', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.5156 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.5156 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.5156 id | content | score ----+------------------+--------- 1 | hello world | -0.5156 2 | goodbye world | -0.5156 4 | world domination | -0.5156 (3 rows) -- Validate BM25 scoring for 'world' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'world', 'english', 1.2, 0.75) as world_bulk_valid; world_bulk_valid ------------------ t (1 row) -- Bulk mode query 3: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.8374 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.8374 id | content | score ----+---------------+--------- 2 | goodbye world | -0.8374 3 | hello goodbye | -0.8374 (2 rows) -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; goodbye_bulk_valid -------------------- t (1 row) -- Bulk mode query 4: 'domination' SELECT id, content, ROUND((content <@> 'domination')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'domination', id; NOTICE: BM25 index scan: tid=(0,4), BM25_score=-1.3260 id | content | score ----+------------------+--------- 4 | world domination | -1.3260 (1 row) -- Validate BM25 scoring for 'domination' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'domination', 'english', 1.2, 0.75) as domination_bulk_valid; domination_bulk_valid ----------------------- t (1 row) -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring2_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring2_incr_idx ON scoring2_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring2_incr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents incrementally INSERT INTO scoring2_incr (content) VALUES ('hello world'); INSERT INTO scoring2_incr (content) VALUES ('goodbye world'); INSERT INTO scoring2_incr (content) VALUES ('hello goodbye'); INSERT INTO scoring2_incr (content) VALUES ('world domination'); INSERT INTO scoring2_incr (content) VALUES ('hello'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,5), BM25_score=-0.6588 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.5156 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.5156 id | content | score ----+---------------+--------- 5 | hello | -0.6588 1 | hello world | -0.5156 3 | hello goodbye | -0.5156 (3 rows) -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; hello_incr_valid ------------------ t (1 row) -- Incremental mode query 2: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'world', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.5156 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.5156 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-0.5156 id | content | score ----+------------------+--------- 1 | hello world | -0.5156 2 | goodbye world | -0.5156 4 | world domination | -0.5156 (3 rows) -- Validate BM25 scoring for 'world' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'world', 'english', 1.2, 0.75) as world_incr_valid; world_incr_valid ------------------ t (1 row) -- Incremental mode query 3: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.8374 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.8374 id | content | score ----+---------------+--------- 2 | goodbye world | -0.8374 3 | hello goodbye | -0.8374 (2 rows) -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; goodbye_incr_valid -------------------- t (1 row) -- Incremental mode query 4: 'domination' SELECT id, content, ROUND((content <@> 'domination')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'domination', id; NOTICE: BM25 index scan: tid=(0,4), BM25_score=-1.3260 id | content | score ----+------------------+--------- 4 | world domination | -1.3260 (1 row) -- Validate BM25 scoring for 'domination' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'domination', 'english', 1.2, 0.75) as domination_incr_valid; domination_incr_valid ----------------------- t (1 row) -- Cleanup DROP TABLE scoring2_bulk CASCADE; DROP TABLE scoring2_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring3.out000066400000000000000000000117541525762047700240310ustar00rootroot00000000000000-- Test case: scoring3 -- Generated BM25 test with 3 documents and 2 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring3_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring3_bulk (content) VALUES ('the quick brown fox jumps over the lazy dog'); INSERT INTO scoring3_bulk (content) VALUES ('a short sentence'); INSERT INTO scoring3_bulk (content) VALUES ('this is a medium length sentence that contains several words'); -- Create index after data insertion (bulk build) CREATE INDEX scoring3_bulk_idx ON scoring3_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring3_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=4.67 -- Bulk mode query 1: 'quick' SELECT id, content, ROUND((content <@> to_bm25query('quick', 'scoring3_bulk_idx'))::numeric, 4) as score FROM scoring3_bulk ORDER BY content <@> 'quick', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.8782 id | content | score ----+---------------------------------------------+--------- 1 | the quick brown fox jumps over the lazy dog | -0.8782 (1 row) -- Validate BM25 scoring for 'quick' SELECT validate_bm25_scoring('scoring3_bulk', 'content', 'scoring3_bulk_idx', 'quick', 'english', 1.2, 0.75) as quick_bulk_valid; quick_bulk_valid ------------------ t (1 row) -- Bulk mode query 2: 'sentence' SELECT id, content, ROUND((content <@> to_bm25query('sentence', 'scoring3_bulk_idx'))::numeric, 4) as score FROM scoring3_bulk ORDER BY content <@> 'sentence', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6134 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.4208 id | content | score ----+--------------------------------------------------------------+--------- 2 | a short sentence | -0.6134 3 | this is a medium length sentence that contains several words | -0.4208 (2 rows) -- Validate BM25 scoring for 'sentence' SELECT validate_bm25_scoring('scoring3_bulk', 'content', 'scoring3_bulk_idx', 'sentence', 'english', 1.2, 0.75) as sentence_bulk_valid; sentence_bulk_valid --------------------- t (1 row) -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring3_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring3_incr_idx ON scoring3_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring3_incr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents incrementally INSERT INTO scoring3_incr (content) VALUES ('the quick brown fox jumps over the lazy dog'); INSERT INTO scoring3_incr (content) VALUES ('a short sentence'); INSERT INTO scoring3_incr (content) VALUES ('this is a medium length sentence that contains several words'); -- Incremental mode query 1: 'quick' SELECT id, content, ROUND((content <@> to_bm25query('quick', 'scoring3_incr_idx'))::numeric, 4) as score FROM scoring3_incr ORDER BY content <@> 'quick', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.8782 id | content | score ----+---------------------------------------------+--------- 1 | the quick brown fox jumps over the lazy dog | -0.8782 (1 row) -- Validate BM25 scoring for 'quick' (incremental) SELECT validate_bm25_scoring('scoring3_incr', 'content', 'scoring3_incr_idx', 'quick', 'english', 1.2, 0.75) as quick_incr_valid; quick_incr_valid ------------------ t (1 row) -- Incremental mode query 2: 'sentence' SELECT id, content, ROUND((content <@> to_bm25query('sentence', 'scoring3_incr_idx'))::numeric, 4) as score FROM scoring3_incr ORDER BY content <@> 'sentence', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6134 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.4208 id | content | score ----+--------------------------------------------------------------+--------- 2 | a short sentence | -0.6134 3 | this is a medium length sentence that contains several words | -0.4208 (2 rows) -- Validate BM25 scoring for 'sentence' (incremental) SELECT validate_bm25_scoring('scoring3_incr', 'content', 'scoring3_incr_idx', 'sentence', 'english', 1.2, 0.75) as sentence_incr_valid; sentence_incr_valid --------------------- t (1 row) -- Cleanup DROP TABLE scoring3_bulk CASCADE; DROP TABLE scoring3_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring4.out000066400000000000000000000057751525762047700240400ustar00rootroot00000000000000-- Test case: scoring4 -- Generated BM25 test with 2 documents and 1 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring4_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring4_bulk (content) VALUES ('goodbye world'); INSERT INTO scoring4_bulk (content) VALUES ('goodbyes are hard'); -- Create index after data insertion (bulk build) CREATE INDEX scoring4_bulk_idx ON scoring4_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring4_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=2.00 -- Bulk mode query 1: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring4_bulk ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.1823 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.1823 id | content | score ----+-------------------+--------- 1 | goodbye world | -0.1823 2 | goodbyes are hard | -0.1823 (2 rows) -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring4_bulk', 'content', 'scoring4_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; goodbye_bulk_valid -------------------- t (1 row) -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring4_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring4_incr_idx ON scoring4_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring4_incr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents incrementally INSERT INTO scoring4_incr (content) VALUES ('goodbye world'); INSERT INTO scoring4_incr (content) VALUES ('goodbyes are hard'); -- Incremental mode query 1: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring4_incr ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.1823 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.1823 id | content | score ----+-------------------+--------- 1 | goodbye world | -0.1823 2 | goodbyes are hard | -0.1823 (2 rows) -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring4_incr', 'content', 'scoring4_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; goodbye_incr_valid -------------------- t (1 row) -- Cleanup DROP TABLE scoring4_bulk CASCADE; DROP TABLE scoring4_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring5.out000066400000000000000000000150301525762047700240220ustar00rootroot00000000000000-- Test case: scoring5 -- Generated BM25 test with 3 documents and 4 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring5_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring5_bulk (content) VALUES ('hello world'); INSERT INTO scoring5_bulk (content) VALUES ('goodbye cruel world'); INSERT INTO scoring5_bulk (content) VALUES ('goodbye nerds'); -- Create index after data insertion (bulk build) CREATE INDEX scoring5_bulk_idx ON scoring5_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring5_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=2.33 -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-1.0417 id | content | score ----+-------------+--------- 1 | hello world | -1.0417 (1 row) -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_bulk_valid; hello_bulk_valid ------------------ t (1 row) -- Bulk mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'cruel', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.8782 id | content | score ----+---------------------+--------- 2 | goodbye cruel world | -0.8782 (1 row) -- Validate BM25 scoring for 'cruel' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'cruel', 'english', 1.2, 0.75) as cruel_bulk_valid; cruel_bulk_valid ------------------ t (1 row) -- Bulk mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'world', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.4992 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.4208 id | content | score ----+---------------------+--------- 1 | hello world | -0.4992 2 | goodbye cruel world | -0.4208 (2 rows) -- Validate BM25 scoring for 'world' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'world', 'english', 1.2, 0.75) as world_bulk_valid; world_bulk_valid ------------------ t (1 row) -- Bulk mode query 4: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.4992 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.4208 id | content | score ----+---------------------+--------- 3 | goodbye nerds | -0.4992 2 | goodbye cruel world | -0.4208 (2 rows) -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; goodbye_bulk_valid -------------------- t (1 row) -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring5_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring5_incr_idx ON scoring5_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring5_incr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents incrementally INSERT INTO scoring5_incr (content) VALUES ('hello world'); INSERT INTO scoring5_incr (content) VALUES ('goodbye cruel world'); INSERT INTO scoring5_incr (content) VALUES ('goodbye nerds'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-1.0417 id | content | score ----+-------------+--------- 1 | hello world | -1.0417 (1 row) -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; hello_incr_valid ------------------ t (1 row) -- Incremental mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'cruel', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.8782 id | content | score ----+---------------------+--------- 2 | goodbye cruel world | -0.8782 (1 row) -- Validate BM25 scoring for 'cruel' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'cruel', 'english', 1.2, 0.75) as cruel_incr_valid; cruel_incr_valid ------------------ t (1 row) -- Incremental mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'world', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.4992 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.4208 id | content | score ----+---------------------+--------- 1 | hello world | -0.4992 2 | goodbye cruel world | -0.4208 (2 rows) -- Validate BM25 scoring for 'world' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'world', 'english', 1.2, 0.75) as world_incr_valid; world_incr_valid ------------------ t (1 row) -- Incremental mode query 4: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,3), BM25_score=-0.4992 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.4208 id | content | score ----+---------------------+--------- 3 | goodbye nerds | -0.4992 2 | goodbye cruel world | -0.4208 (2 rows) -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; goodbye_incr_valid -------------------- t (1 row) -- Cleanup DROP TABLE scoring5_bulk CASCADE; DROP TABLE scoring5_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring6.out000066400000000000000000000123131525762047700240240ustar00rootroot00000000000000-- Test case: scoring6 -- Generated BM25 test with 2 documents and 3 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring6_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring6_bulk (content) VALUES ('hello, world!'); INSERT INTO scoring6_bulk (content) VALUES ('goodbye cruel world...'); -- Create index after data insertion (bulk build) CREATE INDEX scoring6_bulk_idx ON scoring6_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring6_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=2.50 -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring6_bulk ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.7549 id | content | score ----+---------------+--------- 1 | hello, world! | -0.7549 (1 row) -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring6_bulk', 'content', 'scoring6_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_bulk_valid; hello_bulk_valid ------------------ t (1 row) -- Bulk mode query 2: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring6_bulk ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6407 id | content | score ----+------------------------+--------- 2 | goodbye cruel world... | -0.6407 (1 row) -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring6_bulk', 'content', 'scoring6_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; goodbye_bulk_valid -------------------- t (1 row) -- Bulk mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring6_bulk ORDER BY content <@> 'world', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.1986 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.1685 id | content | score ----+------------------------+--------- 1 | hello, world! | -0.1986 2 | goodbye cruel world... | -0.1685 (2 rows) -- Validate BM25 scoring for 'world' SELECT validate_bm25_scoring('scoring6_bulk', 'content', 'scoring6_bulk_idx', 'world', 'english', 1.2, 0.75) as world_bulk_valid; world_bulk_valid ------------------ t (1 row) -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring6_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring6_incr_idx ON scoring6_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation scoring6_incr_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert test documents incrementally INSERT INTO scoring6_incr (content) VALUES ('hello, world!'); INSERT INTO scoring6_incr (content) VALUES ('goodbye cruel world...'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring6_incr ORDER BY content <@> 'hello', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.7549 id | content | score ----+---------------+--------- 1 | hello, world! | -0.7549 (1 row) -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring6_incr', 'content', 'scoring6_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; hello_incr_valid ------------------ t (1 row) -- Incremental mode query 2: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring6_incr ORDER BY content <@> 'goodbye', id; NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.6407 id | content | score ----+------------------------+--------- 2 | goodbye cruel world... | -0.6407 (1 row) -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring6_incr', 'content', 'scoring6_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; goodbye_incr_valid -------------------- t (1 row) -- Incremental mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring6_incr ORDER BY content <@> 'world', id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.1986 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-0.1685 id | content | score ----+------------------------+--------- 1 | hello, world! | -0.1986 2 | goodbye cruel world... | -0.1685 (2 rows) -- Validate BM25 scoring for 'world' (incremental) SELECT validate_bm25_scoring('scoring6_incr', 'content', 'scoring6_incr_idx', 'world', 'english', 1.2, 0.75) as world_incr_valid; world_incr_valid ------------------ t (1 row) -- Cleanup DROP TABLE scoring6_bulk CASCADE; DROP TABLE scoring6_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/scoring_injection.out000066400000000000000000000103671525762047700260070ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION dblink; CREATE TEMP TABLE scoring_sessions (name text PRIMARY KEY, pid int); CREATE TABLE scoring_readers (id int, body text) WITH (autovacuum_enabled = false); CREATE INDEX scoring_readers_idx ON scoring_readers USING bm25(body) WITH (text_config = 'simple'); INSERT INTO scoring_readers VALUES (1, 'alpha'), (2, 'needle'); SET enable_seqscan = off; SELECT id FROM scoring_readers ORDER BY body <@> to_bm25query('needle', 'scoring_readers_idx') LIMIT 1; id 2 (1 row) SELECT dblink_connect('reader', format( 'host=%s port=%s dbname=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database())); dblink_connect OK (1 row) INSERT INTO scoring_sessions SELECT 'reader', pid FROM dblink('reader', 'SELECT pg_backend_pid()') AS t(pid int); SELECT dblink_exec('reader', 'SET enable_seqscan = off'); dblink_exec SET (1 row) SELECT * FROM dblink('reader', 'SELECT injection_points_set_local()') AS t(result text); result (1 row) SELECT * FROM dblink('reader', $$SELECT injection_points_attach('pg-textsearch-scoring-source', 'wait')$$ ) AS t(result text); result (1 row) SELECT dblink_send_query('reader', $q$ SELECT id FROM scoring_readers ORDER BY body <@> to_bm25query('alpha', 'scoring_readers_idx') LIMIT 1 $q$); dblink_send_query 1 (1 row) DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '30 seconds'; BEGIN LOOP PERFORM pg_stat_clear_snapshot(); IF EXISTS ( SELECT 1 FROM pg_stat_activity JOIN scoring_sessions USING (pid) WHERE name = 'reader' AND wait_event_type = 'InjectionPoint' AND wait_event = 'pg-textsearch-scoring-source' ) THEN RETURN; END IF; IF dblink_is_busy('reader') = 0 THEN RAISE EXCEPTION 'reader finished without reaching scoring pause'; END IF; IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'timed out waiting for scoring pause'; END IF; PERFORM pg_sleep(0.01); END LOOP; END $$; -- A reader of the unchanged cache must finish while scoring is paused. SELECT dblink_connect('fast', format( 'host=%s port=%s dbname=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database())); dblink_connect OK (1 row) INSERT INTO scoring_sessions SELECT 'fast', pid FROM dblink('fast', 'SELECT pg_backend_pid()') AS t(pid int); SELECT dblink_exec('fast', 'SET enable_seqscan = off'); dblink_exec SET (1 row) SELECT dblink_send_query('fast', $q$ SELECT id FROM scoring_readers ORDER BY body <@> to_bm25query('needle', 'scoring_readers_idx') LIMIT 1 $q$); dblink_send_query 1 (1 row) DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '30 seconds'; wait_type text; wait_name text; BEGIN LOOP EXIT WHEN dblink_is_busy('fast') = 0; PERFORM pg_stat_clear_snapshot(); SELECT wait_event_type, wait_event INTO wait_type, wait_name FROM pg_stat_activity JOIN scoring_sessions USING (pid) WHERE name = 'fast'; -- Detect the regression by its lock wait, not elapsed time. EXIT WHEN wait_type = 'LWLock' AND wait_name = 'tapir_cache_apply_lock'; IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'fast reader made no progress: %/%', wait_type, wait_name; END IF; PERFORM pg_sleep(0.01); END LOOP; END $$; SELECT dblink_is_busy('fast') AS blocked; blocked 0 (1 row) SELECT injection_points_wakeup('pg-textsearch-scoring-source'); injection_points_wakeup (1 row) SELECT * FROM dblink_get_result('fast') AS t(id int); id 2 (1 row) SELECT * FROM dblink_get_result('fast') AS t(id int); id (0 rows) SELECT dblink_disconnect('fast'); dblink_disconnect OK (1 row) SELECT * FROM dblink_get_result('reader') AS t(id int); id 1 (1 row) SELECT * FROM dblink_get_result('reader') AS t(id int); id (0 rows) SELECT dblink_disconnect('reader'); dblink_disconnect OK (1 row) DROP TABLE scoring_sessions; DROP TABLE scoring_readers; DROP EXTENSION dblink; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/security.out000066400000000000000000000100241525762047700241360ustar00rootroot00000000000000-- Test security restrictions on debug and admin functions CREATE EXTENSION pg_textsearch; -- Create test data CREATE TABLE security_test (id serial, content text); INSERT INTO security_test (content) VALUES ('test document'); CREATE INDEX security_test_idx ON security_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation security_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- Create a non-superuser role DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'test_nonsuperuser') THEN EXECUTE 'REASSIGN OWNED BY test_nonsuperuser TO CURRENT_USER'; EXECUTE 'DROP OWNED BY test_nonsuperuser CASCADE'; DROP ROLE test_nonsuperuser; END IF; END $$; CREATE ROLE test_nonsuperuser LOGIN; GRANT ALL ON TABLE security_test TO test_nonsuperuser; GRANT USAGE ON SCHEMA public TO test_nonsuperuser; -- Test as superuser / index owner (should all work) SELECT bm25_dump_index('security_test_idx') IS NOT NULL AS dump_works; dump_works ------------ t (1 row) SELECT bm25_summarize_index('security_test_idx') IS NOT NULL AS summarize_works; summarize_works ----------------- t (1 row) SELECT bm25_spill_index('security_test_idx') IS NULL AS spill_works; spill_works ------------- t (1 row) SELECT bm25_force_merge('security_test_idx'); bm25_force_merge ------------------ (1 row) -- Switch to non-superuser (not the index owner) SET ROLE test_nonsuperuser; -- Debug functions should fail (REVOKE blocks execution) SELECT bm25_dump_index('security_test_idx'); ERROR: permission denied for function bm25_dump_index SELECT bm25_summarize_index('security_test_idx'); ERROR: permission denied for function bm25_summarize_index -- Admin functions should fail with "must be owner" SELECT bm25_spill_index('security_test_idx'); ERROR: must be owner of index security_test_idx SELECT bm25_force_merge('security_test_idx'); ERROR: must be owner of index security_test_idx -- REVOKE should block non-superusers on debug functions SELECT has_function_privilege('test_nonsuperuser', 'bm25_dump_index(text)', 'EXECUTE') AS dump_priv; dump_priv ----------- f (1 row) SELECT has_function_privilege('test_nonsuperuser', 'bm25_summarize_index(text)', 'EXECUTE') AS summarize_priv; summarize_priv ---------------- f (1 row) -- Resource-control GUCs should require superuser SET pg_textsearch.bulk_load_threshold = 1; ERROR: permission denied to set parameter "pg_textsearch.bulk_load_threshold" SET pg_textsearch.memtable_pages_threshold = 0; ERROR: permission denied to set parameter "pg_textsearch.memtable_pages_threshold" SET pg_textsearch.segments_per_level = 2; ERROR: permission denied to set parameter "pg_textsearch.segments_per_level" SET pg_textsearch.max_segment_size = '1MB'; ERROR: permission denied to set parameter "pg_textsearch.max_segment_size" -- Logging GUCs should require superuser (info disclosure in pooled envs) SET pg_textsearch.log_scores = on; ERROR: permission denied to set parameter "pg_textsearch.log_scores" SET pg_textsearch.log_bmw_stats = on; ERROR: permission denied to set parameter "pg_textsearch.log_bmw_stats" -- User-facing GUCs should still be settable SET pg_textsearch.default_limit = 500; SHOW pg_textsearch.default_limit; pg_textsearch.default_limit ----------------------------- 500 (1 row) SET pg_textsearch.compress_segments = off; SHOW pg_textsearch.compress_segments; pg_textsearch.compress_segments --------------------------------- off (1 row) -- Reset to superuser RESET ROLE; SHOW pg_textsearch.max_segment_size; pg_textsearch.max_segment_size -------------------------------- 4095MB (1 row) SET pg_textsearch.max_segment_size = '1MB'; SHOW pg_textsearch.max_segment_size; pg_textsearch.max_segment_size -------------------------------- 1MB (1 row) RESET pg_textsearch.max_segment_size; -- Cleanup DROP TABLE security_test; DROP OWNED BY test_nonsuperuser CASCADE; DROP ROLE test_nonsuperuser; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/security_acl.out000066400000000000000000000206171525762047700247660ustar00rootroot00000000000000-- Standalone BM25 scoring must enforce the invoking role's SELECT -- privilege on the indexed relation and column. Opening a caller-named -- index for scoring bypasses the executor's range-table permission -- checks, so the operator function performs the ACL check itself. CREATE EXTENSION pg_textsearch; -- Victim objects live in a schema the attacker cannot use. CREATE SCHEMA victim; CREATE TABLE victim.docs (id serial, secret text, note text); INSERT INTO victim.docs (secret, note) VALUES ('alpha secret token', 'hello world'), ('beta secret token', 'goodbye world'); CREATE INDEX docs_bm25 ON victim.docs USING bm25(secret) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=3.00 -- Expression index over two columns not indexed directly. CREATE TABLE victim.expr_docs (id serial, a text, b text); INSERT INTO victim.expr_docs (a, b) VALUES ('alpha apple', 'beta banana'); CREATE INDEX expr_bm25 ON victim.expr_docs USING bm25((a || ' ' || b)) WITH (text_config='english'); NOTICE: BM25 index build started for relation expr_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=4.00 -- Partial index whose predicate references a non-indexed column. CREATE TABLE victim.part_docs (id serial, body text, flag boolean); INSERT INTO victim.part_docs (body, flag) VALUES ('alpha apple', true), ('hidden text', false); CREATE INDEX part_bm25 ON victim.part_docs USING bm25(body) WITH (text_config='english') WHERE flag; NOTICE: BM25 index build started for relation part_bm25 NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- Idempotent role setup DO $$ DECLARE r text; BEGIN FOREACH r IN ARRAY ARRAY['bm25_attacker', 'bm25_reader', 'bm25_colreader', 'bm25_wrongcol', 'bm25_nousage', 'bm25_exprreader', 'bm25_predreader'] LOOP IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN EXECUTE format('REASSIGN OWNED BY %I TO CURRENT_USER', r); EXECUTE format('DROP OWNED BY %I CASCADE', r); EXECUTE format('DROP ROLE %I', r); END IF; END LOOP; END $$; -- Connect-only attacker: no schema usage, no table/column SELECT. CREATE ROLE bm25_attacker LOGIN; -- Reader with full table SELECT. CREATE ROLE bm25_reader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_reader; GRANT SELECT ON victim.docs TO bm25_reader; -- Reader with column-level SELECT on the indexed column only. CREATE ROLE bm25_colreader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_colreader; GRANT SELECT (secret) ON victim.docs TO bm25_colreader; -- Reader with column-level SELECT on a non-indexed column only. CREATE ROLE bm25_wrongcol LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_wrongcol; GRANT SELECT (note) ON victim.docs TO bm25_wrongcol; -- Reader holding table SELECT but lacking schema USAGE: an index scan -- would be denied at the schema, so scoring must be too. CREATE ROLE bm25_nousage LOGIN; GRANT SELECT ON victim.docs TO bm25_nousage; -- Reader used to exercise the expression-index column set. CREATE ROLE bm25_exprreader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_exprreader; GRANT SELECT (a) ON victim.expr_docs TO bm25_exprreader; -- Reader used to exercise the partial-index predicate column set. CREATE ROLE bm25_predreader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_predreader; GRANT SELECT (body) ON victim.part_docs TO bm25_predreader; -- Owner baseline: scoring works. SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25'))) < 0 AS owner_present; owner_present --------------- t (1 row) ------------------------------------------------------------------ -- Attacker: every standalone scoring path must be denied. ------------------------------------------------------------------ SET ROLE bm25_attacker; -- Scalar-subquery constructor (hides the constructor from the planner). SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS attacker_present; ERROR: permission denied for schema victim -- Absent term must be denied too (no zero-vs-negative oracle). SELECT 'zzzabsent'::text <@> (SELECT to_bm25query('zzzabsent', 'victim.docs_bm25')) AS attacker_absent; ERROR: permission denied for schema victim -- text[] array scoring path. SELECT ARRAY['alpha']::text[] <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS attacker_array; ERROR: permission denied for schema victim RESET ROLE; ------------------------------------------------------------------ -- Reader with column-level SELECT on a non-indexed column: denied. ------------------------------------------------------------------ SET ROLE bm25_wrongcol; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS wrongcol_present; ERROR: permission denied for column "secret" of relation "docs" RESET ROLE; ------------------------------------------------------------------ -- Reader with table SELECT but no schema USAGE: denied at the schema. ------------------------------------------------------------------ SET ROLE bm25_nousage; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS nousage_present; ERROR: permission denied for schema victim RESET ROLE; ------------------------------------------------------------------ -- Expression index: a role missing SELECT on any referenced column is -- denied; granting the remaining column allows scoring. ------------------------------------------------------------------ SET ROLE bm25_exprreader; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.expr_bm25')) AS expr_denied; ERROR: permission denied for column "b" of relation "expr_docs" RESET ROLE; GRANT SELECT (b) ON victim.expr_docs TO bm25_exprreader; SET ROLE bm25_exprreader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.expr_bm25'))) < 0 AS expr_allowed; expr_allowed -------------- t (1 row) RESET ROLE; ------------------------------------------------------------------ -- Partial index: a role missing SELECT on the predicate column is -- denied; granting it allows scoring. ------------------------------------------------------------------ SET ROLE bm25_predreader; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.part_bm25')) AS pred_denied; ERROR: permission denied for column "flag" of relation "part_docs" RESET ROLE; GRANT SELECT (flag) ON victim.part_docs TO bm25_predreader; SET ROLE bm25_predreader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.part_bm25'))) < 0 AS pred_allowed; pred_allowed -------------- t (1 row) RESET ROLE; ------------------------------------------------------------------ -- Reader with full table SELECT: scoring works. ------------------------------------------------------------------ SET ROLE bm25_reader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25'))) < 0 AS reader_present; reader_present ---------------- t (1 row) SELECT 'zzzabsent'::text <@> (SELECT to_bm25query('zzzabsent', 'victim.docs_bm25')) = 0 AS reader_absent; reader_absent --------------- t (1 row) RESET ROLE; ------------------------------------------------------------------ -- Reader with column-level SELECT on the indexed column: scoring works. ------------------------------------------------------------------ SET ROLE bm25_colreader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25'))) < 0 AS colreader_present; colreader_present ------------------- t (1 row) RESET ROLE; -- Cleanup DROP SCHEMA victim CASCADE; NOTICE: drop cascades to 3 other objects DETAIL: drop cascades to table victim.docs drop cascades to table victim.expr_docs drop cascades to table victim.part_docs DROP OWNED BY bm25_attacker, bm25_reader, bm25_colreader, bm25_wrongcol, bm25_nousage, bm25_exprreader, bm25_predreader CASCADE; DROP ROLE bm25_attacker; DROP ROLE bm25_reader; DROP ROLE bm25_colreader; DROP ROLE bm25_wrongcol; DROP ROLE bm25_nousage; DROP ROLE bm25_exprreader; DROP ROLE bm25_predreader; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/segment.out000066400000000000000000000225441525762047700237430ustar00rootroot00000000000000-- Test case: segment -- Tests segment query functionality with multiple spill cycles and -- post-spill inserts CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Create table and index CREATE TABLE segment_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX segment_test_idx ON segment_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation segment_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -------------------------------------------------------------------------------- -- Phase 1: Initial data in memtable -------------------------------------------------------------------------------- INSERT INTO segment_test (content) VALUES ('hello world'); INSERT INTO segment_test (content) VALUES ('goodbye cruel world'); INSERT INTO segment_test (content) VALUES ('hello goodbye'); INSERT INTO segment_test (content) VALUES ('world peace'); -- Validate BM25 scoring before spill (memtable only) SELECT 'Phase 1: memtable only' AS phase; phase ------------------------ Phase 1: memtable only (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; hello_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; world_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; hello_world_valid ------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 2: First spill to segment -------------------------------------------------------------------------------- SELECT bm25_spill_index('segment_test_idx') IS NOT NULL AS first_spill; first_spill ------------- t (1 row) -- Validate BM25 scoring after first spill (segment only) SELECT 'Phase 2: after first spill (segment only)' AS phase; phase ------------------------------------------- Phase 2: after first spill (segment only) (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; hello_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; world_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; hello_world_valid ------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 3: Post-spill inserts (memtable + segment) -------------------------------------------------------------------------------- INSERT INTO segment_test (content) VALUES ('hello again friend'); INSERT INTO segment_test (content) VALUES ('peaceful world harmony'); INSERT INTO segment_test (content) VALUES ('cruel goodbye forever'); -- Validate BM25 scoring with mixed data (segment + memtable) SELECT 'Phase 3: post-spill inserts (segment + memtable)' AS phase; phase -------------------------------------------------- Phase 3: post-spill inserts (segment + memtable) (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; hello_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; world_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'cruel', 'english', 1.2, 0.75) AS cruel_valid; cruel_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; hello_world_valid ------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 4: Second spill (two segments now) -------------------------------------------------------------------------------- SELECT bm25_spill_index('segment_test_idx') IS NOT NULL AS second_spill; second_spill -------------- t (1 row) -- Validate BM25 scoring after second spill (two segments) SELECT 'Phase 4: after second spill (two segments)' AS phase; phase -------------------------------------------- Phase 4: after second spill (two segments) (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; hello_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; world_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'cruel', 'english', 1.2, 0.75) AS cruel_valid; cruel_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; hello_world_valid ------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 5: More inserts after second spill -------------------------------------------------------------------------------- INSERT INTO segment_test (content) VALUES ('hello hello hello'); INSERT INTO segment_test (content) VALUES ('world world'); INSERT INTO segment_test (content) VALUES ('new unique term'); -- Validate with three data sources (two segments + memtable) SELECT 'Phase 5: more inserts (two segments + memtable)' AS phase; phase ------------------------------------------------- Phase 5: more inserts (two segments + memtable) (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; hello_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; world_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'unique', 'english', 1.2, 0.75) AS unique_valid; unique_valid -------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; hello_world_valid ------------------- t (1 row) -------------------------------------------------------------------------------- -- Phase 6: Third spill and final validation -------------------------------------------------------------------------------- SELECT bm25_spill_index('segment_test_idx') IS NOT NULL AS third_spill; third_spill ------------- t (1 row) SELECT 'Phase 6: after third spill (three segments)' AS phase; phase --------------------------------------------- Phase 6: after third spill (three segments) (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; hello_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; world_valid ------------- t (1 row) SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world cruel peace', 'english', 1.2, 0.75) AS multi_term_valid; multi_term_valid ------------------ t (1 row) -------------------------------------------------------------------------------- -- Verify row counts and final state -------------------------------------------------------------------------------- SELECT COUNT(*) AS total_documents FROM segment_test; total_documents ----------------- 10 (1 row) -- Show final scores for reference (not validated, just for visibility) SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) AS hello_score FROM segment_test ORDER BY content <@> 'hello', id; id | content | hello_score ----+--------------------+------------- 8 | hello hello hello | -1.3468 1 | hello world | -0.9735 3 | hello goodbye | -0.9735 5 | hello again friend | -0.9735 (4 rows) -- Cleanup DROP TABLE segment_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/segment_integrity.out000066400000000000000000000314231525762047700260350ustar00rootroot00000000000000-- Test case: segment_integrity -- Verifies that segment operations preserve exact document scores. -- -- Key insight: BM25 scores legitimately change when documents are added -- (IDF and avgdl change). This test captures scores IMMEDIATELY before -- an operation and verifies they are IDENTICAL after. This catches: -- - Data loss during spill/merge -- - Corruption of posting lists -- - Document length encoding errors -- - Term frequency encoding errors CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Create table with enough documents to be meaningful CREATE TABLE integrity_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX integrity_test_idx ON integrity_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation integrity_test_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert a variety of documents with different term frequencies and lengths INSERT INTO integrity_test (content) VALUES ('hello world'), ('hello hello world'), ('hello hello hello world world'), ('goodbye cruel world'), ('goodbye goodbye'), ('the quick brown fox jumps over the lazy dog'), ('fox fox fox hunting'), ('lazy dog sleeps'), ('world peace and harmony'), ('database indexing and search'), ('full text search with ranking'), ('search search search query'), ('query optimization techniques'), ('hello world database search'), ('unique document here'), ('another unique entry'), ('fox and dog are friends'), ('peace in the world'), ('cruel cruel world'), ('harmony and peace forever'); -------------------------------------------------------------------------------- -- Test 1: Spill integrity (memtable -> segment) -------------------------------------------------------------------------------- -- Capture scores IMMEDIATELY before spill CREATE TEMP TABLE pre_spill_scores AS SELECT id, content, query, score FROM ( SELECT id, content, 'hello'::text AS query, (content <@> to_bm25query('hello', 'integrity_test_idx'))::numeric AS score FROM integrity_test UNION ALL SELECT id, content, 'world'::text, (content <@> to_bm25query('world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'fox'::text, (content <@> to_bm25query('fox', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'search'::text, (content <@> to_bm25query('search', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'hello world'::text, (content <@> to_bm25query('hello world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'peace harmony'::text, (content <@> to_bm25query('peace harmony', 'integrity_test_idx'))::numeric FROM integrity_test ) sub; SELECT 'Test 1: Spill integrity' AS test, COUNT(DISTINCT id) AS docs, COUNT(DISTINCT query) AS queries FROM pre_spill_scores; test | docs | queries -------------------------+------+--------- Test 1: Spill integrity | 20 | 6 (1 row) -- Spill to segment SELECT bm25_spill_index('integrity_test_idx') IS NOT NULL AS spill_ok; spill_ok ---------- t (1 row) -- Verify ALL scores match after spill SELECT CASE WHEN COUNT(*) = (SELECT COUNT(*) FROM pre_spill_scores) THEN 'PASS' ELSE 'FAIL' END AS spill_integrity, COUNT(*) AS matching, (SELECT COUNT(*) FROM pre_spill_scores) AS expected FROM pre_spill_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) <= 0.0001; spill_integrity | matching | expected -----------------+----------+---------- PASS | 120 | 120 (1 row) -- Show any mismatches (should be empty) SELECT 'Spill mismatches:' AS label; label ------------------- Spill mismatches: (1 row) SELECT p.id, p.query, ROUND(p.score, 6) AS before, ROUND((p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric, 6) AS after FROM pre_spill_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) > 0.0001 LIMIT 5; id | query | before | after ----+-------+--------+------- (0 rows) DROP TABLE pre_spill_scores; -------------------------------------------------------------------------------- -- Test 2: Second spill integrity (add docs, then spill again) -------------------------------------------------------------------------------- -- Add more documents INSERT INTO integrity_test (content) VALUES ('new document after first spill'), ('hello from the other side'), ('world domination plans'), ('search and rescue mission'), ('fox news network'); -- Capture scores IMMEDIATELY before second spill CREATE TEMP TABLE pre_spill2_scores AS SELECT id, content, query, score FROM ( SELECT id, content, 'hello'::text AS query, (content <@> to_bm25query('hello', 'integrity_test_idx'))::numeric AS score FROM integrity_test UNION ALL SELECT id, content, 'world'::text, (content <@> to_bm25query('world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'search'::text, (content <@> to_bm25query('search', 'integrity_test_idx'))::numeric FROM integrity_test ) sub; SELECT 'Test 2: Second spill integrity' AS test, COUNT(DISTINCT id) AS docs FROM pre_spill2_scores; test | docs --------------------------------+------ Test 2: Second spill integrity | 25 (1 row) -- Second spill SELECT bm25_spill_index('integrity_test_idx') IS NOT NULL AS spill2_ok; spill2_ok ----------- t (1 row) -- Verify scores match SELECT CASE WHEN COUNT(*) = (SELECT COUNT(*) FROM pre_spill2_scores) THEN 'PASS' ELSE 'FAIL' END AS spill2_integrity, COUNT(*) AS matching, (SELECT COUNT(*) FROM pre_spill2_scores) AS expected FROM pre_spill2_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) <= 0.0001; spill2_integrity | matching | expected ------------------+----------+---------- PASS | 75 | 75 (1 row) DROP TABLE pre_spill2_scores; -------------------------------------------------------------------------------- -- Test 3: Merge integrity -------------------------------------------------------------------------------- -- Configure for merge (2 segments triggers merge) SET pg_textsearch.segments_per_level = 2; -- Add more docs and spill to trigger merge INSERT INTO integrity_test (content) VALUES ('merge test document one'), ('merge test hello world'), ('merge fox and dog'), ('search merge optimization'); -- Capture scores IMMEDIATELY before merge-triggering spill CREATE TEMP TABLE pre_merge_scores AS SELECT id, content, query, score FROM ( SELECT id, content, 'hello'::text AS query, (content <@> to_bm25query('hello', 'integrity_test_idx'))::numeric AS score FROM integrity_test UNION ALL SELECT id, content, 'world'::text, (content <@> to_bm25query('world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'merge'::text, (content <@> to_bm25query('merge', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'search'::text, (content <@> to_bm25query('search', 'integrity_test_idx'))::numeric FROM integrity_test ) sub; SELECT 'Test 3: Merge integrity' AS test, COUNT(DISTINCT id) AS docs FROM pre_merge_scores; test | docs -------------------------+------ Test 3: Merge integrity | 29 (1 row) -- This spill triggers merge SELECT bm25_spill_index('integrity_test_idx') IS NOT NULL AS merge_spill_ok; merge_spill_ok ---------------- t (1 row) -- Verify all scores match after merge SELECT CASE WHEN COUNT(*) = (SELECT COUNT(*) FROM pre_merge_scores) THEN 'PASS' ELSE 'FAIL' END AS merge_integrity, COUNT(*) AS matching, (SELECT COUNT(*) FROM pre_merge_scores) AS expected FROM pre_merge_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) <= 0.0001; merge_integrity | matching | expected -----------------+----------+---------- PASS | 116 | 116 (1 row) -- Show any mismatches (should be empty) SELECT 'Merge mismatches:' AS label; label ------------------- Merge mismatches: (1 row) SELECT p.id, p.query, ROUND(p.score, 6) AS before, ROUND((p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric, 6) AS after FROM pre_merge_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) > 0.0001 LIMIT 5; id | query | before | after ----+-------+--------+------- (0 rows) DROP TABLE pre_merge_scores; -------------------------------------------------------------------------------- -- Test 4: Document reachability -- Verify documents with matching terms are returned by index scan. -- Note: Documents without any queried terms correctly return score=0 -- and may not appear in ORDER BY results - this is expected behavior. -------------------------------------------------------------------------------- SELECT 'Test 4: Document reachability' AS test; test ------------------------------- Test 4: Document reachability (1 row) -- Test that documents with 'hello' are found when searching for 'hello' -- Count docs containing 'hello' in table vs docs returned by index SELECT 'hello' AS term, (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'hello')) AS expected, (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('hello', 'integrity_test_idx') ) t) AS found, CASE WHEN (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'hello')) = (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('hello', 'integrity_test_idx') ) t) THEN 'PASS' ELSE 'FAIL' END AS hello_reachable; term | expected | found | hello_reachable -------+----------+-------+----------------- hello | 6 | 6 | PASS (1 row) SELECT 'world' AS term, (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'world')) AS expected, (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('world', 'integrity_test_idx') ) t) AS found, CASE WHEN (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'world')) = (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('world', 'integrity_test_idx') ) t) THEN 'PASS' ELSE 'FAIL' END AS world_reachable; term | expected | found | world_reachable -------+----------+-------+----------------- world | 10 | 10 | PASS (1 row) SELECT 'search' AS term, (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'search')) AS expected, (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('search', 'integrity_test_idx') ) t) AS found, CASE WHEN (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'search')) = (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('search', 'integrity_test_idx') ) t) THEN 'PASS' ELSE 'FAIL' END AS search_reachable; term | expected | found | search_reachable --------+----------+-------+------------------ search | 6 | 6 | PASS (1 row) -------------------------------------------------------------------------------- -- Summary -------------------------------------------------------------------------------- SELECT 'SUMMARY' AS section, COUNT(*) AS total_documents FROM integrity_test; section | total_documents ---------+----------------- SUMMARY | 29 (1 row) -- Cleanup DROP TABLE integrity_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/segment_reclaim.out000066400000000000000000000310671525762047700254370ustar00rootroot00000000000000-- Deferred, standby-safe reclaim of displaced segment pages (#380). -- A merge must PARK displaced pages in the tombstone chain rather than -- freeing them immediately; a later VACUUM (once the reclaim horizon -- has advanced past the merge stamp) reclaims them. -- -- Temporary relations isolate the reclaim horizon from other backends -- and standby feedback, and prevent autovacuum from draining the pages. -- WAL and standby safety are covered by standby_reclaim.sh. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TEMP TABLE reclaim_docs (id int, body text); CREATE TEMP TABLE reclaim_control (id int, body text); INSERT INTO reclaim_docs SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; INSERT INTO reclaim_control SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; CREATE INDEX reclaim_idx ON reclaim_docs USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation reclaim_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2000 documents, avg_length=5.00 CREATE INDEX reclaim_control_idx ON reclaim_control USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation reclaim_control_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2000 documents, avg_length=5.00 -- Build wrote a segment directly, so the memtable is empty here; the -- next batch fills it so the following spill produces a second segment. INSERT INTO reclaim_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; INSERT INTO reclaim_control SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS spilled; spilled --------- t (1 row) \pset format unaligned SELECT bm25_spill_index('reclaim_control_idx') > 0 AS control_spilled; control_spilled t (1 row) -- Merge the L0 segments into one L1 segment; this displaces the source -- segments' pages, which must be parked (not freed). SELECT bm25_force_merge('reclaim_idx'); bm25_force_merge (1 row) SELECT bm25_force_merge('reclaim_control_idx'); bm25_force_merge (1 row) -- After a merge, displaced pages are parked (> 0), NOT freed. SELECT bm25_pending_free_pages('reclaim_idx') > 0 AS parked_after_merge; parked_after_merge t (1 row) -- The control index used identical construction, so its first detached -- batch must have the same size. Drain only that batch before performing -- identical second merges on both indexes; its later parked count then -- measures the second batch alone. SELECT bm25_pending_free_pages('reclaim_idx') AS first_batch_count \gset SELECT bm25_pending_free_pages('reclaim_control_idx') = :first_batch_count AS control_first_batch_matches; control_first_batch_matches t (1 row) SELECT txid_current() IS NOT NULL AS control_t1; control_t1 t (1 row) SELECT txid_current() IS NOT NULL AS control_t2; control_t2 t (1 row) VACUUM reclaim_control; SELECT bm25_pending_free_pages('reclaim_control_idx') AS control_after_first_drain; control_after_first_drain 0 (1 row) -- Attach another detached batch while the first one is still parked in -- reclaim_idx. If publication replaced the old head, its count would equal -- the control's second batch alone instead of first_batch + control_batch. INSERT INTO reclaim_docs SELECT g, 'alpha beta second merge term' || (g % 50) FROM generate_series(4001, 5000) g; INSERT INTO reclaim_control SELECT g, 'alpha beta second merge term' || (g % 50) FROM generate_series(4001, 5000) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS second_spilled; second_spilled t (1 row) SELECT bm25_spill_index('reclaim_control_idx') > 0 AS control_second_spilled; control_second_spilled t (1 row) SELECT bm25_force_merge('reclaim_idx'); bm25_force_merge (1 row) SELECT bm25_force_merge('reclaim_control_idx'); bm25_force_merge (1 row) SELECT bm25_pending_free_pages('reclaim_idx') = :first_batch_count + bm25_pending_free_pages('reclaim_control_idx') AS preserved_old_tombstones; preserved_old_tombstones t (1 row) \pset format aligned DROP TABLE reclaim_control; -- Advance the session's xid horizon past the merge stamp so the parked -- pages become reclaimable, then VACUUM to drain them. SELECT txid_current() IS NOT NULL AS t1; t1 ---- t (1 row) SELECT txid_current() IS NOT NULL AS t2; t2 ---- t (1 row) VACUUM reclaim_docs; -- All parked pages reclaimed. SELECT bm25_pending_free_pages('reclaim_idx') AS parked_after_vacuum; parked_after_vacuum --------------------- 0 (1 row) -- Sanity: queries still work after reclaim. SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM reclaim_docs ORDER BY body <@> to_bm25query('alpha', 'reclaim_idx') LIMIT 10 ) s; has_hits ---------- t (1 row) -- The drained pages must be genuinely reusable from the FSM, not just -- unlinked from the tombstone chain: allocating a fresh segment must -- pull those freed blocks via GetFreeIndexPage rather than extend the -- index relation. Capture the size, spill a new segment, and assert -- the relation did not grow. A regression that recorded the frees but -- skipped IndexFreeSpaceMapVacuum would still empty the chain above yet -- extend the relation here (the freed blocks would not be discoverable -- by the root-down GetFreeIndexPage search). SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int AS blocks_before_reuse \gset INSERT INTO reclaim_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(5001, 6000) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS reuse_spilled; reuse_spilled --------------- t (1 row) SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int <= :blocks_before_reuse AS reused_freed_pages_no_extension; reused_freed_pages_no_extension --------------------------------- t (1 row) -- VACUUM's own segment replacement/drop path must also park displaced -- pages instead of returning them to the FSM immediately (#413). Build a -- fresh index layout so the observation is independent of the merge case -- above. DROP INDEX reclaim_idx; TRUNCATE reclaim_docs; INSERT INTO reclaim_docs SELECT g, 'vacuum alpha beta term' || (g % 50) FROM generate_series(1, 1500) g; CREATE INDEX reclaim_idx ON reclaim_docs USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation reclaim_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1500 documents, avg_length=4.00 -- Add a second segment through the on-disk memtable, then delete exactly -- those rows so VACUUM drops that all-dead segment. First create an FSM -- free-page pool: unlocked VACUUM preparation may safely allocate its -- detached tombstone pages there instead of extending the relation. INSERT INTO reclaim_docs SELECT g, 'vacuum pool beta term' || (g % 50) FROM generate_series(1501, 3000) g; \pset format unaligned SELECT bm25_spill_index('reclaim_idx') > 0 AS vacuum_pool_spilled; vacuum_pool_spilled t (1 row) SELECT bm25_force_merge('reclaim_idx'); bm25_force_merge (1 row) SELECT txid_current() IS NOT NULL AS vp1; vp1 t (1 row) SELECT txid_current() IS NOT NULL AS vp2; vp2 t (1 row) VACUUM reclaim_docs; SELECT bm25_pending_free_pages('reclaim_idx') AS pool_after_vacuum; pool_after_vacuum 0 (1 row) INSERT INTO reclaim_docs SELECT g, 'vacuum dropped beta term' || (g % 50) FROM generate_series(3001, 3020) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS vacuum_spilled; vacuum_spilled t (1 row) SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int AS blocks_before_vacuum_drop \gset DELETE FROM reclaim_docs WHERE id BETWEEN 3001 AND 3020; VACUUM reclaim_docs; SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int <= :blocks_before_vacuum_drop AS vacuum_tombstone_reused_fsm; vacuum_tombstone_reused_fsm t (1 row) -- VACUUM may retain the pages or reclaim them immediately if concurrent -- transactions already advanced the horizon. A later VACUUM drains any -- retained pages. segment_reclaim_injection.sql pins the horizon with -- an injection point to assert the parking behaviour deterministically. SELECT txid_current() IS NOT NULL AS vt1; vt1 t (1 row) SELECT txid_current() IS NOT NULL AS vt2; vt2 t (1 row) VACUUM reclaim_docs; SELECT bm25_pending_free_pages('reclaim_idx') AS parked_after_vacuum_drain; parked_after_vacuum_drain 0 (1 row) -- The drained pages must be reusable from the FSM without extending the -- index relation. SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int AS blocks_before_vacuum_reuse \gset INSERT INTO reclaim_docs SELECT g, 'vacuum reuse beta term' || (g % 50) FROM generate_series(3021, 3720) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS vacuum_reuse_spilled; vacuum_reuse_spilled t (1 row) SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int <= :blocks_before_vacuum_reuse AS vacuum_reused_freed_pages_no_extension; vacuum_reused_freed_pages_no_extension t (1 row) \pset format aligned -- Force merge may truncate only pages that reclaim has already stamped -- recyclable. Build a free-page pool below a one-page memtable at EOF, then -- let force merge spill into that pool. The retired memtable page remains -- protected by dead_fxid and must survive until VACUUM stamps it -- TP_FREE_PAGE_MAGIC. \pset format unaligned CREATE TEMP TABLE truncate_docs (id int, body text); INSERT INTO truncate_docs SELECT g, 'truncate alpha beta gamma ' || g || ' ' || repeat(md5(g::text), 4) FROM generate_series(1, 2000) g; CREATE INDEX truncate_idx ON truncate_docs USING bm25 (body) WITH (text_config = 'english', compaction = 'off'); NOTICE: BM25 index build started for relation truncate_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2000 documents, avg_length=6.07 INSERT INTO truncate_docs SELECT g, 'truncate alpha beta delta ' || g || ' ' || repeat(md5(g::text), 4) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('truncate_idx') > 0 AS truncate_pool_spilled; truncate_pool_spilled t (1 row) SELECT bm25_force_merge('truncate_idx'); bm25_force_merge (1 row) SELECT bm25_pending_free_pages('truncate_idx') > 0 AS truncate_pool_parked; truncate_pool_parked t (1 row) INSERT INTO truncate_docs VALUES (4001, 'truncate eof marker'); CREATE TEMP TABLE truncate_suffix AS SELECT max(blkno)::bigint AS blkno FROM bm25_memtable_chain('truncate_idx'); SELECT blkno + 1 = pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AS truncate_memtable_at_eof FROM truncate_suffix; truncate_memtable_at_eof t (1 row) SELECT txid_current() IS NOT NULL AS truncate_t1; truncate_t1 t (1 row) SELECT txid_current() IS NOT NULL AS truncate_t2; truncate_t2 t (1 row) VACUUM truncate_docs; SELECT bm25_pending_free_pages('truncate_idx') = 0 AS truncate_pool_drained; truncate_pool_drained t (1 row) SELECT bm25_force_merge('truncate_idx'); bm25_force_merge (1 row) SELECT EXISTS ( SELECT 1 FROM bm25_memtable_dead_pages('truncate_idx') dead, truncate_suffix suffix WHERE dead.blkno = suffix.blkno AND dead.dead_fxid > 0 ) AND ( SELECT blkno < pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int FROM truncate_suffix ) AS force_merge_preserved_unreclaimed_suffix; force_merge_preserved_unreclaimed_suffix t (1 row) SELECT txid_current() IS NOT NULL AS truncate_reclaim_t1; truncate_reclaim_t1 t (1 row) SELECT txid_current() IS NOT NULL AS truncate_reclaim_t2; truncate_reclaim_t2 t (1 row) VACUUM truncate_docs; CREATE TEMP TABLE truncate_recyclable_size AS SELECT pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AS blocks; SELECT bm25_force_merge('truncate_idx'); bm25_force_merge (1 row) SELECT before.blocks > pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AND suffix.blkno >= pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AS force_merge_truncated_recyclable_suffix FROM truncate_recyclable_size before, truncate_suffix suffix; force_merge_truncated_recyclable_suffix t (1 row) DROP TABLE truncate_docs; \pset format aligned DROP TABLE reclaim_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/segment_reclaim_injection.out000066400000000000000000000054451525762047700275020ustar00rootroot00000000000000-- Deferred reclaim asserted with a pinned horizon. -- -- segment_reclaim.sql cannot assert that VACUUM parks a dropped -- segment's pages, because a concurrently advancing xid horizon may let -- the same VACUUM reclaim them immediately. Pin the horizon with an -- injection point so parking is observable, then release it and confirm -- a later VACUUM drains the parked pages. -- A temporary table isolates reclaim from unrelated snapshots. Permanent -- table WAL and standby safety are covered by standby_reclaim.sh. \pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; CREATE TEMP TABLE reclaim_hold_docs (id int, body text); INSERT INTO reclaim_hold_docs SELECT g, 'alpha beta gamma term' || (g % 50) FROM generate_series(1, 1500) g; CREATE INDEX reclaim_hold_idx ON reclaim_hold_docs USING bm25 (body) WITH (text_config = 'english', compaction = 'off'); -- Second segment holding only rows that are about to be deleted, so -- VACUUM drops it wholesale and displaces its pages. INSERT INTO reclaim_hold_docs SELECT g, 'delta epsilon term' || (g % 50) FROM generate_series(3001, 3400) g; SELECT bm25_spill_index('reclaim_hold_idx') > 0 AS second_segment_spilled; second_segment_spilled t (1 row) SELECT bm25_pending_free_pages('reclaim_hold_idx') AS parked_before_vacuum; parked_before_vacuum 0 (1 row) SELECT pg_textsearch_test_attach_reclaim_horizon_hold(); pg_textsearch_test_attach_reclaim_horizon_hold (1 row) DELETE FROM reclaim_hold_docs WHERE id BETWEEN 3001 AND 3400; VACUUM reclaim_hold_docs; -- The horizon is pinned, so the dropped segment's pages must still be -- parked rather than recycled. SELECT bm25_pending_free_pages('reclaim_hold_idx') > 0 AS parked_after_vacuum_drop; parked_after_vacuum_drop t (1 row) -- Repeat VACUUMs must not drain them while the horizon stays pinned. VACUUM reclaim_hold_docs; SELECT bm25_pending_free_pages('reclaim_hold_idx') > 0 AS still_parked_after_second_vacuum; still_parked_after_second_vacuum t (1 row) -- Release the horizon; the next VACUUM drains every parked page. SELECT injection_points_detach('pg-textsearch-reclaim-horizon'); injection_points_detach (1 row) SELECT txid_current() IS NOT NULL AS t1; t1 t (1 row) SELECT txid_current() IS NOT NULL AS t2; t2 t (1 row) VACUUM reclaim_hold_docs; SELECT bm25_pending_free_pages('reclaim_hold_idx') AS parked_after_release; parked_after_release 0 (1 row) -- Queries still work after reclaim. SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM reclaim_hold_docs ORDER BY body <@> to_bm25query('alpha', 'reclaim_hold_idx') LIMIT 10 ) s; has_hits t (1 row) DROP TABLE reclaim_hold_docs; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; \pset format aligned timescale-pg_textsearch-b63fe58/test/expected/strings.out000066400000000000000000000256731525762047700240000ustar00rootroot00000000000000-- Test long string handling including URLs, paths, and long terms -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Setup test table for long strings DROP TABLE IF EXISTS long_string_docs CASCADE; NOTICE: table "long_string_docs" does not exist, skipping CREATE TABLE long_string_docs ( id SERIAL PRIMARY KEY, content TEXT, category VARCHAR(50) ); -- Insert test data with various long string patterns INSERT INTO long_string_docs (content, category) VALUES -- URLs and web content ('Visit our website at https://www.example-company.com/products/database-solutions/postgresql-extensions/full-text-search?utm_source=docs&utm_medium=referral&utm_campaign=tapir-extension for more information about BM25 indexing', 'web'), ('API endpoint: https://api.search-service.example.org/v2/documents/search/bm25?query=full+text+search&limit=100&offset=0&include_scores=true&config=english', 'api'), ('Documentation available at https://docs.postgresql-extensions.community/tapir/getting-started/installation-guide.html#shared-memory-configuration', 'docs'), -- Long file paths and system content ('Error in file /usr/local/lib/postgresql/extensions/tapir/src/posting-list-management/shared-memory/string-interning-hashtable.c line 1547', 'system'), ('Log file: /var/log/postgresql/tapir-extension/performance-monitoring/bm25-scoring-statistics/daily-reports/2024-08-27-full-text-search-analytics.log', 'logs'), -- Long technical terms and identifiers ('PostgreSQL extension configuration parameter shared_preload_libraries_pgfts_bm25_full_text_search_with_relevance_scoring_enabled', 'config'), ('Function name: tp_calculate_bm25_relevance_score_with_term_frequency_inverse_document_frequency_and_document_length_normalization', 'technical'), -- Mixed long content with URLs embedded ('Our advanced information retrieval system uses BM25 scoring algorithm implementation available at https://github.com/enterprise-search/tapir-postgresql-extension/blob/main/src/algorithms/bm25-relevance-scoring.c for processing full-text search queries', 'mixed'), -- Very long single terms (scientific/chemical names) ('The antidisestablishmentarianism movement was characterized by pneumonoultramicroscopicsilicovolcanoconiosispneumonoultramicroscopicsilicovolcanoconiosis research', 'scientific'), -- Long JSON-like structured content ('Configuration: {"search_engine":"tapir","algorithms":{"primary":"bm25","parameters":{"k1":1.2,"b":0.75}},"text_processing":{"stemming":"english","tokenization":"standard"},"performance":{"shared_memory_mb":64,"max_terms":1000000}}', 'json'); -- Create pg_textsearch index on long strings CREATE INDEX long_strings_idx ON long_string_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation long_strings_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 10 documents, avg_length=10.60 -- Test 1: Search for URL components SELECT id, LEFT(content, 80) || '...' as content_preview, ROUND((content <@> 'https website')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'https website' LIMIT 5; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.2147 id | content_preview | score ----+-------------------------------------------------------------------------------------+--------- 1 | Visit our website at https://www.example-company.com/products/database-solutions... | -2.2147 (1 row) -- Test 2: Search for technical terms SELECT * FROM ( SELECT id, category, ROUND((content <@> 'postgresql extension')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'postgresql extension' LIMIT 10 ) AS subquery ORDER BY score, id; NOTICE: BM25 index scan: tid=(0,6), BM25_score=-3.4064 id | category | score ----+----------+--------- 6 | config | -3.4064 (1 row) -- Test 3: Search for long path components SELECT * FROM ( SELECT id, LEFT(content, 60) || '...' as content_preview, ROUND((content <@> 'file log error')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'file log error' LIMIT 10 ) AS subquery ORDER BY score, id; NOTICE: BM25 index scan: tid=(0,5), BM25_score=-4.9159 NOTICE: BM25 index scan: tid=(0,4), BM25_score=-4.4319 id | content_preview | score ----+-----------------------------------------------------------------+--------- 5 | Log file: /var/log/postgresql/tapir-extension/performance-mo... | -4.9159 4 | Error in file /usr/local/lib/postgresql/extensions/tapir/src... | -4.4319 (2 rows) -- Test 4: Test mixed long and short terms SELECT * FROM ( SELECT id, ROUND((content <@> 'algorithm bm25')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'algorithm bm25' LIMIT 10 ) AS subquery ORDER BY score, id; NOTICE: BM25 index scan: tid=(0,8), BM25_score=-1.6423 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-1.3640 NOTICE: BM25 index scan: tid=(0,1), BM25_score=-0.7705 NOTICE: BM25 index scan: tid=(0,6), BM25_score=-0.5925 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-0.5925 id | score ----+--------- 8 | -1.6423 10 | -1.3640 1 | -0.7705 6 | -0.5925 7 | -0.5925 (5 rows) -- Test 5: Performance test with multiple long term queries SELECT * FROM ( SELECT id, category, ROUND((content <@> 'postgresql pg_textsearch extension search')::numeric, 4) as multi_term_score FROM long_string_docs ORDER BY content <@> 'postgresql pg_textsearch extension search' LIMIT 10 ) AS subquery ORDER BY multi_term_score, id; NOTICE: BM25 index scan: tid=(0,6), BM25_score=-4.3853 NOTICE: BM25 index scan: tid=(0,8), BM25_score=-0.8648 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-0.7182 id | category | multi_term_score ----+----------+------------------ 6 | config | -4.3853 8 | mixed | -0.8648 10 | json | -0.7182 (3 rows) -- Test 6: Test that very long terms don't cause memory issues INSERT INTO long_string_docs (content, category) VALUES ('This document contains a ridiculously long compound word: ' || 'antidisestablishmentarianismpneumonoultramicroscopicsilicovolcanoconiosis' || 'supercalifragilisticexpialidociousfloccinaucinihilipilificationhippopotomonstrosesquippedaliophobia' || ' and should still be indexed correctly', 'stress_test'); -- Verify the stress test document was indexed SELECT * FROM ( SELECT id, LEFT(content, 50) || '...' as preview, ROUND((content <@> 'document ridiculously long')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'document ridiculously long' LIMIT 10 ) AS subquery ORDER BY score, id; NOTICE: BM25 index scan: tid=(0,11), BM25_score=-5.5076 NOTICE: BM25 index scan: tid=(0,3), BM25_score=-1.5699 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-1.5143 id | preview | score ----+-------------------------------------------------------+--------- 11 | This document contains a ridiculously long compoun... | -5.5076 3 | Documentation available at https://docs.postgresql... | -1.5699 7 | Function name: tp_calculate_bm25_relevance_score_w... | -1.5143 (3 rows) -- Test 7: Memory usage statistics after indexing long strings SELECT COUNT(*) as total_documents, AVG(LENGTH(content)) as avg_content_length, MAX(LENGTH(content)) as max_content_length, COUNT(CASE WHEN LENGTH(content) > 200 THEN 1 END) as long_documents, COUNT(CASE WHEN LENGTH(content) > 500 THEN 1 END) as very_long_documents FROM long_string_docs; total_documents | avg_content_length | max_content_length | long_documents | very_long_documents -----------------+----------------------+--------------------+----------------+--------------------- 11 | 179.7272727272727273 | 268 | 4 | 0 (1 row) -- Test 8: Test stack/heap threshold boundary (1KB limit for alloca) INSERT INTO long_string_docs (content, category) VALUES -- Just under 1KB threshold (should use stack allocation) ('Short term: ' || REPEAT('stackallocation', 40), 'stack_test'), -- Just over 1KB threshold (should use heap allocation) ('Long term: ' || REPEAT('heapallocationfallback', 50), 'heap_test'), -- Very long content with mixed term lengths ('Mixed content with ' || REPEAT('shortterm ', 20) || ' and ' || REPEAT('verylongtermnameforheapallocationtestingpurposes', 30) || ' to test both stack and heap allocation paths in the same document', 'mixed_allocation'); -- Verify threshold test documents can be searched SELECT id, category, LENGTH(content) as content_length, CASE WHEN LENGTH(content) < 1000 THEN 'Stack allocation likely' ELSE 'Heap allocation likely' END as allocation_type, ROUND((content <@> to_bm25query('term allocation', 'long_strings_idx'))::numeric, 4) as score FROM long_string_docs WHERE category IN ('stack_test', 'heap_test', 'mixed_allocation') ORDER BY content <@> to_bm25query('term allocation', 'long_strings_idx'), content_length; NOTICE: BM25 index scan: tid=(0,12), BM25_score=-1.7085 NOTICE: BM25 index scan: tid=(0,13), BM25_score=-1.7085 NOTICE: BM25 index scan: tid=(0,14), BM25_score=-1.3618 NOTICE: BM25 index scan: tid=(0,7), BM25_score=-1.0380 NOTICE: BM25 index scan: tid=(0,10), BM25_score=-0.7634 id | category | content_length | allocation_type | score ----+------------------+----------------+-------------------------+--------- 12 | stack_test | 612 | Stack allocation likely | -1.7085 13 | heap_test | 1111 | Heap allocation likely | -1.7085 14 | mixed_allocation | 1730 | Heap allocation likely | -1.3618 (3 rows) -- Test 9: Extreme length document (tests PostgreSQL's text limit handling) INSERT INTO long_string_docs (content, category) VALUES ('Extreme test: ' || REPEAT('This is a very long repeated phrase for testing PostgreSQL text limits and our hybrid stack heap allocation strategy. ', 200), 'extreme'); -- Verify extreme length document indexing SELECT 'Extreme length test' as test_name, LENGTH(content) as total_length, ROUND((content <@> to_bm25query('extreme testing', 'long_strings_idx'))::numeric, 4) as score FROM long_string_docs WHERE category = 'extreme' ORDER BY content <@> to_bm25query('extreme testing', 'long_strings_idx'); NOTICE: BM25 index scan: tid=(0,15), BM25_score=-4.2077 NOTICE: BM25 index scan: tid=(0,14), BM25_score=-2.8100 test_name | total_length | score ---------------------+--------------+--------- Extreme length test | 23614 | -4.2254 (1 row) -- Cleanup DROP TABLE long_string_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/temp_table.out000066400000000000000000000070751525762047700244170ustar00rootroot00000000000000-- -- Tests for BM25 indexes on temporary tables. -- Regression test for GitHub issue #247: crash on rollback after -- scanning a BM25 index on a temp table. -- CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Basic: create, populate, query, drop CREATE TEMP TABLE t_basic (id serial, body text); INSERT INTO t_basic VALUES (DEFAULT, 'hello world'), (DEFAULT, 'foo bar baz'); CREATE INDEX t_basic_idx ON t_basic USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation t_basic_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=2.50 SELECT id, body <@> 'hello' AS score FROM t_basic ORDER BY score LIMIT 5; id | score ----+-------------------- 1 | -0.754912793636322 2 | 0 (2 rows) DROP TABLE t_basic; -- Rollback to savepoint after scan (original crash from #247) BEGIN; CREATE TEMP TABLE t_sp (id serial, body text); INSERT INTO t_sp VALUES (DEFAULT, 'hello world'), (DEFAULT, 'foo bar baz'); CREATE INDEX t_sp_idx ON t_sp USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation t_sp_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2 documents, avg_length=2.50 SAVEPOINT sp1; SELECT body <@> 'hello' FROM t_sp LIMIT 1; ?column? -------------------- -0.754912793636322 (1 row) ROLLBACK TO SAVEPOINT sp1; -- Table and index should still be usable after savepoint rollback SELECT body <@> 'hello' FROM t_sp LIMIT 1; ?column? -------------------- -0.754912793636322 (1 row) COMMIT; DROP TABLE t_sp; -- Full rollback after scan (second crash case from #247) BEGIN; CREATE TEMP TABLE t_rb (id serial, body text); INSERT INTO t_rb VALUES (DEFAULT, 'hello world'); CREATE INDEX t_rb_idx ON t_rb USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation t_rb_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 SELECT body <@> 'hello' FROM t_rb LIMIT 1; ?column? ---------------------- -0.28768208622932434 (1 row) ROLLBACK; -- Rollback of CREATE INDEX itself BEGIN; CREATE TEMP TABLE t_rb2 (id serial, body text); INSERT INTO t_rb2 VALUES (DEFAULT, 'hello world'); SAVEPOINT before_idx; CREATE INDEX t_rb2_idx ON t_rb2 USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation t_rb2_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 ROLLBACK TO SAVEPOINT before_idx; COMMIT; DROP TABLE t_rb2; -- ON COMMIT DROP temp table with index BEGIN; CREATE TEMP TABLE t_ocd (id serial, body text) ON COMMIT DROP; INSERT INTO t_ocd VALUES (DEFAULT, 'on commit drop test'); CREATE INDEX t_ocd_idx ON t_ocd USING bm25(body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation t_ocd_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=3.00 SELECT body <@> 'test' FROM t_ocd LIMIT 1; ?column? ---------------------- -0.28768208622932434 (1 row) COMMIT; -- Table should be gone now SELECT * FROM t_ocd; -- expect error ERROR: relation "t_ocd" does not exist LINE 1: SELECT * FROM t_ocd; ^ DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/text_array.out000066400000000000000000000135321525762047700244600ustar00rootroot00000000000000-- Test BM25 index on text[] columns CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Basic text array table and index CREATE TABLE docs_array ( id SERIAL PRIMARY KEY, content text[] ); CREATE INDEX docs_array_bm25_idx ON docs_array USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_array_bm25_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO docs_array (content) VALUES (ARRAY['the quick brown fox', 'jumped over the lazy dog']), (ARRAY['database system design', 'query optimization']), (ARRAY['full text search', 'inverted index structure']), (ARRAY['postgres extensions', 'custom access methods']), (ARRAY['information retrieval', 'relevance ranking']); -- BM25 search using explicit bm25query SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('database', 'docs_array_bm25_idx') LIMIT 3; id | content ----+------------------------------------------------- 2 | {"database system design","query optimization"} (1 row) -- Score equivalence: text[] scores should match scalar concatenation -- Create equivalent scalar table CREATE TABLE docs_scalar ( id SERIAL PRIMARY KEY, content text ); CREATE INDEX docs_scalar_bm25_idx ON docs_scalar USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_scalar_bm25_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO docs_scalar (content) VALUES ('the quick brown fox jumped over the lazy dog'), ('database system design query optimization'), ('full text search inverted index structure'), ('postgres extensions custom access methods'), ('information retrieval relevance ranking'); -- Scores from text[] and text should match for same content SELECT a.id, a.content <@> to_bm25query('database', 'docs_array_bm25_idx') AS array_score, s.content <@> to_bm25query('database', 'docs_scalar_bm25_idx') AS scalar_score FROM docs_array a JOIN docs_scalar s ON a.id = s.id WHERE a.id = 2 ORDER BY a.id; id | array_score | scalar_score ----+---------------------+--------------------- 2 | -1.4084553718566895 | -1.4084553718566895 (1 row) -- NULL handling: NULL elements should be skipped INSERT INTO docs_array (content) VALUES (ARRAY['hello world', NULL, 'test document']); SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('hello', 'docs_array_bm25_idx') LIMIT 1; id | content ----+-------------------------------------- 6 | {"hello world",NULL,"test document"} (1 row) -- NULL array should be skipped (no crash) INSERT INTO docs_array (content) VALUES (NULL); -- Empty array should be handled gracefully INSERT INTO docs_array (content) VALUES (ARRAY[]::text[]); -- Search still works after NULLs and empties SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('database', 'docs_array_bm25_idx') LIMIT 1; id | content ----+------------------------------------------------- 2 | {"database system design","query optimization"} (1 row) -- UPDATE a text[] row and verify search picks up changes UPDATE docs_array SET content = ARRAY['updated database', 'new content'] WHERE id = 1; SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('updated', 'docs_array_bm25_idx') LIMIT 1; id | content ----+------------------------------------ 1 | {"updated database","new content"} (1 row) -- DELETE and verify DELETE FROM docs_array WHERE id = 2; SELECT count(*) FROM docs_array WHERE id = 2; count ------- 0 (1 row) -- Implicit text[] <@> text resolution SELECT id, content FROM docs_array ORDER BY content <@> 'postgres' LIMIT 1; id | content ----+------------------------------------------------- 4 | {"postgres extensions","custom access methods"} (1 row) -- Domains over text[] use the array opclass and must be flattened. CREATE DOMAIN text_array_domain AS text[]; CREATE TABLE docs_domain (id int, content text_array_domain); INSERT INTO docs_domain VALUES (1, ARRAY['domainbuild', 'common']); CREATE INDEX docs_domain_idx ON docs_domain USING bm25 (content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation docs_domain_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 SELECT id, round((content <@> to_bm25query('domainbuild', 'docs_domain_idx'))::numeric, 4) AS score FROM docs_domain ORDER BY content <@> to_bm25query('domainbuild', 'docs_domain_idx') LIMIT 5; id | score ----+--------- 1 | -0.2877 (1 row) INSERT INTO docs_domain VALUES (2, ARRAY['domaininsert', 'common']); SELECT id, round((content <@> to_bm25query('domaininsert', 'docs_domain_idx'))::numeric, 4) AS score FROM docs_domain ORDER BY content <@> to_bm25query('domaininsert', 'docs_domain_idx') LIMIT 5; id | score ----+--------- 2 | -0.6931 1 | 0.0000 (2 rows) UPDATE docs_domain SET content = ARRAY['domainupdate'] WHERE id = 1; SELECT id, round((content <@> to_bm25query('domainupdate', 'docs_domain_idx'))::numeric, 4) AS score FROM docs_domain ORDER BY content <@> to_bm25query('domainupdate', 'docs_domain_idx') LIMIT 5; id | score ----+--------- 1 | -1.1727 2 | 0.0000 (2 rows) -- Clean up DROP TABLE docs_domain; DROP DOMAIN text_array_domain; DROP TABLE docs_array CASCADE; DROP TABLE docs_scalar CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/text_config.out000066400000000000000000000013701525762047700246040ustar00rootroot00000000000000-- Test parameter "text_config" accepts qualified names CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create table CREATE TABLE text_config_tbl ( id SERIAL PRIMARY KEY, content TEXT ); -- Verify that we can use qualified names when specifying "text_config" -- and the text configuration can be found by Postgres CREATE INDEX text_config_idx1 ON text_config_tbl USING bm25 (content) WITH (text_config = 'pg_catalog.english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation text_config_idx1 NOTICE: Using text search configuration: pg_catalog.english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 DROP TABLE text_config_tbl; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/tombstone_bounds_injection.out000066400000000000000000000107351525762047700277260ustar00rootroot00000000000000-- Invalid head/next links must use corruption handling, not fail in I/O. \pset format unaligned SET client_min_messages = error; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; -- Keep the valid prefix parked so recovery must unlink its bad next link. SELECT pg_textsearch_test_attach_reclaim_horizon_hold(); pg_textsearch_test_attach_reclaim_horizon_hold (1 row) CREATE TEMP TABLE expected_pending (n bigint); CREATE FUNCTION pg_temp.corrupt_link(at_head boolean, target text) RETURNS void LANGUAGE plpgsql AS $$ DECLARE parked bigint; bad_block bigint; BEGIN CREATE TEMP TABLE bounds_docs (id int, body text); INSERT INTO bounds_docs SELECT g, 'alpha beta' FROM generate_series(1, 40) g; CREATE INDEX bounds_idx ON bounds_docs USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO bounds_docs SELECT g, 'alpha gamma' FROM generate_series(41, 80) g; PERFORM bm25_spill_index('bounds_idx'); PERFORM bm25_force_merge('bounds_idx'); parked := bm25_pending_free_pages('bounds_idx'); IF parked <= 0 THEN RAISE EXCEPTION 'expected parked source pages'; END IF; INSERT INTO expected_pending VALUES (CASE WHEN at_head THEN 0 ELSE parked END); bad_block := CASE target WHEN 'zero' THEN 0 WHEN 'eof' THEN pg_relation_size('bounds_idx') / current_setting('block_size')::bigint WHEN 'past_eof' THEN pg_relation_size('bounds_idx') / current_setting('block_size')::bigint + 100 ELSE NULL END; IF bad_block IS NULL THEN RAISE EXCEPTION 'unknown corruption target'; END IF; PERFORM pg_textsearch_test_set_tombstone_link( 'bounds_idx', at_head, bad_block); END $$; CREATE FUNCTION pg_temp.check_diagnostic() RETURNS void LANGUAGE plpgsql AS $$ BEGIN -- Repeating the diagnostic must still error: it cannot repair the chain. FOR attempt IN 1..2 LOOP BEGIN PERFORM bm25_pending_free_pages('bounds_idx'); RAISE EXCEPTION 'corrupt link was accepted'; EXCEPTION WHEN data_corrupted THEN IF SQLERRM NOT LIKE 'pg_textsearch: corrupt tombstone page %' THEN RAISE; END IF; END; END LOOP; END $$; CREATE FUNCTION pg_temp.check_recovery() RETURNS void LANGUAGE plpgsql AS $$ DECLARE hits int[]; BEGIN IF bm25_pending_free_pages('bounds_idx') <> (SELECT n FROM expected_pending) THEN RAISE EXCEPTION 'recovery lost the valid prefix or kept the bad link'; END IF; INSERT INTO bounds_docs VALUES (81, 'probe'); SELECT array_agg(id) INTO hits FROM ( SELECT id FROM bounds_docs ORDER BY body <@> to_bm25query('probe', 'bounds_idx') LIMIT 1 ) ranked; IF hits IS DISTINCT FROM ARRAY[81] THEN RAISE EXCEPTION 'index unusable after recovery: %', hits; END IF; DROP TABLE bounds_docs; TRUNCATE expected_pending; END $$; SELECT pg_temp.corrupt_link(true, 'zero'); corrupt_link (1 row) SELECT pg_temp.check_diagnostic(); check_diagnostic (1 row) VACUUM bounds_docs; SELECT pg_temp.check_recovery(); check_recovery (1 row) SELECT pg_temp.corrupt_link(true, 'eof'); corrupt_link (1 row) SELECT pg_temp.check_diagnostic(); check_diagnostic (1 row) VACUUM bounds_docs; SELECT pg_temp.check_recovery(); check_recovery (1 row) SELECT pg_temp.corrupt_link(true, 'past_eof'); corrupt_link (1 row) SELECT pg_temp.check_diagnostic(); check_diagnostic (1 row) VACUUM bounds_docs; SELECT pg_temp.check_recovery(); check_recovery (1 row) SELECT pg_temp.corrupt_link(false, 'zero'); corrupt_link (1 row) SELECT pg_temp.check_diagnostic(); check_diagnostic (1 row) VACUUM bounds_docs; SELECT pg_temp.check_recovery(); check_recovery (1 row) SELECT pg_temp.corrupt_link(false, 'eof'); corrupt_link (1 row) SELECT pg_temp.check_diagnostic(); check_diagnostic (1 row) VACUUM bounds_docs; SELECT pg_temp.check_recovery(); check_recovery (1 row) SELECT pg_temp.corrupt_link(false, 'past_eof'); corrupt_link (1 row) SELECT pg_temp.check_diagnostic(); check_diagnostic (1 row) VACUUM bounds_docs; SELECT pg_temp.check_recovery(); check_recovery (1 row) SELECT injection_points_detach('pg-textsearch-reclaim-horizon'); injection_points_detach (1 row) DROP TABLE expected_pending; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/tombstone_recover.out000066400000000000000000000047141525762047700260370ustar00rootroot00000000000000-- Self-healing recovery of a corrupt deferred-free tombstone chain -- (issue #427). -- -- Prevention (the allocator refusing to reuse a live page) stops NEW -- corruption, but an index already corrupted by an older binary stays -- wedged: every write path triggers a merge/vacuum tombstone drain that -- raises "corrupt tombstone page ..." and aborts. The drain must -- instead recover — drop the unverifiable tail of the chain, warn, and -- let writes proceed — so operators are not forced to REINDEX. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; -- Keep the recovery WARNING out of the golden output (it names a -- block number); the test asserts the post-recovery state instead. SET client_min_messages = error; CREATE TABLE recover_docs (id int, body text) WITH (autovacuum_enabled = false); INSERT INTO recover_docs SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; CREATE INDEX recover_idx ON recover_docs USING bm25 (body) WITH (text_config = 'english'); INSERT INTO recover_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('recover_idx') > 0 AS spilled; spilled --------- t (1 row) SELECT bm25_force_merge('recover_idx'); bm25_force_merge ------------------ (1 row) SELECT bm25_pending_free_pages('recover_idx') > 0 AS parked; parked -------- t (1 row) -- Simulate an already-corrupt chain node, as an older page-reuse bug -- would have left behind. SELECT bm25_test_corrupt_tombstone_head('recover_idx') > 0 AS corrupted; corrupted ----------- t (1 row) -- A VACUUM drives the tombstone drain. It must self-heal (drop the -- corrupt tail with a WARNING) instead of raising and wedging the -- index for every future write. VACUUM recover_docs; -- The chain was reset past the corruption, so the drain no longer -- errors and reports a clean (empty) pending set. SELECT bm25_pending_free_pages('recover_idx') AS pending_after_recovery; pending_after_recovery ------------------------ 0 (1 row) -- Writes and queries proceed normally after recovery. INSERT INTO recover_docs VALUES (999001, 'alpha beta gamma probe'); SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM recover_docs ORDER BY body <@> to_bm25query('probe', 'recover_idx') LIMIT 10 ) s; has_hits ---------- t (1 row) DROP TABLE recover_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/tombstone_reuse.out000066400000000000000000000100471525762047700255110ustar00rootroot00000000000000-- Recycled-page corruption of the deferred-free tombstone chain -- (issues #426, #427). -- -- The deferred-free tombstone chain (metap.pending_free_head) parks -- displaced segment pages until a later VACUUM returns them to the FSM. -- The chain links are WAL-logged, but the index FSM that guards those -- pages is NOT crash-safe and its GetFreeIndexPage() claim is not -- atomic. Either can offer a block that is still referenced by the -- tombstone chain. If an allocator then reuses that block, it -- overwrites a live tombstone page, and the next tombstone drain -- fails with "corrupt tombstone page ..." — permanently wedging every -- write path for the index (issue #427). -- -- This test reproduces that hazard deterministically with an -- internal-only scaffold that returns the live head tombstone page to -- the FSM, then drives a normal allocation. A correct allocator must -- refuse to reuse a block that is still a live pg_textsearch structure -- page. -- -- autovacuum + auto-spill are disabled so the only allocations and -- drains are the explicit ones below, keeping FSM state deterministic. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE reuse_docs (id int, body text) WITH (autovacuum_enabled = false); INSERT INTO reuse_docs SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; CREATE INDEX reuse_idx ON reuse_docs USING bm25 (body) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation reuse_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 2000 documents, avg_length=5.00 -- Build wrote a segment directly, so the memtable is empty here; the -- next batch fills it so the following spill produces a second segment. INSERT INTO reuse_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('reuse_idx') > 0 AS spilled; spilled --------- t (1 row) -- Merge the L0 segments into one L1 segment; this displaces the source -- segments' pages, which are parked in the tombstone chain (not freed). SELECT bm25_force_merge('reuse_idx'); bm25_force_merge ------------------ (1 row) SELECT bm25_pending_free_pages('reuse_idx') > 0 AS parked_after_merge; parked_after_merge -------------------- t (1 row) -- Drain any incidental FSM free pages left by build/merge by allocating -- a fresh segment, so the block recycled below is the single -- deterministic allocation target. Parked pages are not in the FSM, so -- the tombstone chain is untouched by this step. INSERT INTO reuse_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(4001, 6000) g; SELECT bm25_spill_index('reuse_idx') > 0 AS drain_spilled; drain_spilled --------------- t (1 row) SELECT bm25_pending_free_pages('reuse_idx') > 0 AS still_parked; still_parked -------------- t (1 row) -- Hand the still-live head tombstone page back to the (now empty) FSM. SELECT bm25_test_recycle_tombstone_head('reuse_idx') > 0 AS recycled; recycled ---------- t (1 row) -- The memtable is empty (last op was a spill), so this single insert -- bootstraps a fresh memtable page and allocates it from the FSM — -- picking up the recycled block. A correct allocator must NOT reuse -- that block, because it is still a live tombstone page. INSERT INTO reuse_docs VALUES (999001, 'alpha beta gamma probe'); -- Walking the tombstone chain must still succeed. With a broken -- allocator the insert above overwrote the live tombstone page, so -- this raises: ERROR: pg_textsearch: corrupt tombstone page ... SELECT bm25_pending_free_pages('reuse_idx') >= 0 AS chain_intact; chain_intact -------------- t (1 row) -- Sanity: the index still answers queries and the probe row is found. SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM reuse_docs ORDER BY body <@> to_bm25query('probe', 'reuse_idx') LIMIT 10 ) s; has_hits ---------- t (1 row) DROP TABLE reuse_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/unlogged_index.out000066400000000000000000000211021525762047700252610ustar00rootroot00000000000000-- Test unlogged index behavior and initialization fork creation CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create UNLOGGED table and index CREATE UNLOGGED TABLE unlogged_test ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO unlogged_test (content) VALUES ('test document'); CREATE INDEX unlogged_idx ON unlogged_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); NOTICE: BM25 index build started for relation unlogged_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=2.00 -- VERIFICATION: Check if _init fork exists and has valid size -- -- We verify that the init fork contains at least the meta page SELECT relname, -- Main fork should have data (Metapage + DocID page + etc) pg_relation_size(indexrelid, 'main') > current_setting('block_size')::int as main_has_data, -- Init fork must exist and be at least one page size CASE WHEN pg_relation_size(indexrelid, 'init') >= current_setting('block_size')::int THEN 'PASS' ELSE 'FAIL' END as init_check FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid WHERE c.relname = 'unlogged_idx'; relname | main_has_data | init_check --------------+---------------+------------ unlogged_idx | t | PASS (1 row) -- ---------------------------------------------------------------- -- Coverage for level-N → level-(N+1) merge on an UNLOGGED index -- -- merge.c uses GenericXLog for the linkage update on both -- WAL-logged and UNLOGGED / TEMP indexes (issue #374 unified -- the paths). The UNLOGGED case is otherwise unexercised -- by the regression suite -- almost every other test runs against -- WAL-logged indexes -- so drive a real compaction here. -- -- Set segments_per_level = 2 so 3 spills cross the threshold and -- the third spill triggers a level-0 → level-1 merge. -- ---------------------------------------------------------------- CREATE UNLOGGED TABLE unlogged_merge ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX unlogged_merge_idx ON unlogged_merge USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation unlogged_merge_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 SET pg_textsearch.segments_per_level = 2; -- Three explicit spills, all routed through the GenericXLog -- linkage path because the index is UNLOGGED. INSERT INTO unlogged_merge (content) SELECT 'alpha bravo doc ' || i FROM generate_series(1, 50) i; SELECT bm25_spill_index('unlogged_merge_idx') > 0 AS spill1_wrote; spill1_wrote -------------- t (1 row) INSERT INTO unlogged_merge (content) SELECT 'charlie delta doc ' || i FROM generate_series(51, 100) i; SELECT bm25_spill_index('unlogged_merge_idx') > 0 AS spill2_wrote; spill2_wrote -------------- t (1 row) -- Third spill should trigger compaction of L0 → L1 via the -- GenericXLog merge fallback (covers merge.c's else-branch). INSERT INTO unlogged_merge (content) SELECT 'echo foxtrot doc ' || i FROM generate_series(101, 150) i; SELECT bm25_spill_index('unlogged_merge_idx') > 0 AS spill3_wrote; spill3_wrote -------------- t (1 row) -- Verify the merge happened: bm25_summarize_index should report -- at least one segment at level 1 (post-merge target). If the -- merge didn't run, all segments would still be at level 0. SELECT bm25_summarize_index('unlogged_merge_idx') ~ 'L1 Segment' AS l1_has_segment; l1_has_segment ---------------- t (1 row) -- Sanity: a query against the merged corpus still returns rows. SELECT count(*) > 0 AS query_returns_rows FROM ( SELECT id FROM unlogged_merge ORDER BY content <@> to_bm25query('alpha', 'unlogged_merge_idx') LIMIT 10000 ) t; query_returns_rows -------------------- t (1 row) RESET pg_textsearch.segments_per_level; -- ---------------------------------------------------------------- -- Coverage for shrinkage in the GenericXLog merge fallback -- -- The else-branch of merge.c also has to subtract dead-doc count -- and total_tokens from the in-memory atomics and the on-disk -- metapage when V5 alive-bitset shrinkage drops docs at merge -- time. Without a delete+vacuum sequence the shrinkage values -- are zero and the `if (docs_shrinkage > 0)` / `if -- (tokens_shrinkage > 0)` branches stay uncovered. Drive a -- clean delete + VACUUM + force-merge here to exercise them. -- -- Run with the default segments_per_level (= 8) so the L0 spills -- here don't auto-compact: we want the merge to run only via the -- explicit force-merge call after the alive-bitset bits have been -- flipped dead by VACUUM. -- -- HISTORY / FIX (issue #397). -- -- The post-merge docs_persisted assertion has flaked on the PG17/18 -- Sanitizer jobs twice now: -- * first against the legacy `total_docs:` shared-memory atomic -- (fixed in 65b77c52 by switching the assertion to the metapage -- value), -- * then against the new `docs_persisted:` metapage assertion -- itself (issue #397). -- -- Root cause of the #397 flake: every iteration `DROP TABLE` + -- `CREATE UNLOGGED TABLE` + `CREATE INDEX` churns pg_catalog -- (pg_class, pg_attribute, pg_shdepend, ...). That triggers -- autovacuum workers whose `backend_xmin` holds back the cluster's -- `OldestNonRemovableTransactionId`. If our `VACUUM -- unlogged_shrink` runs while such a worker is active, the index -- ambulkdelete callback classifies our DELETE'd tuples as "dead but -- not yet removable", the per-segment alive_bitset never shrinks, -- and `bm25_force_merge` produces a merged segment with all 200 -- source docs instead of the expected 100. Symptom: a clean 2x -- drift (`docs_persisted: 200` and merged-segment `docs=200`). -- -- The diagnostic loop captured exactly this signature when the -- flake fired locally, with the wait DO block below reporting -- `[pid=N,type=autovacuum worker,query=autovacuum: VACUUM ANALYZE -- pg_catalog.pg_shdepend]` as the xmin holder. -- -- Fix: run the post-DELETE workload with a guarded VACUUM cycle: -- -- (a) Wait briefly for any backend with a held xmin to finish. -- Soft timeout (no error) so the loop keeps making forward -- progress even on a heavily-churning cluster. -- (b) Run `VACUUM unlogged_shrink`. -- (c) Wait again, then run `VACUUM` a second time. This closes -- the residual race where a new autovacuum worker spawns -- between the (a) wait-exit and the (b) VACUUM's OldestXmin -- computation. When (b) already shrank the bitset, the -- second VACUUM is a cheap no-op. -- -- The loop count is 8 -- small enough to keep CI wall time short -- and to limit our own catalog churn, but large enough to amplify -- any residual race a single-iteration assertion would miss. Per -- iteration we capture three diagnostic columns: -- -- pre_merge_segs -- scrubbed segment summary AFTER VACUUM, -- BEFORE force_merge. Happy path shows -- both L0 segments annotated `(alive=50, -- dead=50)`; on a stale-bitset flake both -- would still report alive=100. -- docs_persisted -- metapage total_docs after merge (must -- be `docs_persisted: 100`). -- post_merge_segs -- scrubbed segment summary AFTER merge -- (must be one L1 segment with docs=100). -- ---------------------------------------------------------------- CREATE TABLE unlogged_shrink_diag ( iter int PRIMARY KEY, pre_merge_segs text, docs_persisted text, post_merge_segs text ); -- Drive the per-iteration workload via \gexec. Each iteration is -- emitted as multiple rows (one statement per row) so \gexec sends -- each statement as a separate PQexec call -- otherwise psql would -- batch them into one simple-query, which implicitly wraps the -- block in a transaction and rejects VACUUM. Output of the -- generator query and of every executed iteration is suppressed so -- the regression .out file stays focused on the final diagnostic -- SELECTs. \set ECHO none SELECT iter, pre_merge_segs, docs_persisted, post_merge_segs FROM unlogged_shrink_diag WHERE pre_merge_segs ~ 'dead=' AND (docs_persisted IS DISTINCT FROM 'docs_persisted: 100' OR post_merge_segs !~ 'docs=100') ORDER BY iter; iter | pre_merge_segs | docs_persisted | post_merge_segs ------+----------------+----------------+----------------- (0 rows) DROP TABLE unlogged_shrink_diag; DROP TABLE unlogged_shrink; DROP TABLE unlogged_merge; DROP TABLE unlogged_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/unsupported.out000066400000000000000000000143361525762047700246710ustar00rootroot00000000000000-- Test cases for queries that are NOT YET fully supported -- These document known limitations of the current implicit index resolution CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Setup test tables CREATE TABLE docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE categories ( id SERIAL PRIMARY KEY, doc_id INT REFERENCES docs(id), category TEXT ); INSERT INTO docs (content) VALUES ('postgresql database management'), ('machine learning algorithms'), ('text search and retrieval'); INSERT INTO categories (doc_id, category) VALUES (1, 'technology'), (2, 'science'), (3, 'technology'); CREATE INDEX docs_bm25_idx ON docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_bm25_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.00 SET enable_seqscan = off; -- ============================================================================= -- LIMITATION 1: JOINs with implicit scoring don't use the index -- The query works but falls back to seq scan + standalone scoring. -- This means it returns all documents (non-matching get score 0) vs explicit -- to_bm25query() which uses the index and only returns matching documents. -- ============================================================================= \echo 'Test: JOIN with implicit score - works but returns all docs (no index)' Test: JOIN with implicit score - works but returns all docs (no index) SELECT d.id, d.content, c.category, ROUND((d.content <@> 'database')::numeric, 4) as score FROM docs d JOIN categories c ON d.id = c.doc_id ORDER BY d.content <@> 'database' LIMIT 5; id | content | category | score ----+--------------------------------+------------+--------- 1 | postgresql database management | technology | -0.9808 (1 row) -- The workaround is to use explicit to_bm25query() for score projection \echo 'Test: JOIN with explicit to_bm25query - works correctly' Test: JOIN with explicit to_bm25query - works correctly SELECT d.id, d.content, c.category, ROUND((d.content <@> to_bm25query('database', 'docs_bm25_idx'))::numeric, 4) as score FROM docs d JOIN categories c ON d.id = c.doc_id ORDER BY d.content <@> to_bm25query('database', 'docs_bm25_idx') LIMIT 5; id | content | category | score ----+--------------------------------+------------+--------- 1 | postgresql database management | technology | -0.9808 (1 row) -- ============================================================================= -- LIMITATION 2: Multiple BM25 indexes on same column -- Implicit resolution warns and uses one of the indexes; name one -- explicitly with to_bm25query() for control. -- ============================================================================= \echo 'Test: Multiple BM25 indexes on same column' Test: Multiple BM25 indexes on same column CREATE INDEX docs_bm25_simple_idx ON docs USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation docs_bm25_simple_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3 documents, avg_length=3.33 -- Implicit resolution warns; the matched row is the same whichever tied -- index the planner picks. SELECT id FROM docs ORDER BY content <@> 'database' LIMIT 3; WARNING: multiple BM25 indexes exist on the same column HINT: Use explicit to_bm25query('query', 'index_name') to specify which index to use. id ---- 1 (1 row) -- Explicit naming pins a specific index. EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('database', 'docs_bm25_simple_idx') as score FROM docs ORDER BY content <@> to_bm25query('database', 'docs_bm25_simple_idx') LIMIT 3; QUERY PLAN ---------------------------------------------------------------------------- Limit -> Index Scan using docs_bm25_simple_idx on docs Order By: (content <@> 'docs_bm25_simple_idx:database'::bm25query) (3 rows) DROP INDEX docs_bm25_simple_idx; -- ============================================================================= -- LIMITATION 3: Aggregate functions on scores require explicit to_bm25query() -- ============================================================================= \echo 'Test: Aggregate on scores with explicit index' Test: Aggregate on scores with explicit index SELECT COUNT(*) as matching_docs, ROUND(AVG((content <@> to_bm25query('database', 'docs_bm25_idx'))::numeric), 4) as avg_score FROM docs; matching_docs | avg_score ---------------+----------- 3 | -0.3269 (1 row) -- ============================================================================= -- LIMITATION 4: text <@> text operator doesn't use index scans -- The planner hook transforms text <@> text to text <@> bm25query, but the -- transformed expression doesn't match for index ordered scans. -- Workaround: Use to_bm25query() function (with or without index name). -- ============================================================================= \echo 'Test: text <@> text operator vs to_bm25query()' Test: text <@> text operator vs to_bm25query() -- text <@> text: planner transforms but doesn't use index scan EXPLAIN (COSTS OFF) SELECT id, content <@> 'database' as score FROM docs ORDER BY content <@> 'database' LIMIT 3; QUERY PLAN --------------------------------------------------------------------- Limit -> Index Scan using docs_bm25_idx on docs Order By: (content <@> 'docs_bm25_idx:database'::bm25query) (3 rows) -- to_bm25query(): properly uses index scan EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('database') as score FROM docs ORDER BY content <@> to_bm25query('database') LIMIT 3; QUERY PLAN --------------------------------------------------------------------- Limit -> Index Scan using docs_bm25_idx on docs Order By: (content <@> 'docs_bm25_idx:database'::bm25query) (3 rows) -- Cleanup DROP TABLE categories CASCADE; DROP TABLE docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/updates.out000066400000000000000000000117311525762047700237420ustar00rootroot00000000000000-- Test UPDATE operations with BM25 indexes -- This test reproduces the NULL index_state crash reported in production -- Install the extension if not already installed CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Test basic UPDATE operations CREATE TABLE update_test ( id SERIAL PRIMARY KEY, content TEXT, meta JSONB ); -- Create BM25 index CREATE INDEX idx_update_test_content ON update_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation idx_update_test_content NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert initial data INSERT INTO update_test (content, meta) VALUES ('The quick brown fox jumps over the lazy dog', '{"type": "test", "version": 1}'::jsonb), ('PostgreSQL is a powerful database system', '{"type": "database", "version": 1}'::jsonb), ('Full text search with BM25 ranking is efficient', '{"type": "search", "version": 1}'::jsonb); -- Test simple UPDATE UPDATE update_test SET content = 'The quick brown fox runs fast' WHERE id = 1; -- Test UPDATE with RETURNING UPDATE update_test SET content = 'PostgreSQL is an advanced relational database', meta = meta || '{"last_updated": "2025-01-01", "version": 2}'::jsonb WHERE id = 2 RETURNING id, content; id | content ----+----------------------------------------------- 2 | PostgreSQL is an advanced relational database (1 row) -- Test UPDATE matching multiple rows UPDATE update_test SET meta = meta || '{"updated": true}'::jsonb WHERE content ILIKE '%database%'; -- Test UPDATE with complex WHERE clause (similar to production crash) UPDATE update_test SET content = 'Advanced full text search with BM25 scoring', meta = meta || '{"last_updated": "2025-01-15", "num_updates": 3}'::jsonb WHERE meta @> '{"type": "search"}'::jsonb RETURNING id; id ---- 3 (1 row) -- Test UPDATE with NULL content UPDATE update_test SET content = NULL WHERE id = 1; -- Test UPDATE from NULL to non-NULL UPDATE update_test SET content = 'Previously NULL content now has text' WHERE id = 1; -- Test bulk UPDATE UPDATE update_test SET content = content || ' - updated in bulk' WHERE id IN (2, 3); -- Test UPDATE with subquery UPDATE update_test SET content = (SELECT 'Content from subquery for id ' || u.id::text FROM update_test u WHERE u.id = update_test.id) WHERE id = 2; -- Verify data after updates SELECT id, substring(content, 1, 50) as content_preview, meta->>'type' as type FROM update_test ORDER BY id; id | content_preview | type ----+----------------------------------------------------+---------- 1 | Previously NULL content now has text | test 2 | Content from subquery for id 2 | database 3 | Advanced full text search with BM25 scoring - upda | search (3 rows) -- Test UPDATE in transaction with ROLLBACK BEGIN; UPDATE update_test SET content = 'Transaction test' WHERE id = 1; SELECT id, substring(content, 1, 30) as preview FROM update_test WHERE id = 1; id | preview ----+------------------ 1 | Transaction test (1 row) ROLLBACK; -- Verify content was not changed SELECT id, substring(content, 1, 30) as preview FROM update_test WHERE id = 1; id | preview ----+-------------------------------- 1 | Previously NULL content now ha (1 row) -- Test UPDATE in transaction with COMMIT BEGIN; UPDATE update_test SET content = 'Committed transaction update' WHERE id = 1; COMMIT; -- Verify content was changed SELECT id, substring(content, 1, 30) as preview FROM update_test WHERE id = 1; id | preview ----+------------------------------ 1 | Committed transaction update (1 row) -- Test concurrent UPDATE scenario (serialized for single-session test) -- In production, this would be multiple concurrent sessions UPDATE update_test SET content = 'Concurrent update 1' WHERE id = 2; UPDATE update_test SET content = 'Concurrent update 2' WHERE id = 2; -- Test UPDATE with inheritance (should fail gracefully) CREATE TABLE update_parent ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE update_child () INHERITS (update_parent); -- This should fail with a clear error about inheritance not being supported -- We use DO block to catch the error and show it DO $$ BEGIN CREATE INDEX idx_update_parent_content ON update_parent USING bm25 (content) WITH (text_config = 'english'); EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'Expected error: %', SQLERRM; END; $$; NOTICE: BM25 index build started for relation idx_update_parent_content NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Clean up DROP TABLE IF EXISTS update_child; DROP TABLE IF EXISTS update_parent; DROP TABLE IF EXISTS update_test CASCADE; DROP EXTENSION IF EXISTS pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/vacuum.out000066400000000000000000000064071525762047700236010ustar00rootroot00000000000000-- Test VACUUM behavior with BM25 indexes -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Isolate physical cleanup from other backends and standby feedback. CREATE TEMP TABLE vacuum_test ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO vacuum_test (content) VALUES ('document one with test'), ('document two without keyword'), ('document three with test word'), ('document four plain text'), ('document five has test'); -- Create index CREATE INDEX vacuum_idx ON vacuum_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vacuum_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.60 -- Force index scan (small table would otherwise prefer seq scan) SET enable_seqscan = off; -- Check index statistics before deletions -- Extract just the total_docs line from debug output SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_before; total_docs_before ------------------- 5 (1 row) -- Delete some documents DELETE FROM vacuum_test WHERE id IN ( SELECT id FROM vacuum_test WHERE content LIKE '%test%' LIMIT 2 ); -- Check that index statistics haven't changed yet (deletions not processed) SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_after_delete; total_docs_after_delete ------------------------- 5 (1 row) -- Run VACUUM to clean up deleted tuples VACUUM vacuum_test; -- Check index statistics after VACUUM -- Note: Our current implementation may not update stats until rebuild SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_after_vacuum; total_docs_after_vacuum ------------------------- 5 (1 row) -- Search should still work correctly after VACUUM SELECT id, substring(content, 1, 30) as content_preview FROM vacuum_test ORDER BY content <@> to_bm25query('test', 'vacuum_idx') LIMIT 5; id | content_preview ----+------------------------ 5 | document five has test (1 row) -- Test VACUUM FULL (more aggressive cleanup) -- VACUUM FULL rebuilds indexes DELETE FROM vacuum_test WHERE content NOT LIKE '%test%'; -- We use \set VERBOSITY terse to avoid OID-specific error messages \set VERBOSITY terse VACUUM FULL vacuum_test; NOTICE: BM25 index build started for relation vacuum_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=3.00 \set VERBOSITY default -- Check statistics after VACUUM FULL SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_after_vacuum_full; total_docs_after_vacuum_full ------------------------------ 1 (1 row) -- Verify search still works SELECT id, substring(content, 1, 30) as content_preview FROM vacuum_test ORDER BY content <@> to_bm25query('test', 'vacuum_idx'); id | content_preview ----+------------------------ 5 | document five has test (1 row) -- Clean up DROP TABLE vacuum_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/vacuum_bitmap.out000066400000000000000000000355731525762047700251430ustar00rootroot00000000000000-- Test alive bitset tombstone tracking during VACUUM CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Temporary relations keep VACUUM's horizon independent of other -- backends and standby feedback, so deleted tuples can be removed. -- ============================================================= -- Test 1: Basic VACUUM with bitmap marking -- ============================================================= CREATE TEMP TABLE vb_test (id serial PRIMARY KEY, content text); INSERT INTO vb_test (content) SELECT 'document number ' || i || ' about databases' FROM generate_series(1, 100) i; CREATE INDEX vb_idx ON vb_test USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 100 documents, avg_length=4.00 -- Verify initial query works SELECT count(*) FROM ( SELECT id FROM vb_test ORDER BY content <@> to_bm25query('databases', 'vb_idx') LIMIT 1000 ) q; count ------- 100 (1 row) -- Delete some rows DELETE FROM vb_test WHERE id <= 20; -- VACUUM to apply bitmap marking VACUUM vb_test; -- Query should return only live rows SELECT count(*) FROM ( SELECT id FROM vb_test ORDER BY content <@> to_bm25query('databases', 'vb_idx') LIMIT 1000 ) q; count ------- 80 (1 row) -- Verify no deleted rows appear SELECT count(*) FROM ( SELECT id FROM vb_test ORDER BY content <@> to_bm25query('databases', 'vb_idx') LIMIT 1000 ) q WHERE id <= 20; count ------- 0 (1 row) DROP TABLE vb_test; -- ============================================================= -- Test 2: Full segment delete — segment should be dropped -- ============================================================= CREATE TEMP TABLE vb_full (id serial PRIMARY KEY, content text); INSERT INTO vb_full (content) SELECT 'fulldelete document ' || i FROM generate_series(1, 50) i; CREATE INDEX vb_full_idx ON vb_full USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_full_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=3.00 DELETE FROM vb_full; VACUUM vb_full; -- Should return 0 results SELECT count(*) FROM ( SELECT id FROM vb_full ORDER BY content <@> to_bm25query('fulldelete', 'vb_full_idx') LIMIT 1000 ) q; count ------- 0 (1 row) -- Index should still be usable after inserting new rows INSERT INTO vb_full (content) VALUES ('fresh document about topics'); SELECT count(*) FROM ( SELECT id FROM vb_full ORDER BY content <@> to_bm25query('fresh', 'vb_full_idx') LIMIT 1000 ) q; count ------- 1 (1 row) DROP TABLE vb_full; -- ============================================================= -- Test 3: CTID reuse correctness -- ============================================================= CREATE TEMP TABLE vb_reuse (id serial PRIMARY KEY, content text); INSERT INTO vb_reuse (content) SELECT 'alpha keyword document ' || i FROM generate_series(1, 50) i; CREATE INDEX vb_reuse_idx ON vb_reuse USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_reuse_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=4.00 -- Verify alpha matches SELECT count(*) > 0 AS has_alpha FROM ( SELECT id FROM vb_reuse ORDER BY content <@> to_bm25query('alpha', 'vb_reuse_idx') LIMIT 1000 ) q; has_alpha ----------- t (1 row) -- Delete all, VACUUM (frees CTIDs for reuse) DELETE FROM vb_reuse; VACUUM vb_reuse; -- Insert new rows (Postgres may reuse CTIDs) INSERT INTO vb_reuse (content) SELECT 'beta keyword document ' || i FROM generate_series(1, 50) i; -- alpha should return nothing SELECT count(*) FROM ( SELECT id FROM vb_reuse ORDER BY content <@> to_bm25query('alpha', 'vb_reuse_idx') LIMIT 1000 ) q; count ------- 0 (1 row) -- beta should return results SELECT count(*) > 0 AS has_beta FROM ( SELECT id FROM vb_reuse ORDER BY content <@> to_bm25query('beta', 'vb_reuse_idx') LIMIT 1000 ) q; has_beta ---------- t (1 row) DROP TABLE vb_reuse; -- ============================================================= -- Test 4: Multiple segments with mixed deletes -- ============================================================= CREATE TEMP TABLE vb_multi (id serial PRIMARY KEY, content text); INSERT INTO vb_multi (content) SELECT 'first batch document ' || i || ' with search term' FROM generate_series(1, 50) i; CREATE INDEX vb_multi_idx ON vb_multi USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation vb_multi_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=6.00 -- Force first batch to segment SELECT bm25_spill_index('vb_multi_idx'); bm25_spill_index ------------------ (1 row) -- Insert second batch INSERT INTO vb_multi (content) SELECT 'second batch document ' || i || ' with search term' FROM generate_series(1, 50) i; -- Force second batch to segment SELECT bm25_spill_index('vb_multi_idx'); bm25_spill_index ------------------ 4 (1 row) -- Delete from first batch only DELETE FROM vb_multi WHERE id <= 25; VACUUM vb_multi; -- Should find docs from both batches (minus deleted) SELECT count(*) FROM ( SELECT id FROM vb_multi ORDER BY content <@> to_bm25query('search', 'vb_multi_idx') LIMIT 1000 ) q; count ------- 75 (1 row) DROP TABLE vb_multi; -- ============================================================= -- Test 5: Merge after VACUUM cleans up dead docs -- ============================================================= CREATE TEMP TABLE vb_merge (id serial PRIMARY KEY, content text); INSERT INTO vb_merge (content) SELECT 'merge test document ' || i FROM generate_series(1, 50) i; CREATE INDEX vb_merge_idx ON vb_merge USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation vb_merge_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=4.00 -- Delete half DELETE FROM vb_merge WHERE id <= 25; VACUUM vb_merge; -- Force merge to clean up dead docs SELECT bm25_force_merge('vb_merge_idx'); bm25_force_merge ------------------ (1 row) -- Query should still work correctly SELECT count(*) FROM ( SELECT id FROM vb_merge ORDER BY content <@> to_bm25query('merge', 'vb_merge_idx') LIMIT 1000 ) q; count ------- 25 (1 row) DROP TABLE vb_merge; -- ============================================================= -- Test 6: Score correctness after deletes (multi-term BMW) -- ============================================================= CREATE TEMP TABLE vb_scores (id serial PRIMARY KEY, content text); INSERT INTO vb_scores (content) VALUES ('database systems performance tuning'), ('performance optimization techniques'), ('database indexing strategies'), ('query optimization in databases'), ('systems programming fundamentals'), ('advanced database concepts'), ('delete me not relevant'), ('also delete me irrelevant'), ('performance database benchmark'), ('indexing performance metrics'); CREATE INDEX vb_scores_idx ON vb_scores USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_scores_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 10 documents, avg_length=3.00 -- Delete some rows DELETE FROM vb_scores WHERE content LIKE '%delete me%'; VACUUM vb_scores; -- Multi-term query via BMW index scan SELECT id, content <@> to_bm25query('performance database', 'vb_scores_idx') AS score FROM vb_scores ORDER BY content <@> to_bm25query('performance database', 'vb_scores_idx') LIMIT 5; id | score ----+--------------------- 9 | -1.5869650840759277 1 | -1.3965292572975159 2 | -0.8938179016113281 10 | -0.8938179016113281 3 | -0.6931471824645996 (5 rows) DROP TABLE vb_scores; -- ============================================================= -- Test 7: Many deletes (>64 per segment, exercises realloc) -- ============================================================= CREATE TEMP TABLE vb_many (id serial PRIMARY KEY, content text); INSERT INTO vb_many (content) SELECT 'manydelete document ' || i || ' searchable' FROM generate_series(1, 200) i; CREATE INDEX vb_many_idx ON vb_many USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_many_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 200 documents, avg_length=4.00 -- Delete 150 of 200 rows (well past the initial 64-element array) DELETE FROM vb_many WHERE id <= 150; VACUUM vb_many; SELECT count(*) FROM ( SELECT id FROM vb_many ORDER BY content <@> to_bm25query('searchable', 'vb_many_idx') LIMIT 1000 ) q; count ------- 50 (1 row) DROP TABLE vb_many; -- ============================================================= -- Test 8: Multi-segment merge with dead docs -- ============================================================= CREATE TEMP TABLE vb_multimerge (id serial PRIMARY KEY, content text); INSERT INTO vb_multimerge (content) SELECT 'batch1 doc ' || i || ' target' FROM generate_series(1, 50) i; CREATE INDEX vb_mm_idx ON vb_multimerge USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation vb_mm_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 50 documents, avg_length=4.00 SELECT bm25_spill_index('vb_mm_idx'); bm25_spill_index ------------------ (1 row) INSERT INTO vb_multimerge (content) SELECT 'batch2 doc ' || i || ' target' FROM generate_series(1, 50) i; SELECT bm25_spill_index('vb_mm_idx'); bm25_spill_index ------------------ 4 (1 row) INSERT INTO vb_multimerge (content) SELECT 'batch3 doc ' || i || ' target' FROM generate_series(1, 50) i; SELECT bm25_spill_index('vb_mm_idx'); bm25_spill_index ------------------ 7 (1 row) -- Delete from each batch DELETE FROM vb_multimerge WHERE id <= 10; DELETE FROM vb_multimerge WHERE id > 50 AND id <= 60; DELETE FROM vb_multimerge WHERE id > 100 AND id <= 110; VACUUM vb_multimerge; -- Merge all segments — exercises N-way merge with dead docs SELECT bm25_force_merge('vb_mm_idx'); bm25_force_merge ------------------ (1 row) -- 150 total - 30 deleted = 120 live SELECT count(*) FROM ( SELECT id FROM vb_multimerge ORDER BY content <@> to_bm25query('target', 'vb_mm_idx') LIMIT 1000 ) q; count ------- 120 (1 row) DROP TABLE vb_multimerge; -- ============================================================= -- Test 9: All docs dead across segments then merge -- ============================================================= CREATE TEMP TABLE vb_alldead (id serial PRIMARY KEY, content text); INSERT INTO vb_alldead (content) SELECT 'alldead doc ' || i FROM generate_series(1, 30) i; CREATE INDEX vb_alldead_idx ON vb_alldead USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_alldead_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 30 documents, avg_length=3.00 SELECT bm25_spill_index('vb_alldead_idx'); bm25_spill_index ------------------ (1 row) INSERT INTO vb_alldead (content) SELECT 'alldead second ' || i FROM generate_series(1, 30) i; SELECT bm25_spill_index('vb_alldead_idx'); bm25_spill_index ------------------ 4 (1 row) DELETE FROM vb_alldead; VACUUM vb_alldead; -- Both segments should be dropped — merge is a no-op SELECT bm25_force_merge('vb_alldead_idx'); bm25_force_merge ------------------ (1 row) DO $$ DECLARE summary text := bm25_summarize_index('vb_alldead_idx'); BEGIN IF summary !~ E'Segments:\n \\(none\\)' THEN RAISE EXCEPTION 'all-dead force merge left a segment: %', summary; END IF; END $$; SELECT count(*) FROM ( SELECT id FROM vb_alldead ORDER BY content <@> to_bm25query('alldead', 'vb_alldead_idx') LIMIT 1000 ) q; count ------- 0 (1 row) -- Insert fresh data to verify index still works INSERT INTO vb_alldead (content) VALUES ('revival document here'); SELECT count(*) FROM ( SELECT id FROM vb_alldead ORDER BY content <@> to_bm25query('revival', 'vb_alldead_idx') LIMIT 1000 ) q; count ------- 1 (1 row) DROP TABLE vb_alldead; -- ============================================================= -- Test 10: Dump shows alive bitset info -- ============================================================= CREATE TEMP TABLE vb_dump (id serial PRIMARY KEY, content text); INSERT INTO vb_dump (content) SELECT 'dump test ' || i FROM generate_series(1, 10) i; CREATE INDEX vb_dump_idx ON vb_dump USING bm25 (content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vb_dump_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 10 documents, avg_length=3.00 -- Verify dump output includes alive bitset info SELECT bm25_dump_index('vb_dump_idx') LIKE '%Alive: 10 / 10 docs%' AS has_alive_info; has_alive_info ---------------- t (1 row) DROP TABLE vb_dump; -- ============================================================= -- Test 11: Batched CTID reads across pages and segments -- ============================================================= CREATE TEMP TABLE vb_batched (id serial PRIMARY KEY, content text); INSERT INTO vb_batched (content) SELECT 'bounded batch ' || i FROM generate_series(1, 3000) i; CREATE INDEX vb_batched_idx ON vb_batched USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); NOTICE: BM25 index build started for relation vb_batched_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 3000 documents, avg_length=3.00 INSERT INTO vb_batched (content) SELECT 'bounded batch ' || i FROM generate_series(3001, 6000) i; SELECT bm25_spill_index('vb_batched_idx') > 0 AS spilled; spilled --------- t (1 row) DELETE FROM vb_batched WHERE id % 3 = 0; VACUUM vb_batched; SELECT count(*) FROM ( SELECT id FROM vb_batched ORDER BY content <@> to_bm25query('bounded', 'vb_batched_idx') LIMIT 6000 ) q; count ------- 4000 (1 row) DROP TABLE vb_batched; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/expected/vacuum_extended.out000066400000000000000000000550551525762047700254640ustar00rootroot00000000000000-- Extended vacuum tests for BM25 indexes -- Exercises vacuum paths with segments, bulk deletes, and empty indexes CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================================= -- Test 1: Vacuum with segments present -- ============================================================================= CREATE TABLE vacuum_seg_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_seg_idx ON vacuum_seg_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation vacuum_seg_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert data and force segment creation INSERT INTO vacuum_seg_test (content) SELECT 'vacuum segment test document number ' || i FROM generate_series(1, 200) AS i; SELECT bm25_spill_index('vacuum_seg_idx'); bm25_spill_index ------------------ 4 (1 row) -- Delete half the documents DELETE FROM vacuum_seg_test WHERE id <= 100; -- VACUUM with segments on disk VACUUM vacuum_seg_test; -- Verify search still works after vacuum with segments SELECT count(*) FROM ( SELECT 1 FROM vacuum_seg_test ORDER BY content <@> to_bm25query('vacuum', 'vacuum_seg_idx') ) sub; count ------- 100 (1 row) -- ============================================================================= -- Test 2: VACUUM FULL with segments (forces index rebuild) -- ============================================================================= SET client_min_messages = warning; VACUUM FULL vacuum_seg_test; RESET client_min_messages; -- Verify search works after VACUUM FULL rebuild SELECT count(*) FROM ( SELECT 1 FROM vacuum_seg_test ORDER BY content <@> to_bm25query('vacuum', 'vacuum_seg_idx') ) sub; count ------- 100 (1 row) -- The rebuild must also produce complete corpus accounting. The -- notice suppression above hides the build-count line, which is the -- only place the rebuild reported it, so assert it here instead. -- 100 is the live-only rebuild; 200 appears when a concurrent -- snapshot keeps the 100 deleted rows alive through VACUUM FULL. -- Every doc is exactly 6 tokens, so avg_doc_len pins the token side -- of the accounting for either doc count: a rebuild that dropped or -- double-counted tokens moves avg_doc_len off 6.00. SELECT bm25_summarize_index('vacuum_seg_idx') ~ E'total_docs: (100|200)\n' AND bm25_summarize_index('vacuum_seg_idx') ~ E'avg_doc_len: 6\\.00\n' AS rebuild_accounting_valid; rebuild_accounting_valid -------------------------- t (1 row) DROP TABLE vacuum_seg_test; -- ============================================================================= -- Test 3: Vacuum on empty index -- ============================================================================= CREATE TABLE vacuum_empty_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_empty_idx ON vacuum_empty_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation vacuum_empty_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Vacuum empty index (should be a no-op) VACUUM vacuum_empty_test; -- Insert one doc, delete it, vacuum INSERT INTO vacuum_empty_test (content) VALUES ('single document'); DELETE FROM vacuum_empty_test; VACUUM vacuum_empty_test; DROP TABLE vacuum_empty_test; -- ============================================================================= -- Test 4: Bulk delete from memtable (exercises tp_bulkdelete_memtable_ctids) -- ============================================================================= CREATE TABLE vacuum_bulk_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_bulk_idx ON vacuum_bulk_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation vacuum_bulk_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert docs (stays in memtable) INSERT INTO vacuum_bulk_test (content) SELECT 'bulk delete test term' || (i % 10) || ' document ' || i FROM generate_series(1, 50) AS i; -- Delete most of them DELETE FROM vacuum_bulk_test WHERE id > 5; -- Vacuum to clean up memtable entries VACUUM vacuum_bulk_test; -- Verify remaining docs are searchable SELECT count(*) FROM ( SELECT 1 FROM vacuum_bulk_test ORDER BY content <@> to_bm25query('bulk', 'vacuum_bulk_idx') ) sub; count ------- 5 (1 row) DROP TABLE vacuum_bulk_test; -- ============================================================================= -- Test 5: REINDEX with unflushed memtable data -- -- Verifies that REINDEX correctly rebuilds from the heap even when -- the memtable has unflushed entries from recent INSERTs. -- ============================================================================= CREATE TABLE reindex_memtable_test ( id serial PRIMARY KEY, content text ); CREATE INDEX reindex_memtable_idx ON reindex_memtable_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation reindex_memtable_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Phase 1: Initial data via CREATE INDEX (already built above) INSERT INTO reindex_memtable_test (content) SELECT 'original document about databases number ' || i FROM generate_series(1, 50) AS i; -- Force a spill so we have segments on disk SELECT bm25_spill_index('reindex_memtable_idx'); bm25_spill_index ------------------ 2 (1 row) -- Verify baseline: all 50 docs are searchable SELECT count(*) AS pre_reindex_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; pre_reindex_count ------------------- 50 (1 row) -- Phase 2: Insert more rows that stay in memtable (unflushed) INSERT INTO reindex_memtable_test (content) SELECT 'additional document about databases number ' || i FROM generate_series(51, 75) AS i; -- Verify memtable scan finds all 75 docs SELECT count(*) AS with_memtable_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; with_memtable_count --------------------- 75 (1 row) -- Phase 3: REINDEX discards memtable, rebuilds from heap \set VERBOSITY terse REINDEX INDEX reindex_memtable_idx; NOTICE: BM25 index build started for relation reindex_memtable_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 75 documents, avg_length=5.00 \set VERBOSITY default -- All 75 docs must still be found (rebuilt from heap, not memtable) SELECT count(*) AS post_reindex_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; post_reindex_count -------------------- 75 (1 row) -- Phase 4: Verify INSERTs still work after REINDEX INSERT INTO reindex_memtable_test (content) VALUES ('final document about databases after reindex'); SELECT count(*) AS final_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; final_count ------------- 76 (1 row) DROP TABLE reindex_memtable_test; -- ============================================================================= -- Test 6: Bulk load with unflushed memtable data -- -- Verifies that a large INSERT into an already-indexed table correctly -- triggers auto-spills from the memtable while preserving earlier -- unflushed memtable entries. Uses a low spill threshold so the test -- triggers multiple spills without needing millions of rows. -- ============================================================================= -- Lower spill threshold BEFORE creating the table so it applies to inserts SET pg_textsearch.memtable_pages_threshold = 1; CREATE TABLE bulk_memtable_test ( id serial PRIMARY KEY, content text ); CREATE INDEX bulk_memtable_idx ON bulk_memtable_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bulk_memtable_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Phase 1: Small insert that stays in memtable INSERT INTO bulk_memtable_test (content) SELECT 'initial document about networking topic ' || i FROM generate_series(1, 10) AS i; SELECT count(*) AS initial_count FROM ( SELECT 1 FROM bulk_memtable_test ORDER BY content <@> to_bm25query('networking', 'bulk_memtable_idx') ) sub; initial_count --------------- 10 (1 row) -- Phase 2: Bulk insert triggers multiple auto-spills INSERT INTO bulk_memtable_test (content) SELECT 'bulk loaded document about networking and databases number ' || i FROM generate_series(11, 5000) AS i; -- All 5000 docs must be found (initial memtable + spilled segments + tail) SELECT count(*) AS after_bulk_count FROM ( SELECT 1 FROM bulk_memtable_test ORDER BY content <@> to_bm25query('networking', 'bulk_memtable_idx') ) sub; after_bulk_count ------------------ 5000 (1 row) -- Verify segments were created by the auto-spill SELECT bm25_summarize_index('bulk_memtable_idx') ~ 'Total:.*segments' AS has_segments; has_segments -------------- t (1 row) -- Phase 3: More inserts after bulk load still work INSERT INTO bulk_memtable_test (content) VALUES ('final document about networking after bulk load'); SELECT count(*) AS final_count FROM ( SELECT 1 FROM bulk_memtable_test ORDER BY content <@> to_bm25query('networking', 'bulk_memtable_idx') ) sub; final_count ------------- 5001 (1 row) RESET pg_textsearch.memtable_pages_threshold; DROP TABLE bulk_memtable_test; -- ============================================================================= -- Test 7: VACUUM spills un-spilled memtable on insert-only tables (issue #333) -- -- On insert-only workloads ambulkdelete is skipped (no dead tuples), so the -- memtable spill that used to live only there never ran. The memtable -- chain pages then grew unbounded and queries had to walk the entire -- chain. This test exercises the amvacuumcleanup path that now performs -- the spill. -- ============================================================================= CREATE TABLE vacuum_insert_only_test ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_insert_only_idx ON vacuum_insert_only_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation vacuum_insert_only_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Populate the memtable chain above TP_MIN_SPILL_POSTINGS (1000) but -- below pg_textsearch.bulk_load_threshold so the spill only happens -- via amvacuumcleanup. 2000 docs * ~4 terms each ≈ 8000 postings. INSERT INTO vacuum_insert_only_test (content) SELECT 'insert only document number ' || i FROM generate_series(1, 2000) AS i; -- Chain should be populated before VACUUM. SELECT count(*) > 0 AS chain_populated_before_vacuum FROM bm25_memtable_chain('vacuum_insert_only_idx'); chain_populated_before_vacuum ------------------------------- t (1 row) -- No dead tuples here, so ambulkdelete is not invoked; only -- amvacuumcleanup runs. It must still drain the memtable chain. VACUUM vacuum_insert_only_test; -- Chain must be fully drained and the memtable contents spilled to a segment. SELECT count(*) = 0 AS chain_empty_after_vacuum FROM bm25_memtable_chain('vacuum_insert_only_idx'); chain_empty_after_vacuum -------------------------- t (1 row) -- Search still finds all 2000 docs. SELECT count(*) AS post_vacuum_count FROM ( SELECT 1 FROM vacuum_insert_only_test ORDER BY content <@> to_bm25query('document', 'vacuum_insert_only_idx') ) sub; post_vacuum_count ------------------- 2000 (1 row) DROP TABLE vacuum_insert_only_test; -- ============================================================================= -- Test 8: tiny memtable stays un-spilled across VACUUM (issue #333 guard) -- ============================================================================= -- -- Below TP_MIN_SPILL_POSTINGS the VACUUM-cleanup spill is a no-op, -- because writing a runt L0 segment would cost more in subsequent -- compaction than chain replay saves. CREATE TABLE vacuum_tiny_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_tiny_idx ON vacuum_tiny_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation vacuum_tiny_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- ~50 docs * ~4 terms = ~200 postings, well below the guard. INSERT INTO vacuum_tiny_test (content) SELECT 'tiny doc ' || i FROM generate_series(1, 50) AS i; VACUUM vacuum_tiny_test; -- Chain is still present; no runt L0 segment was produced. SELECT count(*) > 0 AS chain_preserved_after_vacuum FROM bm25_memtable_chain('vacuum_tiny_idx'); chain_preserved_after_vacuum ------------------------------ t (1 row) DROP TABLE vacuum_tiny_test; -- ============================================================================= -- Test 9: dropping a memtable-spilled L0 segment keeps total_len sane -- ============================================================================= -- -- Regression test. tp_write_segment used to leave header.total_tokens -- at the cumulative shared-atomic value instead of the per-segment -- sum. When VACUUM later dropped that segment (all docs dead) the -- inflated header.total_tokens was subtracted from metap->total_len -- and the atomic, clamping them to 0 (or wrapping the atomic) and -- breaking avgdl in BM25 scoring for the surviving docs. -- -- We populate two L0 segments with distinct id ranges, delete every -- row that landed in the second one, and then run VACUUM. Afterwards -- a search over the surviving corpus must return real matches. CREATE TABLE l0_total_len_test (id serial PRIMARY KEY, content text); CREATE INDEX l0_total_len_idx ON l0_total_len_test USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation l0_total_len_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Segment 1: ids 1..50 (the survivors). INSERT INTO l0_total_len_test (content) SELECT 'alpha common term doc ' || i FROM generate_series(1, 50) AS i; SELECT bm25_spill_index('l0_total_len_idx'); bm25_spill_index ------------------ 2 (1 row) -- Segment 2: ids 51..100 (to be fully deleted). INSERT INTO l0_total_len_test (content) SELECT 'beta common term doc ' || i FROM generate_series(51, 100) AS i; SELECT bm25_spill_index('l0_total_len_idx'); bm25_spill_index ------------------ 5 (1 row) -- Wipe every row that went into segment 2, then VACUUM to drop it. DELETE FROM l0_total_len_test WHERE id > 50; VACUUM l0_total_len_test; -- total_len must reflect only the surviving segment's tokens. The -- pre-fix path inflated header.total_tokens to the cumulative atomic -- so the subtraction clamped total_len to 0. A healthy avg_doc_len -- is the sharpest regression signal. SELECT bm25_summarize_index('l0_total_len_idx') ~ E'total_len: (250|500)\n' AND bm25_summarize_index('l0_total_len_idx') ~ E'avg_doc_len: 5\\.00\n' AS l0_accounting_valid; l0_accounting_valid --------------------- t (1 row) DROP TABLE l0_total_len_test; -- ============================================================================= -- Test 10: merged segments carry raw total_tokens (long docs + VACUUM) -- ============================================================================= -- -- Regression test. merge.c used to recompute header.total_tokens as -- Σ decode_fieldnorm(fieldnorms[d]), which is exponentially quantized -- for doc_length > 39 (e.g. 100→96). metap->total_len stayed raw -- (set from the atomic), so the drift was latent until a VACUUM drop -- of the merged segment fed the quantized header value back into -- metap via tp_apply_vacuum_shrinkage, leaving phantom tokens. -- -- Fix: merge starts from Σ source.header.total_tokens (raw) minus a -- dead-only fieldnorm approximation. For all-live sources the merged -- total_tokens is exact, so the VACUUM subtraction cancels to zero. SET pg_textsearch.segments_per_level = 2; CREATE TABLE merge_long_docs (id serial PRIMARY KEY, content text); CREATE INDEX merge_long_idx ON merge_long_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation merge_long_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Each doc has 100 unique tokens. decode(encode(100)) = 96. INSERT INTO merge_long_docs (content) SELECT string_agg('term' || (j + i*1000), ' ') FROM generate_series(1, 30) i, generate_series(1, 100) j GROUP BY i; SELECT bm25_spill_index('merge_long_idx'); bm25_spill_index ------------------ 6 (1 row) INSERT INTO merge_long_docs (content) SELECT string_agg('other' || (j + i*1000), ' ') FROM generate_series(1, 30) i, generate_series(1, 100) j GROUP BY i; SELECT bm25_spill_index('merge_long_idx'); bm25_spill_index ------------------ 35 (1 row) -- Sanity: both batches merged into a single L1 segment. SELECT bm25_summarize_index('merge_long_idx') ~ E'L1 Segment' AS has_l1_segment; has_l1_segment ---------------- t (1 row) -- Delete every row so VACUUM drops the merged segment. With the fix -- the L1 header carries raw 6000 tokens; subtracting that from the -- atomic/metapage total_len (also 6000) leaves 0. Pre-fix the L1 -- header was 60 × 96 = 5760 (quantized), leaving 240 phantom tokens -- that would bias avg_doc_len for any surviving docs and persist -- across the next tp_sync_metapage_stats. DELETE FROM merge_long_docs; VACUUM merge_long_docs; WITH summary AS ( SELECT bm25_summarize_index('merge_long_idx') AS value ) SELECT (value ~ E'total_len: 0\n' AND value ~ E'total_docs: 0\n') OR (value ~ E'total_len: 6000\n' AND value ~ E'total_docs: 60\n') AS merged_drop_accounting_valid FROM summary; merged_drop_accounting_valid ------------------------------ t (1 row) RESET pg_textsearch.segments_per_level; DROP TABLE merge_long_docs; -- ============================================================================= -- Test 11: merge drops V5-bitset-dead docs and applies shrinkage -- ============================================================================= -- -- Regression test. tp_merge_level_segments used to update -- level_heads/level_counts but leave metap->total_docs / total_len -- at the pre-merge value when the merged segment dropped docs that -- VACUUM had previously marked dead in the source bitsets. After -- the merge, Σ segment.num_docs (= live count) was strictly less -- than metap->total_docs, violating the primary invariant. -- -- Setup: two L0 spills below segments_per_level so no compaction -- fires yet, VACUUM to flip dead bits, then a third spill that -- triggers an L0→L1 compaction. The merged L1 segment excludes the -- dead docs; metap must shrink to match. SET pg_textsearch.segments_per_level = 3; CREATE TABLE merge_drop_dead (id serial PRIMARY KEY, content text); CREATE INDEX merge_drop_idx ON merge_drop_dead USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation merge_drop_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Spills 1 and 2. INSERT INTO merge_drop_dead (content) SELECT 'one alpha common term doc ' || i FROM generate_series(1, 30) AS i; SELECT bm25_spill_index('merge_drop_idx'); bm25_spill_index ------------------ 2 (1 row) INSERT INTO merge_drop_dead (content) SELECT 'two beta common term doc ' || i FROM generate_series(31, 60) AS i; SELECT bm25_spill_index('merge_drop_idx'); bm25_spill_index ------------------ 5 (1 row) -- Delete half the rows and VACUUM to flip dead bits in the two L0s. DELETE FROM merge_drop_dead WHERE id % 2 = 0; VACUUM merge_drop_dead; -- Third spill triggers compaction (segments_per_level=3); merge -- drops the bitset-dead docs. INSERT INTO merge_drop_dead (content) SELECT 'three gamma common term doc ' || i FROM generate_series(61, 90) AS i; SELECT bm25_spill_index('merge_drop_idx') > 0 AS third_spill_ok; third_spill_ok ---------------- t (1 row) -- 30 + 30 + 30 = 90 inserted; 30 deleted. Which state is reachable -- depends on whether VACUUM could reclaim the deleted rows: -- -- * no competing snapshot: dead bits flip, the merge drops them, -- and both metap->total_docs and Σ segment.num_docs are 60; -- * a held snapshot pins the deleted rows: no dead bits, the merge -- keeps everything, and both counts are 90. -- -- Either way the invariant total_docs = Σ segment.num_docs must -- hold. The pre-fix bug produced neither state: metap->total_docs -- stayed at 90 (summed sources' num_docs) while the merged segment -- carried only the 60 live docs. Comparing the two counts against -- each other -- not just the metapage count against a set of -- allowed values -- is what rejects that. WITH summary AS ( SELECT bm25_summarize_index('merge_drop_idx') AS value ), parsed AS ( SELECT (regexp_match(value, 'docs_persisted: (\d+)'))[1]::bigint AS metap_docs, (regexp_match(value, 'Total: \d+ segments, \d+ pages \([0-9.]+MB\), ' '\d+ terms, (\d+) docs'))[1]::bigint AS segment_docs, value ~ 'L1 Segment' AS compacted FROM summary ) SELECT compacted AS compaction_occurred, metap_docs = segment_docs AS metap_matches_segments, metap_docs IN (60, 90) AS docs_in_expected_states FROM parsed; compaction_occurred | metap_matches_segments | docs_in_expected_states ---------------------+------------------------+------------------------- t | t | t (1 row) RESET pg_textsearch.segments_per_level; DROP TABLE merge_drop_dead; -- Clean up DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/vacuum_rebuild.out000066400000000000000000000150531525762047700253040ustar00rootroot00000000000000-- Test: VACUUM correctly removes dead index entries so CTID reuse -- does not produce false positives. -- -- The bug only manifests via ORDER BY <@> LIMIT queries, which use -- the index scan path. Seq scans re-evaluate the <@> operator on -- the heap tuple and catch stale results. We use enable_seqscan=off -- to ensure the index scan is used even for small tables. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -- ================================================================ -- Test 1: Basic CTID reuse after VACUUM -- ================================================================ CREATE TABLE vacuum_rebuild_test ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_rebuild_idx ON vacuum_rebuild_test USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vacuum_rebuild_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert documents with distinctive terms INSERT INTO vacuum_rebuild_test (content) VALUES ('alpha bravo charlie'), ('delta echo foxtrot'), ('golf hotel india'); -- Force to segments so VACUUM has something to rebuild SELECT bm25_spill_index('vacuum_rebuild_idx'); bm25_spill_index ------------------ 2 (1 row) -- Verify baseline: index scan for 'alpha' returns the right row SELECT id, content FROM vacuum_rebuild_test ORDER BY content <@> to_bm25query('alpha', 'vacuum_rebuild_idx') LIMIT 1; id | content ----+--------------------- 1 | alpha bravo charlie (1 row) -- Delete rows containing 'alpha' and 'delta', then VACUUM to free -- their CTIDs for reuse. DELETE FROM vacuum_rebuild_test WHERE id <= 2; VACUUM vacuum_rebuild_test; -- Insert new rows. Postgres will reuse the freed CTIDs (0,1) and -- (0,2) for these new rows. INSERT INTO vacuum_rebuild_test (content) VALUES ('juliet kilo lima'), ('mike november oscar'); -- Index scan for 'alpha' must return no rows. Without the fix, -- the stale segment entry for 'alpha' points to reused CTID (0,1) -- and the index scan returns 'juliet kilo lima' as a false match. SELECT id, content FROM vacuum_rebuild_test ORDER BY content <@> to_bm25query('alpha', 'vacuum_rebuild_idx') LIMIT 10; id | content ----+--------- (0 rows) -- Search for new terms must work via index scan SELECT id, content FROM vacuum_rebuild_test ORDER BY content <@> to_bm25query('juliet', 'vacuum_rebuild_idx') LIMIT 1; id | content ----+------------------ 4 | juliet kilo lima (1 row) DROP TABLE vacuum_rebuild_test; -- ================================================================ -- Test 2: All docs in segment deleted (segment removed from chain) -- ================================================================ CREATE TABLE vacuum_allgone ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_allgone_idx ON vacuum_allgone USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vacuum_allgone_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO vacuum_allgone (content) VALUES ('papaya mango'), ('papaya kiwi'); SELECT bm25_spill_index('vacuum_allgone_idx'); bm25_spill_index ------------------ 2 (1 row) -- Delete everything, vacuum DELETE FROM vacuum_allgone; VACUUM vacuum_allgone; -- Insert fresh data INSERT INTO vacuum_allgone (content) VALUES ('strawberry banana'); -- Must NOT find phantom 'papaya' results via index scan SELECT id, content FROM vacuum_allgone ORDER BY content <@> to_bm25query('papaya', 'vacuum_allgone_idx') LIMIT 10; id | content ----+--------- (0 rows) -- Must find the new row SELECT id, content FROM vacuum_allgone ORDER BY content <@> to_bm25query('strawberry', 'vacuum_allgone_idx') LIMIT 1; id | content ----+------------------- 3 | strawberry banana (1 row) DROP TABLE vacuum_allgone; -- ================================================================ -- Test 3: Multi-level segments with deletions -- ================================================================ SET pg_textsearch.memtable_pages_threshold = 1; CREATE TABLE vacuum_multilevel ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_multilevel_idx ON vacuum_multilevel USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vacuum_multilevel_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert enough to create L0 segments and trigger compaction to L1 INSERT INTO vacuum_multilevel (content) SELECT 'multilevel test document number ' || i || ' with keyword searchterm' FROM generate_series(1, 2000) AS i; -- Verify we have segments at multiple levels SELECT bm25_summarize_index('vacuum_multilevel_idx') ~ 'L1' AS has_l1_segments; has_l1_segments ----------------- t (1 row) -- Delete half DELETE FROM vacuum_multilevel WHERE id <= 1000; VACUUM vacuum_multilevel; -- Verify remaining docs are searchable via index scan SELECT count(*) AS remaining FROM ( SELECT id FROM vacuum_multilevel ORDER BY content <@> to_bm25query('searchterm', 'vacuum_multilevel_idx') LIMIT 2000 ) sub; remaining ----------- 1000 (1 row) RESET pg_textsearch.memtable_pages_threshold; DROP TABLE vacuum_multilevel; -- ================================================================ -- Test 4: VACUUM with no dead tuples (no-op) -- ================================================================ CREATE TABLE vacuum_noop ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_noop_idx ON vacuum_noop USING bm25(content) WITH (text_config = 'english'); NOTICE: BM25 index build started for relation vacuum_noop_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 INSERT INTO vacuum_noop (content) VALUES ('nothing deleted here'); SELECT bm25_spill_index('vacuum_noop_idx'); bm25_spill_index ------------------ 2 (1 row) VACUUM vacuum_noop; SELECT id, content FROM vacuum_noop ORDER BY content <@> to_bm25query('deleted', 'vacuum_noop_idx') LIMIT 1; id | content ----+---------------------- 1 | nothing deleted here (1 row) DROP TABLE vacuum_noop; RESET enable_seqscan; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/vacuum_rebuild_injection.out000066400000000000000000000270761525762047700273560ustar00rootroot00000000000000\pset format unaligned \set VERBOSITY terse SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET enable_seqscan = off; -- Temporary fixtures isolate immediate cleanup from other backends' xmin. -- Legacy token totals are rebuilt from heap data. CREATE TEMP TABLE vacuum_legacy_stats ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_legacy_stats_idx ON vacuum_legacy_stats USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_legacy_stats (content) SELECT 'legacy inflated token document ' || i FROM generate_series(1, 10) AS i; SELECT pg_textsearch_test_attach_legacy_segment(1000000); pg_textsearch_test_attach_legacy_segment (1 row) SELECT bm25_spill_index('vacuum_legacy_stats_idx') > 0 AS legacy_header_inflated; legacy_header_inflated t (1 row) SELECT injection_points_detach('pg-textsearch-legacy-segment'); injection_points_detach (1 row) DELETE FROM vacuum_legacy_stats WHERE id <= 5; VACUUM vacuum_legacy_stats; SELECT bm25_level_counts('vacuum_legacy_stats_idx'::regclass) AS levels_after_legacy_vacuum; levels_after_legacy_vacuum {1,0,0,0,0,0,0,0} (1 row) SELECT bm25_summarize_index('vacuum_legacy_stats_idx') ~ E'total_docs: 5\n' AND bm25_summarize_index('vacuum_legacy_stats_idx') ~ E'total_len: 25\n' AS legacy_totals_rebuilt; legacy_totals_rebuilt t (1 row) SELECT bm25_dump_index('vacuum_legacy_stats_idx') LIKE '%Alive: 5 / 5 docs%' AS legacy_replacement_published; legacy_replacement_published t (1 row) DROP TABLE vacuum_legacy_stats; -- Legacy replacement rebases a changed text configuration. CREATE TEXT SEARCH CONFIGURATION public.vacuum_mutable_cfg (COPY = english); CREATE TEMP TABLE vacuum_mutable_config ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_mutable_config_idx ON vacuum_mutable_config USING bm25(content) WITH (text_config = 'public.vacuum_mutable_cfg', compaction = 'off'); INSERT INTO vacuum_mutable_config (content) VALUES ('the the the alpha'), ('the the the alpha'); SELECT pg_textsearch_test_attach_legacy_segment(2); pg_textsearch_test_attach_legacy_segment (1 row) SELECT bm25_spill_index('vacuum_mutable_config_idx') > 0 AS mutable_legacy_created; mutable_legacy_created t (1 row) SELECT injection_points_detach('pg-textsearch-legacy-segment'); injection_points_detach (1 row) ALTER TEXT SEARCH CONFIGURATION public.vacuum_mutable_cfg ALTER MAPPING FOR asciiword WITH simple; DELETE FROM vacuum_mutable_config WHERE id = 1; VACUUM vacuum_mutable_config; SELECT bm25_summarize_index('vacuum_mutable_config_idx') ~ E'total_docs: 1\n' AND bm25_summarize_index('vacuum_mutable_config_idx') ~ E'total_len: 4\n' AS mutable_config_totals_rebased; mutable_config_totals_rebased t (1 row) DROP TABLE vacuum_mutable_config; DROP TEXT SEARCH CONFIGURATION public.vacuum_mutable_cfg; -- Inconsistent multi-segment legacy totals are fully rebased. CREATE TEMP TABLE vacuum_multi_legacy ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_multi_legacy_idx ON vacuum_multi_legacy USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_multi_legacy (content) SELECT 'multi legacy token document ' || i FROM generate_series(1, 5) AS i; SELECT pg_textsearch_test_attach_legacy_segment(1000000); pg_textsearch_test_attach_legacy_segment (1 row) SELECT bm25_spill_index('vacuum_multi_legacy_idx') > 0 AS first_legacy_created; first_legacy_created t (1 row) SELECT injection_points_detach('pg-textsearch-legacy-segment'); injection_points_detach (1 row) INSERT INTO vacuum_multi_legacy (content) SELECT 'multi legacy token document ' || i FROM generate_series(6, 10) AS i; SELECT pg_textsearch_test_attach_legacy_segment(2000000); pg_textsearch_test_attach_legacy_segment (1 row) SELECT bm25_spill_index('vacuum_multi_legacy_idx') > 0 AS second_legacy_created; second_legacy_created t (1 row) SELECT injection_points_detach('pg-textsearch-legacy-segment'); injection_points_detach (1 row) SELECT pg_textsearch_test_attach_vacuum_total_len(4000000); pg_textsearch_test_attach_vacuum_total_len (1 row) DELETE FROM vacuum_multi_legacy WHERE id = 1; VACUUM vacuum_multi_legacy; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); injection_points_detach (1 row) SELECT bm25_summarize_index('vacuum_multi_legacy_idx') ~ E'total_docs: 9\n' AND bm25_summarize_index('vacuum_multi_legacy_idx') ~ E'total_len: 45\n' AS multi_legacy_totals_rebased; multi_legacy_totals_rebased t (1 row) SELECT bm25_dump_index('vacuum_multi_legacy_idx') NOT LIKE '%Version: 4%' AS all_legacy_segments_rebuilt; all_legacy_segments_rebuilt t (1 row) DROP TABLE vacuum_multi_legacy; -- Impossible current-format totals fail closed. CREATE TEMP TABLE vacuum_current_corrupt ( id serial PRIMARY KEY, content text ); INSERT INTO vacuum_current_corrupt (content) VALUES ('current token document one'), ('current token document two'); CREATE INDEX vacuum_current_corrupt_idx ON vacuum_current_corrupt USING bm25(content) WITH (text_config = 'english'); SELECT pg_textsearch_test_attach_vacuum_total_len(0); pg_textsearch_test_attach_vacuum_total_len (1 row) DELETE FROM vacuum_current_corrupt WHERE id = 1; VACUUM vacuum_current_corrupt; ERROR: index token statistics are inconsistent SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); injection_points_detach (1 row) DROP TABLE vacuum_current_corrupt; CREATE TEMP TABLE vacuum_current_inflated ( id serial PRIMARY KEY, content text ); INSERT INTO vacuum_current_inflated (content) VALUES ('inflated token document one'), ('inflated token document two'); CREATE INDEX vacuum_current_inflated_idx ON vacuum_current_inflated USING bm25(content) WITH (text_config = 'english'); SELECT pg_textsearch_test_attach_vacuum_total_len(1000); pg_textsearch_test_attach_vacuum_total_len (1 row) VACUUM vacuum_current_inflated; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); injection_points_detach (1 row) DROP TABLE vacuum_current_inflated; -- Mixed graphs fail closed when a current-format header is corrupt. CREATE TEMP TABLE vacuum_mixed_current_corrupt ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_mixed_current_corrupt_idx ON vacuum_mixed_current_corrupt USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_mixed_current_corrupt (content) VALUES ('legacy mixed document one'), ('legacy mixed document two'); SELECT pg_textsearch_test_attach_legacy_segment(8); pg_textsearch_test_attach_legacy_segment (1 row) SELECT bm25_spill_index('vacuum_mixed_current_corrupt_idx') > 0 AS mixed_legacy_created; mixed_legacy_created t (1 row) SELECT injection_points_detach('pg-textsearch-legacy-segment'); injection_points_detach (1 row) INSERT INTO vacuum_mixed_current_corrupt (content) VALUES ('current mixed document three'), ('current mixed document four'); SELECT pg_textsearch_test_attach_v5_segment_total_len(1); pg_textsearch_test_attach_v5_segment_total_len (1 row) SELECT bm25_spill_index('vacuum_mixed_current_corrupt_idx') > 0 AS mixed_current_created; mixed_current_created t (1 row) SELECT injection_points_detach('pg-textsearch-v5-segment-total-len'); injection_points_detach (1 row) VACUUM vacuum_mixed_current_corrupt; ERROR: current segment token statistics are inconsistent SELECT bm25_summarize_index('vacuum_mixed_current_corrupt_idx') ~ E'total_docs: 4\n' AND bm25_summarize_index('vacuum_mixed_current_corrupt_idx') ~ E'total_len: 16\n' AS mixed_corruption_preserved_totals; mixed_corruption_preserved_totals t (1 row) SELECT bm25_dump_index('vacuum_mixed_current_corrupt_idx') LIKE '%Version: 4%' AS mixed_corruption_preserved_legacy; mixed_corruption_preserved_legacy t (1 row) DROP TABLE vacuum_mixed_current_corrupt; CREATE TEMP TABLE vacuum_mixed_no_dead ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_mixed_no_dead_idx ON vacuum_mixed_no_dead USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_mixed_no_dead (content) VALUES ('mixed legacy document one'), ('mixed legacy document two'); SELECT pg_textsearch_test_attach_legacy_segment(1000); pg_textsearch_test_attach_legacy_segment (1 row) SELECT bm25_spill_index('vacuum_mixed_no_dead_idx') > 0 AS no_dead_legacy_created; no_dead_legacy_created t (1 row) SELECT injection_points_detach('pg-textsearch-legacy-segment'); injection_points_detach (1 row) INSERT INTO vacuum_mixed_no_dead (content) VALUES ('mixed current document three'), ('mixed current document four'); SELECT bm25_spill_index('vacuum_mixed_no_dead_idx') > 0 AS no_dead_current_spilled; no_dead_current_spilled t (1 row) SELECT pg_textsearch_test_attach_vacuum_total_len(100); pg_textsearch_test_attach_vacuum_total_len (1 row) VACUUM vacuum_mixed_no_dead; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); injection_points_detach (1 row) SELECT bm25_summarize_index('vacuum_mixed_no_dead_idx') ~ E'total_docs: 4\n' AND bm25_summarize_index('vacuum_mixed_no_dead_idx') ~ E'total_len: 16\n' AS no_dead_mixed_totals_rebased; no_dead_mixed_totals_rebased t (1 row) SELECT bm25_dump_index('vacuum_mixed_no_dead_idx') NOT LIKE '%Version: 4%' AS no_dead_legacy_rebuilt; no_dead_legacy_rebuilt t (1 row) DROP TABLE vacuum_mixed_no_dead; -- Spill cannot compact corrupt legacy totals before VACUUM repairs them. CREATE TEMP TABLE vacuum_spill_legacy ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_spill_legacy_idx ON vacuum_spill_legacy USING bm25(content) WITH (text_config = 'english', compaction = 'off'); DO $$ BEGIN FOR batch IN 1..4 LOOP INSERT INTO vacuum_spill_legacy (content) SELECT 'alpha beta gamma delta' FROM generate_series(1, 10); PERFORM pg_textsearch_test_attach_legacy_segment(batch * 1000); PERFORM bm25_spill_index('vacuum_spill_legacy_idx'); PERFORM injection_points_detach('pg-textsearch-legacy-segment'); END LOOP; END $$; SELECT pg_textsearch_test_attach_vacuum_total_len(5000); pg_textsearch_test_attach_vacuum_total_len (1 row) -- ALTER's managed lifecycle rejects temporary indexes; test storage here. UPDATE pg_class SET reloptions = array_replace(reloptions, 'compaction=off', 'compaction=inline') WHERE oid = 'vacuum_spill_legacy_idx'::regclass; SET pg_textsearch.segments_per_level = 2; INSERT INTO vacuum_spill_legacy (content) VALUES ('alpha beta gamma delta'); DELETE FROM vacuum_spill_legacy WHERE id = 1; VACUUM vacuum_spill_legacy; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); injection_points_detach (1 row) SELECT bm25_summarize_index('vacuum_spill_legacy_idx') ~ E'total_docs: 40\n' AND bm25_summarize_index('vacuum_spill_legacy_idx') ~ E'total_len: 160\n' AS spill_legacy_totals_rebased; spill_legacy_totals_rebased t (1 row) SELECT bm25_dump_index('vacuum_spill_legacy_idx') NOT LIKE '%Version: 4%' AS spill_legacy_segments_rebuilt; spill_legacy_segments_rebuilt t (1 row) SELECT (bm25_level_counts( 'vacuum_spill_legacy_idx'::regclass))[2] > 0 AS deferred_inline_compaction_ran; deferred_inline_compaction_ran t (1 row) RESET pg_textsearch.segments_per_level; DROP TABLE vacuum_spill_legacy; RESET enable_seqscan; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; \set VERBOSITY default \pset format aligned timescale-pg_textsearch-b63fe58/test/expected/vector.out000066400000000000000000000116101525762047700235730ustar00rootroot00000000000000-- Test bm25vector type and operators functionality -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Cleanup any existing state first DROP TABLE IF EXISTS test_docs CASCADE; NOTICE: table "test_docs" does not exist, skipping -- Setup test table CREATE TABLE test_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO test_docs (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz'), ('hello world example'), ('postgresql full text search'); -- Create pg_textsearch index CREATE INDEX docs_vector_idx ON test_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation docs_vector_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=3.20 -- Test bm25vector I/O functions -- Test input/output with index name format SELECT 'docs_vector_idx:{hello:2,world:1}'::bm25vector; bm25vector ----------------------------------- docs_vector_idx:{hello:2,world:1} (1 row) SELECT 'docs_vector_idx:{}'::bm25vector; bm25vector -------------------- docs_vector_idx:{} (1 row) -- Test pg_textsearch scoring using text <@> bm25query operations SELECT content, ROUND((content <@> to_bm25query('hello world', 'docs_vector_idx'))::numeric, 4) as score FROM test_docs ORDER BY content <@> to_bm25query('hello world', 'docs_vector_idx') LIMIT 10; NOTICE: BM25 index scan: tid=(0,4), BM25_score=-2.8453 content | score ---------------------+--------- hello world example | -2.8453 (1 row) -- Test different query terms with multi-term query SELECT content, ROUND((content <@> to_bm25query('postgresql search', 'docs_vector_idx'))::numeric, 4) as score FROM test_docs ORDER BY content <@> to_bm25query('postgresql search', 'docs_vector_idx') LIMIT 10; NOTICE: BM25 index scan: tid=(0,5), BM25_score=-2.5153 content | score -----------------------------+--------- postgresql full text search | -2.5153 (1 row) -- Test bm25vector equality operator SELECT 'docs_vector_idx:{hello:1,world:2}'::bm25vector = 'docs_vector_idx:{hello:1,world:2}'::bm25vector; ?column? ---------- t (1 row) SELECT 'docs_vector_idx:{hello:1,world:2}'::bm25vector = 'docs_vector_idx:{world:2,hello:1}'::bm25vector; ?column? ---------- t (1 row) -- Test scoring with real documents SELECT id, content, ROUND((content <@> to_bm25query('quick fox', 'docs_vector_idx'))::numeric, 4) as relevance FROM test_docs ORDER BY content <@> to_bm25query('quick fox', 'docs_vector_idx') -- BM25 scoring for ranking LIMIT 3; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-2.8453 id | content | relevance ----+---------------------+----------- 1 | the quick brown fox | -2.8453 (1 row) -- Test with another index using simple config CREATE INDEX docs_simple_idx ON test_docs USING bm25(content) WITH (text_config='simple'); NOTICE: BM25 index build started for relation docs_simple_idx NOTICE: Using text search configuration: simple NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 5 documents, avg_length=4.00 -- Test scoring consistency: ORDER BY vs standalone scoring \echo 'Testing ORDER BY vs standalone scoring consistency' Testing ORDER BY vs standalone scoring consistency WITH order_by_scores AS ( SELECT id, content, ROUND((content <@> to_bm25query('fox dog', 'docs_simple_idx'))::numeric, 6) as order_by_score FROM test_docs ORDER BY content <@> to_bm25query('fox dog', 'docs_simple_idx') LIMIT 3 ), standalone_scores AS ( -- Force standalone scoring by using all documents and calculating scores explicitly SELECT id, content, ROUND((content <@> to_bm25query('fox dog', 'docs_simple_idx'))::numeric, 6) as standalone_score FROM test_docs -- No WHERE clause to force sequential scan and standalone scoring for all rows ) SELECT o.id, o.order_by_score, s.standalone_score, CASE WHEN abs(o.order_by_score - s.standalone_score) < 0.000001 THEN '✓ SCORES MATCH' ELSE '✗ SCORES DIFFER' END as score_consistency FROM order_by_scores o JOIN standalone_scores s ON o.id = s.id ORDER BY o.id; NOTICE: BM25 index scan: tid=(0,1), BM25_score=-1.3863 NOTICE: BM25 index scan: tid=(0,2), BM25_score=-1.2577 id | order_by_score | standalone_score | score_consistency ----+----------------+------------------+------------------- 1 | -1.386294 | -1.386294 | ✓ SCORES MATCH 2 | -1.257669 | -1.257669 | ✓ SCORES MATCH (2 rows) -- Cleanup DROP INDEX docs_vector_idx; DROP INDEX docs_simple_idx; DROP TABLE test_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/vector_v1_rejected.out000066400000000000000000000162671525762047700260630ustar00rootroot00000000000000-- Test that legacy v1 bm25vector binary input is detected and -- rejected with a clear error (rather than silently misinterpreted -- as v2 and producing garbage results or crashes). -- -- pg_textsearch < 1.2.0 used a different on-disk bm25vector layout. -- bm25vector values are almost always transient (built by -- to_bm25vector for a query, consumed during scan) — actually -- persisting them in user tables was uncommon — so 1.2.0 dropped -- v1 read support. Anyone with persisted v1 data must recompute -- via to_bm25vector(text, index_name) from the source text. -- -- This test pins the rejection behavior: the error message must -- direct users to the correct recovery path. -- -- Also covers the varint multi-byte encode/decode paths in v2 at -- byte-width boundaries (1/2/3/4-byte freq, 1/2-byte lex_len) — -- these aren't exercised by typical tokenized text where almost -- all frequencies are 1. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE compat_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO compat_docs (content) VALUES ('seed document for index'); CREATE INDEX compat_idx ON compat_docs USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation compat_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 1 documents, avg_length=3.00 -- ======================================================================== -- v1 input rejection -- ======================================================================== CREATE TABLE v1_reject (id int, v bm25vector); -- Should fail with a clear error pointing at the version mismatch. \set VERBOSITY terse \copy v1_reject FROM PROGRAM 'python3 test/scripts/gen_v1_bm25vector.py' WITH (FORMAT binary) ERROR: bm25vector value uses unsupported legacy format \set VERBOSITY default -- Confirm nothing was inserted. SELECT count(*) AS rows_after_rejection FROM v1_reject; rows_after_rejection ---------------------- 0 (1 row) -- ======================================================================== -- Embedded-NUL lexeme rejection -- -- Lexemes are rendered verbatim into a cstring by tpvector_out, so an -- embedded NUL byte would truncate the text output at that byte and -- drop every following entry. A NUL can never come from tokenized text -- or the text input parser (both operate on NUL-terminated cstrings), -- so such a value is only reachable via crafted binary input. It must -- be rejected at the boundary, not silently accepted. -- ======================================================================== CREATE TABLE nul_reject (id int, v bm25vector); -- Should fail: entry 0 lexeme contains an embedded NUL byte. \set VERBOSITY terse \copy nul_reject FROM PROGRAM 'python3 test/scripts/gen_nul_lexeme_bm25vector.py' WITH (FORMAT binary) ERROR: v2 bm25vector entry 0 lexeme contains an embedded NUL byte \set VERBOSITY default -- Confirm nothing was inserted. SELECT count(*) AS rows_after_nul_rejection FROM nul_reject; rows_after_nul_rejection -------------------------- 0 (1 row) -- ======================================================================== -- Varint encoding boundary coverage -- -- The v2 per-entry header uses LEB128 varint for both `frequency` -- and `lexeme_len`. The 1-byte path covers values 0..127; values -- ≥ 128 hit the multi-byte encode/decode path. Almost all -- naturally-occurring frequencies are 1 (POSDATALEN of a tsvector -- word), so without explicit fixtures the multi-byte varint path -- isn't exercised. These pin both the encode side (tpvector_in → -- create_tpvector_from_strings → tpvector_varint_encode) and the -- decode side (tpvector_out → tpvector_entry_decode → -- tpvector_varint_decode) at every byte-width transition. -- ======================================================================== -- Single-entry boundaries SELECT 'compat_idx:{lex:0}'::bm25vector::text AS freq_0; -- 1 byte freq_0 -------------------- compat_idx:{lex:0} (1 row) SELECT 'compat_idx:{lex:1}'::bm25vector::text AS freq_1; -- 1 byte freq_1 -------------------- compat_idx:{lex:1} (1 row) SELECT 'compat_idx:{lex:127}'::bm25vector::text AS freq_127; -- 1 byte (max) freq_127 ---------------------- compat_idx:{lex:127} (1 row) SELECT 'compat_idx:{lex:128}'::bm25vector::text AS freq_128; -- 2 bytes (min) freq_128 ---------------------- compat_idx:{lex:128} (1 row) SELECT 'compat_idx:{lex:16383}'::bm25vector::text AS freq_16383; -- 2 bytes (max) freq_16383 ------------------------ compat_idx:{lex:16383} (1 row) SELECT 'compat_idx:{lex:16384}'::bm25vector::text AS freq_16384; -- 3 bytes (min) freq_16384 ------------------------ compat_idx:{lex:16384} (1 row) SELECT 'compat_idx:{lex:2097151}'::bm25vector::text AS freq_2097151; -- 3 bytes (max) freq_2097151 -------------------------- compat_idx:{lex:2097151} (1 row) SELECT 'compat_idx:{lex:2097152}'::bm25vector::text AS freq_2097152; -- 4 bytes (min) freq_2097152 -------------------------- compat_idx:{lex:2097152} (1 row) -- Multi-entry walk across mixed-width frequencies. Verifies -- get_tpvector_next_entry correctly advances past 1-, 2-, and -- 3-byte freq encodings. SELECT 'compat_idx:{a:1,b:127,c:128,d:16383,e:16384}'::bm25vector::text AS mixed_freq_widths; mixed_freq_widths ---------------------------------------------- compat_idx:{a:1,b:127,c:128,d:16383,e:16384} (1 row) -- Equality through varint boundaries. WITH v AS ( SELECT 'compat_idx:{x:50000,y:1}'::bm25vector AS orig ) SELECT orig::text AS rendered, orig = orig::text::bm25vector AS round_trip_eq FROM v; rendered | round_trip_eq --------------------------+--------------- compat_idx:{x:50000,y:1} | t (1 row) -- Long-lexeme path (multi-byte lex_len varint). WITH s AS ( SELECT 'compat_idx:{' || repeat('a', 200) || ':1}' AS literal ) SELECT length(literal) AS literal_bytes, literal::bm25vector::text = literal AS roundtrip_200char_lex FROM s; literal_bytes | roundtrip_200char_lex ---------------+----------------------- 215 | t (1 row) -- ======================================================================== -- Large entry counts -- -- entry_count is caller-controlled and bounded only by the varlena -- size, so tpvector_out must not consume stack per entry: a -- per-entry alloca() is released when the function returns, not at -- end of block, and grew the frame until the backend took SIGSEGV. -- Lexemes are zero-padded so input order matches the canonical -- (byte-sorted) entry order. -- ======================================================================== SELECT ('compat_idx:{' || s || '}')::bm25vector::text = 'compat_idx:{' || s || '}' AS roundtrip_1m_entries FROM (SELECT string_agg('l' || lpad(g::text, 7, '0') || ':1', ',' ORDER BY g) FROM generate_series(1, 1000000) g) t(s); roundtrip_1m_entries ---------------------- t (1 row) -- Cleanup DROP TABLE v1_reject; DROP TABLE nul_reject; DROP TABLE compat_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/expected/wand.out000066400000000000000000000140121525762047700232210ustar00rootroot00000000000000-- BMW Multi-term WAND Traversal Test -- -- Regression test for doc-ID ordered traversal in multi-term queries. -- Ensures documents at different block positions across posting lists -- are correctly scored. Uses validation.sql to verify scores match -- the reference BM25 implementation. CREATE EXTENSION pg_textsearch; \set ECHO none -- ============================================================ -- TEST: Multi-term documents score correctly across block boundaries -- ============================================================ CREATE TABLE bmw_bug (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX bmw_bug_idx ON bmw_bug USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation bmw_bug_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Insert in order that puts multi-term doc at different block positions: -- 5 alpha-only docs, then 1 multi-term, then 200 beta-only docs INSERT INTO bmw_bug (content) SELECT 'alpha only ' || i FROM generate_series(1, 5) i; INSERT INTO bmw_bug (content) VALUES ('alpha beta both terms here'); -- id=6 INSERT INTO bmw_bug (content) SELECT 'beta only ' || i FROM generate_series(7, 206) i; -- Spill to segment (creates single segment with all 206 docs) SELECT bm25_spill_index('bmw_bug_idx'); bm25_spill_index ------------------ 4 (1 row) -- Posting list layout in segment: -- "alpha": 6 postings (docs 1-5, 6) - fits in single block -- "beta": 201 postings (doc 6, docs 7-206) -- - Block 0: docs 6-133 -- - Block 1: docs 134-206 -- -- Doc 6 is the only multi-term document. -- Validate BMW produces correct scores for multi-term query SELECT validate_bm25_scoring('bmw_bug', 'content', 'bmw_bug_idx', 'alpha beta', 'english', 1.2, 0.75) as two_term_valid; two_term_valid ---------------- t (1 row) DROP TABLE bmw_bug; -- ============================================================ -- TEST: 3-term query with multiple blocks (buffer management) -- ============================================================ -- This tests a bug where zero-copy buffer pins were shared with the -- reader's cached buffer. When initializing multiple term iterators, -- reading the dictionary for term N could release the buffer that -- term N-1's iterator was still using. CREATE TABLE three_term (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX three_term_idx ON three_term USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation three_term_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Create layout to test multi-term scoring INSERT INTO three_term (content) SELECT 'alpha only ' || i FROM generate_series(1, 3) i; INSERT INTO three_term (content) VALUES ('alpha beta two terms'); -- id=4 INSERT INTO three_term (content) SELECT 'beta only ' || i FROM generate_series(5, 7) i; INSERT INTO three_term (content) VALUES ('alpha beta gamma three terms here'); -- id=8 INSERT INTO three_term (content) SELECT 'gamma only ' || i FROM generate_series(9, 11) i; SELECT bm25_spill_index('three_term_idx'); bm25_spill_index ------------------ 2 (1 row) -- Validate BMW produces correct scores for 3-term query SELECT validate_bm25_scoring('three_term', 'content', 'three_term_idx', 'alpha beta gamma', 'english', 1.2, 0.75) as three_term_valid; three_term_valid ------------------ t (1 row) DROP TABLE three_term; -- ============================================================ -- TEST: 8+ term queries with WAND pivot selection -- ============================================================ -- Validates that WAND pivot selection produces correct results -- for many-term queries across multiple blocks. CREATE TABLE wand_many_terms (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX wand_many_terms_idx ON wand_many_terms USING bm25(content) WITH (text_config='english'); NOTICE: BM25 index build started for relation wand_many_terms_idx NOTICE: Using text search configuration: english NOTICE: Using index options: k1=1.20, b=0.75 NOTICE: BM25 index build completed: 0 documents, avg_length=0.00 -- Create 500 docs with varying term coverage INSERT INTO wand_many_terms (content) SELECT CASE WHEN i % 100 = 0 THEN 'alpha beta gamma delta epsilon zeta eta theta' WHEN i % 50 = 0 THEN 'alpha beta gamma delta epsilon zeta' WHEN i % 10 = 0 THEN 'alpha beta gamma delta' WHEN i % 5 = 0 THEN 'alpha beta gamma' WHEN i % 2 = 0 THEN 'alpha beta' ELSE 'alpha' END || ' document ' || i FROM generate_series(1, 500) i; SELECT bm25_spill_index('wand_many_terms_idx'); bm25_spill_index ------------------ 8 (1 row) -- 8-term query: WAND vs exhaustive (LIMIT 10 avoids tie boundary) WITH bmw AS ( SELECT id, content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query as score FROM wand_many_terms ORDER BY content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query as score FROM wand_many_terms ORDER BY content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query ) x LIMIT 10 ) SELECT 'wand-8-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; test | result -------------+-------- wand-8-term | PASS (1 row) -- Validate scores match reference BM25 computation SELECT validate_bm25_scoring('wand_many_terms', 'content', 'wand_many_terms_idx', 'alpha beta gamma delta epsilon zeta eta theta', 'english', 1.2, 0.75) as eight_term_valid; eight_term_valid ------------------ t (1 row) DROP TABLE wand_many_terms; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/lsan.supp000066400000000000000000000021011525762047700216000ustar00rootroot00000000000000# LeakSanitizer suppressions for pg_textsearch tests # # These suppress known PostgreSQL internal allocations that are # intentionally long-lived and not actual memory leaks. # PostgreSQL memory contexts - allocations live for process lifetime leak:AllocSetContextCreate leak:MemoryContextAlloc leak:palloc leak:repalloc # PostgreSQL frontend utilities (pg_config, etc.) allocate memory and # exit without freeing - normal for short-lived command-line tools leak:pg_strdup leak:get_configdata # OpenSSL one-time initialization leak:libssl leak:libcrypto # PostgreSQL GUC and catalog caches leak:GUC leak:CatalogCache # PostgreSQL process title (argv) saving - intentionally never freed leak:save_ps_display_args # PostgreSQL initdb - short-lived tool doesn't free memory before exit leak:escape_single_quotes_ascii leak:escape_quotes_bki leak:pretty_wal_size leak:pg_malloc_internal leak:setup_config leak:bootstrap_template1 # System library allocations in short-lived tools leak:getaddrinfo # PostgreSQL server startup allocations - live for process lifetime leak:PostmasterMain timescale-pg_textsearch-b63fe58/test/modules/000077500000000000000000000000001525762047700214105ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/test/modules/pg_textsearch_test/000077500000000000000000000000001525762047700253075ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/test/modules/pg_textsearch_test/Makefile000066400000000000000000000003111525762047700267420ustar00rootroot00000000000000MODULES = pg_textsearch_test EXTENSION = pg_textsearch_test DATA = pg_textsearch_test--1.0.sql PG_CPPFLAGS = -I../../../src PG_CONFIG ?= pg_config PGXS := $(shell $(PG_CONFIG) --pgxs) include $(PGXS) timescale-pg_textsearch-b63fe58/test/modules/pg_textsearch_test/pg_textsearch_test--1.0.sql000066400000000000000000000037651525762047700323130ustar00rootroot00000000000000\echo Use "CREATE EXTENSION pg_textsearch_test" to load this file. \quit CREATE FUNCTION pg_textsearch_test_move_level( idx regclass, source_level integer, destination_level integer) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_move_level' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_set_tombstone_link( idx regclass, at_head boolean, block bigint) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_set_tombstone_link' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_panic(point text DEFAULT NULL) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_panic' LANGUAGE C PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_worker_panic( point text DEFAULT NULL) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_worker_panic' LANGUAGE C PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_segment_limit(limit_value integer) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_segment_limit' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_reclaim_horizon_hold() RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_reclaim_horizon_hold' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_query_hint_roundtrip(query text, k bigint) RETURNS boolean AS 'MODULE_PATHNAME', 'pg_textsearch_test_query_hint_roundtrip' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_legacy_segment(total_tokens bigint) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_legacy_segment' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_v5_segment_total_len( total_tokens bigint) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_v5_segment_total_len' LANGUAGE C STRICT PARALLEL UNSAFE; CREATE FUNCTION pg_textsearch_test_attach_vacuum_total_len( total_tokens bigint) RETURNS void AS 'MODULE_PATHNAME', 'pg_textsearch_test_attach_vacuum_total_len' LANGUAGE C STRICT PARALLEL UNSAFE; timescale-pg_textsearch-b63fe58/test/modules/pg_textsearch_test/pg_textsearch_test.c000066400000000000000000000252461525762047700313630ustar00rootroot00000000000000#include #include #include #include #include #include #include #include #include #include #include #include #include #include "debug/injection.h" #include "index/metapage.h" #include "segment/format.h" #include "segment/tombstone.h" #include "types/query.h" PG_MODULE_MAGIC; PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_panic); PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_worker_panic); PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_segment_limit); PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_reclaim_horizon_hold); PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_legacy_segment); PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_v5_segment_total_len); PG_FUNCTION_INFO_V1(pg_textsearch_test_attach_vacuum_total_len); PG_FUNCTION_INFO_V1(pg_textsearch_test_set_tombstone_link); PG_FUNCTION_INFO_V1(pg_textsearch_test_move_level); Datum pg_textsearch_test_move_level(PG_FUNCTION_ARGS) { Relation index; int from = PG_GETARG_INT32(1); int to = PG_GETARG_INT32(2); Buffer buffers[MAX_GENERIC_XLOG_PAGES]; uint32 count; BlockNumber root; TpIndexMetaPage meta; GenericXLogState *state; Page copy; if (!superuser()) elog(ERROR, "must be superuser to move test segments"); if (from < 0 || from >= TP_MAX_LEVELS || to < 0 || to >= TP_MAX_LEVELS || from == to) elog(ERROR, "invalid test levels"); index = index_open(PG_GETARG_OID(0), AccessExclusiveLock); buffers[0] = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(buffers[0], BUFFER_LOCK_EXCLUSIVE); meta = (TpIndexMetaPage)PageGetContents(BufferGetPage(buffers[0])); if (meta->magic != TP_METAPAGE_MAGIC || meta->version != TP_METAPAGE_VERSION) elog(ERROR, "expected a current BM25 metapage"); count = meta->level_counts[from]; if (count == 0 || count >= MAX_GENERIC_XLOG_PAGES || meta->level_counts[to] != 0) elog(ERROR, "expected a short source chain and empty destination"); root = meta->level_heads[from]; for (uint32 i = 1; i <= count; i++) { TpSegmentHeader *header; buffers[i] = ReadBuffer(index, root); LockBuffer(buffers[i], BUFFER_LOCK_EXCLUSIVE); header = (TpSegmentHeader *)PageGetContents(BufferGetPage(buffers[i])); if (header->magic != TP_SEGMENT_MAGIC || header->version != TP_SEGMENT_FORMAT_VERSION || header->level != (uint32)from) elog(ERROR, "expected a current segment on the source level"); root = header->next_segment; } if (BlockNumberIsValid(root)) elog(ERROR, "unexpected test segment chain tail"); state = GenericXLogStart(index); copy = GenericXLogRegisterBuffer(state, buffers[0], 0); meta = (TpIndexMetaPage)PageGetContents(copy); meta->level_heads[to] = meta->level_heads[from]; meta->level_counts[to] = meta->level_counts[from]; meta->level_heads[from] = InvalidBlockNumber; meta->level_counts[from] = 0; for (uint32 i = 1; i <= count; i++) { copy = GenericXLogRegisterBuffer(state, buffers[i], 0); ((TpSegmentHeader *)PageGetContents(copy))->level = (uint32)to; } GenericXLogFinish(state); for (uint32 i = 0; i <= count; i++) UnlockReleaseBuffer(buffers[i]); index_close(index, AccessExclusiveLock); PG_RETURN_VOID(); } Datum pg_textsearch_test_set_tombstone_link(PG_FUNCTION_ARGS) { Oid index_oid = PG_GETARG_OID(0); bool at_head = PG_GETARG_BOOL(1); int64 block = PG_GETARG_INT64(2); Relation index; Buffer buf; Page page; TpIndexMetaPage metap; BlockNumber head; GenericXLogState *state; if (!superuser()) elog(ERROR, "must be superuser to corrupt a tombstone link"); if (block < 0 || block > MaxBlockNumber) elog(ERROR, "invalid test block number"); index = index_open(index_oid, AccessExclusiveLock); buf = ReadBuffer(index, TP_METAPAGE_BLKNO); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); metap = (TpIndexMetaPage)PageGetContents(BufferGetPage(buf)); if (metap->magic != TP_METAPAGE_MAGIC || metap->version != TP_METAPAGE_VERSION) elog(ERROR, "expected a current BM25 metapage"); head = metap->pending_free_head; if (head == InvalidBlockNumber) elog(ERROR, "expected a nonempty tombstone chain"); if (!at_head) { TpTombstonePage t; UnlockReleaseBuffer(buf); buf = ReadBuffer(index, head); LockBuffer(buf, BUFFER_LOCK_EXCLUSIVE); t = tp_tombstone_page(BufferGetPage(buf)); if (t->magic != TP_TOMBSTONE_MAGIC || t->next_page != InvalidBlockNumber) elog(ERROR, "expected a single tombstone page"); } state = GenericXLogStart(index); page = GenericXLogRegisterBuffer(state, buf, GENERIC_XLOG_FULL_IMAGE); if (at_head) ((TpIndexMetaPage)PageGetContents(page))->pending_free_head = (BlockNumber)block; else tp_tombstone_page(page)->next_page = (BlockNumber)block; GenericXLogFinish(state); UnlockReleaseBuffer(buf); index_close(index, AccessExclusiveLock); PG_RETURN_VOID(); } Datum pg_textsearch_test_attach_panic(PG_FUNCTION_ARGS) { TpInjectionCondition condition = {.pid = MyProcPid}; const char *point = TP_INJECTION_AFTER_SPILL_FINALIZE; if (!PG_ARGISNULL(0)) point = text_to_cstring(PG_GETARG_TEXT_PP(0)); InjectionPointAttach( point, "pg_textsearch", "tp_injection_panic", &condition, sizeof(condition)); PG_RETURN_VOID(); } Datum pg_textsearch_test_attach_worker_panic(PG_FUNCTION_ARGS) { TpInjectionCondition condition = {.pid = MyProcPid, .worker_only = true}; const char *point = TP_INJECTION_AFTER_SPILL_FINALIZE; if (!PG_ARGISNULL(0)) point = text_to_cstring(PG_GETARG_TEXT_PP(0)); InjectionPointAttach( point, "pg_textsearch", "tp_injection_panic", &condition, sizeof(condition)); PG_RETURN_VOID(); } Datum pg_textsearch_test_attach_segment_limit(PG_FUNCTION_ARGS) { TpInjectionSegmentCountLimit condition = {.pid = MyProcPid, .limit = PG_GETARG_INT32(0)}; if (condition.limit < 1 || condition.limit > PG_UINT16_MAX) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("segment count limit must be between 1 and %u", PG_UINT16_MAX))); InjectionPointAttach( TP_INJECTION_SEGMENT_COUNT_LIMIT, "pg_textsearch", "tp_injection_segment_count_limit", &condition, sizeof(condition)); PG_RETURN_VOID(); } Datum pg_textsearch_test_attach_reclaim_horizon_hold(PG_FUNCTION_ARGS) { TpInjectionCondition condition = {.pid = MyProcPid}; InjectionPointAttach( TP_INJECTION_RECLAIM_HORIZON, "pg_textsearch", "tp_injection_reclaim_horizon_hold", &condition, sizeof(condition)); PG_RETURN_VOID(); } /* Return the hinted datum from the direct Limit -> IndexScan under test. */ static TpQuery * test_plan_query(PlannedStmt *stmt) { IndexScan *scan; OpExpr *op; Const *value; if (!IsA(stmt->planTree, Limit) || !IsA(stmt->planTree->lefttree, IndexScan)) elog(ERROR, "expected Limit over IndexScan"); scan = (IndexScan *)stmt->planTree->lefttree; if (list_length(scan->indexorderby) != 1) elog(ERROR, "expected one index ordering expression"); op = castNode(OpExpr, linitial(scan->indexorderby)); value = castNode(Const, lsecond(op->args)); if (value->constisnull || value->constbyval || value->constlen != -1) elog(ERROR, "expected a variable-length query constant"); return (TpQuery *)DatumGetPointer(value->constvalue); } PG_FUNCTION_INFO_V1(pg_textsearch_test_query_hint_roundtrip); Datum pg_textsearch_test_query_hint_roundtrip(PG_FUNCTION_ARGS) { char *sql = text_to_cstring(PG_GETARG_TEXT_PP(0)); int64 expected_k = PG_GETARG_INT64(1); SPIPlanPtr prepared; CachedPlan *cached; PlannedStmt *stmt; TpQuery *query; TpQuery *copied; TpQuery *restored; TpQuerySeedHint hint; TpQuerySeedHint canonical; Size base; bool valid; SPI_connect(); prepared = SPI_prepare(sql, 0, NULL); if (prepared == NULL) elog(ERROR, "could not prepare hint test query"); cached = SPI_plan_get_cached_plan(prepared); if (cached == NULL || list_length(cached->stmt_list) != 1) elog(ERROR, "expected one cached statement"); stmt = linitial_node(PlannedStmt, cached->stmt_list); query = test_plan_query(stmt); base = offsetof(TpQuery, data) + query->query_text_len + 1; if (!(query->flags & TPQUERY_FLAG_SEED_HINT) || VARSIZE(query) != base + sizeof(hint)) elog(ERROR, "expected a complete seed hint"); memcpy(&hint, (char *)query + base, sizeof(hint)); memset(&canonical, 0, sizeof(canonical)); canonical.magic = TPQUERY_SEED_HINT_MAGIC; canonical.k = expected_k; canonical.selectivity = hint.selectivity; copied = test_plan_query(copyObject(stmt)); restored = test_plan_query(stringToNode(nodeToString(stmt))); valid = hint.selectivity > 0 && hint.selectivity < 1 && memcmp(&hint, &canonical, sizeof(hint)) == 0 && VARSIZE(query) == VARSIZE(copied) && VARSIZE(query) == VARSIZE(restored) && memcmp(query, copied, VARSIZE(query)) == 0 && memcmp(query, restored, VARSIZE(query)) == 0; /* SPI_prepare produces an unsaved plan: no ResourceOwner reference. */ ReleaseCachedPlan(cached, NULL); SPI_freeplan(prepared); SPI_finish(); PG_RETURN_BOOL(valid); } Datum pg_textsearch_test_attach_legacy_segment(PG_FUNCTION_ARGS) { int64 total_tokens = PG_GETARG_INT64(0); TpInjectionTokenTotal condition; if (total_tokens < 0) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("legacy segment token total must be nonnegative"))); condition.pid = MyProcPid; condition.total_tokens = (uint64)total_tokens; InjectionPointAttach( TP_INJECTION_LEGACY_SEGMENT, "pg_textsearch", "tp_injection_legacy_segment", &condition, sizeof(condition)); PG_RETURN_VOID(); } Datum pg_textsearch_test_attach_v5_segment_total_len(PG_FUNCTION_ARGS) { int64 total_tokens = PG_GETARG_INT64(0); TpInjectionTokenTotal condition; if (total_tokens < 0) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("V5 segment token total must be nonnegative"))); condition.pid = MyProcPid; condition.total_tokens = (uint64)total_tokens; InjectionPointAttach( TP_INJECTION_V5_SEGMENT_TOTAL_LEN, "pg_textsearch", "tp_injection_v5_segment_total_len", &condition, sizeof(condition)); PG_RETURN_VOID(); } Datum pg_textsearch_test_attach_vacuum_total_len(PG_FUNCTION_ARGS) { int64 total_tokens = PG_GETARG_INT64(0); TpInjectionTokenTotal condition; if (total_tokens < 0) ereport(ERROR, (errcode(ERRCODE_NUMERIC_VALUE_OUT_OF_RANGE), errmsg("VACUUM token total must be nonnegative"))); condition.pid = MyProcPid; condition.total_tokens = (uint64)total_tokens; InjectionPointAttach( TP_INJECTION_VACUUM_TOTAL_LEN, "pg_textsearch", "tp_injection_vacuum_total_len", &condition, sizeof(condition)); PG_RETURN_VOID(); } timescale-pg_textsearch-b63fe58/test/modules/pg_textsearch_test/pg_textsearch_test.control000066400000000000000000000002261525762047700326100ustar00rootroot00000000000000comment = 'Test support for pg_textsearch injection points' default_version = '1.0' module_pathname = '$libdir/pg_textsearch_test' relocatable = true timescale-pg_textsearch-b63fe58/test/postgres-asan-instrumentation-PG18GE.patch000066400000000000000000000034461525762047700277100ustar00rootroot00000000000000diff --git a/src/backend/utils/misc/stack_depth.c b/src/backend/utils/misc/stack_depth.c index 8f7cf531fbc..2f7b1cebbe6 100644 --- a/src/backend/utils/misc/stack_depth.c +++ b/src/backend/utils/misc/stack_depth.c @@ -108,6 +108,12 @@ check_stack_depth(void) bool stack_is_too_deep(void) { + /* + * Pointer arithmetics to determine stack depth doesn't work under + * AddressSanitizer. + */ + return false; + char stack_top_loc; ssize_t stack_depth; diff --git a/src/include/utils/memdebug.h b/src/include/utils/memdebug.h index e88b4c6e8e..4ccbbf0146 100644 --- a/src/include/utils/memdebug.h +++ b/src/include/utils/memdebug.h @@ -19,6 +19,31 @@ #ifdef USE_VALGRIND #include + +#elif __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) + +#include + +#define VALGRIND_MAKE_MEM_DEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_NOACCESS(addr, size) \ + ASAN_POISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_UNDEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_ALLOC(context, addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_FREE(context, addr) \ + ASAN_POISON_MEMORY_REGION(addr, 1 /* Length unknown, poison first byte. */) + +#define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) +#define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) +#define VALGRIND_DESTROY_MEMPOOL(context) do {} while (0) +#define VALGRIND_MEMPOOL_CHANGE(context, optr, nptr, size) do {} while (0) + #else #define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) #define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) timescale-pg_textsearch-b63fe58/test/postgres-asan-instrumentation.patch000066400000000000000000000033051525762047700270110ustar00rootroot00000000000000diff --git a/src/backend/tcop/postgres.c b/src/backend/tcop/postgres.c index 2c50575b37..11b6c688c7 100644 --- a/src/backend/tcop/postgres.c +++ b/src/backend/tcop/postgres.c @@ -3492,6 +4476,12 @@ check_stack_depth(void) bool stack_is_too_deep(void) { + /* + * Pointer arithmetics to determine stack depth doesn't work under + * AddressSanitizer. + */ + return false; + char stack_top_loc; long stack_depth; diff --git a/src/include/utils/memdebug.h b/src/include/utils/memdebug.h index e88b4c6e8e..4ccbbf0146 100644 --- a/src/include/utils/memdebug.h +++ b/src/include/utils/memdebug.h @@ -19,6 +19,31 @@ #ifdef USE_VALGRIND #include + +#elif __has_feature(address_sanitizer) || defined(__SANITIZE_ADDRESS__) + +#include + +#define VALGRIND_MAKE_MEM_DEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_NOACCESS(addr, size) \ + ASAN_POISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MAKE_MEM_UNDEFINED(addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_ALLOC(context, addr, size) \ + ASAN_UNPOISON_MEMORY_REGION(addr, size) + +#define VALGRIND_MEMPOOL_FREE(context, addr) \ + ASAN_POISON_MEMORY_REGION(addr, 1 /* Length unknown, poison first byte. */) + +#define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) +#define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) +#define VALGRIND_DESTROY_MEMPOOL(context) do {} while (0) +#define VALGRIND_MEMPOOL_CHANGE(context, optr, nptr, size) do {} while (0) + #else #define VALGRIND_CHECK_MEM_IS_DEFINED(addr, size) do {} while (0) #define VALGRIND_CREATE_MEMPOOL(context, redzones, zeroed) do {} while (0) timescale-pg_textsearch-b63fe58/test/scripts/000077500000000000000000000000001525762047700214275ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/test/scripts/boolean_concurrent_merge.sh000077500000000000000000000116141525762047700270310ustar00rootroot00000000000000#!/bin/bash # # Exercise Boolean segment snapshots while spill, compaction, and VACUUM # replace and reclaim the captured segment roots. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55460 TEST_DB=boolean_concurrent_merge_test DATA_DIR="${SCRIPT_DIR}/../tmp_boolean_concurrent_merge" LOGFILE="${DATA_DIR}/postgres.log" ERR_DIR="${DATA_DIR}/client_logs" cleanup() { local exit_code=$? jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi if [ "$exit_code" -ne 0 ]; then tail -n 40 "${ERR_DIR}"/*.log "${LOGFILE}" 2>/dev/null || true fi rm -rf "${DATA_DIR}" exit "$exit_code" } trap cleanup EXIT INT TERM rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null mkdir -p "${ERR_DIR}" cat >>"${DATA_DIR}/postgresql.conf" </dev/null createdb -h "${DATA_DIR}" -p "${TEST_PORT}" "${TEST_DB}" PSQL=( psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 ) "${PSQL[@]}" <<'SQL' >/dev/null CREATE EXTENSION pg_textsearch; CREATE TABLE docs (id bigserial PRIMARY KEY, body text NOT NULL); CREATE INDEX docs_bm25 ON docs USING bm25(body) WITH (text_config='simple'); SET pg_textsearch.segments_per_level = 64; DO $$ BEGIN FOR batch IN 1..16 LOOP INSERT INTO docs (body) SELECT 'common token ' || batch || ' ' || gs FROM generate_series(1, 500) gs; PERFORM bm25_spill_index('docs_bm25'); END LOOP; END $$; SQL plan="$("${PSQL[@]}" -c " SET enable_seqscan=off; EXPLAIN (COSTS off) SELECT count(*) FROM docs WHERE body @@ to_tsquery('simple', '!missing')")" if ! grep -q "Index Scan using docs_bm25" <<<"${plan}"; then echo "Boolean concurrency test did not select docs_bm25" >&2 echo "${plan}" >&2 exit 1 fi reader() { local tag=$1 local result for _ in $(seq 1 40); do result="$("${PSQL[@]}" -c " SET enable_seqscan=off; SET statement_timeout='60s'; SELECT (SELECT count(*) FROM docs) = (SELECT count(*) FROM docs WHERE body @@ to_tsquery('simple', '!missing'))" \ 2>>"${ERR_DIR}/reader_${tag}.log")" if [ "${result}" != "t" ]; then echo "Boolean result mismatch: ${result}" \ >>"${ERR_DIR}/reader_${tag}.log" return 40 fi done } writer() { for batch in $(seq 1 40); do "${PSQL[@]}" -c " SET pg_textsearch.segments_per_level=64; INSERT INTO docs (body) SELECT 'common writer ${batch} ' || gs FROM generate_series(1, 100) gs" \ >>"${ERR_DIR}/writer.log" 2>&1 sleep 0.01 "${PSQL[@]}" -c " SET pg_textsearch.segments_per_level=64; SELECT bm25_spill_index('docs_bm25')" \ >>"${ERR_DIR}/writer.log" 2>&1 done } merger() { for _ in $(seq 1 50); do "${PSQL[@]}" -c " SET pg_textsearch.segments_per_level=2; SET statement_timeout='60s'; DO \$merge\$ BEGIN PERFORM bm25_force_merge('docs_bm25'); EXCEPTION WHEN lock_not_available THEN NULL; END \$merge\$; SELECT bm25_pending_free_pages('docs_bm25')" \ >>"${ERR_DIR}/merger.log" 2>&1 "${PSQL[@]}" -c "VACUUM docs" >>"${ERR_DIR}/merger.log" 2>&1 done } reader a & reader_a=$! reader b & reader_b=$! writer & writer_pid=$! merger & merger_pid=$! failed=0 wait "${reader_a}" || failed=1 wait "${reader_b}" || failed=1 wait "${writer_pid}" || failed=1 wait "${merger_pid}" || failed=1 if grep -REq \ "invalid segment header|could not open BM25 segment|memtable chain ended|memtable tail page|terminated by signal" \ "${ERR_DIR}" "${LOGFILE}"; then echo "Boolean scan observed a reclaimed or invalid segment" >&2 exit 1 fi if [ "${failed}" -ne 0 ]; then echo "A Boolean concurrency worker failed" >&2 exit 1 fi if ! grep -Eq '^[1-9][0-9]*$' "${ERR_DIR}/merger.log"; then echo "Concurrency test did not observe displaced segment pages" >&2 exit 1 fi expected="$("${PSQL[@]}" -c "SELECT count(*) FROM docs")" actual="$("${PSQL[@]}" -c " SET enable_seqscan=off; SELECT count(*) FROM docs WHERE body @@ to_tsquery('simple', '!missing')")" if [ "${actual}" != "${expected}" ]; then echo "Boolean result mismatch: expected ${expected}, got ${actual}" >&2 exit 1 fi echo "Boolean scans survived concurrent spill, merge, and VACUUM" timescale-pg_textsearch-b63fe58/test/scripts/build_memory_benchmark_test.py000066400000000000000000000212261525762047700275440ustar00rootroot00000000000000#!/usr/bin/env python3 """Tests for build-memory measurement and failure reporting.""" import importlib.util import json import os from pathlib import Path import subprocess import sys import tempfile import unittest from unittest.mock import patch ROOT = Path(__file__).resolve().parents[2] RUNNER = ROOT / "benchmarks/run_build_memory.py" benchmark = None if RUNNER.exists(): spec = importlib.util.spec_from_file_location("build_memory", RUNNER) benchmark = importlib.util.module_from_spec(spec) spec.loader.exec_module(benchmark) class BuildMemoryTest(unittest.TestCase): def setUp(self): self.assertIsNotNone(benchmark, "build-memory runner not implemented") def test_process_memory_uses_pss_not_shared_rss(self): with tempfile.TemporaryDirectory() as directory: process = Path(directory) / "123" process.mkdir() (process / "smaps_rollup").write_text( "00400000-00500000 ---p 00000000 00:00 0 [rollup]\n" "Rss: 100 kB\nPss: 60 kB\n" "Private_Clean: 4 kB\nPrivate_Dirty: 16 kB\n" "Shared_Clean: 80 kB\n" ) self.assertEqual( benchmark.read_memory(123, Path(directory)), {"rss": 102400, "pss": 61440, "private": 20480}, ) def test_exited_process_is_not_reported_as_zero_memory(self): with tempfile.TemporaryDirectory() as directory: self.assertIsNone( benchmark.read_memory(123, Path(directory)) ) def test_malformed_memory_sample_is_an_error(self): with tempfile.TemporaryDirectory() as directory: process = Path(directory) / "123" process.mkdir() (process / "smaps_rollup").write_text("Rss: 100 kB\n") with self.assertRaises(ValueError): benchmark.read_memory(123, Path(directory)) def test_peak_is_maximum_concurrent_sum_not_sum_of_peaks(self): samples = [ {"phase": "loading", "leader_private": 10, "worker_private": 90, "build_pss": 100}, {"phase": "writing", "leader_private": 80, "worker_private": 5, "build_pss": 85}, {"phase": "writing", "leader_private": 0, "worker_private": 0, "build_pss": None}, ] summary = benchmark.summarize_samples(samples) self.assertEqual(summary["peak_build_pss_bytes"], 100) self.assertEqual( summary["phases"]["writing"]["peak_build_pss_bytes"], 85 ) self.assertEqual(summary["incomplete_samples"], 1) def test_empty_measurements_are_not_published_as_zero(self): summary = benchmark.summarize_samples([]) self.assertIsNone(summary["peak_build_pss_bytes"]) self.assertEqual(summary["samples"], 0) def test_cgroup_snapshot_preserves_limits_and_oom_events(self): with tempfile.TemporaryDirectory() as directory: path = Path(directory) for name, value in { "memory.current": "4096", "memory.max": "max", "memory.stat": "anon 1024\nfile 2048\nshmem 512\n", "memory.events": "low 0\nhigh 0\nmax 3\noom 1\n" "oom_kill 1\noom_group_kill 0\n", }.items(): (path / name).write_text(value) result = benchmark.read_cgroup(path) self.assertEqual(result["current"], 4096) self.assertEqual(result["max"], "max") self.assertEqual(result["anon"], 1024) self.assertEqual(result["events"]["oom_kill"], 1) def test_invalid_options_fail_before_creating_output(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "results" for arguments in ( ["--rows", "0"], ["--workers", "-1"], ["--interval", "nan"], ["--rows", "100", "--workers", "2"], ["--rows", "100", "--case", "serial", "--case", "unique"], ["--terms", "0"], ["--maintenance-work-mem", "64MB'; SELECT 1; --"], ): with self.subTest(arguments=arguments): result = subprocess.run( [sys.executable, str(RUNNER), "--output", str(output), *arguments], capture_output=True, text=True, ) self.assertNotEqual(result.returncode, 0) self.assertFalse(output.exists()) def test_small_serial_case_does_not_require_parallel_rows(self): with patch.object(sys, "argv", [ str(RUNNER), "--case", "serial", "--rows", "100", ]): args = benchmark.parse_args() self.assertEqual(args.cases, ["serial"]) self.assertEqual(args.rows, 100) self.assertEqual(args.workers, 2) @unittest.skipUnless(sys.platform == "linux", "Linux runner") def test_missing_postgres_tools_record_failure(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "results" result = subprocess.run( [sys.executable, str(RUNNER), "--output", str(output), "--pg-config", str(Path(directory) / "missing")], capture_output=True, text=True, ) self.assertNotEqual(result.returncode, 0) summary = json.loads((output / "summary.json").read_text()) self.assertEqual(summary["status"], "error") self.assertIn("missing", summary["error"]) self.assertFalse((output / "data").exists()) @unittest.skipUnless(os.environ.get("BUILD_MEMORY_PG_CONFIG"), "set BUILD_MEMORY_PG_CONFIG for cluster tests") class ClusterIntegrationTest(unittest.TestCase): def test_default_parallel_degree_is_not_reduced_for_small_heap(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "results" result = subprocess.run( [sys.executable, str(RUNNER), "--output", str(output), "--pg-config", os.environ["BUILD_MEMORY_PG_CONFIG"], "--case", "repeated"], capture_output=True, text=True, ) self.assertEqual(result.returncode, 0, result.stderr) summary = json.loads((output / "summary.json").read_text()) self.assertEqual(summary["status"], "ok") case = summary["cases"][0] self.assertLess(case["heap_bytes"], 24 * 2**20) self.assertEqual(case["workers_requested"], 2) self.assertEqual(case["workers_launched"], 2) self.assertEqual(case["matching_docs"], 100000) def test_success_records_metrics_and_cleans_up(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "results" result = subprocess.run( [sys.executable, str(RUNNER), "--output", str(output), "--pg-config", os.environ["BUILD_MEMORY_PG_CONFIG"], "--case", "unique", "--workers", "0", "--terms", "1"], capture_output=True, text=True, ) self.assertEqual(result.returncode, 0, result.stderr) summary = json.loads((output / "summary.json").read_text()) self.assertEqual(summary["status"], "ok") case = summary["cases"][0] self.assertEqual(case["matching_docs"], 100000) self.assertEqual(case["workers_launched"], 0) self.assertEqual(case["text_bytes"], 1700000) self.assertGreater(case["peak_build_pss_bytes"], 0) self.assertGreater(case["build_seconds"], 0) self.assertTrue((output / "unique.csv").exists()) self.assertFalse((output / "data").exists()) def test_timeout_records_failure_and_cleans_up(self): with tempfile.TemporaryDirectory() as directory: output = Path(directory) / "results" result = subprocess.run( [sys.executable, str(RUNNER), "--output", str(output), "--pg-config", os.environ["BUILD_MEMORY_PG_CONFIG"], "--case", "unique", "--timeout", "0.001"], capture_output=True, text=True, ) self.assertNotEqual(result.returncode, 0) summary = json.loads((output / "summary.json").read_text()) self.assertEqual(summary["status"], "error") self.assertIn("timed out", summary["error"]) self.assertNotIn("cleanup_error", summary) self.assertFalse((output / "data").exists()) if __name__ == "__main__": unittest.main() timescale-pg_textsearch-b63fe58/test/scripts/cic.sh000077500000000000000000000441211525762047700225260ustar00rootroot00000000000000#!/bin/bash # # Test CREATE INDEX CONCURRENTLY with concurrent table operations # Verifies that CIC works correctly while the table is being modified # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55436 TEST_DB=pg_textsearch_cic_test DATA_DIR="${SCRIPT_DIR}/../tmp_cic_test" LOGFILE="${DATA_DIR}/postgres.log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } cleanup() { local exit_code=$? log "Cleaning up CIC test environment (exit code: $exit_code)..." jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast &>/dev/null || pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up CIC test PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 local lib_name="pg_textsearch" cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 50 shared_buffers = 128MB unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' log_min_messages = warning shared_preload_libraries = '${lib_name}' # Lower thresholds to trigger multiple memtable spills during CIC tests pg_textsearch.bulk_load_threshold = 10000 pg_textsearch.memtable_pages_threshold = 2 EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w || error "Failed to start PostgreSQL" createdb -h "${DATA_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "CREATE EXTENSION pg_textsearch;" >/dev/null } run_sql() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } run_sql_quiet() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } # Test 1: Basic CIC with concurrent inserts test_cic_with_inserts() { log "Test 1: CREATE INDEX CONCURRENTLY with concurrent INSERTs" # Create table with initial data - use enough rows to trigger memtable spills # bulk_load_threshold is 100000 terms, so we need lots of documents with many terms run_sql_quiet "DROP TABLE IF EXISTS cic_test CASCADE;" run_sql_quiet "CREATE TABLE cic_test (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "INSERT INTO cic_test (content) SELECT 'initial document number ' || i || ' about database search optimization with ' || 'additional text to make indexing trigger memtable spills and exercise the ' || 'concurrent index build code paths thoroughly term' || (i % 1000) FROM generate_series(1, 100000) AS i;" info "Created table with 100000 initial documents" # Start CIC in background local cic_output="${DATA_DIR}/cic_output.txt" { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "CREATE INDEX CONCURRENTLY cic_test_idx ON cic_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" } > "$cic_output" 2>&1 & local cic_pid=$! info "Started CREATE INDEX CONCURRENTLY (PID: $cic_pid)" # Concurrent inserts while CIC is running local insert_count=0 while kill -0 $cic_pid 2>/dev/null; do run_sql_quiet "INSERT INTO cic_test (content) VALUES ('concurrent insert $insert_count about search optimization');" insert_count=$((insert_count + 1)) sleep 0.01 done wait $cic_pid local cic_exit=$? info "CIC completed, performed $insert_count concurrent inserts" if [ $cic_exit -ne 0 ]; then error "CREATE INDEX CONCURRENTLY failed" cat "$cic_output" fi # Verify index is valid local is_valid=$(run_sql "SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_test_idx'::regclass;" | grep -E "^\s*t\s*$") if [ -z "$is_valid" ]; then error "Index is not valid after CIC" fi # Verify no duplicates - count documents matching 'database' local table_count=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content LIKE '%database%';" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local index_count=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content <@> to_bm25query('database', 'cic_test_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Table count for 'database': $table_count, Index count: $index_count" info "Performed $insert_count concurrent inserts during CIC" if [ "$table_count" != "$index_count" ]; then error "Mismatch: table has $table_count rows with 'database', index returns $index_count" fi if [ "$insert_count" -lt 5 ]; then warn "Only $insert_count concurrent inserts - CIC may have been too fast" fi log "✅ Test 1 passed: CIC with concurrent inserts works correctly" run_sql_quiet "DROP TABLE cic_test CASCADE;" } # Test 2: CIC with concurrent updates test_cic_with_updates() { log "Test 2: CREATE INDEX CONCURRENTLY with concurrent UPDATEs" run_sql_quiet "DROP TABLE IF EXISTS cic_test CASCADE;" run_sql_quiet "CREATE TABLE cic_test (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "INSERT INTO cic_test (content) SELECT 'document ' || i || ' original content about searching with more words to trigger spills term' || (i % 500) FROM generate_series(1, 100000) AS i;" info "Created table with 100000 documents" # Start CIC in background local cic_output="${DATA_DIR}/cic_output.txt" { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "CREATE INDEX CONCURRENTLY cic_test_idx ON cic_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" } > "$cic_output" 2>&1 & local cic_pid=$! info "Started CREATE INDEX CONCURRENTLY (PID: $cic_pid)" # Concurrent updates while CIC is running local update_count=0 while kill -0 $cic_pid 2>/dev/null; do local id=$((RANDOM % 100000 + 1)) run_sql_quiet "UPDATE cic_test SET content = 'updated document $id with modified search terms' WHERE id = $id;" update_count=$((update_count + 1)) sleep 0.005 done wait $cic_pid local cic_exit=$? info "CIC completed, performed $update_count concurrent updates" if [ $cic_exit -ne 0 ]; then error "CREATE INDEX CONCURRENTLY failed" cat "$cic_output" fi # Verify index is valid local is_valid=$(run_sql "SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_test_idx'::regclass;" | grep -E "^\s*t\s*$") if [ -z "$is_valid" ]; then error "Index is not valid after CIC" fi # Verify updated content is searchable local updated_count=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content LIKE '%modified%';" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local index_updated=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content <@> to_bm25query('modified', 'cic_test_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Updated rows: $updated_count, Index finds: $index_updated" if [ "$updated_count" != "$index_updated" ]; then error "Mismatch: $updated_count rows have 'modified', index returns $index_updated" fi if [ "$update_count" -lt 5 ]; then warn "Only $update_count concurrent updates - CIC may have been too fast" fi log "✅ Test 2 passed: CIC with concurrent updates works correctly" run_sql_quiet "DROP TABLE cic_test CASCADE;" } # Test 3: CIC with concurrent deletes test_cic_with_deletes() { log "Test 3: CREATE INDEX CONCURRENTLY with concurrent DELETEs" run_sql_quiet "DROP TABLE IF EXISTS cic_test CASCADE;" run_sql_quiet "CREATE TABLE cic_test (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "INSERT INTO cic_test (content) SELECT 'document ' || i || ' deletable content about search with extra text to trigger spills term' || (i % 500) FROM generate_series(1, 100000) AS i;" info "Created table with 100000 documents" # Start CIC in background local cic_output="${DATA_DIR}/cic_output.txt" { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "CREATE INDEX CONCURRENTLY cic_test_idx ON cic_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" } > "$cic_output" 2>&1 & local cic_pid=$! info "Started CREATE INDEX CONCURRENTLY (PID: $cic_pid)" # Concurrent deletes while CIC is running local delete_count=0 while kill -0 $cic_pid 2>/dev/null; do # Delete some rows run_sql_quiet "DELETE FROM cic_test WHERE id % 7 = 0 LIMIT 3;" || true delete_count=$((delete_count + 1)) sleep 0.02 done wait $cic_pid || true local cic_exit=$? info "CIC completed (exit: $cic_exit), performed $delete_count delete batches" if [ $cic_exit -ne 0 ]; then warn "CREATE INDEX CONCURRENTLY had non-zero exit" cat "$cic_output" || true fi # Verify index is valid local is_valid=$(run_sql "SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_test_idx'::regclass;" | grep -E "^\s*t\s*$") if [ -z "$is_valid" ]; then error "Index is not valid after CIC" fi # Verify counts match local table_count=$(run_sql "SELECT COUNT(*) FROM cic_test;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local index_count=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content <@> to_bm25query('search', 'cic_test_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Table count: $table_count, Index count for 'search': $index_count" # All remaining rows should be found (they all contain 'search') if [ "$table_count" != "$index_count" ]; then warn "Mismatch: table has $table_count rows, index returns $index_count for 'search'" else log "✅ Test 3 passed: CIC with concurrent deletes works correctly" fi run_sql_quiet "DROP TABLE cic_test CASCADE;" } # Test 4: CIC with mixed concurrent operations test_cic_with_mixed_ops() { log "Test 4: CREATE INDEX CONCURRENTLY with mixed concurrent operations" run_sql_quiet "DROP TABLE IF EXISTS cic_test CASCADE;" run_sql_quiet "CREATE TABLE cic_test (id SERIAL PRIMARY KEY, content TEXT, category INT);" run_sql_quiet "INSERT INTO cic_test (content, category) SELECT 'document ' || i || ' mixed content about indexing with additional terms term' || (i % 500), i % 10 FROM generate_series(1, 100000) AS i;" info "Created table with 100000 documents" # Start CIC in background local cic_output="${DATA_DIR}/cic_output.txt" { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "CREATE INDEX CONCURRENTLY cic_test_idx ON cic_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" } > "$cic_output" 2>&1 & local cic_pid=$! info "Started CREATE INDEX CONCURRENTLY (PID: $cic_pid)" # Start multiple concurrent workers local pids=() # Insert worker { local count=0 while kill -0 $cic_pid 2>/dev/null; do run_sql_quiet "INSERT INTO cic_test (content, category) VALUES ('inserted during cic $count about searching', $((count % 10)));" count=$((count + 1)) sleep 0.02 done info "Insert worker: added $count rows" } & pids+=($!) # Update worker { local count=0 while kill -0 $cic_pid 2>/dev/null; do local id=$((RANDOM % 100000 + 1)) run_sql_quiet "UPDATE cic_test SET content = 'updated during cic $count for testing' WHERE id = $id;" count=$((count + 1)) sleep 0.01 done info "Update worker: updated $count rows" } & pids+=($!) # Delete worker { local count=0 while kill -0 $cic_pid 2>/dev/null; do run_sql_quiet "DELETE FROM cic_test WHERE category = $((count % 10)) AND id > 90000 LIMIT 1;" count=$((count + 1)) sleep 0.02 done info "Delete worker: deleted up to $count rows" } & pids+=($!) # Select worker (queries during CIC) { local count=0 while kill -0 $cic_pid 2>/dev/null; do # Sequential scan query (index not ready yet) run_sql "SELECT COUNT(*) FROM cic_test WHERE content LIKE '%indexing%';" >/dev/null 2>&1 count=$((count + 1)) sleep 0.01 done info "Select worker: ran $count queries" } & pids+=($!) # Wait for CIC to complete wait $cic_pid local cic_exit=$? # Wait for workers to notice CIC is done sleep 0.5 for pid in "${pids[@]}"; do wait $pid 2>/dev/null || true done info "CIC and all workers completed" if [ $cic_exit -ne 0 ]; then error "CREATE INDEX CONCURRENTLY failed" cat "$cic_output" fi # Verify index is valid local is_valid=$(run_sql "SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_test_idx'::regclass;" | grep -E "^\s*t\s*$") if [ -z "$is_valid" ]; then error "Index is not valid after CIC" fi # Verify index works correctly local table_count=$(run_sql "SELECT COUNT(*) FROM cic_test;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Final table count: $table_count" # Test various searches local search1=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content <@> to_bm25query('indexing', 'cic_test_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local search2=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content <@> to_bm25query('inserted', 'cic_test_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local search3=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content <@> to_bm25query('updated', 'cic_test_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Search results - 'indexing': $search1, 'inserted': $search2, 'updated': $search3" # Cross-check with sequential scan local seq_indexing=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content LIKE '%indexing%';" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local seq_inserted=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content LIKE '%inserted%';" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local seq_updated=$(run_sql "SELECT COUNT(*) FROM cic_test WHERE content LIKE '%updated%';" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Sequential scan - 'indexing': $seq_indexing, 'inserted': $seq_inserted, 'updated': $seq_updated" # Verify counts match (allowing for stemming differences) if [ "$search1" != "$seq_indexing" ]; then warn "Mismatch for 'indexing': index=$search1, seqscan=$seq_indexing" fi if [ "$search2" != "$seq_inserted" ]; then warn "Mismatch for 'inserted': index=$search2, seqscan=$seq_inserted" fi if [ "$search3" != "$seq_updated" ]; then warn "Mismatch for 'updated': index=$search3, seqscan=$seq_updated" fi log "✅ Test 4 passed: CIC with mixed concurrent operations works correctly" run_sql_quiet "DROP TABLE cic_test CASCADE;" } # Test 5: Multiple CIC on different tables test_multiple_cic() { log "Test 5: Multiple concurrent CREATE INDEX CONCURRENTLY operations" # Create multiple tables with enough data to trigger spills for i in 1 2 3; do run_sql_quiet "DROP TABLE IF EXISTS cic_multi_$i CASCADE;" run_sql_quiet "CREATE TABLE cic_multi_$i (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "INSERT INTO cic_multi_$i (content) SELECT 'table $i document ' || j || ' with searchable content and extra terms term' || (j % 500) FROM generate_series(1, 50000) AS j;" done info "Created 3 tables with 50000 documents each" # Start CIC on all tables concurrently local pids="" for i in 1 2 3; do { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "CREATE INDEX CONCURRENTLY cic_multi_${i}_idx ON cic_multi_$i USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" >/dev/null 2>&1 } & local pid=$! pids="$pids $pid" info "Started CIC on table cic_multi_$i (PID: $pid)" done # Wait for all CIC to complete local failures=0 local idx=1 for pid in $pids; do if ! wait $pid; then warn "CIC on table cic_multi_$idx failed" failures=$((failures + 1)) fi idx=$((idx + 1)) done if [ $failures -gt 0 ]; then error "$failures CIC operations failed" fi # Verify all indexes are valid for i in 1 2 3; do local is_valid=$(run_sql "SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_multi_${i}_idx'::regclass;" | grep -E "^\s*t\s*$") if [ -z "$is_valid" ]; then error "Index cic_multi_${i}_idx is not valid" fi local count=$(run_sql "SELECT COUNT(*) FROM cic_multi_$i WHERE content <@> to_bm25query('searchable', 'cic_multi_${i}_idx') < 0;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') info "Table cic_multi_$i: $count documents found via index" if [ "$count" != "50000" ]; then error "Expected 50000 documents in cic_multi_$i, found $count" fi done log "✅ Test 5 passed: Multiple concurrent CIC operations work correctly" for i in 1 2 3; do run_sql_quiet "DROP TABLE cic_multi_$i CASCADE;" done } run_cic_tests() { log "Starting CREATE INDEX CONCURRENTLY tests" test_cic_with_inserts test_cic_with_updates test_cic_with_deletes test_cic_with_mixed_ops test_multiple_cic log "🎉 All CIC tests completed successfully!" } main() { log "Starting pg_textsearch CREATE INDEX CONCURRENTLY testing..." command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found in PATH" command -v psql >/dev/null 2>&1 || error "psql not found in PATH" setup_test_db run_cic_tests exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/compaction_recovery.sh000077500000000000000000000507471525762047700260550ustar00rootroot00000000000000#!/bin/bash # # Verify compaction publication crash recovery and standby mutator guards. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55464 STANDBY_PORT=55465 TEST_DB=compaction_recovery_test PRIMARY_DIR="${SCRIPT_DIR}/../tmp_compaction_recovery_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_compaction_recovery_standby" # Avoid Unix-socket path limits in deeply nested worktrees. REPL_HOST=127.0.0.1 REPL_SOCKET_DIR= # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM primary_value() { primary_sql_quiet "$1" } primary_ranked_value() { PGOPTIONS="-c enable_seqscan=off" primary_sql_quiet "$1" } wait_for_postmaster_exit() { local pid=$1 local label=$2 local deadline=$((SECONDS + 10)) while kill -0 "${pid}" 2>/dev/null; do ((SECONDS < deadline)) || error "${label} postmaster ${pid} did not exit" sleep 0.05 done } restart_primary() { rm -f "${PRIMARY_DIR}/postmaster.pid" pg_ctl start -D "${PRIMARY_DIR}" \ -l "${PRIMARY_DIR}/postgres.log" -w primary_value "SELECT 1;" >/dev/null } log_contains() { local marker=$1 grep -Fq "${marker}" "${PRIMARY_DIR}/postgres.log" \ "${PRIMARY_DIR}/log/postgres.log" 2>/dev/null } wait_for_backend() { local app_name=$1 local deadline=$((SECONDS + 10)) local pid while ((SECONDS < deadline)); do pid=$(primary_value " SELECT pid FROM pg_stat_activity WHERE application_name = '${app_name}' ORDER BY backend_start DESC LIMIT 1;") if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return fi sleep 0.05 done error "backend ${app_name} did not appear" } wait_for_injection() { local point=$1 local backend=$2 local deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do if [ "$(primary_value " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = ${backend} AND wait_event_type = 'InjectionPoint' AND wait_event = '${point}' );")" = "t" ]; then log "Backend ${backend} is waiting at ${point}" return fi sleep 0.05 done error "backend ${backend} did not reach injection point ${point}" } create_crash_fixture() { local prefix=$1 local spilled local graph primary_sql " CREATE TABLE ${prefix}_docs ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO ${prefix}_docs SELECT g, 'crashmarker recovery document ' || g FROM generate_series(1, 300) g; CREATE INDEX ${prefix}_idx ON ${prefix}_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO ${prefix}_docs SELECT g, 'crashmarker recovery document ' || g FROM generate_series(301, 600) g; " >/dev/null spilled=$(primary_value \ "SELECT bm25_spill_index('${prefix}_idx') > 0;") [ "${spilled}" = "t" ] || error "${prefix}: second segment was not spilled" graph=$(primary_value \ "SELECT bm25_level_counts('${prefix}_idx'::regclass)::text;") [ "${graph}" = "{2,0,0,0,0,0,0,0}" ] || error "${prefix}: fixture graph is ${graph}" primary_sql "CHECKPOINT;" >/dev/null } crash_after_detached_wal() { local prefix=$1 local app_name="pgts-${prefix}" local output="${PRIMARY_DIR}/${prefix}.out" local client_pid local backend local postmaster_pid PGAPPNAME="${app_name}" \ psql -X -v ON_ERROR_STOP=1 -p "${PRIMARY_PORT}" -d "${TEST_DB}" \ -c " SELECT injection_points_set_local(); SELECT injection_points_attach( 'pg-textsearch-compaction-after-restamp', 'wait'); SELECT bm25_compact_step('${prefix}_idx'::regclass); " >"${output}" 2>&1 & client_pid=$! backend=$(wait_for_backend "${app_name}") wait_for_injection \ 'pg-textsearch-compaction-after-restamp' "${backend}" postmaster_pid=$(head -1 "${PRIMARY_DIR}/postmaster.pid") pg_ctl stop -D "${PRIMARY_DIR}" -m immediate -w >/dev/null wait_for_postmaster_exit "${postmaster_pid}" "${prefix}" wait "${client_pid}" 2>/dev/null || true restart_primary } trigger_publication_panic() { local prefix=$1 local point=$2 local postmaster_pid local output local rc postmaster_pid=$(head -1 "${PRIMARY_DIR}/postmaster.pid") set +e output=$(timeout 30s psql -X -v ON_ERROR_STOP=1 \ -p "${PRIMARY_PORT}" -d "${TEST_DB}" -c " SELECT pg_textsearch_test_attach_panic('${point}'); SELECT bm25_compact_step('${prefix}_idx'::regclass); " 2>&1) rc=$? set -e [ "${rc}" -ne 0 ] || error "${prefix}: compaction survived the ${point} panic" [ "${rc}" -ne 124 ] || error "${prefix}: timed out waiting for PANIC: ${output}" wait_for_postmaster_exit "${postmaster_pid}" "${prefix}" log_contains "panic triggered for injection point ${point}" || error "${prefix}: missing PANIC marker for ${point}" restart_primary } create_legacy_vacuum_fixture() { local prefix=$1 local legacy_created primary_sql " SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE ${prefix}_docs ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); CREATE INDEX ${prefix}_idx ON ${prefix}_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO ${prefix}_docs (body) SELECT 'legacy recovery document ' || gs || ' ' || repeat(md5(gs::text), 4) FROM generate_series(1, 20000) gs; " >/dev/null primary_sql " SELECT pg_textsearch_test_attach_legacy_segment(1000000); SELECT bm25_spill_index('${prefix}_idx'); SELECT injection_points_detach( 'pg-textsearch-legacy-segment'); " >/dev/null legacy_created=$(primary_value " SELECT bm25_dump_index('${prefix}_idx') LIKE '%Version: 4%';") [ "${legacy_created}" = "t" ] || error "${prefix}: legacy segment was not injected" primary_sql " DELETE FROM ${prefix}_docs WHERE id <= 5000; CHECKPOINT; " >/dev/null } legacy_root() { local prefix=$1 primary_value "SELECT bm25_summarize_index('${prefix}_idx');" | sed -n 's/.*L0 Segment 1: block=\([0-9][0-9]*\).*/\1/p' } index_stat() { local prefix=$1 local field=$2 primary_value "SELECT bm25_summarize_index('${prefix}_idx');" | sed -n "s/^ ${field}: \\([0-9][0-9]*\\)$/\\1/p" } trigger_legacy_vacuum_panic() { local prefix=$1 local point=$2 local postmaster_pid local output local rc postmaster_pid=$(head -1 "${PRIMARY_DIR}/postmaster.pid") set +e output=$(PGOPTIONS="-c min_parallel_index_scan_size=0" \ timeout 30s psql -X -v ON_ERROR_STOP=1 \ -p "${PRIMARY_PORT}" -d "${TEST_DB}" \ -c "SELECT pg_textsearch_test_attach_panic('${point}');" \ -c "VACUUM (PARALLEL 1, VERBOSE) ${prefix}_docs;" 2>&1) rc=$? set -e [ "${rc}" -ne 0 ] || error "${prefix}: legacy VACUUM survived the ${point} panic" [ "${rc}" -ne 124 ] || error "${prefix}: timed out waiting for PANIC: ${output}" grep -q "launched 1 parallel vacuum worker" <<<"${output}" || error "${prefix}: legacy VACUUM did not launch a parallel worker: ${output}" wait_for_postmaster_exit "${postmaster_pid}" "${prefix}" log_contains "panic triggered for injection point ${point}" || error "${prefix}: missing PANIC marker for ${point}" restart_primary } test_parallel_worker_pid_matching() { local output local postmaster_pid local rc if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping parallel worker PID matching: no injection points" return fi log "Testing parallel worker injection matching..." primary_sql " SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE worker_match_docs ( id bigint GENERATED ALWAYS AS IDENTITY, body text NOT NULL ) WITH (autovacuum_enabled = false); CREATE INDEX worker_match_a_idx ON worker_match_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); CREATE INDEX worker_match_b_idx ON worker_match_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO worker_match_docs (body) SELECT 'parallel worker matching document ' || gs FROM generate_series(1, 20000) gs; DELETE FROM worker_match_docs WHERE id <= 5000; CHECKPOINT; " >/dev/null postmaster_pid=$(head -1 "${PRIMARY_DIR}/postmaster.pid") set +e output=$(PGOPTIONS="-c min_parallel_index_scan_size=0" \ timeout 30s psql -X -v ON_ERROR_STOP=1 \ -p "${PRIMARY_PORT}" -d "${TEST_DB}" \ -c "SELECT pg_textsearch_test_attach_worker_panic( 'pg-textsearch-compaction-source-estimate');" \ -c "SELECT injection_points_set_local(); SELECT injection_points_attach( 'pg-textsearch-parallel-vacuum-leader-estimate', 'wait');" \ -c "VACUUM (PARALLEL 1, VERBOSE) worker_match_docs;" 2>&1) rc=$? set -e [ "${rc}" -ne 0 ] || error "parallel worker matching VACUUM survived the panic: ${output}" [ "${rc}" -ne 124 ] || error "timed out waiting for parallel worker panic: ${output}" grep -q "launched 1 parallel vacuum worker" <<<"${output}" || error "parallel worker matching did not launch a worker: ${output}" wait_for_postmaster_exit "${postmaster_pid}" "worker_match" log_contains \ "parallel worker panic triggered for injection point pg-textsearch-compaction-source-estimate" || error "parallel worker did not match its lock group leader" restart_primary primary_sql "DROP TABLE worker_match_docs;" >/dev/null log "PASS: parallel worker matched its lock group leader" } assert_legacy_vacuum_recovery() { local prefix=$1 local old_root=$2 local expect_replaced=$3 local expect_parked=$4 local expected_total_docs=$5 local expected_total_len=$6 local heap_count local new_root local parked local plan local remaining local summary local total_docs local total_len new_root=$(legacy_root "${prefix}") [ -n "${new_root}" ] || error "${prefix}: recovered graph has no L0 segment root" if [ "${expect_replaced}" = "yes" ]; then [ "${new_root}" != "${old_root}" ] || error "${prefix}: legacy root ${old_root} was not replaced" else [ "${new_root}" = "${old_root}" ] || error "${prefix}: unpublished replacement changed root to ${new_root}" fi parked=$(primary_value \ "SELECT bm25_pending_free_pages('${prefix}_idx');") if [ "${expect_parked}" = "yes" ]; then [ "${parked}" -gt 0 ] || error "${prefix}: replaced legacy root has no pending-free pages" else [ "${parked}" = "0" ] || error "${prefix}: unpublished tombstones became reachable" fi remaining=$(primary_ranked_value " SELECT count(*) FROM ( SELECT id FROM ${prefix}_docs ORDER BY body <@> to_bm25query( 'legacy recovery', '${prefix}_idx') ) ranked;") if [ "${remaining}" != "15000" ]; then heap_count=$(primary_value "SELECT count(*) FROM ${prefix}_docs;") plan=$(primary_ranked_value " EXPLAIN (COSTS off) SELECT id FROM ${prefix}_docs ORDER BY body <@> to_bm25query( 'legacy recovery', '${prefix}_idx');") summary=$(primary_value \ "SELECT bm25_summarize_index('${prefix}_idx');") error "${prefix}: recovered ranked scan returned ${remaining}/15000 rows; heap=${heap_count}; plan=${plan}; summary=${summary}" fi summary=$(primary_value \ "SELECT bm25_summarize_index('${prefix}_idx');") total_docs=$(sed -n 's/^ total_docs: \([0-9][0-9]*\)$/\1/p' \ <<<"${summary}") total_len=$(sed -n 's/^ total_len: \([0-9][0-9]*\)$/\1/p' \ <<<"${summary}") if [ "${expect_replaced}" = "no" ]; then [ "${total_docs}" = "${expected_total_docs}" ] || error "${prefix}: recovered total_docs is ${total_docs}, expected ${expected_total_docs}" [ "${total_len}" = "${expected_total_len}" ] || error "${prefix}: recovered total_len is ${total_len}, expected ${expected_total_len}" fi if [ "${expect_replaced}" = "yes" ]; then primary_value "SELECT txid_current();" >/dev/null primary_value "SELECT txid_current();" >/dev/null primary_sql "VACUUM ${prefix}_docs;" >/dev/null parked=$(primary_value \ "SELECT bm25_pending_free_pages('${prefix}_idx');") [ "${parked}" -gt 0 ] || error "${prefix}: provisional reclaim batch drained on activation" primary_value "SELECT txid_current();" >/dev/null primary_value "SELECT txid_current();" >/dev/null primary_sql "VACUUM ${prefix}_docs;" >/dev/null parked=$(primary_value \ "SELECT bm25_pending_free_pages('${prefix}_idx');") [ "${parked}" = "0" ] || error "${prefix}: activated reclaim batch remained parked (${parked})" primary_sql "REINDEX INDEX ${prefix}_idx;" >/dev/null expected_total_docs=$(index_stat "${prefix}" total_docs) expected_total_len=$(index_stat "${prefix}" total_len) [ "${total_docs}" = "${expected_total_docs}" ] || error "${prefix}: recovered total_docs ${total_docs} differs from REINDEX ${expected_total_docs}" [ "${total_len}" = "${expected_total_len}" ] || error "${prefix}: recovered total_len ${total_len} differs from REINDEX ${expected_total_len}" fi log "PASS: ${prefix} recovered root ${new_root}, pending ${parked}" } assert_crash_recovery() { local prefix=$1 local expected_graph=$2 local expected_segments=$3 local expect_parked=$4 local plan local expected_ids local actual_ids local graph local summary local parked plan=$(primary_ranked_value " EXPLAIN (COSTS off) SELECT id FROM ${prefix}_docs ORDER BY body <@> to_bm25query( 'crashmarker', '${prefix}_idx') LIMIT 600;") grep -Fq \ "Index Scan using ${prefix}_idx on ${prefix}_docs" <<<"${plan}" || error "${prefix}: recovery assertion did not use BM25 index: ${plan}" expected_ids=$(primary_value " SELECT string_agg(id::text, ',' ORDER BY id) FROM ${prefix}_docs;") actual_ids=$(primary_ranked_value " SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM ${prefix}_docs ORDER BY body <@> to_bm25query( 'crashmarker', '${prefix}_idx') LIMIT 600 ) ranked;") [ "${actual_ids}" = "${expected_ids}" ] || error "${prefix}: recovered ranked IDs differ from heap IDs ($(tr ',' '\n' <<<"${actual_ids}" | wc -l)/$(tr ',' '\n' <<<"${expected_ids}" | wc -l))" graph=$(primary_value \ "SELECT bm25_level_counts('${prefix}_idx'::regclass)::text;") summary=$(primary_value \ "SELECT bm25_summarize_index('${prefix}_idx');") [ "${graph}" = "${expected_graph}" ] || error "${prefix}: graph is ${graph}, expected ${expected_graph}; ${summary}" [[ "${summary}" == *"Total: ${expected_segments} segments"* ]] || error "${prefix}: invalid graph summary: ${summary}" parked=$(primary_value \ "SELECT bm25_pending_free_pages('${prefix}_idx');") case "${parked}" in ''|*[!0-9]*) error "${prefix}: invalid pending count '${parked}'" ;; esac if [ "${expect_parked}" = "yes" ]; then [ "${parked}" -gt 0 ] || error "${prefix}: published graph has no pending-free chain" else [ "${parked}" = "0" ] || error "${prefix}: unpublished tombstones became reachable" fi log "PASS: ${prefix} recovered IDs, graph ${graph}, pending ${parked}" } test_publication_crash_recovery() { if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping publication crash recovery: no injection points" return fi log "Testing compaction publication crash recovery..." create_crash_fixture detached_wal crash_after_detached_wal detached_wal assert_crash_recovery \ detached_wal "{2,0,0,0,0,0,0,0}" 2 no create_crash_fixture before_publish trigger_publication_panic \ before_publish \ pg-textsearch-before-compaction-publish assert_crash_recovery \ before_publish "{2,0,0,0,0,0,0,0}" 2 no create_crash_fixture after_publish trigger_publication_panic \ after_publish \ pg-textsearch-after-compaction-publish assert_crash_recovery \ after_publish "{0,1,0,0,0,0,0,0}" 1 yes } test_legacy_vacuum_crash_recovery() { local old_root local old_total_docs local old_total_len if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping legacy VACUUM crash recovery: no injection points" return fi log "Testing parallel legacy VACUUM publication crash recovery..." create_legacy_vacuum_fixture legacy_before_publish old_root=$(legacy_root legacy_before_publish) old_total_docs=$(index_stat legacy_before_publish total_docs) old_total_len=$(index_stat legacy_before_publish total_len) [ -n "${old_root}" ] || error "legacy_before_publish: fixture has no L0 segment root" trigger_legacy_vacuum_panic \ legacy_before_publish \ pg-textsearch-before-compaction-publish assert_legacy_vacuum_recovery \ legacy_before_publish "${old_root}" no no \ "${old_total_docs}" "${old_total_len}" create_legacy_vacuum_fixture legacy_after_publish old_root=$(legacy_root legacy_after_publish) [ -n "${old_root}" ] || error "legacy_after_publish: fixture has no L0 segment root" trigger_legacy_vacuum_panic \ legacy_after_publish \ pg-textsearch-after-compaction-publish assert_legacy_vacuum_recovery \ legacy_after_publish "${old_root}" yes yes "" "" } expect_recovery_rejection() { local function_name=$1 local output local expected_message="cannot compact a bm25 index during recovery" if output=$(psql -v ON_ERROR_STOP=1 -v VERBOSITY=verbose \ -p "${STANDBY_PORT}" -d "${TEST_DB}" \ -c "SELECT ${function_name}( 'compaction_recovery_idx'::regclass);" 2>&1); then error "${function_name} unexpectedly succeeded during recovery" fi if [[ "${output}" != *"25006"* ]] || [[ "${output}" != *"${expected_message}"* ]]; then error "${function_name} returned the wrong recovery error: ${output}" fi log "PASS: ${function_name} rejected execution during recovery" } test_standby_guards() { primary_sql " CREATE TABLE compaction_recovery ( id integer PRIMARY KEY, body text ); INSERT INTO compaction_recovery VALUES (1, 'standby compaction guard'); CREATE INDEX compaction_recovery_idx ON compaction_recovery USING bm25(body) WITH (text_config = 'english'); " >/dev/null setup_standby wait_for_standby_catchup expect_recovery_rejection bm25_compact expect_recovery_rejection bm25_compact_step } main() { log "Starting compaction recovery tests..." check_required_tools setup_primary cat >>"${PRIMARY_DIR}/postgresql.conf" < #undef NDEBUG #include #include #include "segment/compression.h" /* Use the real codec and format types without a running backend. */ #undef fprintf #undef printf #undef Assert #define Assert(condition) assert(condition) #undef ereport #define ereport(level, details) decode_error() #define palloc(size) malloc(size) #define pfree(pointer) free(pointer) static jmp_buf error_jump; static bool expect_error; static void decode_error(void) { if (expect_error) longjmp(error_jump, 1); fprintf(stderr, "unexpected codec error\n"); abort(); } #ifdef TP_TEST_NO_AVX2 #define __builtin_cpu_supports(feature) 0 #endif #include "../../src/segment/compression.c" static uint32 width_mask(uint8 bits) { return bits == 32 ? UINT32_MAX : (UINT32_C(1) << bits) - 1; } /* Independent, bit-at-a-time encoder for the existing wire format. */ static uint32 pack_values(const uint32 *values, uint32 count, uint8 bits, uint8 *output) { uint32 bit_offset = 0; for (uint32 i = 0; i < count; i++) for (uint8 bit = 0; bit < bits; bit++, bit_offset++) output[bit_offset / 8] |= ((values[i] >> bit) & 1U) << (bit_offset % 8); return (bit_offset + 7) / 8; } static uint32 test_value(uint32 i, uint8 bits) { uint32 mask = width_mask(bits); switch (i % 4) { case 0: return 0; case 1: return mask; case 2: return 1; default: return (i * UINT32_C(2654435761)) & mask; } } static void check_block(uint8 doc_bits, uint8 freq_bits, uint32 count, uint32 alignment) { uint8 *storage = calloc(1, TP_MAX_COMPRESSED_BLOCK_SIZE + alignment); uint8 *compressed; uint32 deltas[TP_BLOCK_SIZE]; uint32 frequencies[TP_BLOCK_SIZE]; TpBlockPosting postings[TP_BLOCK_SIZE + 1]; uint32 pos; uint32 previous = UINT32_MAX - 15; assert(storage != NULL); compressed = storage + alignment; compressed[0] = doc_bits; compressed[1] = freq_bits; pos = sizeof(TpCompressedBlockHeader); for (uint32 i = 0; i < count; i++) { deltas[i] = test_value(i, doc_bits); frequencies[i] = test_value(i + 1, freq_bits); } pos += pack_values(deltas, count, doc_bits, compressed + pos); pos += pack_values(frequencies, count, freq_bits, compressed + pos); for (uint32 i = 0; i < count; i++) compressed[pos + i] = (uint8)(i * 37 + alignment); memset(postings, 0xcc, sizeof(postings)); tp_decompress_block(compressed, count, previous, postings); for (uint32 i = 0; i < count; i++) { previous += deltas[i]; assert(postings[i].doc_id == previous); assert(postings[i].frequency == frequencies[i]); assert(postings[i].fieldnorm == (uint8)(i * 37 + alignment)); assert(postings[i].reserved == 0); } for (size_t i = 0; i < sizeof(TpBlockPosting); i++) assert(((uint8 *)&postings[count])[i] == 0xcc); free(storage); } static void check_invalid(uint8 doc_bits, uint8 freq_bits, uint32 count) { uint8 compressed[TP_MAX_COMPRESSED_BLOCK_SIZE] = {0}; TpBlockPosting postings[TP_BLOCK_SIZE]; compressed[0] = doc_bits; compressed[1] = freq_bits; expect_error = true; if (setjmp(error_jump) == 0) { tp_decompress_block(compressed, count, 0, postings); fprintf(stderr, "invalid compressed block was accepted\n"); abort(); } expect_error = false; } static void check_decoder(const char *name) { uint32 cases = 0; for (uint32 alignment = 0; alignment < 8; alignment++) for (uint32 count = 1; count <= TP_BLOCK_SIZE; count++) for (uint8 doc_bits = 1; doc_bits <= 32; doc_bits++) for (uint8 freq_bits = 1; freq_bits <= 16; freq_bits++) { check_block(doc_bits, freq_bits, count, alignment); cases++; } tp_decompress_block(NULL, 0, 0, NULL); check_invalid(1, 1, TP_BLOCK_SIZE + 1); check_invalid(0, 1, 1); check_invalid(33, 1, 1); check_invalid(1, 0, 1); check_invalid(1, 17, 1); printf("validated %u compression cases (%s)\n", cases, name); } int main(void) { check_block(25, 3, TP_BLOCK_SIZE, 0); #ifdef TP_HAVE_AVX2 assert(decompress_impl == (__builtin_cpu_supports("avx2") ? decompress_block_avx2 : decompress_block_scalar)); #else assert(decompress_impl == decompress_block_scalar); #endif check_decoder("runtime dispatch"); if (decompress_impl != decompress_block_scalar) { decompress_impl = decompress_block_scalar; check_decoder("scalar fallback"); } return 0; } timescale-pg_textsearch-b63fe58/test/scripts/concurrency.sh000077500000000000000000000737101525762047700243300ustar00rootroot00000000000000#!/bin/bash # # Real concurrency stress test for pg_textsearch extension # Tests actual concurrent operations from multiple PostgreSQL sessions # set -e # Global test size multiplier - scales number of operations, not timing # 1.0 = default size, 2.0 = twice as many operations, 0.5 = half as many TEST_SIZE_MULTIPLIER=${TEST_SIZE_MULTIPLIER:-2.0} SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55434 TEST_DB=pg_textsearch_concurrency_test DATA_DIR="${SCRIPT_DIR}/../tmp_concurrent_test" LOGFILE="${DATA_DIR}/postgres.log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}" } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}" } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}" exit 1 } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}" } # Helper function to scale operation counts (rounds to nearest integer) scaled_count() { local base_count=$1 local scaled=$(awk "BEGIN {printf \"%.0f\", $base_count * $TEST_SIZE_MULTIPLIER}") echo "$scaled" } # Helper function for jittered sleep (±50% random variation) jitter_sleep() { local base_duration=$1 local jitter=$(awk "BEGIN {printf \"%.6f\", $base_duration * (0.5 + $RANDOM / 32767.0)}") sleep "$jitter" } cleanup() { local exit_code=$? log "Cleaning up concurrent test environment (exit code: $exit_code)..." # Kill all background jobs jobs -p | xargs -r kill 2>/dev/null || true # Stop PostgreSQL if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast &>/dev/null || pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi # Clean up temp directory rm -rf "${DATA_DIR}" # Preserve the original exit code exit $exit_code } # Set up cleanup trap trap cleanup EXIT INT TERM setup_test_db() { log "Setting up concurrent test PostgreSQL instance..." # Clean up any existing test data and create fresh directory rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" # Initialize test cluster first log "Initializing test database cluster..." initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 # Now check if pg_textsearch extension is available and can be loaded local pg_config_path=$(which pg_config 2>/dev/null || echo "") if [ -n "$pg_config_path" ]; then local lib_dir=$($pg_config_path --pkglibdir 2>/dev/null || echo "") if [ -n "$lib_dir" ] && [ -f "$lib_dir/pg_textsearch.so" ]; then log "Found pg_textsearch extension at $lib_dir/pg_textsearch.so" # Check library dependencies log "Checking pg_textsearch library dependencies:" ldd "$lib_dir/pg_textsearch.so" 2>/dev/null | head -10 || log "ldd not available or failed" # Check if library is readable and has correct permissions ls -la "$lib_dir/pg_textsearch.so" # Test basic library loading with a minimal postgres process (now that DATA_DIR exists) log "Testing pg_textsearch library loading..." echo "LOAD 'pg_textsearch';" | timeout 10 postgres --single -D "${DATA_DIR}" template1 2>&1 | head -10 || warn "pg_textsearch library loading test had issues" else warn "pg_textsearch extension not found at expected location: $lib_dir/pg_textsearch.so" if [ -n "$lib_dir" ]; then log "Available extensions:" ls -la "$lib_dir"/*.so 2>/dev/null || log "No .so files found in $lib_dir" fi fi fi local lib_name="pg_textsearch" # Configure for concurrent testing cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 50 shared_buffers = 128MB work_mem = 4MB # Use local socket directory to avoid permission issues unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' # Logging configuration log_min_messages = info log_line_prefix = '[%p] %t %u@%d: ' # Load pg_textsearch extension shared_preload_libraries = '${lib_name}' EOF # Start PostgreSQL log "Starting test PostgreSQL instance on port ${TEST_PORT}..." log "Data directory: ${DATA_DIR}" log "Log file: ${LOGFILE}" # Check if port is available if netstat -ln 2>/dev/null | grep -q ":${TEST_PORT} "; then warn "Port ${TEST_PORT} appears to be in use" netstat -ln 2>/dev/null | grep ":${TEST_PORT} " || true fi # Try to start PostgreSQL with more verbose error reporting log "Running: pg_ctl start -D \"${DATA_DIR}\" -l \"${LOGFILE}\" -w" if ! pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w; then log "Failed to start PostgreSQL. Checking for issues..." # Give the log file a moment to be created sleep 2 # Check if log file exists and show contents log "Checking for log file at: ${LOGFILE}" if [ -f "${LOGFILE}" ]; then log "PostgreSQL log contents:" cat "${LOGFILE}" else log "Log file ${LOGFILE} not found after startup failure" log "Contents of data directory:" ls -la "${DATA_DIR}/" # Look for PostgreSQL log files in various locations log "Looking for any log files in data directory:" find "${DATA_DIR}" -name "*.log" -exec echo "Found log: {}" \; -exec cat {} \; 2>/dev/null || log "No log files found" # Check if PostgreSQL is using a different log location due to our config if [ -d "${DATA_DIR}/log" ]; then log "Found log directory, checking contents:" ls -la "${DATA_DIR}/log/" find "${DATA_DIR}/log" -name "*.log" -exec echo "Found log: {}" \; -exec cat {} \; 2>/dev/null || true fi fi # Try to start postgres directly in foreground to see immediate errors log "Attempting direct postgres startup for debugging:" log "Starting postgres with: postgres -D ${DATA_DIR}" timeout 15 postgres -D "${DATA_DIR}" 2>&1 | head -50 || log "Direct postgres startup failed or timed out" error "PostgreSQL startup failed - check output above for details" fi # Create test database and extension createdb -h "${DATA_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "CREATE EXTENSION pg_textsearch;" >/dev/null # Create base test table psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c " CREATE TABLE stress_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL, session_id INTEGER, created_at TIMESTAMP DEFAULT NOW() );" >/dev/null } run_sql() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" 2>/dev/null } run_sql_quiet() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } # Concurrent insert worker concurrent_insert_worker() { local session_id=$1 local num_docs=$2 local base_terms=("database" "search" "concurrent" "index" "query" "text" "system" "performance") info "Session $session_id: Starting concurrent insert worker ($num_docs documents)" for i in $(seq 1 $num_docs); do # Generate random document content local term1=${base_terms[$((RANDOM % ${#base_terms[@]}))]} local term2=${base_terms[$((RANDOM % ${#base_terms[@]}))]} local term3=${base_terms[$((RANDOM % ${#base_terms[@]}))]} local content="session $session_id document $i discusses $term1 and $term2 with $term3 features" run_sql_quiet "INSERT INTO stress_docs (content, session_id) VALUES ('$content', $session_id);" # Small delay to encourage interleaving jitter_sleep 0.001 done info "Session $session_id: Insert worker completed" } # Concurrent search worker concurrent_search_worker() { local session_id=$1 local num_searches=$2 local index_name=$3 local search_terms=("database" "search" "concurrent" "performance" "system") info "Session $session_id: Starting concurrent search worker ($num_searches searches)" local results_file="/tmp/search_results_${session_id}.txt" > "$results_file" for i in $(seq 1 $num_searches); do local term=${search_terms[$((RANDOM % ${#search_terms[@]}))]} local query_result=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('$term', '$index_name')) < -0.01;" 2>&1) local count=$(echo "$query_result" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') if [ -z "$count" ]; then warn "Search query failed for term '$term': $(echo "$query_result" | head -3 | tr '\n' '; ')" count=0 fi echo "Search $i: term='$term' results=$count" >> "$results_file" jitter_sleep 0.002 done local total_results=$(awk -F'results=' '{sum += $2} END {print sum+0}' "$results_file") info "Session $session_id: Search worker completed (total results: $total_results)" rm -f "$results_file" } # Concurrent index building worker concurrent_index_worker() { local session_id=$1 local table_suffix=$2 info "Session $session_id: Creating concurrent index on table $table_suffix" # Create index with slight variations local k1=$(awk "BEGIN {printf \"%.2f\", 1.0 + $session_id * 0.1}") local b=$(awk "BEGIN {printf \"%.2f\", 0.7 + $session_id * 0.02}") run_sql_quiet "CREATE INDEX stress_idx_${table_suffix} ON stress_docs USING bm25(content) WITH (text_config='english', k1=$k1, b=$b);" info "Session $session_id: Index creation completed" } test_concurrent_inserts() { log "Test 1: Concurrent inserts with overlapping terms" # Start multiple insert workers local num_workers=$(scaled_count 5) local docs_per_worker=$(scaled_count 20) local pids=() for i in $(seq 1 $num_workers); do concurrent_insert_worker $i $docs_per_worker & pids+=($!) done # Wait for all workers to complete for pid in "${pids[@]}"; do wait $pid done # Verify results local total_docs=$(run_sql "SELECT COUNT(*) FROM stress_docs;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') local expected=$((num_workers * docs_per_worker)) if [ "$total_docs" -eq "$expected" ]; then log "✅ Concurrent inserts test passed: $total_docs documents inserted" else error "❌ Concurrent inserts test failed: expected $expected, got $total_docs" fi } test_concurrent_index_creation() { log "Test 2: Concurrent index creation" # Create the main index run_sql_quiet "CREATE INDEX stress_main_idx ON stress_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" log "✅ Main index created successfully" # Test that concurrent operations work with existing index local search_count=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('database', 'stress_main_idx')) < -0.01;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") log "✅ Initial search found $search_count results" } test_concurrent_mixed_operations() { log "Test 3: Mixed concurrent operations (inserts + searches)" local num_inserters=$(scaled_count 3) local num_searchers=$(scaled_count 3) local docs_per_inserter=$(scaled_count 15) local searches_per_searcher=$(scaled_count 25) local pids=() # Start inserters for i in $(seq 1 $num_inserters); do concurrent_insert_worker $((100 + i)) $docs_per_inserter & pids+=($!) done # Start searchers for i in $(seq 1 $num_searchers); do concurrent_search_worker $((200 + i)) $searches_per_searcher "stress_main_idx" & pids+=($!) done # Wait for all workers for pid in "${pids[@]}"; do wait $pid || true # Don't fail if a worker has issues done # Verify final state local final_count=$(run_sql "SELECT COUNT(*) FROM stress_docs;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") log "✅ Mixed operations test completed: $final_count total documents" } test_string_interning_consistency() { log "Test 4: String interning consistency under concurrent load" # Create documents with exactly the same terms from different sessions local num_workers=$(scaled_count 4) local pids=() for i in $(seq 1 $num_workers); do ( for j in $(seq 1 $(scaled_count 10)); do run_sql_quiet "INSERT INTO stress_docs (content, session_id) VALUES ('identical database search terms for consistency testing', $((300 + i)));" jitter_sleep 0.001 done info "Session $((300 + i)): Identical term insertion completed" ) & pids+=($!) done # Wait for all workers for pid in "${pids[@]}"; do wait $pid || true # Don't fail if a worker has issues done # Verify search finds all documents local identical_count=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('identical database search', 'stress_main_idx')) < -0.01;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") log "✅ String interning test: found $identical_count documents with identical terms" } test_update_delete_concurrency() { log "Test 5: Concurrent updates and deletes" # Concurrent updates ( for i in $(seq 1 $(scaled_count 20)); do run_sql_quiet "UPDATE stress_docs SET content = 'updated concurrent document number $i with new search terms' WHERE id = $i;" || true jitter_sleep 0.002 done info "Update worker completed" ) & local update_pid=$! # Concurrent deletes ( # Delete some of the stress test documents run_sql_quiet "DELETE FROM stress_docs WHERE session_id BETWEEN 101 AND 103 AND id % 3 = 0;" || true info "Delete worker completed" ) & local delete_pid=$! # Wait for both operations wait $update_pid || true wait $delete_pid || true # Verify index integrity local remaining_count=$(run_sql "SELECT COUNT(*) FROM stress_docs;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") local search_count=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('updated concurrent', 'stress_main_idx')) < -0.01;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") log "✅ Update/Delete test: $remaining_count documents remaining, $search_count match updated terms" } test_high_load_stress() { log "Test 6: High-load stress test" local num_concurrent_sessions=$(scaled_count 8) local operations_per_session=$(scaled_count 30) local pids=() info "Starting $num_concurrent_sessions concurrent sessions with $operations_per_session operations each" # Temporarily disable exit on error for this test to allow graceful handling set +e for i in $(seq 1 $num_concurrent_sessions); do ( local session_id=$((400 + i)) log "High-load session $session_id: Starting $operations_per_session operations" # Mix of operations per session for j in $(seq 1 $operations_per_session); do case $((j % 4)) in 0) # Insert run_sql_quiet "INSERT INTO stress_docs (content, session_id) VALUES ('stress test document $j from session $session_id with performance data', $session_id);" || true ;; 1) # Search - simplified from complex BM25 query to improve reliability run_sql "SELECT COUNT(*) FROM stress_docs WHERE session_id = $session_id;" >/dev/null 2>&1 || true ;; 2) # Update (if document exists) - simplified query run_sql_quiet "UPDATE stress_docs SET content = 'updated stress document from session $session_id operation $j' WHERE session_id = $session_id LIMIT 1;" 2>/dev/null || true ;; 3) # Another simple query run_sql "SELECT MAX(id) FROM stress_docs WHERE session_id = $session_id;" >/dev/null 2>&1 || true ;; esac # Small delay to encourage interleaving jitter_sleep 0.005 done log "High-load session $session_id: Completed $operations_per_session operations" ) & pids+=($!) done log "Started all $num_concurrent_sessions high-load sessions (PIDs: ${pids[*]})" log "Waiting for sessions to complete..." # Wait for all high-load sessions local completed=0 local failed=0 for i in "${!pids[@]}"; do local pid="${pids[$i]}" local session_id=$((401 + i)) if wait $pid 2>/dev/null; then ((completed++)) log "High-load session $session_id (PID $pid) completed successfully" else local exit_code=$? ((failed++)) warn "High-load session $session_id (PID $pid) failed with exit code $exit_code" fi done info "High-load sessions completed: $completed/$num_concurrent_sessions successful, $failed failed" # Check if too many sessions failed if [ $failed -gt $((num_concurrent_sessions / 2)) ]; then warn "More than half of high-load sessions failed ($failed/$num_concurrent_sessions)" warn "This may indicate resource constraints in CI environment" fi # Final verification with simpler queries log "Running final verification queries..." local final_total=$(run_sql "SELECT COUNT(*) FROM stress_docs;" 2>/dev/null | tail -1 | tr -d ' ' || echo "0") if [[ ! "$final_total" =~ ^[0-9]+$ ]]; then final_total="unknown" fi log "✅ High-load stress test completed: $final_total total documents, $completed/$num_concurrent_sessions sessions successful" # Re-enable exit on error set -e } # Test 7: Concurrent index drop with cached state test_concurrent_index_drop() { log "Test 7: Concurrent index drop with cached local state" # Create a test table and index run_sql "CREATE TABLE IF NOT EXISTS drop_test (id SERIAL PRIMARY KEY, content TEXT);" run_sql "INSERT INTO drop_test (content) VALUES ('first test document'), ('second test document'), ('third test document'), ('fourth test document'), ('fifth test document');" run_sql "CREATE INDEX drop_test_idx ON drop_test USING bm25 (content) WITH (text_config = 'english');" # Session 1: Access the index to cache it locally, then hold connection info "Session 1: Caching index state and holding connection..." local session1_output="${TMP_DIR:-/tmp}/session_501.out" { echo "-- First query to cache the index" echo "SELECT COUNT(*) FROM drop_test WHERE content <@> to_bm25query('test', 'drop_test_idx') < -0.01;" echo "SELECT pg_sleep(3);" # Hold connection while index is dropped echo "-- Second query after index should be dropped" echo "SELECT COUNT(*) FROM drop_test WHERE content <@> to_bm25query('test', 'drop_test_idx') < -0.01;" # Try to use dropped index } | psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" > "$session1_output" 2>&1 & local session1_pid=$! # Give session 1 time to cache the index sleep 1 # Session 2: Drop the index info "Session 2: Dropping the index while session 1 has it cached..." run_sql "DROP INDEX drop_test_idx;" # Wait for session 1 to complete wait $session1_pid 2>/dev/null || true # Check if session 1 handled the drop gracefully local session1_contents=$(cat "$session1_output" 2>/dev/null || echo "") # Look for errors in the output if echo "$session1_contents" | grep -q "ERROR"; then # This is expected - the index was dropped log "✅ Session 1 correctly reported error when using dropped index" info "Error details: $(echo "$session1_contents" | grep ERROR | head -1)" elif echo "$session1_contents" | grep -q "count"; then # This shouldn't happen - queries should error when index is dropped log "❌ FAIL: Session 1 completed queries without error after index drop" info "This is incorrect behavior - should error when index doesn't exist" info "Query results: $(echo "$session1_contents" | grep -E "count|^\s*[0-9]")" error "Test failed - queries should error when using dropped index" else log "Unexpected output from Session 1" info "Full output: $session1_contents" fi # Clean up run_sql "DROP TABLE IF EXISTS drop_test CASCADE;" log "✅ Concurrent index drop test completed" } # Test 8: Use-after-free bug - reader accessing posting list during reallocation test_read_during_posting_list_growth() { log "Test 8: Concurrent read during posting list reallocation" # Create a table with exactly 10 documents containing "racecondition" term # This is below the initial posting list capacity of 16 run_sql "CREATE TABLE IF NOT EXISTS race_test (id SERIAL PRIMARY KEY, content TEXT);" for i in $(seq 1 10); do run_sql "INSERT INTO race_test (content) VALUES ('document $i with racecondition term');" done # Create index - posting list for "racecondition" will have capacity 16, count 10 run_sql "CREATE INDEX race_test_idx ON race_test USING bm25(content) WITH (text_config='english');" info "Created index with 10 documents for 'racecondition' (below capacity 16)" # Start a slow SELECT that reads through the posting list # Use pg_sleep to make it slow enough for concurrent inserts to happen local session1_output="${TMP_DIR:-/tmp}/session_race_reader.out" local session1_pid { # This query will scan the posting list for "racecondition" # ORDER BY forces it to read all results before returning echo "SELECT id, content, pg_sleep(0.01) FROM race_test WHERE content <@> to_bm25query('racecondition', 'race_test_idx') < -0.01 ORDER BY content <@> to_bm25query('racecondition', 'race_test_idx');" } | psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" > "$session1_output" 2>&1 & session1_pid=$! info "Reader: Started SELECT scanning posting list (PID: $session1_pid)" # Give reader time to start scanning sleep 0.05 # Now rapidly insert 25 more documents with the same term # Document 17 will trigger reallocation from capacity 16 -> 32 # This frees the old array that the reader may still be accessing info "Writer: Inserting 25 documents to trigger posting list reallocation..." for i in $(seq 11 35); do run_sql_quiet "INSERT INTO race_test (content) VALUES ('document $i with racecondition term');" done info "Writer: Inserts complete (should have triggered reallocation at doc 17)" # Wait for reader to complete (or crash) local reader_exit_code=0 wait $session1_pid 2>/dev/null || reader_exit_code=$? # Check results local session1_contents=$(cat "$session1_output" 2>/dev/null || echo "") if [ $reader_exit_code -ne 0 ]; then error "❌ CRASH: Reader process crashed during concurrent writes (exit code: $reader_exit_code)" info "Reader output: $session1_contents" info "This likely indicates use-after-free when posting list was reallocated" return 1 elif echo "$session1_contents" | grep -qi "ERROR\|FATAL\|server closed\|terminated"; then error "❌ ERROR: Reader encountered error during concurrent writes" info "Reader output: $session1_contents" return 1 else # Count results returned local result_count=$(echo "$session1_contents" | grep -c "racecondition" || echo "0") info "Reader completed successfully: $result_count results returned" if [ "$result_count" -ge 10 ]; then log "✅ Read during posting list growth test passed (no crash)" else warn "Reader returned fewer results than expected: $result_count (expected >= 10)" fi fi # Clean up run_sql "DROP TABLE IF EXISTS race_test CASCADE;" } # Test 9: Multi-reallocation race - long write transaction with concurrent reads test_long_transaction_reallocation_race() { log "Test 9: Long write transaction with multiple reallocations during concurrent reads" # Create table and index run_sql "CREATE TABLE IF NOT EXISTS chicken_race (id SERIAL PRIMARY KEY, content TEXT);" run_sql "CREATE INDEX chicken_idx ON chicken_race USING bm25(content) WITH (text_config='english');" info "Starting long write transaction that will trigger multiple posting list reallocations..." # Start a LONG write transaction that inserts 10000 documents with "chicken" # This will trigger reallocations at documents: 17, 33, 65, 129, 257, 513, 1025, 2049, 4097, 8193 local writer_output="${TMP_DIR:-/tmp}/chicken_writer.out" { echo "BEGIN;" echo "SELECT pg_sleep(0.1);" # Give readers time to start for i in $(seq 1 10000); do echo "INSERT INTO chicken_race (content) VALUES ('document $i about chicken farming');" # Small delay between inserts to increase chance of reader overlap if [ $((i % 5)) -eq 0 ]; then echo "SELECT pg_sleep(0.01);" fi done echo "COMMIT;" } | psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" > "$writer_output" 2>&1 & local writer_pid=$! info "Writer: Long transaction started (PID: $writer_pid), will insert 10000 documents with 'chicken'" # Give writer time to start and insert first few documents sleep 0.2 # Now start 3 concurrent readers that repeatedly query "chicken" # CRITICAL: Use ORDER BY to trigger index scan which accesses posting list entries! # WHERE clause only accesses doc_count, but ORDER BY actually reads entries_dp array local reader_pids=() declare -a reader_pids for reader_id in 1 2 3; do local reader_output="${TMP_DIR:-/tmp}/chicken_reader_${reader_id}.out" { for query_num in $(seq 1 200); do echo "SELECT id FROM chicken_race ORDER BY content <@> to_bm25query('chicken', 'chicken_idx') LIMIT 10;" sleep 0.01 # Small delay between queries done } | psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" > "$reader_output" 2>&1 & local last_pid=$!; reader_pids+=($last_pid); info "Reader $reader_id: Started (PID: $last_pid), will query 'chicken' 200 times" done # Wait for writer to complete local writer_exit_code=0 wait $writer_pid 2>/dev/null || writer_exit_code=$? if [ $writer_exit_code -ne 0 ]; then error "❌ Writer transaction failed (exit code: $writer_exit_code)" cat "$writer_output" return 1 fi info "Writer: Completed successfully" # Wait for all readers and check for crashes local reader_failures=0 local reader_idx=0 for reader_pid in "${reader_pids[@]}"; do reader_idx=$((reader_idx + 1)) local reader_exit_code=0 wait $reader_pid 2>/dev/null || reader_exit_code=$? local reader_output="${TMP_DIR:-/tmp}/chicken_reader_${reader_idx}.out" local reader_contents=$(cat "$reader_output" 2>/dev/null || echo "") if [ $reader_exit_code -ne 0 ]; then error "❌ CRASH: Reader $reader_idx crashed (exit code: $reader_exit_code)" info "Reader $reader_idx output: $reader_contents" reader_failures=$((reader_failures + 1)) elif echo "$reader_contents" | grep -qi "ERROR\|FATAL\|server closed\|terminated"; then error "❌ ERROR: Reader $reader_idx encountered error" info "Reader $reader_idx output: $reader_contents" reader_failures=$((reader_failures + 1)) else info "Reader $reader_idx: Completed successfully" fi done if [ $reader_failures -gt 0 ]; then error "❌ Test FAILED: $reader_failures reader(s) crashed or encountered errors" info "This indicates use-after-free when posting list was reallocated during concurrent reads" return 1 fi # Verify final state local final_count=$(run_sql "SELECT COUNT(*) FROM chicken_race;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ') if [ "$final_count" -eq 10000 ]; then log "✅ Long transaction reallocation race test passed: All readers survived, $final_count documents inserted" else warn "Unexpected document count: $final_count (expected 10000)" fi # Clean up run_sql "DROP TABLE IF EXISTS chicken_race CASCADE;" } run_concurrent_tests() { log "Starting comprehensive concurrent stress tests for pg_textsearch extension" test_concurrent_inserts test_concurrent_index_creation test_concurrent_mixed_operations test_string_interning_consistency test_update_delete_concurrency test_high_load_stress test_concurrent_index_drop test_read_during_posting_list_growth test_long_transaction_reallocation_race # Final system state check log "Final system state verification:" local total_docs=$(run_sql "SELECT COUNT(*) FROM stress_docs;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") local unique_sessions=$(run_sql "SELECT COUNT(DISTINCT session_id) FROM stress_docs WHERE session_id IS NOT NULL;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") info "Total documents: $total_docs" info "Unique sessions: $unique_sessions" # Test a few searches to make sure index is still functional local search1=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('database', 'stress_main_idx')) < -0.01;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") local search2=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('concurrent', 'stress_main_idx')) < -0.01;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") local search3=$(run_sql "SELECT COUNT(*) FROM stress_docs WHERE (content <@> to_bm25query('performance stress', 'stress_main_idx')) < -0.01;" | grep -E "^\s*[0-9]+\s*$" | tr -d ' ' || echo "0") info "Final search verification:" info " 'database': $search1 results" info " 'concurrent': $search2 results" info " 'performance stress': $search3 results" } main() { log "Starting pg_textsearch concurrent stress testing..." # Check prerequisites command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found in PATH" command -v psql >/dev/null 2>&1 || error "psql not found in PATH" setup_test_db run_concurrent_tests log "🎉 All concurrent stress tests completed successfully!" log "The pg_textsearch extension handled concurrent operations correctly." exit 0 } # Only run if executed directly if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/concurrent_duplicate_read.sh000077500000000000000000000166271525762047700272110ustar00rootroot00000000000000#!/bin/bash # # Regression test for duplicate CTIDs during concurrent top-k index scans. # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55459 TEST_DB=concurrent_duplicate_read_test DATA_DIR="${SCRIPT_DIR}/../tmp_concurrent_duplicate_read" LOGFILE="${DATA_DIR}/postgres.log" ERR_DIR="${DATA_DIR}/client_logs" # Tunables. QUERY_LIMIT=3000 # top-k limit; must be < corpus to force re-execution SEED_ROWS=3500 # initial corpus, just above QUERY_LIMIT DURATION=45 # seconds of concurrent insert + read activity N_INSERTERS=6 N_READERS=2 # Per-session safety nets for long-running CALL/DO statements. STMT_TIMEOUT="$((DURATION + 30))s" LOCK_TIMEOUT="30s" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } cleanup() { local exit_code=$? log "Cleaning up (exit code: $exit_code)..." jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 mkdir -p "${ERR_DIR}" cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' shared_preload_libraries = 'pg_textsearch' logging_collector = on log_directory = '.' log_filename = 'postgres.log' # Keep this focused on scan re-execution, not VACUUM races. autovacuum = off # Churn the index under concurrent inserts. pg_textsearch.memtable_pages_threshold = 2 pg_textsearch.segments_per_level = 2 pg_textsearch.bulk_load_threshold = 0 EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w -o "-p ${TEST_PORT}" || \ error "Failed to start PostgreSQL" createdb -h "${DATA_DIR}" -p ${TEST_PORT} ${TEST_DB} psql -h "${DATA_DIR}" -p ${TEST_PORT} -d ${TEST_DB} \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null log "Test database ready" } PSQL="psql -h ${DATA_DIR} -p ${TEST_PORT} -d ${TEST_DB} -qAt -v ON_ERROR_STOP=1" seed_data() { log "Creating index, seeding ${SEED_ROWS} docs, and defining helpers..." $PSQL </dev/null SET client_min_messages=warning; CREATE TABLE docs (id bigserial PRIMARY KEY, content text NOT NULL); CREATE INDEX docs_idx ON docs USING bm25(content) WITH (text_config='english'); -- Match the whole corpus while preserving some score variation. INSERT INTO docs(content) SELECT 'alpha w' || (gs % 50) FROM generate_series(1, ${SEED_ROWS}) gs; -- Reader-visible failure signal. CREATE TABLE dup_signal (detected_at timestamptz, extra int, sample_query text); -- Commit frequently so readers can re-score a changing corpus. CREATE PROCEDURE insert_docs(seconds int) LANGUAGE plpgsql AS \$\$ DECLARE deadline timestamptz := clock_timestamp() + make_interval(secs => seconds); i bigint := 0; BEGIN LOOP INSERT INTO docs(content) VALUES ('alpha w' || (i % 50)); i := i + 1; IF i % 5 = 0 THEN COMMIT; END IF; EXIT WHEN clock_timestamp() > deadline; END LOOP; COMMIT; END \$\$; SQL # This test must exercise ORDER BY ... LIMIT via index scan. local plan plan=$($PSQL -c "EXPLAIN (COSTS off) SELECT ctid FROM docs ORDER BY content <@> to_bm25query('alpha', 'docs_idx') LIMIT ${QUERY_LIMIT}") if ! echo "$plan" | grep -qi "Index Scan"; then warn "Reader plan was not an Index Scan:" echo "$plan" error "TEST SETUP FAILED: reader no longer exercises the index-scan path" fi } inserter() { PGOPTIONS="-c statement_timeout=${STMT_TIMEOUT} -c lock_timeout=${LOCK_TIMEOUT}" \ $PSQL -c "CALL insert_docs(${DURATION});" \ >>"${ERR_DIR}/inserter_$1.log" 2>&1 || return 10 } # Repeatedly check each top-k result set for duplicate CTIDs. reader() { local tag=$1 PGOPTIONS="-c statement_timeout=${STMT_TIMEOUT} -c lock_timeout=${LOCK_TIMEOUT}" \ $PSQL >>"${ERR_DIR}/reader_${tag}.log" 2>&1 < ${DURATION}); BEGIN LOOP SELECT count(*) - count(DISTINCT ctid) INTO d FROM ( SELECT ctid FROM docs ORDER BY content <@> to_bm25query('alpha', 'docs_idx') LIMIT ${QUERY_LIMIT} ) s; IF d > 0 THEN INSERT INTO dup_signal(detected_at, extra, sample_query) VALUES (clock_timestamp(), d, 'reader ${tag}'); RAISE WARNING 'DUPLICATE DETECTED by reader ${tag}: % extra row(s)', d; RETURN; END IF; EXIT WHEN clock_timestamp() > deadline; END LOOP; END \$\$; SQL } run_test() { log "Running ${N_INSERTERS} inserters + ${N_READERS} readers for ~${DURATION}s..." local pids=() for i in $(seq 1 ${N_INSERTERS}); do inserter "$i" & pids+=("$!") done for i in $(seq 1 ${N_READERS}); do reader "$i" & pids+=("$!") done local failed=0 for p in "${pids[@]}"; do wait "$p" || { warn "a client (pid $p) exited non-zero"; failed=1; } done local dups if ! dups=$($PSQL -c "SELECT count(*) FROM dup_signal;"); then error "TEST FAILED: could not query dup_signal (server down?)" fi if [ "${dups:-0}" -gt 0 ]; then warn "Duplicate ctids observed in a single index-scan result set:" $PSQL -c "SELECT detected_at, extra, sample_query FROM dup_signal;" || true error "TEST FAILED: BM25 index scan returned the same row more than once" fi if [ "$failed" -ne 0 ]; then warn "Client logs:" tail -n 20 "${ERR_DIR}"/*.log 2>/dev/null || true error "TEST FAILED: a concurrent client exited with an error" fi log "TEST PASSED: no duplicate ctids across concurrent top-k reads" } # Quiescent sanity checks after the concurrent phase. verify_postrun() { local corpus expected total distinct corpus=$($PSQL -c "SELECT count(*) FROM docs;") # Ensure readers could hit the limit-doubling path. if [ "${corpus:-0}" -le "${QUERY_LIMIT}" ]; then error "TEST INCONCLUSIVE: corpus (${corpus}) did not exceed QUERY_LIMIT (${QUERY_LIMIT}); re-execution path not exercised" fi # Static top-k should be complete and duplicate-free. expected=${QUERY_LIMIT} read -r total distinct <<<"$($PSQL -c " SELECT count(*), count(DISTINCT ctid) FROM ( SELECT ctid FROM docs ORDER BY content <@> to_bm25query('alpha', 'docs_idx') LIMIT ${QUERY_LIMIT} ) s;" | tr '|' ' ')" if [ "${total}" != "${expected}" ]; then error "TEST FAILED: top-k returned ${total} rows, expected ${expected} (corpus=${corpus}) -- under-return/miss" fi if [ "${distinct}" != "${total}" ]; then error "TEST FAILED: top-k returned ${total} rows but only ${distinct} distinct ctids -- duplicate" fi log "POST-RUN PASSED: top-k returns ${total} distinct rows on a ${corpus}-row corpus" } # Main setup_test_db seed_data run_test verify_postrun log "All tests passed!" exit 0 timescale-pg_textsearch-b63fe58/test/scripts/crash_safety_spill.sh000077500000000000000000000126641525762047700256550ustar00rootroot00000000000000#!/bin/bash # # Crash safety test for memtable spill ordering # # Verifies that the finalize-before-mark-dead ordering prevents FSM # corruption when a crash occurs during spill. After recovery, VACUUM # should not free pages still in the live chain, and subsequent # inserts/spills should not corrupt the index. # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55437 TEST_DB=crash_safety_spill_test DATA_DIR="${SCRIPT_DIR}/../tmp_crash_safety_spill" LOGFILE="${DATA_DIR}/postgres.log" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } cleanup() { log "Cleaning up..." if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} shared_preload_libraries = 'pg_textsearch' logging_collector = on log_directory = '.' log_filename = 'postgres.log' fsync = on full_page_writes = on wal_level = replica max_wal_senders = 2 EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w -o "-p ${TEST_PORT}" || \ error "Failed to start PostgreSQL" createdb -p ${TEST_PORT} ${TEST_DB} psql -p ${TEST_PORT} -d ${TEST_DB} </dev/null CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; EOF log "Test database ready" } run_test() { log "Creating test table and index..." psql -p ${TEST_PORT} -d ${TEST_DB} </dev/null CREATE TABLE docs (id serial PRIMARY KEY, content text); CREATE INDEX docs_bm25 ON docs USING bm25(content) WITH (text_config = 'english'); EOF log "Inserting initial data..." psql -p ${TEST_PORT} -d ${TEST_DB} </dev/null INSERT INTO docs (content) SELECT 'document number ' || i || ' with some searchable content words' FROM generate_series(1, 1000) i; EOF log "Forcing spill to create segment..." psql -p ${TEST_PORT} -d ${TEST_DB} -c "SELECT bm25_spill_index('docs_bm25');" >/dev/null log "Verifying initial data..." local count1=$(psql -p ${TEST_PORT} -d ${TEST_DB} -t -c \ "SELECT count(*) FROM (SELECT 1 FROM docs ORDER BY content <@> to_bm25query('document', 'docs_bm25') LIMIT 1000) sub;") count1=$(echo $count1 | tr -d ' ') [ "$count1" -eq "1000" ] || error "Initial count mismatch: expected 1000, got $count1" log "Initial verification passed: $count1 documents" log "Inserting data and triggering crash during spill (same transaction)..." local crash_output="${DATA_DIR}/crash_output.log" if psql -p ${TEST_PORT} -d ${TEST_DB} >"${crash_output}" 2>&1 </dev/null || true sleep 1 # Clean up stale pid file if still present if [ -f "${DATA_DIR}/postmaster.pid" ]; then local old_pid=$(head -1 "${DATA_DIR}/postmaster.pid") if ! kill -0 $old_pid 2>/dev/null; then rm -f "${DATA_DIR}/postmaster.pid" else # Force kill if still running kill -9 $old_pid 2>/dev/null || true sleep 1 rm -f "${DATA_DIR}/postmaster.pid" fi fi log "Restarting after crash..." pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w -o "-p ${TEST_PORT}" || \ error "Failed to restart PostgreSQL after crash" log "Running VACUUM to trigger FSM reclaim..." psql -p ${TEST_PORT} -d ${TEST_DB} -c "VACUUM docs;" >/dev/null log "Inserting more data after recovery..." psql -p ${TEST_PORT} -d ${TEST_DB} </dev/null INSERT INTO docs (content) SELECT 'post recovery document ' || i || ' additional searchable text' FROM generate_series(1, 500) i; EOF log "Forcing another spill..." psql -p ${TEST_PORT} -d ${TEST_DB} -c "SELECT bm25_spill_index('docs_bm25');" >/dev/null log "Verifying data integrity..." # 1000 initial + 500 post-recovery = 1500 # The 500 pre-crash docs are rolled back (same transaction as PANIC) local count2=$(psql -p ${TEST_PORT} -d ${TEST_DB} -t -c \ "SELECT count(*) FROM (SELECT 1 FROM docs ORDER BY content <@> to_bm25query('document', 'docs_bm25') LIMIT 2000) sub;") count2=$(echo $count2 | tr -d ' ') [ "$count2" -eq "1500" ] || error "Final count mismatch: expected 1500, got $count2" log "Final verification passed: $count2 documents" log "TEST PASSED: Crash safety spill ordering verified" } setup_test_db run_test log "All tests passed!" exit 0 timescale-pg_textsearch-b63fe58/test/scripts/cross_database_registry.sh000077500000000000000000000625671525762047700267130ustar00rootroot00000000000000#!/bin/bash # # Regression test for issue #464: index OIDs are database-local, while the # pg_textsearch registry is cluster-wide. Template clones make matching OIDs # deterministic and let this test verify that registry and cache state remain # independent across databases. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_DIR="$(cd "${SCRIPT_DIR}/.." && pwd -P)" TEST_PORT="${TEST_PORT:-55463}" TEST_HOST=127.0.0.1 TOP_K=3 SEED_DB=pg_textsearch_registry_seed DB_A=pg_textsearch_registry_a DB_B=pg_textsearch_registry_b DB_DROP=pg_textsearch_registry_drop DB_FORCE=pg_textsearch_registry_force DATA_DIR="${TEST_DIR}/tmp_cross_database_registry_${TEST_PORT}_$$" LOGFILE="${DATA_DIR}/postgres.log" PRESERVED_LOGFILE="${TEST_DIR}/tmp_cross_database_registry_${TEST_PORT}_$$_postgres.log" COMMAND_TIMEOUT_SECONDS=20 HOLDER_TIMEOUT_SECONDS=60 PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG}" --bindir)" RED='\033[0;31m' GREEN='\033[0;32m' BLUE='\033[0;34m' NC='\033[0m' PASS_COUNT=0 FAIL_COUNT=0 DATA_DIR_OWNED=0 DATA_DIR_ID= HOLDER_PID= HOLDER_APPLICATION_NAME= log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } pass() { PASS_COUNT=$((PASS_COUNT + 1)) log "PASS: $1" } fail() { FAIL_COUNT=$((FAIL_COUNT + 1)) echo -e "${RED}[$(date '+%H:%M:%S')] FAIL: $1${NC}" } postmaster_pid_is_alive() { local pid data_dir_is_owned || return 1 [ -f "${DATA_DIR}/postmaster.pid" ] || return 1 pid=$(sed -n '1p' "${DATA_DIR}/postmaster.pid") [[ "${pid}" =~ ^[0-9]+$ ]] || return 1 kill -0 "${pid}" 2>/dev/null } data_dir_is_owned() { local current_id [ "${DATA_DIR_OWNED}" -eq 1 ] || return 1 [ -d "${DATA_DIR}" ] && [ ! -L "${DATA_DIR}" ] || return 1 current_id=$(stat -c '%d:%i' -- "${DATA_DIR}") || return 1 [ "${current_id}" = "${DATA_DIR_ID}" ] } postmaster_matches_data_dir() { local pid_data_dir data_dir_is_owned || return 1 [ -f "${DATA_DIR}/postmaster.pid" ] || return 1 pid_data_dir=$(sed -n '2p' "${DATA_DIR}/postmaster.pid") [ "${pid_data_dir}" = "${DATA_DIR}" ] || return 1 "${PGBINDIR}/pg_ctl" status -D "${DATA_DIR}" >/dev/null 2>&1 } preserve_failure_log() { data_dir_is_owned || return 1 ln -T -- "${LOGFILE}" "${PRESERVED_LOGFILE}" } verify_test_postmaster() { local actual_data_dir if ! postmaster_matches_data_dir; then fail "PostgreSQL postmaster is not running from ${DATA_DIR}" return 1 fi if ! actual_data_dir=$( "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d postgres -v ON_ERROR_STOP=1 -tA \ -c "SHOW data_directory;" 2>&1 ); then fail "Could not verify the test postmaster: ${actual_data_dir}" return 1 fi if [ "${actual_data_dir}" != "${DATA_DIR}" ]; then fail "Port ${TEST_PORT} belongs to ${actual_data_dir}, not ${DATA_DIR}" return 1 fi } cleanup() { local exit_code=$? local preserve_data_dir=0 trap - EXIT trap '' INT TERM set +e if ! stop_database_holder; then fail "Could not stop the active database-holder process" exit_code=1 fi if [ "${DATA_DIR_OWNED}" -eq 0 ]; then exit "${exit_code}" fi if ! data_dir_is_owned; then fail "Owned PGDATA identity changed; skipping cleanup" exit 1 fi log "Cleaning up cross-database registry test (exit code: ${exit_code})..." if postmaster_matches_data_dir; then if ! timeout --signal=TERM --kill-after=2s \ "${COMMAND_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m fast -w; then info "Fast shutdown failed; trying immediate shutdown" if ! timeout --signal=TERM --kill-after=2s \ "${COMMAND_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" \ -m immediate -w; then fail "PostgreSQL shutdown failed" preserve_data_dir=1 exit_code=1 fi fi elif postmaster_pid_is_alive; then fail "Cannot confirm ownership of the live postmaster in ${DATA_DIR}" preserve_data_dir=1 exit_code=1 fi if [ "${preserve_data_dir}" -eq 0 ] && { postmaster_matches_data_dir || postmaster_pid_is_alive; }; then fail "PostgreSQL postmaster is still running after shutdown" preserve_data_dir=1 exit_code=1 fi if [ "${preserve_data_dir}" -eq 0 ] && [ "${exit_code}" -ne 0 ] && [ -f "${LOGFILE}" ]; then if preserve_failure_log; then info "Preserved server log: ${PRESERVED_LOGFILE}" else fail "Could not safely preserve the server log; preserving ${DATA_DIR}" preserve_data_dir=1 exit_code=1 fi fi if [ "${preserve_data_dir}" -ne 0 ]; then info "Preserved PGDATA and server log: ${DATA_DIR}" elif ! data_dir_is_owned; then fail "Owned PGDATA identity changed; skipping removal" exit_code=1 else rm -rf "${DATA_DIR}" fi exit "${exit_code}" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM run_sql_quiet() { local database="$1" local sql="$2" timeout --signal=TERM --kill-after=2s "${COMMAND_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${database}" -v ON_ERROR_STOP=1 -q -c "${sql}" >/dev/null } run_sql_value() { local database="$1" local sql="$2" timeout --signal=TERM --kill-after=2s "${COMMAND_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${database}" -v ON_ERROR_STOP=1 -tA -c "${sql}" 2>&1 } run_createdb() { timeout --signal=TERM --kill-after=2s "${COMMAND_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/createdb" -h "${TEST_HOST}" -p "${TEST_PORT}" "$@" } run_dropdb() { timeout --signal=TERM --kill-after=2s "${COMMAND_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/dropdb" -h "${TEST_HOST}" -p "${TEST_PORT}" "$@" } holder_process_is_running() { local state [ -n "${HOLDER_PID}" ] || return 1 state=$(ps -o stat= -p "${HOLDER_PID}" 2>/dev/null) || return 1 [[ "${state}" != Z* ]] } wait_for_holder_exit() { local deadline=$((SECONDS + 10)) local holder_pid="${HOLDER_PID}" [ -n "${holder_pid}" ] || return 0 while holder_process_is_running; do if [ "${SECONDS}" -ge "${deadline}" ]; then return 1 fi sleep 0.1 done wait "${holder_pid}" 2>/dev/null || true HOLDER_PID= HOLDER_APPLICATION_NAME= } start_database_holder() { local database="$1" local application_name="$2" local connected if [ -n "${HOLDER_PID}" ]; then fail "Refusing to replace an active database-holder process" return 1 fi PGAPPNAME="${application_name}" \ timeout --signal=TERM --kill-after=2s \ "${HOLDER_TIMEOUT_SECONDS}s" \ "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${database}" -v ON_ERROR_STOP=1 -q \ -c "SELECT pg_sleep(300);" >/dev/null 2>&1 & HOLDER_PID=$! HOLDER_APPLICATION_NAME="${application_name}" for _ in $(seq 1 100); do connected=$(run_sql_value postgres " SELECT count(*) FROM pg_stat_activity WHERE datname = '${database}' AND application_name = '${application_name}'; ") if [ "${connected}" = "1" ]; then pass "${database} has an active test connection" return 0 fi if ! holder_process_is_running; then wait_for_holder_exit || true fail "${database} test connection exited before registering" return 1 fi sleep 0.05 done fail "${database} test connection did not register" return 1 } stop_database_holder() { local holder_pid="${HOLDER_PID}" [ -n "${holder_pid}" ] || return 0 if postmaster_matches_data_dir && [ -n "${HOLDER_APPLICATION_NAME}" ]; then run_sql_quiet postgres " SELECT pg_terminate_backend(pid) FROM pg_stat_activity WHERE application_name = '${HOLDER_APPLICATION_NAME}'; " || true fi if wait_for_holder_exit; then return 0 fi kill "${holder_pid}" 2>/dev/null || true if wait_for_holder_exit; then return 0 fi kill -KILL "${holder_pid}" 2>/dev/null || true wait "${holder_pid}" 2>/dev/null || true HOLDER_PID= HOLDER_APPLICATION_NAME= return 1 } assert_equals() { local label="$1" local expected="$2" local actual="$3" if [ "${actual}" = "${expected}" ]; then pass "${label}" else fail "${label}: expected '${expected}', got '${actual}'" fi } cache_cold_build() { local database="$1" cache_cold_build_index "${database}" docs_idx } cache_cold_build_index() { local database="$1" local index_name="$2" run_sql_value "${database}" " SELECT result || '|' || records_applied || '|' || cursor_seq || '|' || estimated_bytes FROM bm25_cache_cold_build('${index_name}'); " } cache_apply() { local database="$1" cache_apply_index "${database}" docs_idx } cache_apply_index() { local database="$1" local index_name="$2" run_sql_value "${database}" " SELECT result || '|' || records_applied || '|' || cursor_seq || '|' || estimated_bytes FROM bm25_cache_apply_to_tail('${index_name}'); " } top_hits() { local database="$1" local term="$2" run_sql_value "${database}" " SELECT count(*) || '|' || bool_and(body LIKE '${term} marker %') || '|' || string_agg(id::text, ',' ORDER BY score, id) FROM ( SELECT id, body, body <@> to_bm25query('${term}', 'docs_idx') AS score FROM docs ORDER BY body <@> to_bm25query('${term}', 'docs_idx') LIMIT ${TOP_K} ) hits " } setup_cluster() { log "Setting up isolated PostgreSQL cluster..." if ! mkdir -m 700 -- "${DATA_DIR}" 2>/dev/null; then fail "Refusing to replace existing PGDATA: ${DATA_DIR}" return 1 fi if ! DATA_DIR_ID=$(stat -c '%d:%i' -- "${DATA_DIR}"); then fail "Could not record ownership of PGDATA: ${DATA_DIR}" return 1 fi DATA_DIR_OWNED=1 if ! "${PGBINDIR}/initdb" -D "${DATA_DIR}" \ --auth-local=trust --auth-host=trust >/dev/null 2>&1; then fail "initdb failed" return 1 fi cat >>"${DATA_DIR}/postgresql.conf" </dev/null 2>&1 || { fail "pg_ctl not found" return 1 } command -v timeout >/dev/null 2>&1 || { fail "timeout not found" return 1 } setup_cluster || return 1 oid_a=$(run_sql_value "${DB_A}" "SELECT 'docs_idx'::regclass::oid;") oid_b=$(run_sql_value "${DB_B}" "SELECT 'docs_idx'::regclass::oid;") if [[ "${oid_a}" =~ ^[0-9]+$ ]] && [ "${oid_a}" = "${oid_b}" ]; then pass "template clones have matching BM25 index OIDs (${oid_a})" else fail "matching-OID setup failed: db_a=${oid_a}, db_b=${oid_b}" fi run_sql_quiet "${DB_A}" " INSERT INTO docs(body) SELECT 'alpha marker ' || lpad(g::text, 2, '0') || repeat(' alpha', 10 - g) FROM generate_series(1, 4) g; " run_sql_quiet "${DB_B}" " INSERT INTO docs(body) SELECT 'bravo marker ' || lpad(g::text, 2, '0') || repeat(' bravo', 10 - g) FROM generate_series(1, 9) g; " cold_a=$(cache_cold_build "${DB_A}") IFS='|' read -r result records a_cursor a_bytes <<<"${cold_a}" assert_equals "database A cold-build result" "OK" "${result}" assert_equals "database A cache record count" "5" "${records}" if [[ "${a_bytes}" =~ ^[0-9]+$ ]] && [ "${a_bytes}" -gt 0 ]; then pass "database A cache has independent nonzero accounting" else fail "database A cache accounting is invalid: '${a_bytes}'" fi cold_b=$(cache_cold_build "${DB_B}") IFS='|' read -r result records b_cursor b_bytes <<<"${cold_b}" assert_equals "database B cold-build is independent" "OK" "${result}" assert_equals "database B cache record count" "10" "${records}" if [[ "${b_bytes}" =~ ^[0-9]+$ ]] && [ "${b_bytes}" -gt 0 ]; then pass "database B cache has independent nonzero accounting" else fail "database B cache accounting is invalid: '${b_bytes}'" fi a_after_b_cache=$(cache_apply "${DB_A}") assert_equals "database B cache build leaves A cache unchanged" \ "OK|0|${a_cursor}|${a_bytes}" "${a_after_b_cache}" global_a=$(run_sql_value "${DB_A}" \ "SELECT bm25_cache_global_estimated_bytes();") global_b=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();") assert_equals "global cache accounting is cluster-wide" \ "${global_a}" "${global_b}" if [[ "${a_bytes}" =~ ^[0-9]+$ ]] && [[ "${b_bytes}" =~ ^[0-9]+$ ]]; then expected_global=$((a_bytes + b_bytes)) assert_equals "global accounting includes both databases" \ "${expected_global}" "${global_a}" else fail "cannot validate global accounting: A=${a_bytes}, B=${b_bytes}" fi a_top_hits=$(top_hits "${DB_A}" "alpha") b_top_hits=$(top_hits "${DB_B}" "bravo") assert_equals "database A returns its ranked top ${TOP_K}" \ "3|true|2,3,4" "${a_top_hits}" assert_equals "database B returns its ranked top ${TOP_K}" \ "3|true|2,3,4" "${b_top_hits}" # The eviction budget remains cluster-wide. With B as the caller, A is # the only eligible victim even though both indexes have the same OID. evict_result=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_evict_largest('docs_idx');") assert_equals "cross-database eviction distinguishes the caller key" \ "evicted" "${evict_result}" a_after_evict=$(cache_apply "${DB_A}") b_after_evict=$(cache_apply "${DB_B}") assert_equals "cross-database eviction selected database A" \ "NOT_INITIALIZED|0|0|0" "${a_after_evict}" assert_equals "cross-database eviction preserved caller B" \ "OK|0|${b_cursor}|${b_bytes}" "${b_after_evict}" global_after_evict=$(run_sql_value "${DB_A}" \ "SELECT bm25_cache_global_estimated_bytes();") assert_equals "cross-database eviction drains global accounting" \ "${b_bytes}" "${global_after_evict}" cold_a=$(cache_cold_build "${DB_A}") IFS='|' read -r result records a_cursor a_bytes <<<"${cold_a}" assert_equals "database A cache rebuild after eviction" "OK" "${result}" assert_equals "database A rebuilt cache record count" "5" "${records}" a_before_reindex=$(cache_apply "${DB_A}") if run_sql_quiet "${DB_B}" "REINDEX INDEX docs_idx;"; then pass "database B REINDEX completed" else fail "database B REINDEX failed" fi a_after_reindex=$(cache_apply "${DB_A}") assert_equals "database B REINDEX leaves A registry/cache state unchanged" \ "${a_before_reindex}" "${a_after_reindex}" a_top_hits=$(top_hits "${DB_A}" "alpha") b_top_hits=$(top_hits "${DB_B}" "bravo") assert_equals "database A top ${TOP_K} survives B REINDEX" \ "3|true|2,3,4" "${a_top_hits}" assert_equals "database B top ${TOP_K} survives its REINDEX" \ "3|true|2,3,4" "${b_top_hits}" baseline_global=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();") b_before_drop=$(cache_apply "${DB_B}") run_createdb --template="${DB_A}" "${DB_DROP}" run_sql_quiet "${DB_DROP}" " INSERT INTO docs(body) SELECT 'charlie marker ' || lpad(g::text, 2, '0') || repeat(' charlie', 10 - g) FROM generate_series(1, 6) g; CREATE TABLE extra_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX extra_docs_idx ON extra_docs USING bm25 (body) WITH (text_config = 'english'); INSERT INTO extra_docs(body) SELECT 'delta marker ' || lpad(g::text, 2, '0') || repeat(' delta', 8 - g) FROM generate_series(1, 4) g; " drop_cache=$(cache_cold_build "${DB_DROP}") IFS='|' read -r result records _ drop_bytes <<<"${drop_cache}" assert_equals "drop-test database cold-build result" "OK" "${result}" assert_equals "drop-test database cache record count" "11" "${records}" if [[ "${drop_bytes}" =~ ^[0-9]+$ ]] && [ "${drop_bytes}" -gt 0 ]; then pass "drop-test database cache has nonzero accounting" else fail "drop-test database cache accounting is invalid: '${drop_bytes}'" fi drop_extra_cache=$(cache_cold_build_index "${DB_DROP}" extra_docs_idx) IFS='|' read -r result records _ drop_extra_bytes <<<"${drop_extra_cache}" assert_equals "drop-test second index cold-build result" "OK" "${result}" assert_equals "drop-test second index cache record count" "4" "${records}" if [[ "${drop_extra_bytes}" =~ ^[0-9]+$ ]] && [ "${drop_extra_bytes}" -gt 0 ]; then pass "drop-test second index has nonzero accounting" else fail "drop-test second index accounting is invalid: '${drop_extra_bytes}'" fi global_with_drop=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();") assert_equals "drop-test database contributes both registry entries" \ "$((baseline_global + drop_bytes + drop_extra_bytes))" \ "${global_with_drop}" drop_before_failed=$(cache_apply "${DB_DROP}") drop_extra_before_failed=$(cache_apply_index "${DB_DROP}" extra_docs_idx) start_database_holder "${DB_DROP}" pgts_drop_holder set +e run_dropdb "${DB_DROP}" >/dev/null 2>&1 drop_status=$? set -e if [ "${drop_status}" -eq 0 ]; then fail "ordinary DROP DATABASE unexpectedly succeeded with an active connection" elif [ "${drop_status}" -eq 124 ] || [ "${drop_status}" -eq 137 ]; then fail "ordinary DROP DATABASE timed out instead of failing promptly" else pass "failed DROP DATABASE preserves the populated database" fi assert_equals "failed DROP DATABASE preserves global accounting" \ "${global_with_drop}" \ "$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();")" drop_after_failed=$(cache_apply "${DB_DROP}") assert_equals "failed DROP DATABASE preserves registry/cache state" \ "${drop_before_failed}" "${drop_after_failed}" drop_extra_after_failed=$(cache_apply_index "${DB_DROP}" extra_docs_idx) assert_equals "failed DROP DATABASE preserves every registry entry" \ "${drop_extra_before_failed}" "${drop_extra_after_failed}" stop_database_holder || fail "failed DROP DATABASE holder did not stop within 10 seconds" if run_dropdb "${DB_DROP}"; then pass "successful DROP DATABASE completed" else fail "successful DROP DATABASE failed" fi global_after_drop=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();") assert_equals "successful DROP DATABASE releases only its accounting" \ "${baseline_global}" "${global_after_drop}" assert_equals "successful DROP DATABASE removes both cache charges" \ "$((drop_bytes + drop_extra_bytes))" \ "$((global_with_drop - global_after_drop))" b_after_drop=$(cache_apply "${DB_B}") assert_equals "successful DROP DATABASE preserves database B cache" \ "${b_before_drop}" "${b_after_drop}" assert_equals "successful DROP DATABASE preserves database B queries" \ "3|true|2,3,4" "$(top_hits "${DB_B}" "bravo")" run_createdb --template="${DB_A}" "${DB_FORCE}" run_sql_quiet "${DB_FORCE}" " INSERT INTO docs(body) SELECT 'foxtrot marker ' || lpad(g::text, 2, '0') || repeat(' foxtrot', 12 - g) FROM generate_series(1, 7) g; CREATE TABLE force_extra_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX force_extra_docs_idx ON force_extra_docs USING bm25 (body) WITH (text_config = 'english'); INSERT INTO force_extra_docs(body) SELECT 'golf marker ' || lpad(g::text, 2, '0') || repeat(' golf', 8 - g) FROM generate_series(1, 4) g; " force_cache=$(cache_cold_build "${DB_FORCE}") IFS='|' read -r result records _ force_bytes <<<"${force_cache}" assert_equals "force-drop database cold-build result" "OK" "${result}" assert_equals "force-drop database cache record count" "12" "${records}" force_extra_cache=$( cache_cold_build_index "${DB_FORCE}" force_extra_docs_idx ) IFS='|' read -r result records _ force_extra_bytes <<<"${force_extra_cache}" assert_equals "force-drop second index cold-build result" "OK" "${result}" assert_equals "force-drop second index cache record count" "4" "${records}" if [[ "${force_bytes}" =~ ^[0-9]+$ ]] && [[ "${force_extra_bytes}" =~ ^[0-9]+$ ]] && [ "${force_bytes}" -gt 0 ] && [ "${force_extra_bytes}" -gt 0 ]; then pass "force-drop database has two nonzero cache charges" else fail "force-drop cache accounting is invalid: ${force_bytes}, ${force_extra_bytes}" fi global_with_force=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();") assert_equals "force-drop database contributes both registry entries" \ "$((baseline_global + force_bytes + force_extra_bytes))" \ "${global_with_force}" start_database_holder "${DB_FORCE}" pgts_force_holder set +e run_dropdb --force "${DB_FORCE}" >/dev/null 2>&1 force_status=$? set -e if [ "${force_status}" -eq 0 ]; then pass "DROP DATABASE WITH (FORCE) completed" elif [ "${force_status}" -eq 124 ] || [ "${force_status}" -eq 137 ]; then fail "DROP DATABASE WITH (FORCE) timed out" else fail "DROP DATABASE WITH (FORCE) failed with status ${force_status}" fi if wait_for_holder_exit; then pass "forced DROP DATABASE reaped its terminated client" else fail "forced DROP DATABASE client did not exit within 10 seconds" stop_database_holder || true fi global_after_force=$(run_sql_value "${DB_B}" \ "SELECT bm25_cache_global_estimated_bytes();") assert_equals "DROP DATABASE WITH (FORCE) releases only its accounting" \ "${baseline_global}" "${global_after_force}" assert_equals "DROP DATABASE WITH (FORCE) removes both cache charges" \ "$((force_bytes + force_extra_bytes))" \ "$((global_with_force - global_after_force))" assert_equals "DROP DATABASE WITH (FORCE) preserves database B cache" \ "${b_before_drop}" "$(cache_apply "${DB_B}")" assert_equals "DROP DATABASE WITH (FORCE) preserves database B queries" \ "3|true|2,3,4" "$(top_hits "${DB_B}" "bravo")" info "Assertions: ${PASS_COUNT} passed, ${FAIL_COUNT} failed" if [ "${FAIL_COUNT}" -ne 0 ]; then return 1 fi log "All cross-database registry tests passed" } main "$@" timescale-pg_textsearch-b63fe58/test/scripts/drop_index_rollback.sh000066400000000000000000000276261525762047700260040ustar00rootroot00000000000000#!/bin/bash # Shared state must outlive rolled-back drops and concurrent readers (#506). set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" TEST_DIR="$(mktemp -d "${ROOT}/test/tmp_drop_rollback.XXXXXX")" SOCKET_DIR="$(mktemp -d /tmp/pgts-drop.XXXXXX)" export PGHOST="${SOCKET_DIR}" export PGPORT="${TEST_PORT:-55463}" export PGDATABASE=postgres READER_PID= DROPPER_PID= cleanup() { local status=$? trap - EXIT for pid in "${READER_PID}" "${DROPPER_PID}"; do if [ -n "${pid}" ]; then kill "${pid}" 2>/dev/null || true wait "${pid}" 2>/dev/null || true fi done if [ -f "${TEST_DIR}/data/postmaster.pid" ]; then if ! pg_ctl -D "${TEST_DIR}/data" -m immediate -w stop >/dev/null; then echo "Could not stop test server; preserving ${TEST_DIR}" >&2 exit 1 fi fi if [ "${status}" -ne 0 ]; then cat "${TEST_DIR}/postgres.log" >&2 for output in reader.out dropper.out; do if [ -f "${TEST_DIR}/${output}" ]; then cat "${TEST_DIR}/${output}" >&2 fi done fi rm -rf "${TEST_DIR}" "${SOCKET_DIR}" exit "${status}" } trap cleanup EXIT trap 'exit 1' INT TERM initdb -D "${TEST_DIR}/data" --auth=trust >/dev/null cat >>"${TEST_DIR}/data/postgresql.conf" < to_bm25query('drug trial', 'docs_idx') LIMIT 30000 ) hits; IF actual <> expected THEN RAISE EXCEPTION 'expected % hits, got %', expected, actual; END IF; END; $$; SQL cat >"${TEST_DIR}/writer.sql" <<'SQL' INSERT INTO docs SELECT i, 'drug trial late ' || i FROM generate_series(20002, 20101) i; SQL fixture() { sql <<'SQL' DROP TABLE IF EXISTS docs; CREATE TABLE docs (id int, txt text); INSERT INTO docs SELECT i, 'drug trial phase ' || i || ' result ' || md5(i::text) FROM generate_series(1, 20000) i; CREATE INDEX docs_idx ON docs USING bm25 (txt) WITH (text_config = 'english'); INSERT INTO docs VALUES (20001, 'drug trial cached'); SQL } wait_for_reader() { local marker=$1 for _ in $(seq 1 300); do if grep -Fxq "${marker}" "${TEST_DIR}/reader.out"; then return fi if ! kill -0 "${READER_PID}" 2>/dev/null; then echo "Reader exited before ${marker}" >&2 return 1 fi sleep 0.1 done echo "Timed out waiting for ${marker}" >&2 return 1 } concurrent_drop_case() { local outcome=$1 reader_fd cache_bytes waiting=false echo "Testing concurrent drop with active reader: ${outcome}" fixture mkfifo "${TEST_DIR}/reader-${outcome}.sql" exec {reader_fd}<>"${TEST_DIR}/reader-${outcome}.sql" PGAPPNAME=drop-rollback-reader \ psql -X -v ON_ERROR_STOP=1 <"${TEST_DIR}/reader-${outcome}.sql" \ >"${TEST_DIR}/reader.out" 2>&1 & READER_PID=$! cat >&"${reader_fd}" <<'SQL' SET enable_seqscan = off; BEGIN; SELECT assert_hits(20001); DECLARE live_cursor CURSOR FOR SELECT id FROM docs ORDER BY txt <@> to_bm25query('drug trial', 'docs_idx') LIMIT 5; FETCH 1 FROM live_cursor; \echo reader_ready SQL wait_for_reader reader_ready cache_bytes="$(sql -Atc "SELECT bm25_cache_global_estimated_bytes();")" PGAPPNAME=drop-rollback-dropper \ psql -X -v ON_ERROR_STOP=1 -c "DROP INDEX CONCURRENTLY docs_idx;" \ >"${TEST_DIR}/dropper.out" 2>&1 & DROPPER_PID=$! for _ in $(seq 1 300); do waiting="$(sql -Atc "SELECT EXISTS (SELECT FROM pg_index WHERE indexrelid = 'docs_idx'::regclass AND NOT indisvalid) AND EXISTS (SELECT FROM pg_stat_activity WHERE application_name = 'drop-rollback-dropper' AND wait_event_type = 'Lock');")" [ "${waiting}" = "t" ] && break sleep 0.1 done if [ "${waiting}" != "t" ]; then echo "Concurrent drop did not reach its reader wait" >&2 return 1 fi if [ "${cache_bytes}" -eq 0 ] || [ "$(sql -Atc "SELECT bm25_cache_global_estimated_bytes();")" \ != "${cache_bytes}" ]; then echo "Concurrent drop freed shared state before readers finished" >&2 return 1 fi sql -c "INSERT INTO docs VALUES (20002, 'drug trial late');" if [ "${outcome}" = cancel ]; then sql -c "SELECT pg_cancel_backend(pid) FROM pg_stat_activity WHERE application_name = 'drop-rollback-dropper';" if wait "${DROPPER_PID}"; then echo "Cancelled concurrent drop unexpectedly succeeded" >&2 return 1 fi DROPPER_PID= grep -Fq "canceling statement due to user request" \ "${TEST_DIR}/dropper.out" fi cat >&"${reader_fd}" <<'SQL' FETCH ALL FROM live_cursor; CLOSE live_cursor; SELECT assert_hits(20002); COMMIT; \echo reader_finished SQL wait_for_reader reader_finished if [ "${outcome}" = cancel ]; then sql -c "REINDEX INDEX docs_idx;" printf 'SELECT assert_hits(20002);\n' >&"${reader_fd}" else wait "${DROPPER_PID}" DROPPER_PID= sql <<'SQL' DO $$ BEGIN IF bm25_cache_global_estimated_bytes() <> 0 THEN RAISE EXCEPTION 'concurrent drop leaked shared cache'; END IF; END; $$; SQL fi printf '\\q\n' >&"${reader_fd}" wait "${READER_PID}" READER_PID= exec {reader_fd}>&- } rollback_case() { local name="$1" ddl="$2" echo "Testing ${name}" fixture printf '%s\n' "${ddl}" >"${TEST_DIR}/drop.sql" # \! runs the dropper and writer while the original reader stays connected. sql < 0 AND bm25_cache_global_estimated_bytes() = :cache_bytes AS retained \gset \\if :retained \\else DO \$\$ BEGIN RAISE EXCEPTION 'rollback freed shared cache'; END \$\$; \\endif \\! psql -X -v ON_ERROR_STOP=1 -f "${TEST_DIR}/writer.sql" \\if :SHELL_ERROR DO \$\$ BEGIN RAISE EXCEPTION 'writer failed'; END \$\$; \\endif SELECT assert_hits(20101); -- The dropping backend must also retain a usable wrapper after rollback. BEGIN; DROP INDEX docs_idx; ROLLBACK; SELECT assert_hits(20101); SQL } rollback_case "transaction rollback" \ "BEGIN; DROP INDEX docs_idx; ROLLBACK;" rollback_case "savepoint rollback followed by commit" \ "BEGIN; SAVEPOINT s; DROP INDEX docs_idx; ROLLBACK TO s; COMMIT;" rollback_case "released savepoint followed by rollback" \ "BEGIN; SAVEPOINT s; DROP INDEX docs_idx; RELEASE s; ROLLBACK;" rollback_case "nested release followed by parent rollback" \ "BEGIN; SAVEPOINT a; SAVEPOINT b; DROP INDEX docs_idx;\ RELEASE b; ROLLBACK TO a; COMMIT;" rollback_case "cascading table drop rollback" \ "BEGIN; DROP TABLE docs; ROLLBACK;" rollback_case "extension drop rollback" \ "BEGIN; DROP EXTENSION pg_textsearch CASCADE; ROLLBACK;" rollback_case "caught statement error" \ 'DO $$ BEGIN DROP INDEX docs_idx; PERFORM 1 / 0; EXCEPTION WHEN division_by_zero THEN NULL; END $$;' concurrent_drop_case commit concurrent_drop_case cancel echo "Testing concurrent drop failure after physical deletion" fixture sql <<'SQL' SELECT assert_hits(20001); SELECT bm25_cache_global_estimated_bytes() AS cache_bytes \gset CREATE FUNCTION reject_index_drop() RETURNS event_trigger LANGUAGE plpgsql AS $$ BEGIN RAISE EXCEPTION 'reject concurrent drop'; END; $$; CREATE EVENT TRIGGER reject_index_drop ON sql_drop WHEN TAG IN ('DROP INDEX') EXECUTE FUNCTION reject_index_drop(); \set ON_ERROR_STOP off DROP INDEX CONCURRENTLY docs_idx; \set drop_state :SQLSTATE \set ON_ERROR_STOP on DROP EVENT TRIGGER reject_index_drop; DROP FUNCTION reject_index_drop(); SELECT :'drop_state' = 'P0001' AND to_regclass('docs_idx') IS NOT NULL AND :cache_bytes > 0 AND bm25_cache_global_estimated_bytes() = :cache_bytes AS retained \gset \if :retained \else DO $$ BEGIN RAISE EXCEPTION 'failed drop lost shared state'; END $$; \endif REINDEX INDEX docs_idx; SELECT assert_hits(20001); SQL echo "Testing PREPARE rejection and savepoint queue cleanup" fixture sql <<'SQL' SELECT assert_hits(20001); BEGIN; DROP INDEX docs_idx; \set ON_ERROR_STOP off PREPARE TRANSACTION 'drop_rejected'; \set prepare_state :SQLSTATE \set ON_ERROR_STOP on ROLLBACK; SELECT :'prepare_state' = '0A000' AS rejected \gset \if :rejected \else DO $$ BEGIN RAISE EXCEPTION 'PREPARE accepted pending drop'; END $$; \endif SELECT assert_hits(20001); BEGIN; SAVEPOINT s; DROP INDEX docs_idx; ROLLBACK TO s; PREPARE TRANSACTION 'drop_rolled_back'; COMMIT PREPARED 'drop_rolled_back'; SELECT assert_hits(20001); SQL echo "Testing committed drop cleanup and create/drop ownership" for ddl in \ "DROP INDEX docs_idx" \ "DROP INDEX CONCURRENTLY docs_idx" \ "DROP TABLE docs" \ "BEGIN; SAVEPOINT a; SAVEPOINT b; DROP INDEX docs_idx;\ RELEASE b; RELEASE a; COMMIT" do fixture sql < 0 AS populated \\gset \\if :populated \\else DO \$\$ BEGIN RAISE EXCEPTION 'cache was not populated'; END \$\$; \\endif ${ddl}; DO \$\$ BEGIN IF bm25_cache_global_estimated_bytes() <> 0 THEN RAISE EXCEPTION 'committed drop leaked cache memory'; END IF; END; \$\$; SQL done echo "Testing mixed parent and savepoint drops" fixture sql <<'SQL' CREATE TABLE survivor (txt text); CREATE INDEX survivor_idx ON survivor USING bm25 (txt) WITH (text_config = 'english'); INSERT INTO survivor VALUES ('drug trial'); SELECT txt FROM survivor ORDER BY txt <@> to_bm25query('drug trial', 'survivor_idx') LIMIT 1; SELECT bm25_cache_global_estimated_bytes() AS survivor_bytes \gset SELECT assert_hits(20001); BEGIN; DROP INDEX docs_idx; SAVEPOINT s; DROP INDEX survivor_idx; ROLLBACK TO s; COMMIT; SELECT bm25_cache_global_estimated_bytes() = :survivor_bytes AS retained \gset \if :retained \else DO $$ BEGIN RAISE EXCEPTION 'wrong drop survived savepoint'; END $$; \endif SELECT txt FROM survivor ORDER BY txt <@> to_bm25query('drug trial', 'survivor_idx') LIMIT 1; CREATE INDEX docs_idx ON docs USING bm25 (txt) WITH (text_config = 'english'); INSERT INTO docs VALUES (20002, 'drug trial'); SELECT assert_hits(20002); DROP TABLE docs, survivor; DO $$ BEGIN IF bm25_cache_global_estimated_bytes() <> 0 THEN RAISE EXCEPTION 'multiple drops leaked cache memory'; END IF; END; $$; SQL sql <<'SQL' DROP TABLE IF EXISTS docs; CREATE TABLE docs (id int, txt text); INSERT INTO docs VALUES (1, 'drug trial'); BEGIN; CREATE INDEX docs_idx ON docs USING bm25 (txt) WITH (text_config = 'english'); INSERT INTO docs VALUES (2, 'drug trial'); SELECT assert_hits(2); DROP INDEX docs_idx; ROLLBACK; BEGIN; CREATE INDEX docs_idx ON docs USING bm25 (txt) WITH (text_config = 'english'); INSERT INTO docs VALUES (2, 'drug trial'); SELECT assert_hits(2); DROP INDEX docs_idx; COMMIT; BEGIN; PREPARE TRANSACTION 'after_drop'; COMMIT PREPARED 'after_drop'; DO $$ BEGIN IF bm25_cache_global_estimated_bytes() <> 0 THEN RAISE EXCEPTION 'create/drop leaked cache memory'; END IF; END; $$; SQL echo "DROP rollback tests passed" timescale-pg_textsearch-b63fe58/test/scripts/durable_compaction.sh000077500000000000000000017433731525762047700256420ustar00rootroot00000000000000#!/bin/bash # # End-to-end permission and lifecycle tests for owner-scoped workflows. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" DATA_DIR="${REPO_ROOT}/test/tmp_durable_compaction" KEEP_DIR="${REPO_ROOT}/test/tmp_durable_compaction_logs" SOCKET_DIR="${REPO_ROOT}" LOGFILE="${DATA_DIR}/postgres.log" TEST_PORT=55447 TEST_DB=durable_compaction_test REMOTE_DB=durable_compaction_remote ROLLBACK_DB=durable_compaction_rollback PG_CONFIG_BIN="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG_BIN}" --bindir)" PKGLIBDIR="$("${PG_CONFIG_BIN}" --pkglibdir)" SHAREDIR="$("${PG_CONFIG_BIN}" --sharedir)" PG_DURABLE_VERSION="${PG_DURABLE_VERSION:-0.2.8}" PG_DURABLE_PACKAGE_DIR="${PG_DURABLE_PACKAGE_DIR:-}" PACKAGE_BACKUP_DIR="${REPO_ROOT}/test/tmp_durable_package_backup" DURABLE_PACKAGE_LIBDIR= STAGED_SHARE_FILES=() log() { printf '[durable] %s\n' "$1"; } error() { printf '[durable] ERROR: %s\n' "$1" >&2; exit 1; } restore_durable_package() { local staged source for staged in "${STAGED_SHARE_FILES[@]}"; do rm -f "${staged}" done if [ -d "${PACKAGE_BACKUP_DIR}" ]; then while IFS= read -r source; do cp -p "${source}" "${SHAREDIR}/extension/" done < <(find "${PACKAGE_BACKUP_DIR}" -maxdepth 1 -type f -print) rm -rf "${PACKAGE_BACKUP_DIR}" fi } cleanup() { local exit_code=$? trap - EXIT INT TERM if [ "${exit_code}" -ne 0 ] && [ -d "${DATA_DIR}" ]; then rm -rf "${KEEP_DIR}" mkdir -p "${KEEP_DIR}" if ! PGCONNECT_TIMEOUT=5 PGOPTIONS='-c statement_timeout=5000' \ sql_super -x -c "SELECT * FROM df.instances; SELECT * FROM df.nodes; SELECT pid, state, wait_event_type, wait_event, query FROM pg_stat_activity WHERE datname = current_database();" \ >"${KEEP_DIR}/workflow-state.log" 2>&1; then log "Could not capture all workflow failure diagnostics" fi fi if [ -f "${DATA_DIR}/postmaster.pid" ]; then "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m immediate \ >/dev/null 2>&1 || true fi if [ "${exit_code}" -ne 0 ] && [ -d "${DATA_DIR}" ]; then cp "${LOGFILE}" "${KEEP_DIR}/" 2>/dev/null || true fi rm -rf "${DATA_DIR}" restore_durable_package exit "${exit_code}" } trap cleanup EXIT INT TERM sql_in_db_as() { local database=$1 role=$2 shift 2 "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U "${role}" -d "${database}" -qAt -v ON_ERROR_STOP=1 "$@" } sql_as() { local role=$1 shift sql_in_db_as "${TEST_DB}" "${role}" "$@" } sql_super() { sql_as postgres "$@" } reindex_database() { local command=$1 output for _ in $(seq 1 5); do if output="$(sql_super -c "${command}" 2>&1)"; then printf '%s' "${output}" return 0 fi if ! grep -Fq "deadlock detected" <<<"${output}"; then printf '%s\n' "${output}" >&2 return 1 fi sleep 0.2 done printf '%s\n' "${output}" >&2 return 1 } sql_remote_as() { local role=$1 shift sql_in_db_as "${REMOTE_DB}" "${role}" "$@" } sql_remote_super() { sql_remote_as postgres "$@" } assert_eq() { local description=$1 expected=$2 actual=$3 if [ "${actual}" != "${expected}" ]; then error "${description}: expected '${expected}', got '${actual}'" fi log "PASS: ${description}" } assert_ne() { local description=$1 unexpected=$2 actual=$3 if [ "${actual}" = "${unexpected}" ]; then error "${description}: did not expect '${unexpected}'" fi log "PASS: ${description}" } helper_privileges_for_role() { local role=$1 sql_super -c "SELECT pg_catalog.concat_ws( ':', pg_catalog.has_function_privilege( '${role}', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE'), pg_catalog.has_function_privilege( '${role}', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE'));" } assert_rejected_background_alter() { local description=$1 owner=$2 executor=$3 prefix=$4 expected_error=$5 local alter_error dependencies_before jobs_before privileges_before local table_name="${prefix}_docs" local index_name="${prefix}_idx" sql_super -c "CREATE TABLE ${table_name} (body text); ALTER TABLE ${table_name} OWNER TO ${owner}; CREATE INDEX ${index_name} ON ${table_name} USING bm25(body) WITH (text_config = 'english', compaction = 'manual');" assert_eq "${description} test index has the expected owner" \ "${owner}" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = '${index_name}'::regclass;")" jobs_before="$(managed_job_count)" dependencies_before="$(dependency_count)" privileges_before="$(helper_privileges_for_role "${owner}")" if [ "${executor}" = "set-role" ]; then if alter_error="$(sql_super -c "SET ROLE ${owner}; ALTER INDEX public.${index_name} SET (compaction = 'background');" 2>&1)"; then error "${description} ALTER unexpectedly succeeded" fi elif alter_error="$(sql_as "${executor}" -c " ALTER INDEX public.${index_name} SET (compaction = 'background');" 2>&1)"; then error "${description} ALTER unexpectedly succeeded" fi if ! grep -Fq "${expected_error}" <<<"${alter_error}"; then error "${description} ALTER did not fail admission: ${alter_error}" fi assert_eq "${description} ALTER preserves manual reloption" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_catalog.pg_class WHERE oid = '${index_name}'::regclass;")" assert_eq "${description} ALTER creates no managed job" \ "${jobs_before}" "$(managed_job_count)" assert_eq "${description} ALTER leaves dependencies unchanged" \ "${dependencies_before}" "$(dependency_count)" assert_eq "${description} ALTER leaves helper privileges unchanged" \ "${privileges_before}" "$(helper_privileges_for_role "${owner}")" sql_super -c "DROP TABLE ${table_name};" } test_alter_preflight_rejections() { assert_rejected_background_alter \ "NOLOGIN owner" durable_nologin set-role alter_nologin \ "index owner must have LOGIN for background compaction" assert_rejected_background_alter \ "disallowed superuser owner" postgres postgres alter_superuser \ "pg_durable superuser instances are disabled" assert_rejected_background_alter \ "owner without pg_durable table privileges" durable_usage_only \ durable_usage_only alter_usage_only \ "index owner lacks required pg_durable privileges" assert_rejected_background_alter \ "owner without database CONNECT" durable_no_connect durable_actor \ alter_no_connect \ "index owner cannot connect for background compaction" assert_rejected_background_alter \ "owner without pg_textsearch schema USAGE" \ durable_no_textsearch_schema durable_writer \ alter_no_textsearch_schema \ "index owner lacks required pg_textsearch schema privilege" } test_cic_owner_privilege_preflight() { local create_error jobs_before schema_error schema_error='Role "durable_no_textsearch_schema" lacks USAGE privilege' schema_error+=' on schema public.' sql_super -c "REVOKE CONNECT ON DATABASE durable_compaction_test FROM PUBLIC; GRANT CONNECT ON DATABASE durable_compaction_test TO postgres, durable_owner, durable_owner_two, durable_usage_only, durable_read_only, durable_bypass, durable_actor, durable_writer, durable_maintainer;" jobs_before="$(managed_job_count)" sql_super -c "CREATE TABLE no_connect_docs (body text); ALTER TABLE no_connect_docs OWNER TO durable_no_connect;" if create_error="$(sql_as durable_actor -c " CREATE INDEX CONCURRENTLY no_connect_docs_idx ON no_connect_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC accepted an owner without database CONNECT" fi assert_eq "rejected no-CONNECT CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('no_connect_docs_idx') IS NULL;")" if ! grep -Fq \ "index owner cannot connect for background compaction" \ <<<"${create_error}"; then error "no-CONNECT owner did not fail deterministic CIC preflight: \ ${create_error}" fi assert_eq "no-CONNECT CIC creates no probe workflow" "${jobs_before}" \ "$(managed_job_count)" sql_super -c "DROP TABLE no_connect_docs;" sql_super -c "GRANT CONNECT ON DATABASE durable_compaction_test TO durable_no_textsearch_schema; REVOKE USAGE ON SCHEMA public FROM PUBLIC; GRANT USAGE ON SCHEMA public TO postgres, durable_owner, durable_owner_two, durable_nologin, durable_usage_only, durable_read_only, durable_bypass, durable_no_connect, durable_actor, durable_writer, durable_maintainer;" jobs_before="$(managed_job_count)" sql_super -c "CREATE TABLE no_textsearch_schema_docs (body text); ALTER TABLE no_textsearch_schema_docs OWNER TO durable_no_textsearch_schema;" if create_error="$(sql_as durable_writer -c " CREATE INDEX CONCURRENTLY no_textsearch_schema_docs_idx ON no_textsearch_schema_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC accepted an owner without pg_textsearch \ schema USAGE" fi assert_eq "rejected no-schema-USAGE CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass( 'no_textsearch_schema_docs_idx') IS NULL;")" if ! grep -Fq \ "index owner lacks required pg_textsearch schema privilege" \ <<<"${create_error}"; then error "no-schema-USAGE owner did not fail stable privilege check: \ ${create_error}" fi if ! grep -Fq "${schema_error}" <<<"${create_error}"; then error "no-schema-USAGE owner error did not identify public: \ ${create_error}" fi if ! grep -Fq \ 'Grant access with GRANT USAGE ON SCHEMA public TO' \ <<<"${create_error}"; then error "no-schema-USAGE owner error did not provide an actionable \ hint: ${create_error}" fi assert_eq "no-schema-USAGE CIC creates no probe workflow" \ "${jobs_before}" "$(managed_job_count)" assert_eq "no-schema-USAGE CIC leaves no sticky dependency" "0" \ "$(dependency_count)" assert_eq "no-schema-USAGE CIC leaves no private-helper grant" "f" \ "$(sql_super -c "SELECT pg_catalog.has_function_privilege( 'durable_no_textsearch_schema', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') OR pg_catalog.has_function_privilege( 'durable_no_textsearch_schema', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE');")" sql_super -c "DROP TABLE no_textsearch_schema_docs;" assert_eq "usage-only owner can resolve df.start" "t" \ "$(sql_super -c "SELECT pg_catalog.has_function_privilege( 'durable_usage_only', procedure.oid, 'EXECUTE') FROM pg_catalog.pg_proc AS procedure JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = procedure.pronamespace JOIN pg_catalog.pg_depend AS dependency ON dependency.classid = 'pg_catalog.pg_proc'::regclass AND dependency.objid = procedure.oid AND dependency.deptype = 'e' JOIN pg_catalog.pg_extension AS extension ON extension.oid = dependency.refobjid WHERE namespace.nspname = 'df' AND procedure.proname = 'start' AND extension.extname = 'pg_durable';")" assert_eq "usage-only owner can resolve df.explain" "t" \ "$(sql_super -c "SELECT pg_catalog.has_function_privilege( 'durable_usage_only', 'df.explain(text)', 'EXECUTE');")" assert_eq "usage-only owner cannot read df.instances" "f" \ "$(sql_super -c "SELECT pg_catalog.has_table_privilege( 'durable_usage_only', 'df.instances', 'SELECT');")" jobs_before="$(managed_job_count)" sql_super -c "CREATE TABLE usage_only_docs (body text); ALTER TABLE usage_only_docs OWNER TO durable_usage_only;" if create_error="$(sql_as durable_usage_only -c " CREATE INDEX CONCURRENTLY usage_only_docs_idx ON usage_only_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC accepted an owner without durable table access" fi assert_eq "rejected usage-only CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('usage_only_docs_idx') IS NULL;")" if ! grep -Fq "index owner lacks required pg_durable privileges" \ <<<"${create_error}"; then error "usage-only owner did not fail deterministic CIC preflight: \ ${create_error}" fi assert_eq "usage-only CIC creates no probe workflow" "${jobs_before}" \ "$(managed_job_count)" sql_super -c "DROP TABLE usage_only_docs;" assert_eq "read-only owner cannot insert df.instances" "f" \ "$(sql_super -c "SELECT pg_catalog.has_table_privilege( 'durable_read_only', 'df.instances', 'INSERT');")" jobs_before="$(managed_job_count)" sql_super -c "CREATE TABLE read_only_docs (body text); ALTER TABLE read_only_docs OWNER TO durable_read_only;" if create_error="$(sql_as durable_read_only -c " CREATE INDEX CONCURRENTLY read_only_docs_idx ON read_only_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC accepted an owner without durable INSERT" fi assert_eq "rejected read-only CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('read_only_docs_idx') IS NULL;")" if ! grep -Fq "index owner lacks required pg_durable privileges" \ <<<"${create_error}"; then error "read-only owner did not fail deterministic CIC preflight: \ ${create_error}" fi assert_eq "read-only CIC creates no probe workflow" "${jobs_before}" \ "$(managed_job_count)" sql_super -c "DROP TABLE read_only_docs;" } test_defaulted_start_arity() { local create_output index_oid instance_id start_shape local shimmed=false start_shape="$(sql_super -c "SELECT procedure.pronargs || ':' || procedure.pronargdefaults FROM pg_catalog.pg_proc AS procedure JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = procedure.pronamespace JOIN pg_catalog.pg_depend AS dependency ON dependency.classid = 'pg_catalog.pg_proc'::regclass AND dependency.objid = procedure.oid AND dependency.deptype = 'e' JOIN pg_catalog.pg_extension AS extension ON extension.oid = dependency.refobjid WHERE namespace.nspname = 'df' AND procedure.proname = 'start' AND extension.extname = 'pg_durable';")" case "${start_shape}" in 4:*) sql_super <<'SQL' >/dev/null ALTER FUNCTION df.start(text, text, text, text) RENAME TO start_v027; CREATE FUNCTION df.start( fut text, label text, database text, transaction_mode text, max_attempts integer DEFAULT 1, max_backoff interval DEFAULT interval '16 seconds', on_failure text DEFAULT 'fail') RETURNS text LANGUAGE sql SET search_path = pg_catalog, pg_temp AS $body$ SELECT df.start_v027(fut, label, database, transaction_mode) $body$; ALTER EXTENSION pg_durable ADD FUNCTION df.start(text, text, text, text, integer, interval, text); SQL shimmed=true ;; 7:*) ;; *) error "unexpected extension-owned df.start shape: ${start_shape}" ;; esac assert_eq "defaulted start probe is callable with four arguments" \ "7:true" \ "$(sql_super -c "SELECT pronargs || ':' || (pronargdefaults >= pronargs - 4) FROM pg_catalog.pg_proc WHERE oid = 'df.start(text,text,text,text,integer,interval,text)' ::pg_catalog.regprocedure;")" sql_super -c "CREATE TABLE arity_docs (body text); ALTER TABLE arity_docs OWNER TO durable_owner;" if ! create_output="$(sql_as durable_owner -c " CREATE INDEX CONCURRENTLY arity_docs_idx ON arity_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "four-argument call rejected defaulted df.start: \ ${create_output}" fi index_oid="$(sql_super -c "SELECT 'arity_docs_idx'::regclass::oid;")" instance_id="$(active_job_id_for_owner \ "${index_oid}" durable_owner)" [ -n "${instance_id}" ] || error "defaulted df.start probe created no owner workflow" sql_as durable_owner -c \ "SELECT df.cancel('${instance_id}', 'arity probe complete');" \ >/dev/null wait_for_terminal "${instance_id}" 30 sql_super -c "DROP TABLE arity_docs;" if "${shimmed}"; then sql_super -c "ALTER EXTENSION pg_durable DROP FUNCTION df.start(text, text, text, text, integer, interval, text); DROP FUNCTION df.start(text, text, text, text, integer, interval, text); ALTER FUNCTION df.start_v027(text, text, text, text) RENAME TO start;" fi log "PASS: four-argument adapter accepts trailing defaulted arguments" } stage_durable_package() { local package_library package_control package_sql source destination local -a package_libraries package_controls package_sql_files if [ -z "${PG_DURABLE_PACKAGE_DIR}" ]; then error "PG_DURABLE_PACKAGE_DIR must name one pg_durable package" fi mapfile -t package_libraries < <( find "${PG_DURABLE_PACKAGE_DIR}" -type f -name pg_durable.so -print ) mapfile -t package_controls < <( find "${PG_DURABLE_PACKAGE_DIR}" -type f \ -name pg_durable.control -print ) mapfile -t package_sql_files < <( find "${PG_DURABLE_PACKAGE_DIR}" -type f \ -name "pg_durable--${PG_DURABLE_VERSION}.sql" -print ) [ "${#package_libraries[@]}" -eq 1 ] || error "package must contain exactly one pg_durable.so" [ "${#package_controls[@]}" -eq 1 ] || error "package must contain exactly one pg_durable.control" [ "${#package_sql_files[@]}" -eq 1 ] || error "package must contain exactly one pg_durable version SQL file" package_library="${package_libraries[0]}" package_control="${package_controls[0]}" package_sql="${package_sql_files[0]}" if ! grep -Eq \ "^default_version[[:space:]]*=[[:space:]]*'${PG_DURABLE_VERSION}'" \ "${package_control}"; then error "pg_durable package control/version SQL do not match" fi DURABLE_PACKAGE_LIBDIR="$(dirname "${package_library}")" rm -rf "${PACKAGE_BACKUP_DIR}" mkdir -p "${PACKAGE_BACKUP_DIR}" for source in "${package_control}" "${package_sql}"; do destination="${SHAREDIR}/extension/$(basename "${source}")" if [ -f "${destination}" ]; then cp -p "${destination}" "${PACKAGE_BACKUP_DIR}/" fi cp -p "${source}" "${destination}" STAGED_SHARE_FILES+=("${destination}") cmp -s "${source}" "${destination}" || error "failed to stage matching pg_durable package SQL" done log "Using pg_durable ${PG_DURABLE_VERSION} package library: \ $(sha256sum "${package_library}" | cut -d' ' -f1)" log "Using pg_durable ${PG_DURABLE_VERSION} package SQL: \ $(sha256sum "${package_sql}" | cut -d' ' -f1)" } active_jobs_for_index() { local oid=$1 sql_super -c "SELECT count(*) FROM df.instances WHERE label LIKE 'pg_textsearch:bg:v1:%:${oid}:%' AND status IN ('pending', 'running');" } wait_for_no_debt() { local index_name=$1 timeout=$2 local waited=0 while [ "$waited" -lt "$timeout" ]; do if [ "$(sql_super -c "SELECT NOT bm25_needs_compaction( '${index_name}'::regclass);")" = "t" ]; then return 0 fi sleep 1 waited=$((waited + 1)) done error "index ${index_name} retained compaction debt" } wait_for_terminal() { local instance_id=$1 timeout=$2 local waited=0 status= while [ "${waited}" -lt "${timeout}" ]; do status="$(sql_super -c \ "SELECT status FROM df.instances WHERE id = '${instance_id}';")" case "${status}" in completed|failed|cancelled) return 0 ;; esac sleep 1 waited=$((waited + 1)) done error "instance ${instance_id} did not become terminal" } quiesce_durable_worker() { local instance_id while IFS= read -r instance_id; do [ -n "${instance_id}" ] || continue sql_super -c "SELECT df.cancel( id, 'extension dependency test teardown') FROM df.instances WHERE id = '${instance_id}' AND status IN ('pending', 'running');" >/dev/null wait_for_terminal "${instance_id}" 30 done < <(sql_super -c "SELECT id FROM df.instances WHERE status IN ('pending', 'running') ORDER BY id;") assert_eq "extension dependency teardown has no active workflows" "0" \ "$(sql_super -c "SELECT count(*) FROM df.instances WHERE status IN ('pending', 'running');")" PGHOST="${SOCKET_DIR}" "${PGBINDIR}/pg_ctl" restart -D "${DATA_DIR}" \ -l "${LOGFILE}" -w \ -o "-c shared_preload_libraries=pg_textsearch" >/dev/null assert_eq "extension dependency teardown stops the durable worker" \ "pg_textsearch" \ "$(sql_super -c "SHOW shared_preload_libraries;")" } signal_node_is_waiting() { local instance_id=$1 signal_name=${2:-} # A running node is published before its signal subscription is durable. sql_super -c "SELECT EXISTS ( SELECT 1 FROM df.instance_nodes('${instance_id}') n JOIN _duroxide.history h ON n.status_details::jsonb->>'execution_id' = h.instance_id || '::' || h.execution_id WHERE n.node_type = 'SIGNAL' AND n.inferred_status = 'running' AND ('${signal_name}' = '' OR n.query::jsonb->>'signal_name' = '${signal_name}') AND h.event_data::jsonb->>'type' = 'ExternalSubscribed' AND h.event_data::jsonb->>'name' = n.query::jsonb->>'signal_name' AND NOT EXISTS ( SELECT 1 FROM _duroxide.history later WHERE later.instance_id = h.instance_id AND later.execution_id = h.execution_id AND later.event_id > h.event_id AND ((later.event_data::jsonb->>'type' = 'ExternalEvent' AND later.event_data::jsonb->>'name' = n.query::jsonb->>'signal_name') OR (later.event_data::jsonb->>'type' = 'ExternalSubscribedCancelled' AND later.event_data::jsonb->>'source_event_id' = h.event_id::text))));" } wait_for_signal_node() { local instance_id=$1 timeout=$2 signal_name=${3:-} local waited=0 while [ "${waited}" -lt "${timeout}" ]; do if [ "$(signal_node_is_waiting \ "${instance_id}" "${signal_name}")" = "t" ]; then return 0 fi sleep 1 waited=$((waited + 1)) done error "instance ${instance_id} did not begin waiting for a signal" } wait_for_log_message() { local message=$1 timeout=$2 local waited=0 while [ "${waited}" -lt "${timeout}" ]; do if grep -Fq "${message}" "${LOGFILE}"; then return 0 fi sleep 1 waited=$((waited + 1)) done error "server log did not contain: ${message}" } wait_for_audit_rows() { local expected=$1 timeout=$2 local waited=0 count while [ "${waited}" -lt "${timeout}" ]; do count="$(sql_super -c \ "SELECT count(*) FROM public.worker_identity_audit;")" if [ "${count}" -ge "${expected}" ]; then return 0 fi sleep 1 waited=$((waited + 1)) done error "worker identity audit did not reach ${expected} rows" } wait_for_durable_worker() { local attempt id status for attempt in $(seq 1 60); do id="$(sql_as durable_owner -c \ "SELECT df.start('SELECT 1', 'probe-${attempt}', current_database(), 'caller');" \ 2>/dev/null || true)" if [ -n "${id}" ]; then for _ in $(seq 1 20); do status="$(sql_super -c \ "SELECT status FROM df.instances WHERE id = '${id}';" \ 2>/dev/null || true)" if [ "${status}" = "completed" ]; then return 0 fi if [ "${status}" = "failed" ]; then break fi sleep 1 done fi sleep 1 done error "pg_durable worker did not initialize for ${TEST_DB}" } initialize_database() { sql_super -c \ "CREATE EXTENSION pg_durable VERSION '${PG_DURABLE_VERSION}';" assert_eq "loaded pg_durable package version" "${PG_DURABLE_VERSION}" \ "$(sql_super -c "SELECT extversion FROM pg_catalog.pg_extension WHERE extname = 'pg_durable';")" assert_eq "df.instances submitted_by type" "regrole" \ "$(sql_super -c "SELECT pg_catalog.format_type( attribute.atttypid, attribute.atttypmod) FROM pg_catalog.pg_attribute AS attribute WHERE attribute.attrelid = 'df.instances'::regclass AND attribute.attname = 'submitted_by' AND NOT attribute.attisdropped;")" sql_super -c "CREATE EXTENSION pg_textsearch;" sql_super -c "GRANT CREATE ON SCHEMA public TO durable_owner, durable_owner_two, durable_usage_only, durable_read_only, durable_bypass, durable_actor, durable_writer, durable_maintainer;" sql_super -c "GRANT pg_maintain TO durable_maintainer;" sql_super -c "SELECT df.grant_usage('durable_owner');" >/dev/null sql_super -c "SELECT df.grant_usage('durable_owner_two');" >/dev/null sql_super -c "SELECT df.grant_usage('durable_bypass');" >/dev/null sql_super -c "SELECT df.grant_usage('durable_no_connect');" >/dev/null sql_super -c \ "SELECT df.grant_usage('durable_no_textsearch_schema');" >/dev/null sql_super -c "GRANT USAGE ON SCHEMA df TO durable_usage_only, durable_read_only; GRANT SELECT ON df.instances, df.vars TO durable_read_only;" sql_super -c "CREATE SCHEMA owner_shadow AUTHORIZATION durable_owner;" sql_as durable_owner <<'SQL' CREATE FUNCTION owner_shadow.reject_text_equality(left_value text, right_value text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $body$ BEGIN IF pg_catalog.left(left_value, 20) OPERATOR(pg_catalog.=) 'pg_textsearch:bg:v1:' OR left_value OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running', 'completed', 'failed', 'cancelled']::pg_catalog.text[]) THEN RAISE EXCEPTION 'owner search_path resolved shadow text equality'; END IF; RETURN left_value OPERATOR(pg_catalog.=) right_value; END $body$; CREATE FUNCTION owner_shadow.reject_text_like(text, text) RETURNS boolean LANGUAGE plpgsql IMMUTABLE AS $body$ BEGIN RAISE EXCEPTION 'owner search_path resolved shadow text LIKE'; END $body$; CREATE OPERATOR owner_shadow.= ( LEFTARG = pg_catalog.text, RIGHTARG = pg_catalog.text, FUNCTION = owner_shadow.reject_text_equality ); CREATE OPERATOR owner_shadow.~~ ( LEFTARG = pg_catalog.text, RIGHTARG = pg_catalog.text, FUNCTION = owner_shadow.reject_text_like ); SQL sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET search_path = owner_shadow, public, pg_catalog; ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET maintenance_work_mem = '1MB';" wait_for_durable_worker assert_eq "hostile owner search_path is active" \ "owner_shadow, public, pg_catalog" \ "$(sql_as durable_owner -c "SHOW search_path;")" } dependency_count() { sql_super -c "SELECT count(*) FROM pg_catalog.pg_depend AS dep JOIN pg_catalog.pg_am AS am ON dep.classid = 'pg_catalog.pg_am'::regclass AND dep.objid = am.oid JOIN pg_catalog.pg_extension AS ext ON dep.refclassid = 'pg_catalog.pg_extension'::regclass AND dep.refobjid = ext.oid WHERE am.amname = 'bm25' AND ext.extname = 'pg_durable' AND dep.deptype = 'n';" } remote_dependency_snapshot() { sql_remote_super -c "SELECT COALESCE( pg_catalog.string_agg( dep.refobjid::pg_catalog.text, ',' ORDER BY dep.refobjid), '') FROM pg_catalog.pg_depend AS dep JOIN pg_catalog.pg_am AS am ON dep.classid = 'pg_catalog.pg_am'::regclass AND dep.objid = am.oid WHERE am.amname = 'bm25' AND dep.refclassid = 'pg_catalog.pg_extension'::regclass AND dep.deptype = 'n';" } remote_helper_grant() { sql_remote_super -c "SELECT pg_catalog.has_function_privilege( 'durable_owner', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') OR pg_catalog.has_function_privilege( 'durable_owner', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE');" } active_job_id() { local index_oid=$1 sql_super -c "SELECT id FROM df.instances WHERE label LIKE 'pg_textsearch:bg:v1:%:${index_oid}:%' AND status IN ('pending', 'running') ORDER BY created_at DESC, id DESC LIMIT 1;" } latest_job_id() { local index_oid=$1 sql_super -c "SELECT id FROM df.instances WHERE label LIKE 'pg_textsearch:bg:v1:%:${index_oid}:%' ORDER BY created_at DESC, id DESC LIMIT 1;" } active_job_id_for_owner() { local index_oid=$1 owner=$2 sql_super -c "SELECT id FROM df.instances WHERE label LIKE 'pg_textsearch:bg:v1:%:${index_oid}:%' AND submitted_by = '${owner}'::regrole AND status IN ('pending', 'running') ORDER BY created_at DESC, id DESC LIMIT 1;" } current_generation_job_id() { local index_oid=$1 sql_super -c "SELECT instance.id FROM df.instances AS instance JOIN pg_catalog.pg_class AS relation ON relation.oid = ${index_oid} JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE instance.label LIKE pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%%', database.oid, relation.oid, coalesce( nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner) AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status IN ('pending', 'running') ORDER BY instance.created_at DESC, instance.id DESC LIMIT 1;" } current_generation_job_count() { local index_oid=$1 sql_super -c "SELECT count(*) FROM df.instances AS instance JOIN pg_catalog.pg_class AS relation ON relation.oid = ${index_oid} JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE instance.label LIKE pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%%', database.oid, relation.oid, coalesce( nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner) AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status IN ('pending', 'running');" } current_generation_schedule_job_count() { local index_oid=$1 schedule=$2 sql_super -c "SELECT count(*) FROM df.instances AS instance JOIN pg_catalog.pg_class AS relation ON relation.oid = ${index_oid} JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE instance.label LIKE pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%%', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner) AND instance.label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '${schedule}', 'UTF8'), 'hex')) AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status IN ('pending', 'running');" } background_target_is_current() { local index_oid=$1 relfilenumber=$2 owner=$3 sql_as "${owner}" -c "SELECT bm25_background_target_is_current( relation.oid, database.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), ${relfilenumber}::pg_catalog.oid, '${owner}'::pg_catalog.regrole::pg_catalog.oid) FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE relation.oid = ${index_oid};" } managed_job_count() { sql_super -c "SELECT count(*) FROM df.instances WHERE label LIKE 'pg_textsearch:bg:v1:%';" } index_lineage() { local index_name=$1 sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation, LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = '${index_name}'::regclass AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';" } remove_index_lineage() { local index_name=$1 sql_super -c "UPDATE pg_catalog.pg_class SET reloptions = ARRAY( SELECT option FROM pg_catalog.unnest(reloptions) AS option WHERE option OPERATOR(pg_catalog.!~~) 'compaction_lineage=%') WHERE oid = '${index_name}'::regclass;" } create_compaction_debt() { sql_as durable_owner <<'SQL' >/dev/null BEGIN; DO $body$ BEGIN FOR n IN 1..2 LOOP INSERT INTO documents (id, body) SELECT 10000 + n * 100 + i, format('signal round %s document %s filler', n, i) FROM generate_series(1, 20) AS i; PERFORM bm25_spill_index('documents_idx'); END LOOP; END $body$; COMMIT; SQL } create_bypass_compaction_debt() { sql_as durable_bypass <<'SQL' >/dev/null BEGIN; DO $body$ BEGIN FOR n IN 1..2 LOOP INSERT INTO bypass_documents (id, body) SELECT 10000 + n * 100 + i, format('bypass round %s document %s filler', n, i) FROM generate_series(1, 20) AS i; PERFORM bm25_spill_index('bypass_documents_idx'); END LOOP; END $body$; COMMIT; SQL } setup_cluster() { for required in \ "${PKGLIBDIR}/pg_textsearch.so" \ "${DURABLE_PACKAGE_LIBDIR}/pg_durable.so" \ "${SHAREDIR}/extension/pg_durable--${PG_DURABLE_VERSION}.sql"; do [ -f "${required}" ] || error "required install artifact missing: ${required}" done rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" "${PGBINDIR}/initdb" -D "${DATA_DIR}" -U postgres \ --auth-local=trust --auth-host=reject >/dev/null { printf "port = %s\n" "${TEST_PORT}" printf "unix_socket_directories = '%s'\n" "${SOCKET_DIR}" printf "listen_addresses = ''\n" printf "dynamic_library_path = '%s:\$libdir'\n" \ "${DURABLE_PACKAGE_LIBDIR}" printf "shared_preload_libraries = 'pg_durable,pg_textsearch'\n" printf "logging_collector = on\n" printf "log_directory = '.'\n" printf "log_filename = 'postgres.log'\n" printf "pg_durable.database = '%s'\n" "${TEST_DB}" printf "pg_durable.worker_role = 'postgres'\n" printf "pg_durable.max_user_connections = 4\n" printf "pg_textsearch.segments_per_level = 2\n" } >>"${DATA_DIR}/postgresql.conf" PGHOST="${SOCKET_DIR}" "${PGBINDIR}/pg_ctl" start -D "${DATA_DIR}" \ -l "${LOGFILE}" -w >/dev/null "${PGBINDIR}/createdb" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres "${TEST_DB}" sql_super -c "CREATE ROLE durable_owner LOGIN;" sql_super -c "CREATE ROLE durable_owner_two LOGIN;" sql_super -c "CREATE ROLE durable_nologin NOLOGIN;" sql_super -c "CREATE ROLE durable_usage_only LOGIN;" sql_super -c "CREATE ROLE durable_read_only LOGIN;" sql_super -c "CREATE ROLE durable_bypass LOGIN BYPASSRLS;" sql_super -c "CREATE ROLE durable_no_connect LOGIN;" sql_super -c "CREATE ROLE durable_no_textsearch_schema LOGIN;" sql_super -c "CREATE ROLE durable_actor LOGIN;" sql_super -c "CREATE ROLE durable_writer LOGIN;" sql_super -c "CREATE ROLE durable_maintainer LOGIN;" sql_super -c "GRANT durable_no_connect TO durable_actor; GRANT durable_no_textsearch_schema TO durable_writer;" } test_missing_durable_cic() { local create_error sql_super -c "CREATE EXTENSION pg_textsearch; CREATE TABLE missing_durable_docs (body text);" if create_error="$(sql_super -c " CREATE INDEX CONCURRENTLY missing_durable_idx ON missing_durable_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC succeeded without pg_durable" fi if ! grep -Fq \ "background compaction requires pg_durable 0.2.8 or newer" \ <<<"${create_error}"; then error "missing-pg_durable CIC did not use the stable admission message" fi assert_eq "rejected missing-pg_durable CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('missing_durable_idx') IS NULL;")" sql_super -c "DROP TABLE missing_durable_docs; DROP EXTENSION pg_textsearch;" } test_cic_preflight_rejections() { local create_error sql_super -c "CREATE TABLE superuser_docs (body text);" if create_error="$(sql_super -c " CREATE INDEX CONCURRENTLY superuser_docs_idx ON superuser_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC accepted a disallowed superuser owner" fi if ! grep -Fq "pg_durable superuser instances are disabled" \ <<<"${create_error}"; then error "superuser rejection did not happen during CIC preflight" fi assert_eq "rejected superuser CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('superuser_docs_idx') IS NULL;")" sql_super -c "DROP TABLE superuser_docs;" sql_super -c "CREATE TABLE invalid_schedule_docs (body text); ALTER TABLE invalid_schedule_docs OWNER TO durable_owner;" if create_error="$(sql_as durable_owner -c " CREATE INDEX CONCURRENTLY invalid_schedule_docs_idx ON invalid_schedule_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = 'not a cron');" 2>&1)"; then error "background CIC accepted an invalid compaction schedule" fi if ! grep -Fq "Invalid cron expression" <<<"${create_error}"; then error "invalid schedule was not rejected during CIC preflight" fi assert_eq "rejected invalid-schedule CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('invalid_schedule_docs_idx') IS NULL;")" sql_super -c "DROP TABLE invalid_schedule_docs;" } test_create_activation() { local index_oid create_output canonical_label sql_super -c "CREATE TABLE documents (id integer, body text); ALTER TABLE documents OWNER TO durable_owner;" sql_as durable_owner -c "INSERT INTO documents SELECT i, 'document ' || i || ' filler filler filler' FROM generate_series(1, 10000) AS i;" if ! create_output="$(sql_as durable_owner -c " CREATE INDEX CONCURRENTLY documents_idx ON documents USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CIC failed under hostile owner search_path: \ ${create_output}" fi if ! grep -Fq \ "pg_textsearch background compaction is a preview feature" \ <<<"${create_output}"; then error "successful activation did not emit the preview warning" fi index_oid="$(sql_super -c \ "SELECT 'documents_idx'::regclass::oid;")" assert_eq "successful background CIC is valid and ready" "t" \ "$(sql_super -c "SELECT index.indisvalid AND index.indisready FROM pg_catalog.pg_index AS index WHERE index.indexrelid = ${index_oid};")" assert_eq "hostile-path activation admits one active owner job" "1" \ "$(active_jobs_for_index "${index_oid}")" canonical_label="$(sql_super -c "SELECT pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s:%s:%s', database.oid, relation.oid, coalesce( nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, index_catalog.indrelid, pg_catalog.substr( lineage.option, pg_catalog.length('compaction_lineage=') + 1), pg_catalog.encode(pg_catalog.convert_to( pg_catalog.current_setting( 'pg_textsearch.background_compaction_schedule'), 'UTF8'), 'hex')) FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = relation.oid JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() CROSS JOIN LATERAL ( SELECT option FROM pg_catalog.unnest(relation.reloptions) AS option WHERE option OPERATOR(pg_catalog.~~) 'compaction_lineage=%' ) AS lineage WHERE relation.oid = 'documents_idx'::regclass;")" assert_eq "label captures physical identity and full schedule" \ "${canonical_label}" \ "$(sql_super -c "SELECT label FROM df.instances WHERE id = '$(active_job_id "${index_oid}")';")" assert_eq "workflow is submitted as the index owner" "durable_owner" \ "$(sql_super -c "SELECT submitted_by::pg_catalog.text FROM df.instances WHERE id = '$(active_job_id "${index_oid}")';")" wait_for_no_debt documents_idx 60 assert_eq "hostile-path current selection keeps the job active" "1" \ "$(active_jobs_for_index "${index_oid}")" assert_eq "activation pins pg_durable with a normal dependency" "1" \ "$(dependency_count)" local first_instance recovered_instance first_instance="$(active_job_id "${index_oid}")" wait_for_signal_node "${first_instance}" 30 create_compaction_debt wait_for_no_debt documents_idx 30 assert_eq "hostile-path current selection survives spill signaling" \ "${first_instance}" "$(active_job_id "${index_oid}")" sql_as durable_owner -c \ "SELECT df.cancel('${first_instance}', 'test recovery');" >/dev/null wait_for_terminal "${first_instance}" 30 create_compaction_debt wait_for_no_debt documents_idx 30 recovered_instance="$(active_job_id "${index_oid}")" if [ -z "${recovered_instance}" ] || [ "${recovered_instance}" = "${first_instance}" ]; then error "spill did not recover the terminal managed generation" fi assert_eq "recovery admits one replacement owner workflow" "1" \ "$(active_jobs_for_index "${index_oid}")" } test_reindex_nonrelation_passthrough() { local database_job_after database_job_before index_oid local reindex_output schema_job_after schema_job_before sql_super -c " CREATE SCHEMA lifecycle_reindex_scope AUTHORIZATION durable_owner; CREATE TABLE lifecycle_reindex_scope.documents (id integer PRIMARY KEY, body text); INSERT INTO lifecycle_reindex_scope.documents VALUES (1, 'one'), (2, 'two'); ALTER TABLE lifecycle_reindex_scope.documents OWNER TO durable_owner;" >/dev/null sql_as durable_owner -c " CREATE INDEX documents_bm25_idx ON lifecycle_reindex_scope.documents USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'lifecycle_reindex_scope.documents_bm25_idx'::regclass::oid;")" schema_job_before="$(current_generation_job_id "${index_oid}")" reindex_output="$(sql_as durable_owner -c \ "REINDEX SCHEMA lifecycle_reindex_scope;" 2>&1)" if grep -Fq "still active" <<<"${reindex_output}"; then error "REINDEX SCHEMA leaked an active snapshot across commit" fi schema_job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${schema_job_after}" ] || [ "${schema_job_after}" = "${schema_job_before}" ]; then error "REINDEX SCHEMA did not reconcile the managed generation" fi log "PASS: REINDEX SCHEMA reconciles managed indexes" database_job_before="${schema_job_after}" reindex_output="$(reindex_database \ "REINDEX DATABASE ${TEST_DB};")" if grep -Fq "still active" <<<"${reindex_output}"; then error "REINDEX DATABASE leaked an active snapshot across commit" fi database_job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${database_job_after}" ] || [ "${database_job_after}" = "${database_job_before}" ]; then error "REINDEX DATABASE did not reconcile the managed generation" fi log "PASS: REINDEX DATABASE reconciles managed indexes" wait_for_signal_node "${database_job_after}" 30 database_job_before="${database_job_after}" reindex_output="$(reindex_database "REINDEX DATABASE;")" database_job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${database_job_after}" ] || [ "${database_job_after}" = "${database_job_before}" ]; then error "bare REINDEX DATABASE did not reconcile the managed generation" fi log "PASS: bare REINDEX DATABASE reconciles managed indexes" sql_as durable_owner -c \ "DROP SCHEMA lifecycle_reindex_scope CASCADE;" >/dev/null } test_late_bulk_reindex_target() { local file_after file_before index_oid job_after reindex_output sql_super <<'SQL' >/dev/null CREATE SCHEMA lifecycle_reindex_late AUTHORIZATION durable_owner; CREATE TABLE lifecycle_reindex_late.documents (id integer PRIMARY KEY, body text); INSERT INTO lifecycle_reindex_late.documents VALUES (1, 'one'), (2, 'two'); ALTER TABLE lifecycle_reindex_late.documents OWNER TO durable_owner; CREATE INDEX documents_seed_bm25_idx ON lifecycle_reindex_late.documents USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); CREATE TABLE public.lifecycle_reindex_late_capture (index_oid oid, filenumber oid); CREATE FUNCTION public.lifecycle_reindex_late_create() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting( 'lifecycle.reindex_late', true) OPERATOR(pg_catalog.=) 'armed' THEN PERFORM pg_catalog.set_config( 'lifecycle.reindex_late', 'created', false); EXECUTE $command$ REINDEX INDEX lifecycle_reindex_late.documents_seed_bm25_idx $command$; EXECUTE $command$ CREATE INDEX documents_bm25_idx ON lifecycle_reindex_late.documents USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *') $command$; INSERT INTO public.lifecycle_reindex_late_capture SELECT relation.oid, pg_catalog.pg_relation_filenode(relation.oid) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'lifecycle_reindex_late.documents_bm25_idx'::regclass; END IF; END $body$; CREATE EVENT TRIGGER lifecycle_reindex_late_create ON ddl_command_start WHEN TAG IN ('REINDEX') EXECUTE FUNCTION public.lifecycle_reindex_late_create(); SQL sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET lifecycle.reindex_late = 'armed';" >/dev/null reindex_output="$(sql_as durable_owner -c \ "REINDEX SCHEMA lifecycle_reindex_late;" 2>&1)" if grep -Fq "still active" <<<"${reindex_output}"; then error "late-target REINDEX leaked an active snapshot across commit" fi index_oid="$(sql_super -c "SELECT index_oid FROM public.lifecycle_reindex_late_capture;")" file_before="$(sql_super -c "SELECT filenumber FROM public.lifecycle_reindex_late_capture;")" index_oid="$(sql_super -c "SELECT 'lifecycle_reindex_late.documents_bm25_idx'::regclass::oid;")" file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" if [ "${file_after}" = "${file_before}" ]; then error "REINDEX SCHEMA did not rebuild its late managed index" fi job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ]; then error "REINDEX SCHEMA did not reconcile its late managed index" fi log "PASS: REINDEX SCHEMA reconciles a late managed index" sql_super -c " DROP INDEX lifecycle_reindex_late.documents_bm25_idx; TRUNCATE public.lifecycle_reindex_late_capture;" >/dev/null reindex_output="$(sql_as durable_owner -c \ "REINDEX (CONCURRENTLY) SCHEMA lifecycle_reindex_late;" 2>&1)" index_oid="$(sql_super -c "SELECT index_oid FROM public.lifecycle_reindex_late_capture;")" file_before="$(sql_super -c "SELECT filenumber FROM public.lifecycle_reindex_late_capture;")" index_oid="$(sql_super -c "SELECT 'lifecycle_reindex_late.documents_bm25_idx'::regclass::oid;")" file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" if [ "${file_after}" = "${file_before}" ]; then error "concurrent REINDEX SCHEMA did not rebuild its late managed index" fi job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ]; then error "concurrent REINDEX SCHEMA did not reconcile its late managed index" fi log "PASS: concurrent REINDEX SCHEMA reconciles a late managed index" sql_super -c " ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET lifecycle.reindex_late; DROP EVENT TRIGGER lifecycle_reindex_late_create; DROP FUNCTION public.lifecycle_reindex_late_create(); DROP TABLE public.lifecycle_reindex_late_capture; DROP SCHEMA lifecycle_reindex_late CASCADE;" >/dev/null } test_bulk_reindex_concurrent_mode_change() { local blocker_pid file_after file_before index_oid job_after local reindex_output reindex_pid reindex_status=0 sql_super -c "CREATE SCHEMA lifecycle_reindex_mode AUTHORIZATION durable_owner;" >/dev/null sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE lifecycle_reindex_mode.blocker_docs (id integer PRIMARY KEY); CREATE TABLE lifecycle_reindex_mode.late_docs (id integer, body text); INSERT INTO lifecycle_reindex_mode.late_docs VALUES (1, 'one'); CREATE INDEX late_docs_idx ON lifecycle_reindex_mode.late_docs USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); SQL PGAPPNAME=lifecycle-reindex-mode-blocker \ sql_as durable_owner -c " BEGIN; LOCK TABLE lifecycle_reindex_mode.blocker_docs IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/reindex-mode-blocker.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-reindex-mode-blocker' AND relation_lock.relation = 'lifecycle_reindex_mode.blocker_docs'::regclass AND relation_lock.mode = 'AccessExclusiveLock' AND relation_lock.granted;")" = "1" ]; then break fi sleep 0.1 done index_oid="$(sql_super -c "SELECT 'lifecycle_reindex_mode.late_docs_idx'::regclass::oid;")" file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" reindex_output="${DATA_DIR}/reindex-mode.out" PGAPPNAME=lifecycle-reindex-mode \ PGOPTIONS="-c statement_timeout=30s" \ sql_as durable_owner -c \ "REINDEX SCHEMA lifecycle_reindex_mode;" \ >"${reindex_output}" 2>&1 & reindex_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-reindex-mode' AND relation_lock.relation = 'lifecycle_reindex_mode.blocker_docs'::regclass AND NOT relation_lock.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "broad REINDEX reached its post-initial-commit relation wait" \ "1" "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-reindex-mode' AND relation_lock.relation = 'lifecycle_reindex_mode.blocker_docs'::regclass AND NOT relation_lock.granted;")" sql_as durable_owner -c " ALTER INDEX lifecycle_reindex_mode.late_docs_idx SET (compaction = 'background', compaction_schedule = '1 2 3 4 *');" >/dev/null sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-reindex-mode-blocker';" >/dev/null wait "${blocker_pid}" || true wait "${reindex_pid}" || reindex_status=$? if [ "${reindex_status}" -ne 0 ]; then error "broad REINDEX with a concurrent mode change failed: $(cat "${reindex_output}")" fi file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" assert_ne "broad REINDEX rebuilt the newly managed index" \ "${file_before}" "${file_after}" job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ]; then error "broad REINDEX lost a concurrent background mode change" fi log "PASS: broad REINDEX tracks a concurrent background mode change" sql_super -c "DROP SCHEMA lifecycle_reindex_mode CASCADE;" >/dev/null } test_bulk_reindex_concurrent_schedule_change() { local blocker_pid blocker_table file_before index_name index_oid local reindex_output reindex_pid reindex_status=0 schedule_matches local target_table sql_super -c "CREATE SCHEMA lifecycle_reindex_schedule_change AUTHORIZATION durable_owner;" >/dev/null sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE lifecycle_reindex_schedule_change.a_docs (id integer, body text); INSERT INTO lifecycle_reindex_schedule_change.a_docs VALUES (1, 'a'); CREATE INDEX a_docs_idx ON lifecycle_reindex_schedule_change.a_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '1 0 1 1 *'); CREATE TABLE lifecycle_reindex_schedule_change.b_docs (id integer, body text); INSERT INTO lifecycle_reindex_schedule_change.b_docs VALUES (1, 'b'); CREATE INDEX b_docs_idx ON lifecycle_reindex_schedule_change.b_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '1 0 1 1 *'); SQL target_table="$(sql_super -c "SELECT relation.relname FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = 'lifecycle_reindex_schedule_change' AND relation.relname IN ('a_docs', 'b_docs') ORDER BY relation.ctid LIMIT 1;")" blocker_table="$(sql_super -c "SELECT relation.relname FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = 'lifecycle_reindex_schedule_change' AND relation.relname IN ('a_docs', 'b_docs') ORDER BY relation.ctid DESC LIMIT 1;")" index_name="${target_table}_idx" index_oid="$(sql_super -c "SELECT 'lifecycle_reindex_schedule_change.${index_name}' ::regclass::oid;")" file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" PGAPPNAME=lifecycle-reindex-schedule-blocker \ sql_as durable_owner -c " BEGIN; LOCK TABLE lifecycle_reindex_schedule_change.${blocker_table} IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/reindex-schedule-blocker.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-reindex-schedule-blocker' AND relation_lock.relation = 'lifecycle_reindex_schedule_change.${blocker_table}' ::regclass AND relation_lock.mode = 'AccessExclusiveLock' AND relation_lock.granted;")" = "1" ]; then break fi sleep 0.1 done reindex_output="${DATA_DIR}/reindex-schedule-change.out" PGAPPNAME=lifecycle-reindex-schedule-change \ PGOPTIONS="-c statement_timeout=30s" \ sql_as durable_owner -c \ "REINDEX SCHEMA lifecycle_reindex_schedule_change;" \ >"${reindex_output}" 2>&1 & reindex_pid=$! schedule_matches=0 for _ in $(seq 1 200); do if [ "$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" != \ "${file_before}" ] && [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-reindex-schedule-change' AND relation_lock.relation = 'lifecycle_reindex_schedule_change.${blocker_table}' ::regclass AND NOT relation_lock.granted;")" = "1" ]; then schedule_matches="$( current_generation_schedule_job_count \ "${index_oid}" "1 0 1 1 *" )" if [ "${schedule_matches}" = "1" ]; then break fi fi sleep 0.1 done assert_eq "broad REINDEX reconciled before its later relation wait" \ "1" "${schedule_matches}" sql_as durable_owner -c " ALTER INDEX lifecycle_reindex_schedule_change.${index_name} SET (compaction_schedule = '2 0 1 1 *');" >/dev/null sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-reindex-schedule-blocker';" >/dev/null wait "${blocker_pid}" || true wait "${reindex_pid}" || reindex_status=$? if [ "${reindex_status}" -ne 0 ]; then error "broad REINDEX with a concurrent schedule change failed: $(cat "${reindex_output}")" fi schedule_matches="$( current_generation_schedule_job_count "${index_oid}" "2 0 1 1 *" )" assert_eq "broad REINDEX preserves a later committed schedule" \ "1" "${schedule_matches}" sql_super -c " DROP SCHEMA lifecycle_reindex_schedule_change CASCADE;" >/dev/null } test_failed_bulk_reindex_reconciliation() { local failure_value index_oid job_after job_before reindex_error sql_super -c "CREATE SCHEMA lifecycle_reindex_failure AUTHORIZATION durable_owner;" >/dev/null sql_as durable_owner <<'SQL' CREATE TABLE lifecycle_reindex_failure.first_docs (id integer, body text); INSERT INTO lifecycle_reindex_failure.first_docs VALUES (1, 'one'); CREATE INDEX first_docs_idx ON lifecycle_reindex_failure.first_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE FUNCTION lifecycle_reindex_failure.fail_when_enabled(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE AS $body$ BEGIN IF pg_catalog.current_setting( 'lifecycle.fail_reindex', true) = value THEN RAISE EXCEPTION 'intentional later REINDEX failure'; END IF; RETURN value; END $body$; CREATE TABLE lifecycle_reindex_failure.second_docs (body text); INSERT INTO lifecycle_reindex_failure.second_docs VALUES ('two'); CREATE INDEX second_docs_idx ON lifecycle_reindex_failure.second_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX first_docs_fail_idx ON lifecycle_reindex_failure.first_docs (lifecycle_reindex_failure.fail_when_enabled(body)); CREATE INDEX second_docs_fail_idx ON lifecycle_reindex_failure.second_docs (lifecycle_reindex_failure.fail_when_enabled(body)); SQL index_oid="$(sql_super -c " SELECT pg_catalog.format( '%I.%I', namespace.nspname, relation.relname || '_idx' )::regclass::oid FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = 'lifecycle_reindex_failure' AND relation.relname IN ('first_docs', 'second_docs') ORDER BY relation.ctid LIMIT 1;")" failure_value="$(sql_super -c " SELECT CASE relation.relname WHEN 'first_docs' THEN 'one' ELSE 'two' END FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = relation.relnamespace WHERE namespace.nspname = 'lifecycle_reindex_failure' AND relation.relname IN ('first_docs', 'second_docs') ORDER BY relation.ctid DESC LIMIT 1;")" job_before="$(current_generation_job_id "${index_oid}")" if reindex_error="$(PGOPTIONS="-c lifecycle.fail_reindex=${failure_value}" \ sql_as durable_owner -c \ "REINDEX SCHEMA lifecycle_reindex_failure;" 2>&1)"; then error "bulk REINDEX failure test unexpectedly succeeded" fi if ! grep -Fq "intentional later REINDEX failure" \ <<<"${reindex_error}"; then error "bulk REINDEX failed for the wrong reason: ${reindex_error}" fi job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "failed bulk REINDEX stranded an already committed generation" fi log "PASS: failed bulk REINDEX reconciles committed generations" sql_super -c \ "DROP SCHEMA lifecycle_reindex_failure CASCADE;" >/dev/null } test_partitioned_create_activation() { local create_output leaf_jobs parent_oid sql_as durable_owner -c " CREATE TABLE public.lifecycle_partitioned_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_partitioned_docs_low PARTITION OF public.lifecycle_partitioned_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_partitioned_docs_high PARTITION OF public.lifecycle_partitioned_docs FOR VALUES FROM (100) TO (200);" if ! create_output="$(sql_as durable_owner -c " CREATE INDEX lifecycle_partitioned_idx ON public.lifecycle_partitioned_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" 2>&1)"; then error "partitioned background CREATE failed: ${create_output}" fi parent_oid="$(sql_super -c \ "SELECT 'public.lifecycle_partitioned_idx'::regclass::oid;")" assert_eq "partitioned parent has two physical leaf indexes" "2" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child ON child.oid = inheritance.inhrelid WHERE inheritance.inhparent = ${parent_oid} AND child.relkind = 'i';")" assert_eq "partitioned parent has no managed workflow" "0" \ "$(active_jobs_for_index "${parent_oid}")" leaf_jobs="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS relation ON relation.oid = inheritance.inhrelid JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() JOIN df.instances AS instance ON instance.label LIKE pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%%', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner) AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status IN ('pending', 'running') WHERE inheritance.inhparent = ${parent_oid} AND relation.relkind = 'i';")" assert_eq "partitioned CREATE activates every physical leaf" "2" \ "${leaf_jobs}" sql_as durable_owner -c " CREATE TABLE public.lifecycle_partitioned_docs_later PARTITION OF public.lifecycle_partitioned_docs FOR VALUES FROM (200) TO (300);" >/dev/null 2>&1 assert_eq "new partition inherits one managed workflow" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS relation ON relation.oid = inheritance.inhrelid JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = relation.oid WHERE inheritance.inhparent = ${parent_oid} AND relation.relkind = 'i' AND index_catalog.indrelid = 'public.lifecycle_partitioned_docs_later'::regclass AND EXISTS ( SELECT 1 FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid::pg_catalog.text || ':%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" sql_as durable_owner -c " CREATE TABLE public.lifecycle_partitioned_attach (id integer, body text); CREATE INDEX lifecycle_partitioned_attach_idx ON public.lifecycle_partitioned_attach USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); ALTER TABLE public.lifecycle_partitioned_docs ATTACH PARTITION public.lifecycle_partitioned_attach FOR VALUES FROM (300) TO (400);" >/dev/null 2>&1 assert_eq "attached index inherits managed parent options" "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=background'], relation.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *'], relation.reloptions @> ARRAY['compaction_lineage=' || pg_catalog.substr( parent_option, pg_catalog.length('compaction_lineage=') + 1)]) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL ( SELECT option AS parent_option FROM pg_catalog.pg_class AS parent, LATERAL pg_catalog.unnest(parent.reloptions) AS option WHERE parent.oid = ${parent_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%' ) AS lineage WHERE relation.oid = 'public.lifecycle_partitioned_attach_idx'::regclass;")" assert_eq "attached existing index receives a managed workflow" "1" \ "$(active_jobs_for_index \ "$(sql_super -c "SELECT 'public.lifecycle_partitioned_attach_idx'::regclass::oid;")")" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'partitioned lifecycle test complete') FROM df.instances AS instance JOIN pg_catalog.pg_inherits AS inheritance ON instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || inheritance.inhrelid::pg_catalog.text || ':%') WHERE inheritance.inhparent = ${parent_oid} AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_as durable_owner -c \ "DROP TABLE public.lifecycle_partitioned_docs;" } test_direct_index_partition_attach() { local child_oid parent_oid sql_as durable_owner <<'SQL' CREATE TABLE public.lifecycle_direct_attach_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_direct_attach_leaf PARTITION OF public.lifecycle_direct_attach_docs FOR VALUES FROM (0) TO (100); CREATE INDEX lifecycle_direct_attach_parent_idx ON ONLY public.lifecycle_direct_attach_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX lifecycle_direct_attach_child_idx ON public.lifecycle_direct_attach_leaf USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); ALTER INDEX public.lifecycle_direct_attach_parent_idx ATTACH PARTITION public.lifecycle_direct_attach_child_idx; SQL parent_oid="$(sql_super -c "SELECT 'public.lifecycle_direct_attach_parent_idx'::regclass::oid;")" child_oid="$(sql_super -c "SELECT 'public.lifecycle_direct_attach_child_idx'::regclass::oid;")" assert_eq "directly attached index inherits managed parent options" \ "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', child.reloptions @> ARRAY['compaction=background'], child.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *'], child.reloptions @> ARRAY['compaction_lineage=' || pg_catalog.substr( parent_option, pg_catalog.length('compaction_lineage=') + 1)]) FROM pg_catalog.pg_class AS child CROSS JOIN LATERAL ( SELECT option AS parent_option FROM pg_catalog.pg_class AS parent, LATERAL pg_catalog.unnest(parent.reloptions) AS option WHERE parent.oid = ${parent_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%' ) AS lineage WHERE child.oid = ${child_oid};")" assert_eq "directly attached index receives a managed workflow" "1" \ "$(current_generation_job_count "${child_oid}")" sql_super -c \ "DROP TABLE public.lifecycle_direct_attach_docs;" >/dev/null } test_reused_intermediate_partition_options() { local direct_existing_oid direct_intermediate_oid direct_late_oid local direct_lineage table_existing_oid table_intermediate_oid local table_late_oid table_lineage sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_intermediate_table_root (id integer, subid integer, body text) PARTITION BY RANGE (id); CREATE INDEX lifecycle_intermediate_table_root_idx ON ONLY public.lifecycle_intermediate_table_root USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_intermediate_table_branch (id integer, subid integer, body text) PARTITION BY RANGE (subid); CREATE TABLE public.lifecycle_intermediate_table_low PARTITION OF public.lifecycle_intermediate_table_branch FOR VALUES FROM (0) TO (100); CREATE INDEX lifecycle_intermediate_table_branch_idx ON public.lifecycle_intermediate_table_branch USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); ALTER TABLE public.lifecycle_intermediate_table_root ATTACH PARTITION public.lifecycle_intermediate_table_branch FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_intermediate_direct_root (id integer, subid integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_intermediate_direct_branch (id integer, subid integer, body text) PARTITION BY RANGE (subid); ALTER TABLE public.lifecycle_intermediate_direct_root ATTACH PARTITION public.lifecycle_intermediate_direct_branch FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_intermediate_direct_low PARTITION OF public.lifecycle_intermediate_direct_branch FOR VALUES FROM (0) TO (100); CREATE INDEX lifecycle_intermediate_direct_root_idx ON ONLY public.lifecycle_intermediate_direct_root USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX lifecycle_intermediate_direct_branch_idx ON public.lifecycle_intermediate_direct_branch USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); ALTER INDEX public.lifecycle_intermediate_direct_root_idx ATTACH PARTITION public.lifecycle_intermediate_direct_branch_idx; SQL table_intermediate_oid="$(sql_super -c "SELECT 'public.lifecycle_intermediate_table_branch_idx'::regclass::oid;")" table_existing_oid="$(sql_super -c "SELECT 'public.lifecycle_intermediate_table_low_body_idx'::regclass::oid;")" table_lineage="$( index_lineage public.lifecycle_intermediate_table_root_idx )" assert_eq "table attach aligns reused intermediate options" "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=background'], relation.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *'], relation.reloptions @> ARRAY['compaction_lineage=${table_lineage}']) FROM pg_catalog.pg_class AS relation WHERE relation.oid = ${table_intermediate_oid};")" assert_eq "table attach activates existing physical leaves" "1" \ "$(current_generation_job_count "${table_existing_oid}")" sql_as durable_owner -c " CREATE TABLE public.lifecycle_intermediate_table_late PARTITION OF public.lifecycle_intermediate_table_branch FOR VALUES FROM (100) TO (200);" >/dev/null 2>&1 table_late_oid="$(sql_super -c "SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_intermediate_table_late'::regclass;")" assert_eq "later table-attached leaf inherits aligned options" "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=background'], relation.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *'], relation.reloptions @> ARRAY['compaction_lineage=${table_lineage}']) FROM pg_catalog.pg_class AS relation WHERE relation.oid = ${table_late_oid};")" assert_eq "later table-attached leaf has one workflow" "1" \ "$(current_generation_job_count "${table_late_oid}")" direct_intermediate_oid="$(sql_super -c "SELECT 'public.lifecycle_intermediate_direct_branch_idx'::regclass::oid;")" direct_existing_oid="$(sql_super -c "SELECT 'public.lifecycle_intermediate_direct_low_body_idx'::regclass::oid;")" direct_lineage="$( index_lineage public.lifecycle_intermediate_direct_root_idx )" assert_eq "direct attach aligns reused intermediate options" "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=background'], relation.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *'], relation.reloptions @> ARRAY['compaction_lineage=${direct_lineage}']) FROM pg_catalog.pg_class AS relation WHERE relation.oid = ${direct_intermediate_oid};")" assert_eq "direct attach activates existing physical leaves" "1" \ "$(current_generation_job_count "${direct_existing_oid}")" sql_as durable_owner -c " CREATE TABLE public.lifecycle_intermediate_direct_late PARTITION OF public.lifecycle_intermediate_direct_branch FOR VALUES FROM (100) TO (200);" >/dev/null 2>&1 direct_late_oid="$(sql_super -c "SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_intermediate_direct_late'::regclass;")" assert_eq "later directly attached leaf inherits aligned options" \ "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=background'], relation.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *'], relation.reloptions @> ARRAY['compaction_lineage=${direct_lineage}']) FROM pg_catalog.pg_class AS relation WHERE relation.oid = ${direct_late_oid};")" assert_eq "later directly attached leaf has one workflow" "1" \ "$(current_generation_job_count "${direct_late_oid}")" sql_super -c " DROP TABLE public.lifecycle_intermediate_table_root, public.lifecycle_intermediate_direct_root;" >/dev/null } test_partition_detach_lineage() { local branch_index_oid concurrent_child_index_oid concurrent_detach_output local concurrent_detach_pid concurrent_new_lineage concurrent_old_job local concurrent_old_lineage concurrent_reader_pid detach_backend_pid local detach_dump detached_job detached_lineage detached_low_index_oid local old_job old_lineage restore_output root_index_oid sibling_index_oid local restored_detached_lineage restored_parent_lineage detach_dump="${DATA_DIR}/partition-detach-lineage.sql" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_detach_root (id integer, subid integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_detach_branch (id integer, subid integer, body text) PARTITION BY RANGE (subid); ALTER TABLE public.lifecycle_detach_root ATTACH PARTITION public.lifecycle_detach_branch FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_detach_branch_low PARTITION OF public.lifecycle_detach_branch FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_detach_sibling PARTITION OF public.lifecycle_detach_root FOR VALUES FROM (100) TO (200); CREATE INDEX lifecycle_detach_root_idx ON public.lifecycle_detach_root USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL root_index_oid="$(sql_super -c "SELECT 'public.lifecycle_detach_root_idx'::regclass::oid;")" branch_index_oid="$(sql_super -c "SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_detach_branch'::regclass;")" detached_low_index_oid="$(sql_super -c "SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_detach_branch_low'::regclass;")" sibling_index_oid="$(sql_super -c "SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_detach_sibling'::regclass;")" old_lineage="$(index_lineage public.lifecycle_detach_root_idx)" old_job="$(current_generation_job_id "${detached_low_index_oid}")" sql_as durable_owner -c " ALTER TABLE public.lifecycle_detach_root DETACH PARTITION public.lifecycle_detach_branch;" >/dev/null 2>&1 detached_lineage="$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${branch_index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_ne "plain detach assigns a fresh hierarchy lineage" \ "${old_lineage}" "${detached_lineage}" assert_eq "plain detach lineage remains 128-bit" "32" \ "${#detached_lineage}" assert_eq "detached physical leaves share the fresh lineage" \ "${detached_lineage}" \ "$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${detached_low_index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "former parent hierarchy retains its lineage" \ "${old_lineage}:${old_lineage}" \ "$(index_lineage public.lifecycle_detach_root_idx):$( sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${sibling_index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';" )" detached_job="$(current_generation_job_id "${detached_low_index_oid}")" assert_ne "plain detach replaces the detached workflow" \ "${old_job}" "${detached_job}" assert_eq "plain detach preserves the workflow schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '0 0 1 1 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${detached_job}';")" "${PGBINDIR}/pg_dump" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" --schema-only --no-owner \ --table=public.lifecycle_detach_root \ --table=public.lifecycle_detach_branch \ --table=public.lifecycle_detach_branch_low \ --table=public.lifecycle_detach_sibling >"${detach_dump}" sql_as durable_owner -c " DROP TABLE public.lifecycle_detach_root, public.lifecycle_detach_branch;" >/dev/null if ! restore_output="$(sql_as durable_owner -f "${detach_dump}" 2>&1)"; then error "detached hierarchies failed schema-only restore: \ ${restore_output}" fi restored_parent_lineage="$( index_lineage public.lifecycle_detach_root_idx )" restored_detached_lineage="$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ( SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_detach_branch'::regclass) AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "restored parent lineage remains 128-bit" "32" \ "${#restored_parent_lineage}" assert_eq "restored detached lineage remains 128-bit" "32" \ "${#restored_detached_lineage}" assert_ne "restored detached hierarchies retain distinct lineages" \ "${restored_parent_lineage}" "${restored_detached_lineage}" sql_as durable_owner -c " DROP TABLE public.lifecycle_detach_root, public.lifecycle_detach_branch;" >/dev/null sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_detach_concurrent_root (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_detach_concurrent_low PARTITION OF public.lifecycle_detach_concurrent_root FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_detach_concurrent_high PARTITION OF public.lifecycle_detach_concurrent_root FOR VALUES FROM (100) TO (200); CREATE INDEX lifecycle_detach_concurrent_idx ON public.lifecycle_detach_concurrent_root USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL concurrent_child_index_oid="$(sql_super -c "SELECT indexrelid FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_detach_concurrent_high'::regclass;")" concurrent_old_lineage="$( index_lineage public.lifecycle_detach_concurrent_idx )" concurrent_old_job="$( current_generation_job_id "${concurrent_child_index_oid}" )" PGAPPNAME=lifecycle-detach-reader sql_as durable_owner <<'SQL' \ >/dev/null 2>&1 & BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM public.lifecycle_detach_concurrent_root; SELECT pg_catalog.pg_sleep(30); COMMIT; SQL concurrent_reader_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-detach-reader' AND query OPERATOR(pg_catalog.~~) '%pg_sleep%';")" = "1" ]; then break fi sleep 0.1 done assert_eq "concurrent detach reader holds an old snapshot" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-detach-reader' AND query OPERATOR(pg_catalog.~~) '%pg_sleep%';")" concurrent_detach_output="${DATA_DIR}/partition-detach-concurrent.out" PGAPPNAME=lifecycle-detach-concurrent sql_as durable_owner -c " ALTER TABLE public.lifecycle_detach_concurrent_root DETACH PARTITION public.lifecycle_detach_concurrent_high CONCURRENTLY;" >"${concurrent_detach_output}" 2>&1 & concurrent_detach_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhrelid = 'public.lifecycle_detach_concurrent_high'::regclass AND inhdetachpending;")" = "1" ]; then break fi sleep 0.1 done assert_eq "concurrent detach reaches pending phase" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhrelid = 'public.lifecycle_detach_concurrent_high'::regclass AND inhdetachpending;")" assert_eq "concurrent detach phase one preserves lineage" \ "${concurrent_old_lineage}" \ "$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${concurrent_child_index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" detach_backend_pid="$(sql_super -c "SELECT pid FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-detach-concurrent';")" sql_super -c \ "SELECT pg_catalog.pg_terminate_backend(${detach_backend_pid});" \ >/dev/null if wait "${concurrent_detach_pid}"; then error "concurrent detach completed before forced finalization" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-detach-reader';" >/dev/null wait "${concurrent_reader_pid}" 2>/dev/null || true assert_eq "interrupted concurrent detach remains pending" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits WHERE inhrelid = 'public.lifecycle_detach_concurrent_high'::regclass AND inhdetachpending;")" sql_as durable_owner -c " ALTER TABLE public.lifecycle_detach_concurrent_root DETACH PARTITION public.lifecycle_detach_concurrent_high FINALIZE;" >/dev/null 2>&1 concurrent_new_lineage="$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${concurrent_child_index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_ne "concurrent finalize assigns a fresh lineage" \ "${concurrent_old_lineage}" "${concurrent_new_lineage}" assert_eq "concurrent finalize leaves parent lineage unchanged" \ "${concurrent_old_lineage}" \ "$(index_lineage public.lifecycle_detach_concurrent_idx)" assert_ne "concurrent finalize replaces the detached workflow" \ "${concurrent_old_job}" \ "$(current_generation_job_id "${concurrent_child_index_oid}")" sql_super -c " DROP TABLE public.lifecycle_detach_concurrent_root, public.lifecycle_detach_concurrent_high;" >/dev/null } test_partitioned_existing_leaf_reconciliation() { local high_oid low_oid parent_lineage parent_oid sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_existing_leaf_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_existing_leaf_low PARTITION OF public.lifecycle_existing_leaf_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_existing_leaf_high PARTITION OF public.lifecycle_existing_leaf_docs FOR VALUES FROM (100) TO (200); CREATE INDEX lifecycle_existing_leaf_low_idx ON public.lifecycle_existing_leaf_low USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); SQL sql_super -c "ALTER TABLE public.lifecycle_existing_leaf_high OWNER TO durable_owner_two;" sql_as durable_owner_two -c " CREATE INDEX lifecycle_existing_leaf_high_idx ON public.lifecycle_existing_leaf_high USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '5 4 3 2 *');" >/dev/null 2>&1 low_oid="$(sql_super -c "SELECT 'public.lifecycle_existing_leaf_low_idx'::regclass::oid;")" high_oid="$(sql_super -c "SELECT 'public.lifecycle_existing_leaf_high_idx'::regclass::oid;")" sql_as durable_owner -c " CREATE INDEX lifecycle_existing_leaf_parent_idx ON public.lifecycle_existing_leaf_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 parent_oid="$(sql_super -c "SELECT 'public.lifecycle_existing_leaf_parent_idx'::regclass::oid;")" parent_lineage="$( index_lineage public.lifecycle_existing_leaf_parent_idx )" assert_eq "partitioned CREATE attaches both existing leaf indexes" \ "2" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_partition_tree(${parent_oid}) AS tree WHERE tree.isleaf AND tree.relid = ANY ( ARRAY[${low_oid}, ${high_oid}]::pg_catalog.oid[]);")" assert_eq "attached leaves inherit parent background options" "2:2:2" \ "$(sql_super -c "SELECT count(*) FILTER ( WHERE relation.reloptions @> ARRAY['compaction=background']) || ':' || count(*) FILTER ( WHERE relation.reloptions @> ARRAY['compaction_schedule=0 0 1 1 *']) || ':' || count(*) FILTER ( WHERE relation.reloptions @> ARRAY['compaction_lineage=${parent_lineage}']) FROM pg_catalog.pg_partition_tree(${parent_oid}) AS tree JOIN pg_catalog.pg_class AS relation ON relation.oid = tree.relid WHERE tree.isleaf AND relation.relkind = 'i';")" assert_eq "attached leaves have one current owner workflow each" "2:2" \ "$(sql_super -c "SELECT count(*) || ':' || count(*) FILTER ( WHERE EXISTS ( SELECT 1 FROM df.instances AS instance JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE instance.label OPERATOR(pg_catalog.~~) pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s:%s:%%', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, index_catalog.indrelid, '${parent_lineage}') AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running'] ::pg_catalog.text[]))) FROM pg_catalog.pg_partition_tree(${parent_oid}) AS tree JOIN pg_catalog.pg_class AS relation ON relation.oid = tree.relid JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = relation.oid WHERE tree.isleaf AND relation.relkind = 'i';")" assert_eq "attached leaves select the parent workflow as current" "2" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_partition_tree(${parent_oid}) AS tree JOIN pg_catalog.pg_class AS relation ON relation.oid = tree.relid JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE tree.isleaf AND relation.relkind = 'i' AND ( SELECT instance.label FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%%', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner) AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running'] ::pg_catalog.text[]) ORDER BY instance.created_at DESC, instance.id DESC LIMIT 1 ) OPERATOR(pg_catalog.~~) ('%:' || '${parent_lineage}' || ':%');")" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'existing leaf reconciliation complete') FROM df.instances AS instance WHERE instance.submitted_by = 'durable_owner'::regrole AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) AND (instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${low_oid}:%');" >/dev/null sql_as durable_owner_two -c "SELECT df.cancel( instance.id, 'existing leaf reconciliation complete') FROM df.instances AS instance WHERE instance.submitted_by = 'durable_owner_two'::regrole AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) AND (instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${high_oid}:%');" >/dev/null sql_super -c \ "DROP TABLE public.lifecycle_existing_leaf_docs;" } test_create_tracking_reentry() { local nested_oid outer_oid sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_create_reentry_docs (body_a text, body_b text); CREATE FUNCTION public.lifecycle_create_reenter() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-create-reentry' AND pg_catalog.to_regclass( 'public.lifecycle_create_reentry_nested_idx') IS NULL THEN EXECUTE 'CREATE INDEX lifecycle_create_reentry_nested_idx ' 'ON public.lifecycle_create_reentry_docs USING bm25(body_b) ' 'WITH (text_config = ''english'', compaction = ''manual'')'; END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_create_reenter ON ddl_command_end WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION public.lifecycle_create_reenter();" >/dev/null PGAPPNAME=lifecycle-create-reentry sql_as durable_owner -c " CREATE INDEX lifecycle_create_reentry_outer_idx ON public.lifecycle_create_reentry_docs USING bm25(body_a) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_super -c "DROP EVENT TRIGGER lifecycle_create_reenter; DROP FUNCTION public.lifecycle_create_reenter();" \ >/dev/null outer_oid="$(sql_super -c "SELECT 'public.lifecycle_create_reentry_outer_idx'::regclass::oid;")" nested_oid="$(sql_super -c "SELECT 'public.lifecycle_create_reentry_nested_idx'::regclass::oid;")" assert_eq "nested CREATE retains its own manual configuration" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_catalog.pg_class WHERE oid = ${nested_oid};")" assert_eq "outer CREATE activates only its own index" "1:0" \ "$(active_jobs_for_index "${outer_oid}"):$( active_jobs_for_index "${nested_oid}")" sql_super -c "DROP TABLE public.lifecycle_create_reentry_docs;" } test_cached_create_uses_fresh_lineage() { local first_lineage second_lineage sql_super -c " CREATE SCHEMA lifecycle_cached_create_a AUTHORIZATION durable_owner; CREATE SCHEMA lifecycle_cached_create_b AUTHORIZATION durable_owner;" \ >/dev/null sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE lifecycle_cached_create_a.docs (body text); CREATE TABLE lifecycle_cached_create_b.docs (body text); CREATE FUNCTION public.lifecycle_cached_create() RETURNS void LANGUAGE plpgsql AS $body$ DECLARE schema_name text; BEGIN FOREACH schema_name IN ARRAY ARRAY[ 'lifecycle_cached_create_a', 'lifecycle_cached_create_b' ] LOOP PERFORM pg_catalog.set_config( 'search_path', pg_catalog.format('%I, public, pg_catalog', schema_name), true); CREATE INDEX lifecycle_cached_create_idx ON docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); END LOOP; END $body$; SELECT public.lifecycle_cached_create(); SQL first_lineage="$( index_lineage lifecycle_cached_create_a.lifecycle_cached_create_idx )" second_lineage="$( index_lineage lifecycle_cached_create_b.lifecycle_cached_create_idx )" if [ "${second_lineage}" = "${first_lineage}" ]; then error "cached CREATE INDEX reused its prior compaction lineage" fi assert_eq "cached CREATE INDEX creates one workflow per heap" "2" \ "$(( $(active_jobs_for_index "$(sql_super -c "SELECT 'lifecycle_cached_create_a.lifecycle_cached_create_idx' ::regclass::oid;")") + $(active_jobs_for_index "$(sql_super -c "SELECT 'lifecycle_cached_create_b.lifecycle_cached_create_idx' ::regclass::oid;")") ))" sql_super -c "DROP SCHEMA lifecycle_cached_create_a CASCADE; DROP SCHEMA lifecycle_cached_create_b CASCADE; DROP FUNCTION public.lifecycle_cached_create();" } test_create_tracking_concurrent() { local blocker_pid concurrent_oid outer_oid outer_output outer_pid local outer_status=0 sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_create_concurrent_docs (body_a text, body_b text); INSERT INTO public.lifecycle_create_concurrent_docs VALUES ('pause', 'other'), ('continue', 'other'); CREATE FUNCTION public.lifecycle_create_concurrent_pause(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF value OPERATOR(pg_catalog.=) 'pause' THEN PERFORM pg_catalog.pg_advisory_xact_lock(478, 8); END IF; RETURN value; END $body$; SQL PGAPPNAME=lifecycle-create-concurrent-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 8); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/create-concurrent-gate.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 8 AND granted;")" = "1" ]; then break fi sleep 0.1 done outer_output="${DATA_DIR}/create-concurrent-outer.out" PGAPPNAME=lifecycle-create-concurrent-outer \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "CREATE INDEX lifecycle_create_concurrent_outer_idx ON public.lifecycle_create_concurrent_docs USING bm25(public.lifecycle_create_concurrent_pause(body_a)) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" \ >"${outer_output}" 2>&1 & outer_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-concurrent-outer' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "first CREATE pauses during its index build" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-concurrent-outer' AND wait_event = 'advisory';")" sql_as durable_owner -c " SET statement_timeout = '10s'; CREATE INDEX lifecycle_create_concurrent_other_idx ON public.lifecycle_create_concurrent_docs USING bm25(body_b) WITH (text_config = 'english', compaction = 'manual');" \ >/dev/null sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-concurrent-gate';" >/dev/null wait "${blocker_pid}" || true wait "${outer_pid}" || outer_status=$? if [ "${outer_status}" -ne 0 ]; then error "tracked concurrent CREATE failed: $(cat "${outer_output}")" fi outer_oid="$(sql_super -c "SELECT 'public.lifecycle_create_concurrent_outer_idx'::regclass::oid;")" concurrent_oid="$(sql_super -c "SELECT 'public.lifecycle_create_concurrent_other_idx'::regclass::oid;")" assert_eq "concurrent CREATE retains its manual configuration" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_catalog.pg_class WHERE oid = ${concurrent_oid};")" assert_eq "tracked CREATE does not absorb a concurrent index" "1:0" \ "$(active_jobs_for_index "${outer_oid}"):$( active_jobs_for_index "${concurrent_oid}")" sql_super -c "DROP TABLE public.lifecycle_create_concurrent_docs; DROP FUNCTION public.lifecycle_create_concurrent_pause(text);" } test_create_tracking_table_rename() { local heap_oid outer_oid outer_output outer_pid outer_status=0 local rename_output rename_pid rename_status=0 snapshot_pid sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_create_rename_docs (body text); INSERT INTO public.lifecycle_create_rename_docs SELECT pg_catalog.format('document %s filler', value) FROM pg_catalog.generate_series(1, 1000) AS value; SQL heap_oid="$(sql_super -c "SELECT 'public.lifecycle_create_rename_docs'::regclass::oid;")" PGAPPNAME=lifecycle-create-rename-snapshot sql_super -c \ "BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM public.lifecycle_create_rename_docs; SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/create-rename-snapshot.out" 2>&1 & snapshot_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-rename-snapshot' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done outer_output="${DATA_DIR}/create-rename-outer.out" PGAPPNAME=lifecycle-create-rename-outer \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "CREATE INDEX CONCURRENTLY lifecycle_create_rename_idx ON public.lifecycle_create_rename_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" \ >"${outer_output}" 2>&1 & outer_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_progress_create_index AS progress JOIN pg_catalog.pg_stat_activity AS activity ON activity.pid = progress.pid WHERE activity.application_name = 'lifecycle-create-rename-outer' AND progress.phase = 'waiting for old snapshots';")" = "1" ]; then break fi sleep 0.1 done assert_eq "concurrent CREATE waits for the old snapshot" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_progress_create_index AS progress JOIN pg_catalog.pg_stat_activity AS activity ON activity.pid = progress.pid WHERE activity.application_name = 'lifecycle-create-rename-outer' AND progress.phase = 'waiting for old snapshots';")" rename_output="${DATA_DIR}/create-rename-other.out" PGAPPNAME=lifecycle-create-rename-other \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "ALTER TABLE public.lifecycle_create_rename_docs RENAME TO lifecycle_create_renamed_docs;" \ >"${rename_output}" 2>&1 & rename_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-rename-other' AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done assert_eq "table rename queues behind concurrent CREATE" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-rename-other' AND wait_event_type = 'Lock';")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-rename-snapshot';" >/dev/null wait "${snapshot_pid}" || true wait "${outer_pid}" || outer_status=$? wait "${rename_pid}" || rename_status=$? if [ "${rename_status}" -ne 0 ]; then error "queued table rename failed: $(cat "${rename_output}")" fi if [ "${outer_status}" -ne 0 ]; then error "CREATE lost its original heap after rename: \ $(cat "${outer_output}")" fi outer_oid="$(sql_super -c "SELECT 'public.lifecycle_create_rename_idx'::regclass::oid;")" assert_eq "tracked CREATE retains the original heap OID" \ "${heap_oid}" \ "$(sql_super -c "SELECT indrelid FROM pg_catalog.pg_index WHERE indexrelid = ${outer_oid};")" assert_eq "renamed-table CREATE activates its created index" "1" \ "$(active_jobs_for_index "${outer_oid}")" sql_super -c "DROP TABLE public.lifecycle_create_renamed_docs;" } test_create_like_regenerates_lineage() { local clone_index clone_lineage create_output source_lineage sql_as durable_owner -c " CREATE TABLE public.lifecycle_like_source (body text); CREATE INDEX lifecycle_like_source_idx ON public.lifecycle_like_source USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 source_lineage="$(index_lineage public.lifecycle_like_source_idx)" if ! create_output="$(sql_as durable_owner -c " CREATE TABLE public.lifecycle_like_clone (LIKE public.lifecycle_like_source INCLUDING INDEXES);" 2>&1)"; then if ! grep -Fq \ "background compaction lineage is already in use" \ <<<"${create_output}"; then error "CREATE TABLE LIKE failed unexpectedly: ${create_output}" fi error "CREATE TABLE LIKE reused the source managed lineage" fi clone_index="$(sql_super -c "SELECT indexrelid::regclass::text FROM pg_catalog.pg_index WHERE indrelid = 'public.lifecycle_like_clone'::regclass;")" clone_lineage="$(index_lineage "${clone_index}")" if [ -z "${clone_lineage}" ] || [ "${clone_lineage}" = "${source_lineage}" ]; then error "CREATE TABLE LIKE did not assign a fresh managed lineage" fi assert_eq "CREATE TABLE LIKE activates the cloned managed index" "1" \ "$(active_jobs_for_index \ "$(sql_super -c "SELECT '${clone_index}'::regclass::oid;")")" sql_super -c "DROP TABLE public.lifecycle_like_clone, public.lifecycle_like_source;" } test_owner_reconciliation() { local alter_error alter_output dependencies_before index_oid job_before local job_after jobs_before filenumber_before filenumber_after local privileges_before sql_as durable_owner -c " CREATE TABLE public.lifecycle_owner_docs (body text); CREATE INDEX lifecycle_owner_idx ON public.lifecycle_owner_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_owner_idx'::regclass::oid;")" filenumber_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_before="$(current_generation_job_id "${index_oid}")" jobs_before="$(managed_job_count)" if ! alter_output="$(sql_super -c " ALTER INDEX public.lifecycle_owner_idx OWNER TO durable_nologin;" 2>&1)"; then error "ALTER INDEX OWNER no-op failed: ${alter_output}" fi if ! grep -Fq "cannot change owner of index" <<<"${alter_output}"; then error "ALTER INDEX OWNER did not preserve the core warning: \ ${alter_output}" fi assert_eq "ALTER INDEX OWNER remains a no-op" "durable_owner" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" assert_eq "ALTER INDEX OWNER no-op creates no workflow" \ "${jobs_before}" "$(managed_job_count)" if alter_error="$(sql_as durable_owner -c " ALTER TABLE public.lifecycle_owner_docs OWNER TO durable_usage_only;" 2>&1)"; then error "ALTER TABLE OWNER bypassed role membership" fi if ! grep -Fq "must be able to SET ROLE \"durable_usage_only\"" \ <<<"${alter_error}"; then error "ALTER TABLE OWNER did not preserve core role-membership \ ordering: ${alter_error}" fi sql_super -c \ "ALTER TABLE public.lifecycle_owner_docs OWNER TO durable_owner_two;" \ >/dev/null 2>&1 filenumber_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_after="$(current_generation_job_id "${index_oid}")" assert_eq "ALTER OWNER preserves the physical generation" \ "${filenumber_before}" "${filenumber_after}" assert_eq "ALTER TABLE OWNER changes the physical index owner" \ "durable_owner_two" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "ALTER TABLE OWNER did not create a new-owner workflow" fi assert_eq "ALTER TABLE OWNER submits the replacement as the new owner" \ "durable_owner_two" \ "$(sql_super -c "SELECT submitted_by::pg_catalog.text FROM df.instances WHERE id = '${job_after}';")" sql_as durable_owner -c " CREATE TABLE public.lifecycle_owner_reject_docs (body text); CREATE INDEX lifecycle_owner_reject_idx ON public.lifecycle_owner_reject_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 jobs_before="$(managed_job_count)" dependencies_before="$(dependency_count)" privileges_before="$(helper_privileges_for_role durable_nologin)" if alter_error="$(sql_as durable_writer -c " ALTER TABLE public.lifecycle_owner_reject_docs OWNER TO durable_nologin;" 2>&1)"; then error "unrelated writer changed background index ownership" fi if ! grep -Fq "must be owner of table lifecycle_owner_reject_docs" \ <<<"${alter_error}"; then error "unauthorized ALTER OWNER bypassed core ownership checks: \ ${alter_error}" fi assert_eq "unauthorized ALTER OWNER creates no workflow" \ "${jobs_before}" "$(managed_job_count)" if alter_error="$(sql_super -c " ALTER TABLE public.lifecycle_owner_reject_docs OWNER TO durable_nologin;" 2>&1)"; then error "ALTER TABLE OWNER accepted an ineligible NOLOGIN owner" fi if ! grep -Fq \ "index owner must have LOGIN for background compaction" \ <<<"${alter_error}"; then error "ALTER TABLE OWNER did not preflight the new owner: \ ${alter_error}" fi assert_eq "rejected ALTER TABLE OWNER preserves the old owner" \ "durable_owner" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_owner_reject_idx'::regclass;")" assert_eq "rejected ALTER TABLE OWNER creates no workflow" \ "${jobs_before}" "$(managed_job_count)" assert_eq "rejected ALTER TABLE OWNER changes no dependency" \ "${dependencies_before}" "$(dependency_count)" assert_eq "rejected ALTER TABLE OWNER changes no helper grants" \ "${privileges_before}" \ "$(helper_privileges_for_role durable_nologin)" sql_super -c " CREATE SCHEMA lifecycle_owner_private AUTHORIZATION durable_owner; GRANT durable_owner_two TO durable_owner;" sql_as durable_owner -c " CREATE TABLE lifecycle_owner_private.docs (body text); CREATE INDEX docs_idx ON lifecycle_owner_private.docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 if alter_error="$(sql_as durable_owner -c " ALTER TABLE lifecycle_owner_private.docs OWNER TO durable_owner_two;" 2>&1)"; then error "ALTER TABLE OWNER bypassed target schema CREATE" fi if ! grep -Fq "permission denied for schema lifecycle_owner_private" \ <<<"${alter_error}"; then error "ALTER TABLE OWNER did not preserve core schema CREATE \ ordering: ${alter_error}" fi assert_eq "schema-rejected ALTER TABLE OWNER preserves the old owner" \ "durable_owner" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = 'lifecycle_owner_private.docs_idx'::regclass;")" sql_super -c " REVOKE durable_owner_two FROM durable_owner; DROP SCHEMA lifecycle_owner_private CASCADE;" >/dev/null sql_as durable_owner_two -c "SELECT df.cancel( '${job_after}', 'owner lifecycle test complete');" >/dev/null sql_as durable_owner -c "SELECT df.cancel( instance.id, 'owner lifecycle test complete') FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${index_oid}:%' AND instance.submitted_by OPERATOR(pg_catalog.=) 'durable_owner'::regrole AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_as durable_owner -c "SELECT df.cancel( instance.id, 'owner lifecycle test complete') FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || 'public.lifecycle_owner_reject_idx'::regclass::oid::text || ':%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_super -c \ "DROP TABLE public.lifecycle_owner_docs, public.lifecycle_owner_reject_docs;" } test_owner_change_preserves_captured_schedule() { local index_oid job_after sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '1 2 3 4 *';" sql_as durable_owner -c " CREATE TABLE public.lifecycle_owner_schedule_docs (body text); CREATE INDEX lifecycle_owner_schedule_idx ON public.lifecycle_owner_schedule_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_owner_schedule_idx'::regclass::oid;")" sql_super -c " SET pg_textsearch.background_compaction_schedule = '5 6 7 8 *'; ALTER TABLE public.lifecycle_owner_schedule_docs OWNER TO durable_owner_two;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" assert_eq "ALTER OWNER preserves the captured default schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${job_after}';")" sql_as durable_owner_two -c \ "SELECT df.cancel('${job_after}', 'owner schedule test complete');" \ >/dev/null sql_super -c "DROP TABLE public.lifecycle_owner_schedule_docs;" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET pg_textsearch.background_compaction_schedule;" } test_reassign_owned_reconciliation() { local create_output index_oid job_after job_before reassign_output sql_super -c " CREATE ROLE lifecycle_reassign_old LOGIN; CREATE ROLE lifecycle_reassign_new LOGIN; GRANT CONNECT ON DATABASE ${TEST_DB} TO lifecycle_reassign_old, lifecycle_reassign_new; GRANT USAGE, CREATE ON SCHEMA public TO lifecycle_reassign_old; GRANT USAGE ON SCHEMA public TO lifecycle_reassign_new; ALTER ROLE lifecycle_reassign_old IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '1 2 3 4 *'; SELECT df.grant_usage('lifecycle_reassign_old'); SELECT df.grant_usage('lifecycle_reassign_new');" >/dev/null if ! create_output="$(sql_as lifecycle_reassign_old -c " CREATE TABLE public.lifecycle_reassign_docs (body text); CREATE INDEX lifecycle_reassign_idx ON public.lifecycle_reassign_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" 2>&1)"; then error "REASSIGN OWNED setup failed: ${create_output}" fi index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_reassign_idx'::regclass::oid;")" job_before="$(current_generation_job_id "${index_oid}")" if ! reassign_output="$(sql_super -c " SET pg_textsearch.background_compaction_schedule = '5 6 7 8 *'; REASSIGN OWNED BY lifecycle_reassign_old TO lifecycle_reassign_new;" 2>&1)"; then error "REASSIGN OWNED failed: ${reassign_output}" fi job_after="$(current_generation_job_id "${index_oid}")" assert_eq "REASSIGN OWNED changes the physical index owner" \ "lifecycle_reassign_new" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "REASSIGN OWNED did not create a new-owner workflow" fi assert_eq "REASSIGN OWNED submits the replacement as the new owner" \ "lifecycle_reassign_new" \ "$(sql_super -c "SELECT submitted_by::pg_catalog.text FROM df.instances WHERE id = '${job_after}';")" assert_eq "REASSIGN OWNED preserves the captured default schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${job_after}';")" sql_as lifecycle_reassign_new -c " INSERT INTO public.lifecycle_reassign_docs SELECT pg_catalog.format('reassigned document %s', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.lifecycle_reassign_idx');" \ >/dev/null 2>&1 assert_eq "reassigned owner can signal the current workflow" "1" \ "$(current_generation_job_count "${index_oid}")" sql_as lifecycle_reassign_new -c \ "SELECT df.cancel('${job_after}', 'reassign lifecycle complete');" \ >/dev/null sql_super -c " DROP OWNED BY lifecycle_reassign_old; DROP OWNED BY lifecycle_reassign_new CASCADE; DROP ROLE lifecycle_reassign_old; DROP ROLE lifecycle_reassign_new;" >/dev/null } test_reindex_reconciliation() { local a_before a_after a_table_after b_before b_after local a_file_before a_file_after a_table_file_before a_table_file_after local b_file_before b_file_after a_oid b_oid sql_as durable_owner -c " CREATE TABLE public.lifecycle_reindex_docs (body_a text, body_b text); CREATE INDEX lifecycle_reindex_a_idx ON public.lifecycle_reindex_docs USING bm25(body_a) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_as durable_owner -c " CREATE INDEX lifecycle_reindex_b_idx ON public.lifecycle_reindex_docs USING bm25(body_b) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 a_oid="$(sql_super -c \ "SELECT 'public.lifecycle_reindex_a_idx'::regclass::oid;")" b_oid="$(sql_super -c \ "SELECT 'public.lifecycle_reindex_b_idx'::regclass::oid;")" a_before="$(current_generation_job_id "${a_oid}")" b_before="$(current_generation_job_id "${b_oid}")" a_file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${a_oid});")" sql_as durable_owner -c \ "REINDEX INDEX public.lifecycle_reindex_a_idx;" >/dev/null 2>&1 a_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${a_oid});")" a_after="$(current_generation_job_id "${a_oid}")" if [ "${a_file_after}" = "${a_file_before}" ]; then error "REINDEX INDEX did not replace the physical generation" fi if [ -z "${a_after}" ] || [ "${a_after}" = "${a_before}" ]; then error "REINDEX INDEX did not reconcile a replacement workflow" fi a_table_after="${a_after}" a_table_file_before="${a_file_after}" b_file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${b_oid});")" sql_as durable_owner -c \ "REINDEX TABLE public.lifecycle_reindex_docs;" >/dev/null 2>&1 a_table_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${a_oid});")" b_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${b_oid});")" a_table_after="$(current_generation_job_id "${a_oid}")" b_after="$(current_generation_job_id "${b_oid}")" if [ "${a_table_file_after}" = "${a_table_file_before}" ] || [ "${b_file_after}" = "${b_file_before}" ]; then error "REINDEX TABLE did not replace every physical generation" fi if [ -z "${a_table_after}" ] || [ "${a_table_after}" = "${a_after}" ] || [ -z "${b_after}" ] || [ "${b_after}" = "${b_before}" ]; then error "REINDEX TABLE did not reconcile every background index" fi assert_eq "REINDEX TABLE selects one current workflow per index" "2" \ "$(( $(current_generation_job_count "${a_oid}") + $(current_generation_job_count "${b_oid}") ))" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'reindex lifecycle test complete') FROM df.instances AS instance WHERE (instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${a_oid}:%' OR instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${b_oid}:%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_as durable_owner -c "DROP TABLE public.lifecycle_reindex_docs;" } test_plain_reindex_preserves_captured_schedule() { local current_after current_before current_oid local legacy_after legacy_before legacy_lineage legacy_oid sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '1 2 3 4 *';" sql_as durable_owner -c " CREATE TABLE public.lifecycle_reindex_schedule_index_docs (body text); INSERT INTO public.lifecycle_reindex_schedule_index_docs VALUES ('current'); CREATE INDEX lifecycle_reindex_schedule_index_idx ON public.lifecycle_reindex_schedule_index_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); CREATE TABLE public.lifecycle_reindex_schedule_table_docs (body text); INSERT INTO public.lifecycle_reindex_schedule_table_docs VALUES ('legacy'); CREATE INDEX lifecycle_reindex_schedule_table_idx ON public.lifecycle_reindex_schedule_table_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" >/dev/null 2>&1 current_oid="$(sql_super -c "SELECT 'public.lifecycle_reindex_schedule_index_idx'::regclass::oid;")" legacy_oid="$(sql_super -c "SELECT 'public.lifecycle_reindex_schedule_table_idx'::regclass::oid;")" current_before="$(current_generation_job_id "${current_oid}")" legacy_before="$(current_generation_job_id "${legacy_oid}")" sql_super -c "UPDATE df.instances AS instance SET label = pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, pg_catalog.encode( pg_catalog.convert_to('1 2 3 4 *', 'UTF8'), 'hex')) FROM pg_catalog.pg_class AS relation, pg_catalog.pg_database AS database WHERE instance.id = '${legacy_before}' AND relation.oid = ${legacy_oid} AND database.datname = pg_catalog.current_database();" remove_index_lineage public.lifecycle_reindex_schedule_table_idx sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '5 6 7 8 *';" sql_as durable_owner -c " REINDEX INDEX public.lifecycle_reindex_schedule_index_idx;" \ >/dev/null 2>&1 current_after="$(current_generation_job_id "${current_oid}")" if [ -z "${current_after}" ] || [ "${current_after}" = "${current_before}" ]; then error "plain REINDEX INDEX created no replacement workflow" fi assert_eq "plain REINDEX INDEX preserves the captured schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${current_after}';")" sql_as durable_owner -c " REINDEX TABLE public.lifecycle_reindex_schedule_table_docs;" \ >/dev/null 2>&1 legacy_after="$(current_generation_job_id "${legacy_oid}")" if [ -z "${legacy_after}" ] || [ "${legacy_after}" = "${legacy_before}" ]; then error "plain REINDEX TABLE created no replacement workflow" fi legacy_lineage="$( index_lineage public.lifecycle_reindex_schedule_table_idx )" assert_eq "plain REINDEX TABLE backfills legacy lineage" "32" \ "${#legacy_lineage}" assert_eq "plain REINDEX TABLE preserves the legacy schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${legacy_after}';")" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET pg_textsearch.background_compaction_schedule;" sql_super -c " DROP TABLE public.lifecycle_reindex_schedule_index_docs, public.lifecycle_reindex_schedule_table_docs;" } test_physical_rewrite_reconciliation() { local excluded_oid external_oid file_after file_before index_oid local job_after job_before local lock_error lock_pid tablespace_dir sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '1 2 3 4 *';" sql_as durable_owner -c " CREATE TABLE public.lifecycle_rewrite_docs (id integer, body text); INSERT INTO public.lifecycle_rewrite_docs SELECT value, pg_catalog.format('document %s', value) FROM pg_catalog.generate_series(1, 100) AS value; CREATE INDEX lifecycle_rewrite_cluster_idx ON public.lifecycle_rewrite_docs(id); CREATE INDEX lifecycle_rewrite_idx ON public.lifecycle_rewrite_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_rewrite_idx'::regclass::oid;")" file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_before="$(current_generation_job_id "${index_oid}")" sql_super -c "ALTER ROLE postgres IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '5 6 7 8 *';" sql_super -c \ "VACUUM (FULL) public.lifecycle_rewrite_docs;" >/dev/null 2>&1 file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_after="$(current_generation_job_id "${index_oid}")" if [ "${file_after}" = "${file_before}" ]; then error "VACUUM FULL did not replace the BM25 physical generation" fi if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "VACUUM FULL did not reconcile the replacement workflow" fi file_before="${file_after}" job_before="${job_after}" sql_super -c "CLUSTER public.lifecycle_rewrite_docs USING lifecycle_rewrite_cluster_idx;" >/dev/null 2>&1 file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_after="$(current_generation_job_id "${index_oid}")" if [ "${file_after}" = "${file_before}" ]; then error "CLUSTER did not replace the BM25 physical generation" fi if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "CLUSTER did not reconcile the replacement workflow" fi tablespace_dir="${DATA_DIR}-lifecycle-rewrite-tablespace" mkdir -p "${tablespace_dir}" sql_super -c " CREATE TABLESPACE lifecycle_rewrite_tablespace OWNER durable_owner LOCATION '${tablespace_dir}';" >/dev/null job_before="${job_after}" sql_super -c " ALTER INDEX public.lifecycle_rewrite_idx SET TABLESPACE lifecycle_rewrite_tablespace;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "ALTER INDEX SET TABLESPACE did not reconcile the workflow" fi job_before="${job_after}" sql_super -c " ALTER INDEX ALL IN TABLESPACE lifecycle_rewrite_tablespace OWNED BY durable_owner SET TABLESPACE pg_default;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "ALTER INDEX ALL IN TABLESPACE did not reconcile the workflow" fi sql_as durable_owner -c " CREATE TABLE public.lifecycle_rewrite_external (body text); CREATE INDEX lifecycle_rewrite_external_idx ON public.lifecycle_rewrite_external USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *') TABLESPACE lifecycle_rewrite_tablespace;" \ >/dev/null 2>&1 external_oid="$(sql_super -c "SELECT 'public.lifecycle_rewrite_external_idx'::regclass::oid;")" remove_index_lineage public.lifecycle_rewrite_external_idx sql_as durable_owner_two -c " CREATE TABLE public.lifecycle_rewrite_excluded (body text); CREATE INDEX lifecycle_rewrite_excluded_idx ON public.lifecycle_rewrite_excluded USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 excluded_oid="$(sql_super -c "SELECT 'public.lifecycle_rewrite_excluded_idx'::regclass::oid;")" remove_index_lineage public.lifecycle_rewrite_excluded_idx PGAPPNAME=lifecycle-tablespace-nowait \ sql_as durable_owner -c " BEGIN; ALTER INDEX public.lifecycle_rewrite_idx SET (compaction_schedule = '1 2 3 4 *'); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/tablespace-nowait-lock.out" 2>&1 & lock_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation = ${index_oid} AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if lock_error="$(sql_super -c " SET statement_timeout = '2s'; ALTER INDEX ALL IN TABLESPACE pg_default OWNED BY durable_owner SET TABLESPACE lifecycle_rewrite_tablespace NOWAIT;" 2>&1)"; then error "ALTER INDEX ALL NOWAIT unexpectedly moved a locked index" fi if grep -Fq "canceling statement due to statement timeout" \ <<<"${lock_error}"; then error "ALTER INDEX ALL NOWAIT blocked behind an extension prelock" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-tablespace-nowait';" >/dev/null wait "${lock_pid}" || true job_before="${job_after}" sql_super -c " ALTER INDEX ALL IN TABLESPACE pg_default OWNED BY durable_owner SET TABLESPACE lifecycle_rewrite_tablespace;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "pg_default bulk move did not reconcile the workflow" fi assert_eq "OWNED BY leaves excluded legacy lineage untouched" "" \ "$(index_lineage "${excluded_oid}")" assert_eq "pg_default move leaves another tablespace untouched" "" \ "$(index_lineage "${external_oid}")" assert_eq "physical rewrites preserve the captured default schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${job_after}';")" sql_as durable_owner -c \ "SELECT df.cancel('${job_after}', 'rewrite lifecycle test complete');" \ >/dev/null sql_super -c "DROP TABLE public.lifecycle_rewrite_docs, public.lifecycle_rewrite_excluded, public.lifecycle_rewrite_external;" sql_super -c "DROP TABLESPACE lifecycle_rewrite_tablespace;" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET pg_textsearch.background_compaction_schedule;" sql_super -c "ALTER ROLE postgres IN DATABASE ${TEST_DB} RESET pg_textsearch.background_compaction_schedule;" } test_database_owner_vacuum_full() { local file_after file_before index_oid job_after job_before local vacuum_output sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_database_vacuum_docs (body text); INSERT INTO public.lifecycle_database_vacuum_docs VALUES ('alpha'); CREATE INDEX lifecycle_database_vacuum_idx ON public.lifecycle_database_vacuum_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_database_vacuum_idx'::regclass::oid;")" file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_before="$(current_generation_job_id "${index_oid}")" sql_super -c "GRANT USAGE ON SCHEMA public TO durable_actor; ALTER DATABASE ${TEST_DB} OWNER TO durable_actor;" \ >/dev/null if ! vacuum_output="$(sql_as durable_actor -c \ "VACUUM (FULL) public.lifecycle_database_vacuum_docs;" 2>&1)"; then sql_super -c "ALTER DATABASE ${TEST_DB} OWNER TO postgres;" \ >/dev/null error "database-owner VACUUM FULL failed: ${vacuum_output}" fi sql_super -c "ALTER DATABASE ${TEST_DB} OWNER TO postgres;" >/dev/null file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" job_after="$(current_generation_job_id "${index_oid}")" if [ "${file_after}" = "${file_before}" ]; then error "database-owner VACUUM FULL did not rewrite the index" fi if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "database-owner VACUUM FULL did not reconcile the workflow" fi log "PASS: database-owner VACUUM FULL reconciles managed indexes" sql_super -c \ "DROP TABLE public.lifecycle_database_vacuum_docs;" >/dev/null } test_vacuum_full_skip_locked() { local blocker_pid index_oid job_before vacuum_output sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_skip_locked_docs (body text); INSERT INTO public.lifecycle_skip_locked_docs VALUES ('alpha'); CREATE INDEX lifecycle_skip_locked_idx ON public.lifecycle_skip_locked_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_skip_locked_idx'::regclass::oid;")" job_before="$(current_generation_job_id "${index_oid}")" PGAPPNAME=lifecycle-vacuum-skip-lock \ sql_as durable_owner -c " BEGIN; LOCK TABLE public.lifecycle_skip_locked_docs IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/vacuum-skip-lock.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_skip_locked_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if ! vacuum_output="$(PGOPTIONS='-c statement_timeout=2s' \ sql_as durable_owner -c \ "VACUUM (FULL, SKIP_LOCKED) public.lifecycle_skip_locked_docs;" 2>&1)"; then kill "${blocker_pid}" 2>/dev/null || true wait "${blocker_pid}" 2>/dev/null || true error "VACUUM FULL SKIP_LOCKED blocked or failed: ${vacuum_output}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-vacuum-skip-lock';" >/dev/null wait "${blocker_pid}" 2>/dev/null || true for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_skip_locked_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "0" ]; then break fi sleep 0.1 done assert_eq "VACUUM FULL SKIP_LOCKED leaves the workflow unchanged" \ "${job_before}" "$(current_generation_job_id "${index_oid}")" log "PASS: VACUUM FULL SKIP_LOCKED preserves core lock skipping" if ! vacuum_output="$(sql_as durable_owner -c \ "VACUUM (FULL, SKIP_LOCKED) public.lifecycle_skip_locked_docs;" 2>&1)"; then error "unlocked VACUUM FULL SKIP_LOCKED failed: ${vacuum_output}" fi if [ "$(current_generation_job_id "${index_oid}")" = "${job_before}" ]; then error "unlocked VACUUM FULL SKIP_LOCKED did not reconcile the workflow" fi log "PASS: unlocked VACUUM FULL SKIP_LOCKED reconciles the workflow" sql_super -c \ "DROP TABLE public.lifecycle_skip_locked_docs;" >/dev/null } test_partition_vacuum_child_authorization() { local blocker_pid child_index child_job_before low_index low_job_after local low_job_before only_output server_version vacuum_output vacuum_pid local vacuum_status=0 server_version="$(sql_super -c \ "SELECT pg_catalog.current_setting('server_version_num')::integer;")" if [ "${server_version}" -lt 180000 ]; then log "PASS: partition-child VACUUM authorization is PG18-only" return fi sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE FUNCTION public.lifecycle_vacuum_auth_pause(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE AS $body$ BEGIN PERFORM pg_catalog.pg_advisory_lock(478, 11); PERFORM pg_catalog.pg_advisory_unlock(478, 11); RETURN value; END $body$; CREATE TABLE public.lifecycle_vacuum_auth_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_vacuum_auth_low PARTITION OF public.lifecycle_vacuum_auth_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_vacuum_auth_high PARTITION OF public.lifecycle_vacuum_auth_docs FOR VALUES FROM (100) TO (200); CREATE INDEX lifecycle_vacuum_auth_idx ON public.lifecycle_vacuum_auth_docs USING bm25(public.lifecycle_vacuum_auth_pause(body)) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); INSERT INTO public.lifecycle_vacuum_auth_low VALUES (1, 'alpha'); SQL sql_super -c " ALTER TABLE public.lifecycle_vacuum_auth_high OWNER TO durable_owner_two; GRANT MAINTAIN ON public.lifecycle_vacuum_auth_low TO durable_writer;" >/dev/null child_index="$(sql_super -c " SELECT child_index.oid::regclass::text FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child_index ON child_index.oid = inheritance.inhrelid JOIN pg_catalog.pg_index AS index_info ON index_info.indexrelid = child_index.oid WHERE inheritance.inhparent = 'public.lifecycle_vacuum_auth_idx'::regclass AND index_info.indrelid = 'public.lifecycle_vacuum_auth_high'::regclass;")" low_index="$(sql_super -c " SELECT child_index.oid::regclass::text FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child_index ON child_index.oid = inheritance.inhrelid JOIN pg_catalog.pg_index AS index_info ON index_info.indexrelid = child_index.oid WHERE inheritance.inhparent = 'public.lifecycle_vacuum_auth_idx'::regclass AND index_info.indrelid = 'public.lifecycle_vacuum_auth_low'::regclass;")" child_job_before="$(current_generation_job_id \ "$(sql_super -c "SELECT '${child_index}'::regclass::oid;")")" low_job_before="$(current_generation_job_id \ "$(sql_super -c "SELECT '${low_index}'::regclass::oid;")")" PGAPPNAME=lifecycle-vacuum-auth-gate \ sql_super -c " SELECT pg_catalog.pg_advisory_lock(478, 11); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/vacuum-auth-gate.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 11 AND granted;")" = "1" ]; then break fi sleep 0.1 done vacuum_output="${DATA_DIR}/vacuum-auth-child.out" PGAPPNAME=lifecycle-vacuum-auth-child \ sql_as durable_writer -c " VACUUM (FULL) public.lifecycle_vacuum_auth_docs;" \ >"${vacuum_output}" 2>&1 & vacuum_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-vacuum-auth-child' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "authorized child VACUUM reaches the rewrite gate" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-vacuum-auth-child' AND wait_event = 'advisory';")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-vacuum-auth-gate';" >/dev/null wait "${blocker_pid}" || true wait "${vacuum_pid}" || vacuum_status=$? if [ "${vacuum_status}" -ne 0 ]; then error "authorized child VACUUM failed: $(cat "${vacuum_output}")" fi low_job_after="$(current_generation_job_id \ "$(sql_super -c "SELECT '${low_index}'::regclass::oid;")")" if [ -z "${low_job_after}" ] || [ "${low_job_after}" = "${low_job_before}" ]; then error "VACUUM did not reconcile an authorized child of an \ unauthorized root" fi assert_eq "VACUUM leaves an unauthorized child workflow unchanged" \ "${child_job_before}" \ "$(current_generation_job_id \ "$(sql_super -c "SELECT '${child_index}'::regclass::oid;")")" sql_super -c "GRANT MAINTAIN ON public.lifecycle_vacuum_auth_docs TO durable_writer;" >/dev/null PGAPPNAME=lifecycle-vacuum-auth-only-lock \ sql_as durable_owner_two -c " BEGIN; ALTER INDEX ${child_index} SET (compaction_schedule = '2 3 4 5 *'); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/vacuum-auth-only-lock.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = '${child_index}'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if ! only_output="$(PGOPTIONS='-c statement_timeout=2s' \ sql_as durable_writer -c " VACUUM (FULL) ONLY public.lifecycle_vacuum_auth_docs;" 2>&1)"; then error "VACUUM ONLY prelocked a child index: ${only_output}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-vacuum-auth-only-lock';" \ >/dev/null wait "${blocker_pid}" || true sql_super -c "DROP TABLE public.lifecycle_vacuum_auth_docs; DROP FUNCTION public.lifecycle_vacuum_auth_pause(text);" \ >/dev/null } test_global_cluster_scope() { local blocker_pid clustered_file_before clustered_index_oid local clustered_job_before cluster_output unclustered_index_oid local unclustered_job_before sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_global_cluster_docs (id integer, body text); INSERT INTO public.lifecycle_global_cluster_docs VALUES (1, 'alpha'); CREATE INDEX lifecycle_global_cluster_order_idx ON public.lifecycle_global_cluster_docs(id); CREATE INDEX lifecycle_global_cluster_idx ON public.lifecycle_global_cluster_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CLUSTER public.lifecycle_global_cluster_docs USING lifecycle_global_cluster_order_idx; CREATE TABLE public.lifecycle_global_unclustered_docs (body text); CREATE INDEX lifecycle_global_unclustered_idx ON public.lifecycle_global_unclustered_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL clustered_index_oid="$(sql_super -c "SELECT 'public.lifecycle_global_cluster_idx'::regclass::oid;")" unclustered_index_oid="$(sql_super -c "SELECT 'public.lifecycle_global_unclustered_idx'::regclass::oid;")" clustered_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${clustered_index_oid});")" clustered_job_before="$( current_generation_job_id "${clustered_index_oid}" )" unclustered_job_before="$( current_generation_job_id "${unclustered_index_oid}" )" PGAPPNAME=lifecycle-global-cluster-lock \ sql_as durable_owner -c " BEGIN; ALTER INDEX public.lifecycle_global_unclustered_idx RENAME TO lifecycle_global_unclustered_locked_idx; SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/global-cluster-lock.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation = ${unclustered_index_oid} AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if ! cluster_output="$(PGOPTIONS='-c statement_timeout=2s' \ sql_as durable_owner -c "CLUSTER;" 2>&1)"; then sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-global-cluster-lock';" \ >/dev/null wait "${blocker_pid}" || true error "global CLUSTER touched an unclustered heap: ${cluster_output}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-global-cluster-lock';" >/dev/null wait "${blocker_pid}" || true if [ "$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${clustered_index_oid});")" = \ "${clustered_file_before}" ]; then error "global CLUSTER did not rewrite the clustered managed index" fi if [ "$(current_generation_job_id "${clustered_index_oid}")" = \ "${clustered_job_before}" ]; then error "global CLUSTER did not reconcile the clustered workflow" fi assert_eq "global CLUSTER ignores unclustered managed indexes" \ "${unclustered_job_before}" \ "$(current_generation_job_id "${unclustered_index_oid}")" sql_super -c "DROP TABLE public.lifecycle_global_cluster_docs, public.lifecycle_global_unclustered_docs;" \ >/dev/null } test_global_cluster_concurrent_mode_change() { local blocker_pid cluster_output cluster_pid cluster_status=0 local file_after file_before index_oid job_after sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_global_cluster_mode_blocker (id integer, body text); INSERT INTO public.lifecycle_global_cluster_mode_blocker VALUES (1, 'blocker'); CREATE INDEX lifecycle_global_cluster_mode_blocker_order_idx ON public.lifecycle_global_cluster_mode_blocker(id); CREATE INDEX lifecycle_global_cluster_mode_blocker_idx ON public.lifecycle_global_cluster_mode_blocker USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); CLUSTER public.lifecycle_global_cluster_mode_blocker USING lifecycle_global_cluster_mode_blocker_order_idx; CREATE TABLE public.lifecycle_global_cluster_mode_target (id integer, body text); INSERT INTO public.lifecycle_global_cluster_mode_target VALUES (1, 'target'); CREATE INDEX lifecycle_global_cluster_mode_target_order_idx ON public.lifecycle_global_cluster_mode_target(id); CREATE INDEX lifecycle_global_cluster_mode_target_idx ON public.lifecycle_global_cluster_mode_target USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); CLUSTER public.lifecycle_global_cluster_mode_target USING lifecycle_global_cluster_mode_target_order_idx; SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_global_cluster_mode_target_idx'::regclass::oid;")" file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" PGAPPNAME=lifecycle-global-cluster-mode-blocker \ sql_as durable_owner -c " BEGIN; ALTER INDEX public.lifecycle_global_cluster_mode_blocker_idx SET (compaction = 'inline'); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/global-cluster-mode-blocker.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_global_cluster_mode_blocker_idx' ::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done cluster_output="${DATA_DIR}/global-cluster-mode.out" PGAPPNAME=lifecycle-global-cluster-mode \ PGOPTIONS="-c statement_timeout=30s" \ sql_as durable_owner -c "CLUSTER;" \ >"${cluster_output}" 2>&1 & cluster_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-global-cluster-mode' AND relation_lock.relation = 'public.lifecycle_global_cluster_mode_blocker_idx' ::regclass AND NOT relation_lock.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "global CLUSTER reached its candidate-discovery wait" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-global-cluster-mode' AND relation_lock.relation = 'public.lifecycle_global_cluster_mode_blocker_idx' ::regclass AND NOT relation_lock.granted;")" sql_as durable_owner -c " ALTER INDEX public.lifecycle_global_cluster_mode_target_idx SET (compaction = 'background', compaction_schedule = '1 2 3 4 *');" >/dev/null sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-global-cluster-mode-blocker';" >/dev/null wait "${blocker_pid}" || true wait "${cluster_pid}" || cluster_status=$? if [ "${cluster_status}" -ne 0 ]; then error "global CLUSTER with a concurrent mode change failed: $(cat "${cluster_output}")" fi file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" assert_ne "global CLUSTER rebuilt the newly managed index" \ "${file_before}" "${file_after}" job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ]; then error "global CLUSTER lost a concurrent background mode change" fi log "PASS: global CLUSTER tracks a concurrent background mode change" sql_super -c " DROP TABLE public.lifecycle_global_cluster_mode_target, public.lifecycle_global_cluster_mode_blocker;" >/dev/null } test_inheritance_vacuum_full_scope() { local child_file_after child_file_before child_index_oid local child_job_after child_job_before parent_file_after local parent_file_before parent_index_oid parent_job_after local parent_job_before local server_version_num server_version_num="$(sql_super -c \ "SHOW server_version_num;")" if [ "${server_version_num}" -lt 180000 ]; then log "PASS: recursive ordinary-inheritance VACUUM is PostgreSQL 18+" return fi sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_vacuum_parent (body text); CREATE TABLE public.lifecycle_vacuum_child () INHERITS (public.lifecycle_vacuum_parent); INSERT INTO public.lifecycle_vacuum_parent VALUES ('parent'); INSERT INTO public.lifecycle_vacuum_child VALUES ('child'); CREATE INDEX lifecycle_vacuum_parent_idx ON public.lifecycle_vacuum_parent USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX lifecycle_vacuum_child_idx ON public.lifecycle_vacuum_child USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL parent_index_oid="$(sql_super -c "SELECT 'public.lifecycle_vacuum_parent_idx'::regclass::oid;")" child_index_oid="$(sql_super -c "SELECT 'public.lifecycle_vacuum_child_idx'::regclass::oid;")" parent_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" parent_job_before="$(current_generation_job_id "${parent_index_oid}")" child_job_before="$(current_generation_job_id "${child_index_oid}")" sql_as durable_owner -c \ "VACUUM (FULL) public.lifecycle_vacuum_parent;" >/dev/null 2>&1 parent_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" parent_job_after="$(current_generation_job_id "${parent_index_oid}")" child_job_after="$(current_generation_job_id "${child_index_oid}")" if [ "${parent_file_after}" = "${parent_file_before}" ] || [ "${child_file_after}" = "${child_file_before}" ]; then error "recursive VACUUM FULL did not rewrite parent and child indexes" fi if [ "${parent_job_after}" = "${parent_job_before}" ] || [ "${child_job_after}" = "${child_job_before}" ]; then error "recursive VACUUM FULL did not reconcile parent and child" fi parent_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" child_job_before="$(current_generation_job_id "${child_index_oid}")" sql_as durable_owner -c \ "VACUUM (FULL) ONLY public.lifecycle_vacuum_parent;" >/dev/null 2>&1 parent_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" if [ "${parent_file_after}" = "${parent_file_before}" ]; then error "VACUUM FULL ONLY did not rewrite the parent index" fi assert_eq "VACUUM FULL ONLY leaves child storage unchanged" \ "${child_file_before}" \ "$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" assert_eq "VACUUM FULL ONLY leaves child workflow unchanged" \ "${child_job_before}" \ "$(current_generation_job_id "${child_index_oid}")" sql_super -c "DROP TABLE public.lifecycle_vacuum_child, public.lifecycle_vacuum_parent;" >/dev/null } test_inheritance_alter_rewrite_scope() { local child_file_after child_file_before child_index_oid local child_job_after child_job_before parent_file_after local parent_file_before parent_index_oid parent_job_after local parent_job_before sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_alter_parent (marker integer, body text); CREATE TABLE public.lifecycle_alter_child () INHERITS (public.lifecycle_alter_parent); INSERT INTO public.lifecycle_alter_parent VALUES (1, 'parent'); INSERT INTO public.lifecycle_alter_child VALUES (2, 'child'); CREATE INDEX lifecycle_alter_parent_idx ON public.lifecycle_alter_parent USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX lifecycle_alter_child_idx ON public.lifecycle_alter_child USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL parent_index_oid="$(sql_super -c "SELECT 'public.lifecycle_alter_parent_idx'::regclass::oid;")" child_index_oid="$(sql_super -c "SELECT 'public.lifecycle_alter_child_idx'::regclass::oid;")" parent_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" parent_job_before="$(current_generation_job_id "${parent_index_oid}")" child_job_before="$(current_generation_job_id "${child_index_oid}")" sql_as durable_owner -c " ALTER TABLE public.lifecycle_alter_parent ALTER COLUMN marker TYPE bigint USING marker::pg_catalog.int8;" >/dev/null 2>&1 parent_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" parent_job_after="$(current_generation_job_id "${parent_index_oid}")" child_job_after="$(current_generation_job_id "${child_index_oid}")" if [ "${parent_file_after}" = "${parent_file_before}" ] || [ "${child_file_after}" = "${child_file_before}" ]; then error "recursive ALTER TABLE did not rewrite parent and child indexes" fi if [ -z "${parent_job_after}" ] || [ "${parent_job_after}" = "${parent_job_before}" ]; then error "recursive ALTER TABLE did not reconcile parent" fi if [ -z "${child_job_after}" ] || [ "${child_job_after}" = "${child_job_before}" ]; then error "recursive ALTER TABLE did not reconcile child" fi parent_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" child_job_before="$(current_generation_job_id "${child_index_oid}")" sql_as durable_owner -c " ALTER TABLE ONLY public.lifecycle_alter_parent SET UNLOGGED;" >/dev/null 2>&1 parent_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" if [ "${parent_file_after}" = "${parent_file_before}" ]; then error "ALTER TABLE ONLY did not rewrite the parent index" fi assert_eq "ALTER TABLE ONLY leaves child storage unchanged" \ "${child_file_before}" \ "$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${child_index_oid});")" assert_eq "ALTER TABLE ONLY leaves child workflow unchanged" \ "${child_job_before}" \ "$(current_generation_job_id "${child_index_oid}")" sql_super -c "DROP TABLE public.lifecycle_alter_child, public.lifecycle_alter_parent;" >/dev/null } test_additional_rewrite_reconciliation() { local index_oid job_after job_before mv_index_oid mv_job_after local mv_job_before sql_as durable_owner -c " CREATE TABLE public.lifecycle_additional_rewrite_docs (id integer, body text); INSERT INTO public.lifecycle_additional_rewrite_docs VALUES (1, 'one'); CREATE INDEX lifecycle_additional_rewrite_idx ON public.lifecycle_additional_rewrite_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_additional_rewrite_idx'::regclass::oid;")" job_before="$(current_generation_job_id "${index_oid}")" sql_as durable_owner -c " ALTER TABLE public.lifecycle_additional_rewrite_docs ALTER COLUMN id TYPE bigint;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "ALTER COLUMN TYPE did not reconcile the workflow" fi sql_as durable_owner -c " ALTER TABLE public.lifecycle_additional_rewrite_docs ADD COLUMN generated_body text GENERATED ALWAYS AS (body) STORED;" >/dev/null 2>&1 job_before="${job_after}" sql_as durable_owner -c " ALTER TABLE public.lifecycle_additional_rewrite_docs ALTER COLUMN generated_body SET EXPRESSION AS (body OPERATOR(pg_catalog.||) ' changed');" \ >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "ALTER COLUMN SET EXPRESSION did not reconcile the workflow" fi job_before="${job_after}" sql_as durable_owner -c \ "TRUNCATE public.lifecycle_additional_rewrite_docs;" \ >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "TRUNCATE did not reconcile the workflow" fi sql_as durable_owner -c " CREATE MATERIALIZED VIEW public.lifecycle_rewrite_mv AS SELECT body FROM public.lifecycle_additional_rewrite_docs; CREATE INDEX lifecycle_rewrite_mv_idx ON public.lifecycle_rewrite_mv USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 mv_index_oid="$(sql_super -c "SELECT 'public.lifecycle_rewrite_mv_idx'::regclass::oid;")" mv_job_before="$(current_generation_job_id "${mv_index_oid}")" sql_as durable_owner -c \ "REFRESH MATERIALIZED VIEW public.lifecycle_rewrite_mv;" \ >/dev/null 2>&1 mv_job_after="$(current_generation_job_id "${mv_index_oid}")" if [ -z "${mv_job_after}" ] || [ "${mv_job_after}" = "${mv_job_before}" ]; then error "REFRESH MATERIALIZED VIEW did not reconcile the workflow" fi sql_super -c "DROP MATERIALIZED VIEW public.lifecycle_rewrite_mv; DROP TABLE public.lifecycle_additional_rewrite_docs;" } test_truncate_cascade_reconciliation() { local child_oid child_job_after child_job_before sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_truncate_parent ( id integer PRIMARY KEY); CREATE TABLE public.lifecycle_truncate_child ( parent_id integer REFERENCES public.lifecycle_truncate_parent(id), body text); INSERT INTO public.lifecycle_truncate_parent VALUES (1); INSERT INTO public.lifecycle_truncate_child VALUES (1, 'one'); CREATE INDEX lifecycle_truncate_child_idx ON public.lifecycle_truncate_child USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL child_oid="$(sql_super -c "SELECT 'public.lifecycle_truncate_child_idx'::regclass::oid;")" child_job_before="$(current_generation_job_id "${child_oid}")" sql_as durable_owner -c \ "TRUNCATE public.lifecycle_truncate_parent CASCADE;" \ >/dev/null 2>&1 child_job_after="$(current_generation_job_id "${child_oid}")" if [ -z "${child_job_after}" ] || [ "${child_job_after}" = "${child_job_before}" ]; then error "TRUNCATE CASCADE did not reconcile the cascaded workflow" fi sql_super -c "DROP TABLE public.lifecycle_truncate_child, public.lifecycle_truncate_parent;" >/dev/null } test_maintenance_privilege_compatibility() { local cluster_oid cluster_job_after cluster_job_before local global_job_after matview_oid matview_job_after matview_job_before local schema_job_after schema_job_before sql_super -c "CREATE SCHEMA lifecycle_maintain AUTHORIZATION durable_owner;" >/dev/null sql_as durable_owner <<'SQL' CREATE TABLE lifecycle_maintain.cluster_docs (id integer, body text); INSERT INTO lifecycle_maintain.cluster_docs VALUES (2, 'two'), (1, 'one'); CREATE INDEX lifecycle_cluster_order_idx ON lifecycle_maintain.cluster_docs(id); CREATE INDEX lifecycle_cluster_bm25_idx ON lifecycle_maintain.cluster_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); ALTER TABLE lifecycle_maintain.cluster_docs CLUSTER ON lifecycle_cluster_order_idx; CREATE MATERIALIZED VIEW lifecycle_maintain.docs_mv AS SELECT body FROM lifecycle_maintain.cluster_docs; CREATE INDEX lifecycle_docs_mv_idx ON lifecycle_maintain.docs_mv USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL sql_super -c "GRANT USAGE ON SCHEMA lifecycle_maintain TO durable_maintainer; GRANT MAINTAIN ON lifecycle_maintain.cluster_docs, lifecycle_maintain.docs_mv TO durable_maintainer;" >/dev/null cluster_oid="$(sql_super -c "SELECT 'lifecycle_maintain.lifecycle_cluster_bm25_idx'::regclass::oid;")" cluster_job_before="$(current_generation_job_id "${cluster_oid}")" sql_as durable_maintainer -c " BEGIN; CLUSTER lifecycle_maintain.cluster_docs USING lifecycle_cluster_order_idx; COMMIT;" >/dev/null cluster_job_after="$(current_generation_job_id "${cluster_oid}")" if [ -z "${cluster_job_after}" ] || [ "${cluster_job_after}" = "${cluster_job_before}" ]; then error "transactional MAINTAIN CLUSTER did not reconcile the workflow" fi matview_oid="$(sql_super -c "SELECT 'lifecycle_maintain.lifecycle_docs_mv_idx'::regclass::oid;")" matview_job_before="$(current_generation_job_id "${matview_oid}")" sql_as durable_maintainer -c \ "REFRESH MATERIALIZED VIEW lifecycle_maintain.docs_mv;" \ >/dev/null matview_job_after="$(current_generation_job_id "${matview_oid}")" if [ -z "${matview_job_after}" ] || [ "${matview_job_after}" = "${matview_job_before}" ]; then error "MAINTAIN REFRESH did not reconcile the workflow" fi schema_job_before="${cluster_job_after}" sql_as durable_maintainer -c \ "REINDEX SCHEMA lifecycle_maintain;" >/dev/null schema_job_after="$(current_generation_job_id "${cluster_oid}")" if [ -z "${schema_job_after}" ] || [ "${schema_job_after}" = "${schema_job_before}" ]; then error "pg_maintain REINDEX SCHEMA did not reconcile the workflow" fi sql_as durable_maintainer -c "CLUSTER;" >/dev/null global_job_after="$(current_generation_job_id "${cluster_oid}")" if [ -z "${global_job_after}" ] || [ "${global_job_after}" = "${schema_job_after}" ]; then error "database-wide MAINTAIN CLUSTER did not reconcile the workflow" fi sql_super -c "DROP SCHEMA lifecycle_maintain CASCADE;" >/dev/null } test_tablespace_move_without_owned_by() { local blocker_pid error_output error_tablespace_dir index_oid local job_after job_before tablespace_dir tablespace_dir="${DATA_DIR}-lifecycle-owner-tablespace" mkdir -p "${tablespace_dir}" sql_super -c "CREATE TABLESPACE lifecycle_owner_tablespace OWNER durable_owner LOCATION '${tablespace_dir}';" \ >/dev/null sql_as durable_owner -c " CREATE TABLE public.lifecycle_owner_move_docs (body text); CREATE INDEX lifecycle_owner_move_idx ON public.lifecycle_owner_move_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *') TABLESPACE lifecycle_owner_tablespace;" \ >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_owner_move_idx'::regclass::oid;")" job_before="$(current_generation_job_id "${index_oid}")" sql_as durable_owner -c " ALTER INDEX ALL IN TABLESPACE lifecycle_owner_tablespace SET TABLESPACE pg_default;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "owner move without OWNED BY did not reconcile the workflow" fi sql_as durable_owner -c " ALTER INDEX public.lifecycle_owner_move_idx SET TABLESPACE lifecycle_owner_tablespace;" >/dev/null 2>&1 job_before="$(current_generation_job_id "${index_oid}")" sql_as durable_owner -c " ALTER INDEX ALL IN TABLESPACE lifecycle_owner_tablespace OWNED BY durable_owner, durable_writer SET TABLESPACE pg_default;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ] || [ "${job_after}" = "${job_before}" ]; then error "mixed OWNED BY move did not reconcile the caller-owned index" fi error_tablespace_dir="${DATA_DIR}-lifecycle-owner-error-tablespace" mkdir -p "${error_tablespace_dir}" sql_super -c " CREATE TABLESPACE lifecycle_owner_error_tablespace OWNER durable_owner LOCATION '${error_tablespace_dir}';" >/dev/null sql_super -c " CREATE TABLE public.lifecycle_owner_conflict_docs (value integer); ALTER TABLE public.lifecycle_owner_conflict_docs OWNER TO durable_writer; CREATE INDEX lifecycle_owner_conflict_idx ON public.lifecycle_owner_conflict_docs(value) TABLESPACE lifecycle_owner_error_tablespace;" >/dev/null sql_as durable_owner -c " CREATE TABLE public.lifecycle_owner_locked_docs (body text); CREATE INDEX lifecycle_owner_locked_idx ON public.lifecycle_owner_locked_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *') TABLESPACE lifecycle_owner_error_tablespace;" >/dev/null 2>&1 PGAPPNAME=lifecycle-owner-mixed-lock \ sql_as durable_owner -c " BEGIN; ALTER INDEX public.lifecycle_owner_locked_idx SET (compaction_schedule = '1 2 3 4 *'); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/owner-mixed-lock.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_owner_locked_idx'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if error_output="$(sql_as durable_owner -c " ALTER INDEX ALL IN TABLESPACE lifecycle_owner_error_tablespace OWNED BY durable_owner, durable_writer SET TABLESPACE pg_default NOWAIT;" 2>&1)"; then error "mixed OWNED BY moved an index owned by another role" fi if ! grep -Fq "must be owner of index lifecycle_owner_conflict_idx" \ <<<"${error_output}" && ! grep -Fq \ 'lock on relation "public.lifecycle_owner_locked_idx" is not available' \ <<<"${error_output}"; then error "mixed OWNED BY did not preserve a core failure: \ ${error_output}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-owner-mixed-lock';" >/dev/null wait "${blocker_pid}" || true sql_super -c "DROP TABLE public.lifecycle_owner_move_docs;" >/dev/null sql_super -c "DROP TABLESPACE lifecycle_owner_tablespace;" >/dev/null sql_super -c "DROP TABLE public.lifecycle_owner_conflict_docs, public.lifecycle_owner_locked_docs;" >/dev/null sql_super -c \ "DROP TABLESPACE lifecycle_owner_error_tablespace;" >/dev/null } test_repeatable_read_direct_activation_snapshot() { local index_oid initial_job reader_output reader_pid sql_as durable_owner -c " CREATE TABLE public.lifecycle_rr_activation_docs (body text); CREATE INDEX lifecycle_rr_activation_idx ON public.lifecycle_rr_activation_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_rr_activation_idx'::regclass::oid;")" initial_job="$(current_generation_job_id "${index_oid}")" sql_as durable_owner -c \ "SELECT df.cancel('${initial_job}', 'repeatable-read setup');" \ >/dev/null wait_for_terminal "${initial_job}" 30 reader_output="${DATA_DIR}/repeatable-read-activation.out" PGAPPNAME=lifecycle-repeatable-read-activation \ sql_as durable_owner <<'SQL' >"${reader_output}" 2>&1 & BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM df.instances; SELECT pg_catalog.pg_sleep(5); ALTER INDEX public.lifecycle_rr_activation_idx SET (compaction_schedule = '1 2 3 4 *'); COMMIT; SQL reader_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-repeatable-read-activation' AND query OPERATOR(pg_catalog.~~) '%pg_sleep%';")" = "1" ]; then break fi sleep 0.1 done sql_as durable_owner -c " ALTER INDEX public.lifecycle_rr_activation_idx SET (compaction_schedule = '1 2 3 4 *');" >/dev/null 2>&1 wait "${reader_pid}" || { cat "${reader_output}" >&2 error "repeatable-read activation failed" } assert_eq "repeatable-read direct activation keeps one workflow" "1" \ "$(current_generation_job_count "${index_oid}")" sql_super -c \ "DROP TABLE public.lifecycle_rr_activation_docs;" >/dev/null } test_owner_repair_without_old_durable_acl() { local index_oid job_after sql_as durable_owner -c " CREATE TABLE public.lifecycle_revoked_owner_docs (body text); CREATE INDEX lifecycle_revoked_owner_idx ON public.lifecycle_revoked_owner_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '1 2 3 4 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_revoked_owner_idx'::regclass::oid;")" sql_super -c "REVOKE USAGE ON SCHEMA df FROM durable_owner; REVOKE ALL ON df.instances FROM durable_owner;" >/dev/null sql_super -c "ALTER TABLE public.lifecycle_revoked_owner_docs OWNER TO durable_owner_two;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" assert_eq "revoked old-owner ACL permits ownership repair" \ "durable_owner_two:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', submitted_by::pg_catalog.text, label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex'))) FROM df.instances WHERE id = '${job_after}';")" sql_super -c "SELECT df.grant_usage('durable_owner');" >/dev/null sql_super -c \ "DROP TABLE public.lifecycle_revoked_owner_docs;" >/dev/null } test_writer_search_path_isolation() { local memtable_threshold_before sql_as durable_owner -c " CREATE TABLE public.lifecycle_search_path_docs (id integer, body text); CREATE INDEX lifecycle_search_path_idx ON public.lifecycle_search_path_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_super <<'SQL' >/dev/null CREATE TABLE public.lifecycle_search_path_audit ( role_name name NOT NULL ); ALTER TABLE public.lifecycle_search_path_audit OWNER TO durable_owner; REVOKE ALL ON public.lifecycle_search_path_audit FROM PUBLIC; CREATE SCHEMA lifecycle_writer_shadow AUTHORIZATION durable_writer; GRANT USAGE ON SCHEMA lifecycle_writer_shadow TO durable_owner; GRANT INSERT ON public.lifecycle_search_path_docs TO durable_writer; SQL sql_as durable_writer <<'SQL' >/dev/null CREATE FUNCTION lifecycle_writer_shadow.hijack_varchar_text( left_value varchar, right_value text) RETURNS boolean LANGUAGE plpgsql VOLATILE AS $body$ BEGIN INSERT INTO public.lifecycle_search_path_audit(role_name) VALUES (current_user); RETURN left_value::pg_catalog.text OPERATOR(pg_catalog.=) right_value; END $body$; CREATE OPERATOR lifecycle_writer_shadow.= ( LEFTARG = pg_catalog.varchar, RIGHTARG = pg_catalog.text, FUNCTION = lifecycle_writer_shadow.hijack_varchar_text ); SQL memtable_threshold_before="$(sql_super -c \ "SHOW pg_textsearch.memtable_pages_threshold;")" sql_super -c "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" >/dev/null sql_super -c "SELECT pg_catalog.pg_reload_conf();" >/dev/null sql_as durable_writer <<'SQL' >/dev/null SET search_path = lifecycle_writer_shadow, pg_catalog, public; INSERT INTO public.lifecycle_search_path_docs SELECT document_number, (SELECT pg_catalog.string_agg( pg_catalog.format( 'searchpath%sterm%s', document_number, term_number), ' ') FROM pg_catalog.generate_series(1, 200) AS term_number) FROM pg_catalog.generate_series(1, 6) AS document_number; SQL sql_super -c "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" >/dev/null sql_super -c "SELECT pg_catalog.pg_reload_conf();" >/dev/null assert_eq "writer spill threshold is restored" \ "${memtable_threshold_before}" \ "$(sql_super -c \ "SHOW pg_textsearch.memtable_pages_threshold;")" assert_eq "writer search_path cannot execute as the index owner" "0" \ "$(sql_super -c \ "SELECT count(*) FROM public.lifecycle_search_path_audit;")" sql_super -c "DROP TABLE public.lifecycle_search_path_docs, public.lifecycle_search_path_audit; DROP SCHEMA lifecycle_writer_shadow CASCADE;" >/dev/null } test_persisted_helper_oid_guard() { local index_oid instance_id sql_as durable_owner -c " CREATE TABLE public.lifecycle_helper_oid_docs (body text); CREATE INDEX lifecycle_helper_oid_idx ON public.lifecycle_helper_oid_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_helper_oid_idx'::regclass::oid;")" instance_id="$(current_generation_job_id "${index_oid}")" wait_for_signal_node "${instance_id}" 90 sql_super <<'SQL' >/dev/null CREATE TABLE public.lifecycle_helper_oid_audit ( role_name name NOT NULL ); ALTER TABLE public.lifecycle_helper_oid_audit OWNER TO durable_owner; REVOKE ALL ON public.lifecycle_helper_oid_audit FROM PUBLIC; GRANT INSERT ON public.lifecycle_helper_oid_audit TO durable_owner; ALTER DATABASE durable_compaction_test OWNER TO durable_actor; SET ROLE durable_actor; ALTER SCHEMA public RENAME TO lifecycle_original_public; CREATE SCHEMA public AUTHORIZATION durable_actor; CREATE FUNCTION public.lifecycle_helper_oid_payload() RETURNS boolean LANGUAGE plpgsql VOLATILE AS $body$ BEGIN INSERT INTO lifecycle_original_public.lifecycle_helper_oid_audit( role_name) VALUES (current_user); RETURN false; END $body$; CREATE FUNCTION public.bm25_background_target_is_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE sql IMMUTABLE AS 'SELECT public.lifecycle_helper_oid_payload()'; GRANT USAGE ON SCHEMA public TO durable_owner; GRANT EXECUTE ON FUNCTION public.lifecycle_helper_oid_payload() TO durable_owner; GRANT EXECUTE ON FUNCTION public.bm25_background_target_is_current(oid, oid, oid, oid, oid) TO durable_owner; RESET ROLE; SQL sql_as durable_owner -c \ "SELECT df.signal('${instance_id}', 'compact', '{}');" >/dev/null wait_for_terminal "${instance_id}" 30 assert_eq "persisted workflow verifies the helper function OID" "0" \ "$(sql_super -c "SELECT count(*) FROM lifecycle_original_public.lifecycle_helper_oid_audit;")" sql_super <<'SQL' >/dev/null DROP SCHEMA public CASCADE; ALTER SCHEMA lifecycle_original_public RENAME TO public; ALTER DATABASE durable_compaction_test OWNER TO postgres; DROP TABLE public.lifecycle_helper_oid_docs, public.lifecycle_helper_oid_audit; SQL } test_reassign_owned_heap_lock_order() { local blocker_pid index_oid reassign_output reassign_pid sql_super -c "CREATE ROLE lifecycle_reassign_lock_old LOGIN; CREATE ROLE lifecycle_reassign_lock_new LOGIN; SELECT df.grant_usage('lifecycle_reassign_lock_old'); SELECT df.grant_usage('lifecycle_reassign_lock_new'); GRANT CONNECT ON DATABASE ${TEST_DB} TO lifecycle_reassign_lock_old, lifecycle_reassign_lock_new; GRANT USAGE ON SCHEMA public TO lifecycle_reassign_lock_old, lifecycle_reassign_lock_new; GRANT CREATE ON SCHEMA public TO lifecycle_reassign_lock_old, lifecycle_reassign_lock_new;" >/dev/null sql_as lifecycle_reassign_lock_old -c " CREATE TABLE public.lifecycle_reassign_lock_docs (body text); CREATE INDEX lifecycle_reassign_lock_idx ON public.lifecycle_reassign_lock_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_reassign_lock_idx'::regclass::oid;")" PGAPPNAME=lifecycle-reassign-lock-blocker \ sql_super -c " BEGIN; LOCK TABLE public.lifecycle_reassign_lock_docs IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/reassign-lock-blocker.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_reassign_lock_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done reassign_output="${DATA_DIR}/reassign-lock-order.out" PGAPPNAME=lifecycle-reassign-lock-order \ sql_super -c " REASSIGN OWNED BY lifecycle_reassign_lock_old TO lifecycle_reassign_lock_new;" \ >"${reassign_output}" 2>&1 & reassign_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-reassign-lock-order' AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done assert_eq "REASSIGN waits for the heap before locking its index" "0" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-reassign-lock-order' AND relation_lock.locktype = 'relation' AND relation_lock.relation = ${index_oid} AND relation_lock.mode = 'ShareUpdateExclusiveLock' AND relation_lock.granted;")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-reassign-lock-blocker';" >/dev/null wait "${blocker_pid}" || true wait "${reassign_pid}" || { cat "${reassign_output}" >&2 error "REASSIGN lock-order test failed" } sql_super -c "DROP TABLE public.lifecycle_reassign_lock_docs; DROP OWNED BY lifecycle_reassign_lock_old, lifecycle_reassign_lock_new; DROP ROLE lifecycle_reassign_lock_old; DROP ROLE lifecycle_reassign_lock_new;" >/dev/null } test_rewrite_preflight_ordering() { local alter_error blocker_pid bulk_error private_blocker_pid local tablespace_dir vacuum_error sql_as durable_owner -c " CREATE TABLE public.lifecycle_preflight_a (body text); CREATE INDEX lifecycle_preflight_a_idx ON public.lifecycle_preflight_a USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_as durable_owner -c " CREATE TABLE public.lifecycle_preflight_b (body text); CREATE INDEX lifecycle_preflight_b_idx ON public.lifecycle_preflight_b USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_super -c " CREATE SCHEMA lifecycle_preflight_private AUTHORIZATION durable_owner;" >/dev/null sql_as durable_owner -c " CREATE TABLE lifecycle_preflight_private.docs (body text); CREATE INDEX docs_idx ON lifecycle_preflight_private.docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 tablespace_dir="${DATA_DIR}-lifecycle-preflight-tablespace" mkdir -p "${tablespace_dir}" sql_super -c " CREATE TABLESPACE lifecycle_preflight_tablespace OWNER durable_writer LOCATION '${tablespace_dir}';" >/dev/null PGAPPNAME=lifecycle-vacuum-preflight \ sql_as durable_owner -c " BEGIN; ALTER INDEX public.lifecycle_preflight_b_idx SET (compaction_schedule = '1 2 3 4 *'); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/vacuum-preflight-lock.out" 2>&1 & blocker_pid=$! PGAPPNAME=lifecycle-private-preflight \ sql_as durable_owner -c " BEGIN; ALTER INDEX lifecycle_preflight_private.docs_idx SET (compaction_schedule = '1 2 3 4 *'); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/private-preflight-lock.out" 2>&1 & private_blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE relation IN ( 'public.lifecycle_preflight_b_idx'::regclass, 'lifecycle_preflight_private.docs_idx'::regclass) AND mode = 'AccessExclusiveLock' AND granted;")" = "2" ]; then break fi sleep 0.1 done if vacuum_error="$(sql_as durable_owner -c " BEGIN; SET LOCAL statement_timeout = '2s'; VACUUM (FULL);" 2>&1)"; then error "VACUUM FULL unexpectedly ran inside a transaction block" fi if ! grep -Fq "cannot run inside a transaction block" \ <<<"${vacuum_error}"; then error "VACUUM FULL prelocked indexes before its transaction check: \ ${vacuum_error}" fi if alter_error="$(sql_as durable_owner -c " SET statement_timeout = '2s'; ALTER MATERIALIZED VIEW public.lifecycle_preflight_b OWNER TO durable_usage_only;" 2>&1)"; then error "ALTER MATERIALIZED VIEW accepted an ordinary table" fi if ! grep -Fq "is not a materialized view" <<<"${alter_error}"; then error "ALTER OWNER target checks preceded relation-kind validation: \ ${alter_error}" fi if alter_error="$(sql_as durable_owner -c " SET statement_timeout = '2s'; ALTER TABLE public.lifecycle_preflight_b OWNER TO durable_usage_only;" 2>&1)"; then error "ALTER OWNER unexpectedly accepted an unauthorized target role" fi if ! grep -Fq \ "must be able to SET ROLE \"durable_usage_only\"" \ <<<"${alter_error}"; then error "ALTER OWNER prelocked indexes before its target-role check: \ ${alter_error}" fi sql_super -c "GRANT durable_owner_two TO durable_owner;" >/dev/null if alter_error="$(sql_as durable_owner -c " SET statement_timeout = '2s'; ALTER TABLE lifecycle_preflight_private.docs OWNER TO durable_owner_two;" 2>&1)"; then error "ALTER OWNER unexpectedly bypassed target schema CREATE" fi if ! grep -Fq \ "permission denied for schema lifecycle_preflight_private" \ <<<"${alter_error}"; then error "ALTER OWNER prelocked indexes before its schema check: \ ${alter_error}" fi sql_super -c "REVOKE durable_owner_two FROM durable_owner;" >/dev/null if bulk_error="$(sql_as durable_owner -c " SET statement_timeout = '2s'; ALTER INDEX ALL IN TABLESPACE pg_default SET TABLESPACE lifecycle_preflight_missing;" 2>&1)"; then error "bulk move unexpectedly accepted a missing tablespace" fi if ! grep -Fq \ 'tablespace "lifecycle_preflight_missing" does not exist' \ <<<"${bulk_error}"; then error "bulk move prelocked indexes before tablespace lookup: \ ${bulk_error}" fi if bulk_error="$(sql_as durable_owner -c " SET statement_timeout = '2s'; ALTER INDEX ALL IN TABLESPACE pg_default SET TABLESPACE lifecycle_preflight_tablespace;" 2>&1)"; then error "bulk move unexpectedly bypassed tablespace CREATE privilege" fi if ! grep -Fq \ "permission denied for tablespace lifecycle_preflight_tablespace" \ <<<"${bulk_error}"; then error "bulk move prelocked indexes before tablespace ACL checks: \ ${bulk_error}" fi if ! bulk_error="$(sql_as durable_owner -c " SET statement_timeout = '2s'; ALTER INDEX ALL IN TABLESPACE pg_default SET TABLESPACE pg_default;" 2>&1)"; then error "bulk move no-op waited for index locks: ${bulk_error}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name IN ( 'lifecycle-vacuum-preflight', 'lifecycle-private-preflight');" >/dev/null wait "${blocker_pid}" || true wait "${private_blocker_pid}" || true sql_super -c "DROP TABLE public.lifecycle_preflight_a, public.lifecycle_preflight_b; DROP SCHEMA lifecycle_preflight_private CASCADE;" \ >/dev/null sql_super -c "DROP TABLESPACE lifecycle_preflight_tablespace;" \ >/dev/null } test_concurrent_reindex_reconciliation() { local a_before a_after a_oid_before a_oid_after local b_before b_after b_oid_before b_oid_after sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '1 2 3 4 *';" sql_as durable_owner -c " CREATE TABLE public.lifecycle_concurrent_docs (body_a text, body_b text); INSERT INTO public.lifecycle_concurrent_docs SELECT pg_catalog.format('document %s', value), pg_catalog.format('other %s', value) FROM generate_series(1, 20) AS value; CREATE INDEX lifecycle_concurrent_a_idx ON public.lifecycle_concurrent_docs USING bm25(body_a) WITH (text_config = 'english', compaction = 'background');" >/dev/null 2>&1 sql_as durable_owner -c " CREATE INDEX lifecycle_concurrent_b_idx ON public.lifecycle_concurrent_docs USING bm25(body_b) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 a_oid_before="$(sql_super -c \ "SELECT 'public.lifecycle_concurrent_a_idx'::regclass::oid;")" a_before="$(current_generation_job_id "${a_oid_before}")" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET pg_textsearch.background_compaction_schedule = '5 6 7 8 *';" sql_as durable_owner -c \ "REINDEX INDEX CONCURRENTLY public.lifecycle_concurrent_a_idx;" >/dev/null 2>&1 a_oid_after="$(sql_super -c \ "SELECT 'public.lifecycle_concurrent_a_idx'::regclass::oid;")" a_after="$(current_generation_job_id "${a_oid_after}")" if [ "${a_oid_after}" = "${a_oid_before}" ]; then error "REINDEX INDEX CONCURRENTLY did not replace the index OID" fi if [ -z "${a_after}" ] || [ "${a_after}" = "${a_before}" ]; then error "REINDEX INDEX CONCURRENTLY did not reconcile the new OID" fi assert_eq "concurrent index replacement has one current workflow" "1" \ "$(current_generation_job_count "${a_oid_after}")" assert_eq "concurrent index replacement preserves captured schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${a_after}';")" a_oid_before="${a_oid_after}" b_oid_before="$(sql_super -c \ "SELECT 'public.lifecycle_concurrent_b_idx'::regclass::oid;")" a_before="${a_after}" b_before="$(current_generation_job_id "${b_oid_before}")" sql_as durable_owner -c \ "REINDEX TABLE CONCURRENTLY public.lifecycle_concurrent_docs;" >/dev/null 2>&1 a_oid_after="$(sql_super -c \ "SELECT 'public.lifecycle_concurrent_a_idx'::regclass::oid;")" b_oid_after="$(sql_super -c \ "SELECT 'public.lifecycle_concurrent_b_idx'::regclass::oid;")" a_after="$(current_generation_job_id "${a_oid_after}")" b_after="$(current_generation_job_id "${b_oid_after}")" if [ "${a_oid_after}" = "${a_oid_before}" ] || [ "${b_oid_after}" = "${b_oid_before}" ]; then error "REINDEX TABLE CONCURRENTLY did not replace every index OID" fi if [ -z "${a_after}" ] || [ "${a_after}" = "${a_before}" ] || [ -z "${b_after}" ] || [ "${b_after}" = "${b_before}" ]; then error "REINDEX TABLE CONCURRENTLY did not reconcile replacement OIDs" fi sql_as durable_owner -c "SELECT df.cancel( instance.id, 'concurrent reindex lifecycle test complete') FROM df.instances AS instance WHERE (instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${a_oid_after}:%' OR instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${b_oid_after}:%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_as durable_owner -c \ "DROP TABLE public.lifecycle_concurrent_docs;" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET pg_textsearch.background_compaction_schedule;" } test_partitioned_reindex_reconciliation() { local leaf_count leaf_jobs parent_index_oid sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_reindex_partitioned_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_reindex_partitioned_low PARTITION OF public.lifecycle_reindex_partitioned_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_reindex_partitioned_high PARTITION OF public.lifecycle_reindex_partitioned_docs FOR VALUES FROM (100) TO (200); INSERT INTO public.lifecycle_reindex_partitioned_docs VALUES (1, 'low partition'), (101, 'high partition'); CREATE INDEX lifecycle_reindex_partitioned_idx ON public.lifecycle_reindex_partitioned_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL parent_index_oid="$(sql_super -c "SELECT 'public.lifecycle_reindex_partitioned_idx'::regclass::oid;")" sql_super </dev/null 2>&1 assert_eq "partitioned REINDEX INDEX keeps both leaf OIDs" "2" \ "$(sql_super -c "SELECT count(*) FROM public.lifecycle_reindex_partitioned_before AS before JOIN pg_catalog.pg_class AS current ON current.oid = before.index_oid WHERE pg_catalog.pg_relation_filenode(current.oid) OPERATOR(pg_catalog.<>) before.filenumber;")" leaf_jobs="$(sql_super -c "SELECT count(*) FROM public.lifecycle_reindex_partitioned_before AS before JOIN pg_catalog.pg_class AS current ON current.oid = before.index_oid JOIN df.instances AS instance ON instance.label OPERATOR(pg_catalog.~~) pg_catalog.format( 'pg_textsearch:bg:v1:%%:%s:%%:%s:%s:%%', current.oid, pg_catalog.pg_relation_filenode(current.oid), current.relowner) AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) WHERE instance.id OPERATOR(pg_catalog.<>) before.instance_id;")" assert_eq "partitioned REINDEX INDEX reconciles every leaf" "2" \ "${leaf_jobs}" sql_super -c "TRUNCATE public.lifecycle_reindex_partitioned_before; INSERT INTO public.lifecycle_reindex_partitioned_before SELECT child.relname, child.oid, pg_catalog.pg_relation_filenode(child.oid), ( SELECT instance.id FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || child.oid::text || ':%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) ORDER BY instance.created_at DESC, instance.id DESC LIMIT 1 ) FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child ON child.oid = inheritance.inhrelid WHERE inheritance.inhparent = ${parent_index_oid} AND child.relkind = 'i';" sql_as durable_owner -c \ "REINDEX TABLE CONCURRENTLY public.lifecycle_reindex_partitioned_docs;" >/dev/null 2>&1 leaf_count="$(sql_super -c "SELECT count(*) FROM public.lifecycle_reindex_partitioned_before AS before JOIN pg_catalog.pg_class AS current ON current.relname = before.relname AND current.relnamespace = 'public'::regnamespace WHERE current.oid OPERATOR(pg_catalog.<>) before.index_oid;")" assert_eq "partitioned concurrent REINDEX resolves replacement OIDs" \ "2" "${leaf_count}" leaf_jobs="$(sql_super -c "SELECT count(*) FROM public.lifecycle_reindex_partitioned_before AS before JOIN pg_catalog.pg_class AS current ON current.relname = before.relname AND current.relnamespace = 'public'::regnamespace JOIN df.instances AS instance ON instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || current.oid::text || ':%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) WHERE instance.id OPERATOR(pg_catalog.<>) before.instance_id;")" assert_eq "partitioned concurrent REINDEX reconciles every leaf" "2" \ "${leaf_jobs}" sql_as durable_owner -c \ "DROP TABLE public.lifecycle_reindex_partitioned_docs;" sql_super -c "DROP TABLE public.lifecycle_reindex_partitioned_before;" } test_partitioned_reindex_failure_reconciliation() { local first_file_before first_file_after first_index_oid local first_job_before first_job_after reindex_error second_file_before local second_file_after second_index_oid sql_as durable_owner <<'SQL' >/dev/null CREATE FUNCTION public.lifecycle_reindex_value(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting( 'lifecycle.reindex_failure', true) OPERATOR(pg_catalog.=) 'on' AND value OPERATOR(pg_catalog.=) 'fail' THEN RAISE EXCEPTION 'intentional partition reindex failure'; END IF; RETURN value; END $body$; CREATE TABLE public.lifecycle_reindex_failure_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_reindex_failure_low PARTITION OF public.lifecycle_reindex_failure_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_reindex_failure_high PARTITION OF public.lifecycle_reindex_failure_docs FOR VALUES FROM (100) TO (200); INSERT INTO public.lifecycle_reindex_failure_docs VALUES (1, 'safe'), (101, 'fail'); CREATE INDEX lifecycle_reindex_failure_idx ON public.lifecycle_reindex_failure_docs USING bm25 (public.lifecycle_reindex_value(body)) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL first_index_oid="$(sql_super -c "SELECT child.oid FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child ON child.oid = inheritance.inhrelid WHERE inheritance.inhparent = 'public.lifecycle_reindex_failure_idx'::regclass ORDER BY child.oid LIMIT 1;")" second_index_oid="$(sql_super -c "SELECT child.oid FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child ON child.oid = inheritance.inhrelid WHERE inheritance.inhparent = 'public.lifecycle_reindex_failure_idx'::regclass ORDER BY child.oid DESC LIMIT 1;")" first_file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${first_index_oid});")" second_file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${second_index_oid});")" first_job_before="$(current_generation_job_id "${first_index_oid}")" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET lifecycle.reindex_failure = 'on';" if reindex_error="$(sql_as durable_owner -c " REINDEX INDEX public.lifecycle_reindex_failure_idx;" 2>&1)"; then error "partitioned REINDEX failure injection unexpectedly succeeded" fi if ! grep -Fq "intentional partition reindex failure" \ <<<"${reindex_error}"; then error "partitioned REINDEX failed for the wrong reason: \ ${reindex_error}" fi first_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${first_index_oid});")" second_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${second_index_oid});")" if [ "${first_file_after}" = "${first_file_before}" ]; then error "partitioned REINDEX did not commit the earlier leaf" fi assert_eq "failed partitioned REINDEX rolls back the failing leaf" \ "${second_file_before}" "${second_file_after}" first_job_after="$(current_generation_job_id "${first_index_oid}")" if [ -z "${first_job_after}" ] || [ "${first_job_after}" = "${first_job_before}" ]; then error "committed partition leaf replacement was not reconciled" fi assert_eq "failed partitioned REINDEX selects the admitted earlier leaf" \ "${first_job_after}" \ "$(current_generation_job_id "${first_index_oid}")" assert_eq "earlier leaf reconciliation does not eagerly cancel history" \ "t" \ "$(sql_super -c "SELECT status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) FROM df.instances WHERE id = '${first_job_before}';")" assert_eq "old physical helper rejects the committed replacement" "f" \ "$(background_target_is_current \ "${first_index_oid}" "${first_file_before}" durable_owner)" sql_as durable_owner -c "SELECT df.signal( '${first_job_before}', 'compact', '{}');" >/dev/null wait_for_terminal "${first_job_before}" 30 log "PASS: stale earlier-leaf workflow retires through its guard" sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET lifecycle.reindex_failure;" sql_as durable_owner -c " DROP TABLE public.lifecycle_reindex_failure_docs; DROP FUNCTION public.lifecycle_reindex_value(text);" } test_partitioned_reindex_rename_reconciliation() { local blocker_pid high_index_oid high_index_name high_file_before local high_file_after high_job_before high_job_after reindex_pid local reindex_output="${DATA_DIR}/partitioned-reindex-rename.out" sql_as durable_owner <<'SQL' >/dev/null CREATE FUNCTION public.lifecycle_reindex_pause(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF value OPERATOR(pg_catalog.=) 'pause' THEN PERFORM pg_catalog.pg_advisory_xact_lock(478, 3); END IF; RETURN value; END $body$; CREATE TABLE public.lifecycle_reindex_rename_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_reindex_rename_low PARTITION OF public.lifecycle_reindex_rename_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_reindex_rename_high PARTITION OF public.lifecycle_reindex_rename_docs FOR VALUES FROM (100) TO (200); INSERT INTO public.lifecycle_reindex_rename_docs VALUES (1, 'pause'), (101, 'continue'); CREATE INDEX lifecycle_reindex_rename_idx ON public.lifecycle_reindex_rename_docs USING bm25 (public.lifecycle_reindex_pause(body)) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL high_index_oid="$(sql_super -c "SELECT index_class.oid FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS index_class ON index_class.oid = inheritance.inhrelid JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = index_class.oid WHERE inheritance.inhparent = 'public.lifecycle_reindex_rename_idx'::regclass AND index_catalog.indrelid = 'public.lifecycle_reindex_rename_high'::regclass;")" high_index_name="$(sql_super -c "SELECT relname FROM pg_catalog.pg_class WHERE oid = ${high_index_oid};")" high_file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${high_index_oid});")" high_job_before="$(current_generation_job_id "${high_index_oid}")" sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 3); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/partitioned-reindex-lock.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 3 AND granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-reindex-rename \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "REINDEX INDEX public.lifecycle_reindex_rename_idx;" \ >"${reindex_output}" 2>&1 & reindex_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-reindex-rename' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "partitioned REINDEX pauses before the second leaf" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-reindex-rename' AND wait_event = 'advisory';")" sql_as durable_owner -c "ALTER INDEX public.${high_index_name} RENAME TO lifecycle_reindex_rename_high_renamed_idx;" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE pid <> pg_catalog.pg_backend_pid() AND query OPERATOR(pg_catalog.~~) 'SELECT pg_catalog.pg_advisory_lock(478, 3)%';" >/dev/null wait "${blocker_pid}" || true if ! wait "${reindex_pid}"; then error "partitioned REINDEX with leaf rename failed: \ $(cat "${reindex_output}")" fi high_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${high_index_oid});")" high_job_after="$(current_generation_job_id "${high_index_oid}")" if [ "${high_file_after}" = "${high_file_before}" ]; then error "partitioned REINDEX did not rebuild the renamed leaf" fi if [ -z "${high_job_after}" ] || [ "${high_job_after}" = "${high_job_before}" ]; then error "renamed pending leaf was not reconciled by original OID" fi sql_as durable_owner -c " DROP TABLE public.lifecycle_reindex_rename_docs; DROP FUNCTION public.lifecycle_reindex_pause(text);" } test_reindex_tracking_reentry() { local leaf_jobs reindex_output sql_as durable_owner <<'SQL' >/dev/null CREATE TABLE public.lifecycle_reentry_outer_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_reentry_outer_low PARTITION OF public.lifecycle_reentry_outer_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_reentry_outer_high PARTITION OF public.lifecycle_reentry_outer_docs FOR VALUES FROM (100) TO (200); INSERT INTO public.lifecycle_reentry_outer_docs VALUES (1, 'low'), (101, 'high'); CREATE INDEX lifecycle_reentry_outer_idx ON public.lifecycle_reentry_outer_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_reentry_inner_docs (body text); INSERT INTO public.lifecycle_reentry_inner_docs VALUES ('inner'); CREATE INDEX lifecycle_reentry_inner_idx ON public.lifecycle_reentry_inner_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL sql_super <<'SQL' >/dev/null CREATE TABLE public.lifecycle_reentry_before AS SELECT child.oid AS index_oid, pg_catalog.pg_relation_filenode(child.oid) AS filenumber, ( SELECT instance.id FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || child.oid::text || ':%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) ORDER BY instance.created_at DESC, instance.id DESC LIMIT 1 ) AS instance_id FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child ON child.oid = inheritance.inhrelid WHERE inheritance.inhparent = 'public.lifecycle_reentry_outer_idx'::regclass; CREATE FUNCTION public.lifecycle_reindex_reenter() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting( 'lifecycle.reindex_reentry', true) OPERATOR(pg_catalog.=) 'outer' THEN PERFORM pg_catalog.set_config( 'lifecycle.reindex_reentry', 'inner', false); EXECUTE 'REINDEX INDEX public.lifecycle_reentry_inner_idx'; END IF; END $body$; CREATE EVENT TRIGGER lifecycle_reindex_reenter ON ddl_command_start WHEN TAG IN ('REINDEX') EXECUTE FUNCTION public.lifecycle_reindex_reenter(); SQL sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} SET lifecycle.reindex_reentry = 'outer';" if ! reindex_output="$(sql_as durable_owner -c " REINDEX INDEX public.lifecycle_reentry_outer_idx;" 2>&1)"; then error "nested ordinary REINDEX broke outer tracking: ${reindex_output}" fi sql_super -c "ALTER ROLE durable_owner IN DATABASE ${TEST_DB} RESET lifecycle.reindex_reentry;" sql_super -c "DROP EVENT TRIGGER lifecycle_reindex_reenter; DROP FUNCTION public.lifecycle_reindex_reenter();" leaf_jobs="$(sql_super -c "SELECT pg_catalog.count(*) FROM public.lifecycle_reentry_before AS before JOIN pg_catalog.pg_class AS current ON current.oid = before.index_oid JOIN df.instances AS instance ON instance.label OPERATOR(pg_catalog.~~) pg_catalog.format( 'pg_textsearch:bg:v1:%%:%s:%%:%s:%s:%%', current.oid, pg_catalog.pg_relation_filenode(current.oid), current.relowner) AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) WHERE pg_catalog.pg_relation_filenode(current.oid) OPERATOR(pg_catalog.<>) before.filenumber AND instance.id OPERATOR(pg_catalog.<>) before.instance_id;")" assert_eq "nested ordinary REINDEX preserves outer tracking" \ "2" "${leaf_jobs}" sql_super -c "DROP TABLE public.lifecycle_reentry_before;" sql_as durable_owner -c " DROP TABLE public.lifecycle_reentry_outer_docs, public.lifecycle_reentry_inner_docs;" } test_ordinary_inheritance_reindex_scope() { local child_file_before child_file_after child_job_before local child_job_after parent_file_before parent_file_after local parent_index_oid sql_as durable_owner -c " CREATE TABLE public.lifecycle_inherit_parent (body text); CREATE TABLE public.lifecycle_inherit_child () INHERITS (public.lifecycle_inherit_parent); CREATE INDEX lifecycle_inherit_parent_idx ON public.lifecycle_inherit_parent USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX lifecycle_inherit_child_idx ON public.lifecycle_inherit_child USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_super -c "ALTER TABLE public.lifecycle_inherit_child OWNER TO durable_owner_two;" >/dev/null 2>&1 parent_index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_inherit_parent_idx'::regclass::oid;")" parent_file_before="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode( 'public.lifecycle_inherit_child_idx'::regclass);")" child_job_before="$(current_generation_job_id \ "$(sql_super -c "SELECT 'public.lifecycle_inherit_child_idx'::regclass::oid;")")" sql_super -c "ALTER ROLE durable_owner_two NOLOGIN;" sql_as durable_owner -c \ "REINDEX TABLE public.lifecycle_inherit_parent;" >/dev/null 2>&1 sql_super -c "ALTER ROLE durable_owner_two LOGIN;" parent_file_after="$(sql_super -c \ "SELECT pg_catalog.pg_relation_filenode(${parent_index_oid});")" child_file_after="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode( 'public.lifecycle_inherit_child_idx'::regclass);")" child_job_after="$(current_generation_job_id \ "$(sql_super -c "SELECT 'public.lifecycle_inherit_child_idx'::regclass::oid;")")" if [ "${parent_file_after}" = "${parent_file_before}" ]; then error "ordinary parent REINDEX did not rebuild its own index" fi assert_eq "ordinary child index is not reindexed with its parent" \ "${child_file_before}" "${child_file_after}" assert_eq "ordinary child workflow is not mistaken for parent work" \ "${child_job_before}" "${child_job_after}" sql_super -c "DROP TABLE public.lifecycle_inherit_child, public.lifecycle_inherit_parent;" } test_legacy_lineage_backfill() { local legacy_instance legacy_lineage legacy_oid owner_job local replacement_instance local owner_instance owner_lineage reindex_lineage reindex_oid_before local reindex_oid_after install_signal_probe sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_legacy_signal_docs (id integer, body text); CREATE INDEX lifecycle_legacy_signal_idx ON public.lifecycle_legacy_signal_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); GRANT INSERT ON public.lifecycle_legacy_signal_docs TO durable_writer; CREATE TABLE public.lifecycle_legacy_owner_docs (body text); CREATE INDEX lifecycle_legacy_owner_idx ON public.lifecycle_legacy_owner_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_legacy_reindex_docs (body text); INSERT INTO public.lifecycle_legacy_reindex_docs VALUES ('legacy'); CREATE INDEX lifecycle_legacy_reindex_idx ON public.lifecycle_legacy_reindex_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL legacy_oid="$(sql_super -c "SELECT 'public.lifecycle_legacy_signal_idx'::regclass::oid;")" legacy_instance="$(current_generation_job_id "${legacy_oid}")" wait_for_signal_node "${legacy_instance}" 30 sql_super -c "UPDATE df.instances AS instance SET label = pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, pg_catalog.encode( pg_catalog.convert_to('0 0 1 1 *', 'UTF8'), 'hex')) FROM pg_catalog.pg_class AS relation, pg_catalog.pg_database AS database WHERE instance.id = '${legacy_instance}' AND relation.oid = ${legacy_oid} AND database.datname = pg_catalog.current_database();" remove_index_lineage public.lifecycle_legacy_signal_idx assert_eq "legacy signal setup removes lineage" "" \ "$(index_lineage public.lifecycle_legacy_signal_idx)" reset_signal_probe sql_as durable_owner <<'SQL' >/dev/null BEGIN; INSERT INTO public.lifecycle_legacy_signal_docs SELECT 1000 + document_number, (SELECT pg_catalog.string_agg( pg_catalog.format( 'legacy%sterm%s', document_number, term_number), ' ') FROM generate_series(1, 200) AS term_number) FROM generate_series(1, 6) AS document_number; SELECT bm25_spill_index('public.lifecycle_legacy_signal_idx'); INSERT INTO public.lifecycle_legacy_signal_docs SELECT 2000 + document_number, (SELECT pg_catalog.string_agg( pg_catalog.format( 'legacy%sterm%s', document_number, term_number), ' ') FROM generate_series(201, 400) AS term_number) FROM generate_series(1, 6) AS document_number; SELECT bm25_spill_index('public.lifecycle_legacy_signal_idx'); COMMIT; SQL legacy_lineage="$(index_lineage public.lifecycle_legacy_signal_idx)" assert_eq "legacy signal lazily assigns a 128-bit lineage" "32" \ "${#legacy_lineage}" assert_eq "legacy signal dispatches the replacement workflow" "1" \ "$(signal_attempt_count)" replacement_instance="$(sql_super -c "SELECT instance_id FROM public.compaction_signal_audit;")" assert_eq "legacy signal selects the replacement workflow" \ "${replacement_instance}" \ "$(current_generation_job_id "${legacy_oid}")" assert_eq "legacy migration does not eagerly cancel history" "t" \ "$(sql_super -c "SELECT status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) FROM df.instances WHERE id = '${legacy_instance}';")" sql_as durable_owner -c "SELECT df.signal_v028( '${legacy_instance}', 'compact', '{}');" >/dev/null wait_for_terminal "${legacy_instance}" 30 log "PASS: stale legacy workflow retires through its current guard" owner_instance="$(current_generation_job_id \ "$(sql_super -c "SELECT 'public.lifecycle_legacy_owner_idx'::regclass::oid;")")" sql_super -c "UPDATE df.instances AS instance SET label = pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, pg_catalog.encode( pg_catalog.convert_to('1 2 3 4 *', 'UTF8'), 'hex')) FROM pg_catalog.pg_class AS relation, pg_catalog.pg_database AS database WHERE instance.id = '${owner_instance}' AND relation.oid = 'public.lifecycle_legacy_owner_idx'::regclass AND database.datname = pg_catalog.current_database();" remove_index_lineage public.lifecycle_legacy_owner_idx sql_super -c "ALTER TABLE public.lifecycle_legacy_owner_docs OWNER TO durable_owner_two;" >/dev/null 2>&1 owner_lineage="$(index_lineage public.lifecycle_legacy_owner_idx)" assert_eq "legacy owner reconciliation backfills lineage" "32" \ "${#owner_lineage}" owner_job="$(current_generation_job_id \ "$(sql_super -c "SELECT 'public.lifecycle_legacy_owner_idx'::regclass::oid;")")" assert_eq "legacy owner reconciliation uses the new owner" \ "durable_owner_two" \ "$(sql_super -c "SELECT submitted_by::pg_catalog.text FROM df.instances WHERE id = '${owner_job}';")" assert_eq "legacy owner reconciliation preserves its schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${owner_job}';")" reindex_oid_before="$(sql_super -c "SELECT 'public.lifecycle_legacy_reindex_idx'::regclass::oid;")" remove_index_lineage public.lifecycle_legacy_reindex_idx sql_as durable_owner -c "REINDEX INDEX CONCURRENTLY public.lifecycle_legacy_reindex_idx;" >/dev/null 2>&1 reindex_oid_after="$(sql_super -c "SELECT 'public.lifecycle_legacy_reindex_idx'::regclass::oid;")" if [ "${reindex_oid_after}" = "${reindex_oid_before}" ]; then error "legacy concurrent REINDEX did not replace the index OID" fi reindex_lineage="$(index_lineage public.lifecycle_legacy_reindex_idx)" assert_eq "legacy concurrent REINDEX backfills lineage" "32" \ "${#reindex_lineage}" if [ -z "$(current_generation_job_id "${reindex_oid_after}")" ]; then error "legacy concurrent REINDEX created no replacement workflow" fi restore_signal_probe sql_super -c "DROP TABLE public.lifecycle_legacy_signal_docs, public.lifecycle_legacy_owner_docs, public.lifecycle_legacy_reindex_docs;" } test_legacy_reconciliation_edges() { local index_oid job_after legacy_instance sql_as durable_owner -c " CREATE TABLE public.lifecycle_legacy_noop_docs (body text); CREATE INDEX lifecycle_legacy_noop_idx ON public.lifecycle_legacy_noop_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_legacy_noop_idx'::regclass::oid;")" legacy_instance="$(current_generation_job_id "${index_oid}")" sql_super -c "UPDATE df.instances AS instance SET label = pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, pg_catalog.encode( pg_catalog.convert_to('1 2 3 4 *', 'UTF8'), 'hex')) FROM pg_catalog.pg_class AS relation, pg_catalog.pg_database AS database WHERE instance.id = '${legacy_instance}' AND relation.oid = ${index_oid} AND database.datname = pg_catalog.current_database();" remove_index_lineage public.lifecycle_legacy_noop_idx sql_as durable_owner -c " ALTER INDEX public.lifecycle_legacy_noop_idx SET TABLESPACE pg_default;" >/dev/null 2>&1 job_after="$(current_generation_job_id "${index_oid}")" if [ -z "${job_after}" ]; then error "no-op rewrite did not migrate the legacy workflow" fi assert_eq "legacy migration preserves the captured schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode(pg_catalog.convert_to( '1 2 3 4 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${job_after}';")" sql_super -c "ALTER ROLE durable_owner NOLOGIN;" >/dev/null if ! sql_super -c " ALTER TABLE public.lifecycle_legacy_noop_docs OWNER TO durable_owner_two;" >/dev/null 2>&1; then sql_super -c "ALTER ROLE durable_owner LOGIN;" >/dev/null error "ineligible old owner prevented ownership repair" fi sql_super -c "ALTER ROLE durable_owner LOGIN;" >/dev/null assert_eq "ineligible old owner can be replaced" "durable_owner_two" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relowner) FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" sql_super -c "DROP TABLE public.lifecycle_legacy_noop_docs;" } test_refresh_selects_requested_workflow() { local first_instance index_oid second_instance selected_instance local rollback_error sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_refresh_docs (body text); CREATE INDEX lifecycle_refresh_idx ON public.lifecycle_refresh_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_refresh_idx'::regclass::oid;")" first_instance="$(current_generation_job_id "${index_oid}")" sql_super -c "REVOKE EXECUTE ON FUNCTION df.cancel(text, text) FROM durable_owner;" if rollback_error="$(sql_as durable_owner <<'SQL' 2>&1 BEGIN; ALTER INDEX public.lifecycle_refresh_idx SET (compaction_schedule = '5 4 3 2 *'); DO $body$ BEGIN RAISE EXCEPTION 'force admitted refresh rollback'; END $body$; COMMIT; SQL )"; then error "admitted workflow refresh unexpectedly committed" fi if ! grep -Fq "force admitted refresh rollback" \ <<<"${rollback_error}"; then error "workflow refresh failed before the rollback probe: \ ${rollback_error}" fi assert_eq "rolled-back refresh preserves the prior current workflow" \ "${first_instance}" \ "$(current_generation_job_id "${index_oid}")" sql_as durable_owner -c "ALTER INDEX public.lifecycle_refresh_idx SET (compaction_schedule = '5 4 3 2 *');" >/dev/null 2>&1 second_instance="$(current_generation_job_id "${index_oid}")" if [ -z "${second_instance}" ] || [ "${second_instance}" = "${first_instance}" ]; then error "schedule refresh did not publish a new current workflow" fi assert_eq "schedule refresh keeps prior workflow for guard retirement" \ "t" \ "$(sql_super -c "SELECT status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) FROM df.instances WHERE id = '${first_instance}';")" sql_as durable_owner -c "ALTER INDEX public.lifecycle_refresh_idx SET (compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 selected_instance="$(current_generation_job_id "${index_oid}")" if [ -z "${selected_instance}" ] || [ "${selected_instance}" = "${first_instance}" ] || [ "${selected_instance}" = "${second_instance}" ]; then error "schedule rollback did not republish the requested workflow" fi assert_eq "latest workflow carries the requested schedule" "t" \ "$(sql_super -c "SELECT label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode( pg_catalog.convert_to('0 0 1 1 *', 'UTF8'), 'hex')) FROM df.instances WHERE id = '${selected_instance}';")" sql_super -c "GRANT EXECUTE ON FUNCTION df.cancel(text, text) TO durable_owner;" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'schedule refresh test complete') FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${index_oid}:%' AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_super -c "DROP TABLE public.lifecycle_refresh_docs;" } test_concurrent_legacy_lineage_backfill() { local final_lineage first_output first_pid first_status=0 local gate_pid index_oid legacy_instance local second_output second_pid second_status=0 first_output="${DATA_DIR}/legacy-backfill-first.out" second_output="${DATA_DIR}/legacy-backfill-second.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_legacy_race_docs (id integer, body text); CREATE INDEX lifecycle_legacy_race_idx ON public.lifecycle_legacy_race_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_legacy_race_idx'::regclass::oid;")" legacy_instance="$(current_generation_job_id "${index_oid}")" wait_for_signal_node "${legacy_instance}" 30 sql_super -c "UPDATE df.instances AS instance SET label = pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%s', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, pg_catalog.encode( pg_catalog.convert_to('0 0 1 1 *', 'UTF8'), 'hex')) FROM pg_catalog.pg_class AS relation, pg_catalog.pg_database AS database WHERE instance.id = '${legacy_instance}' AND relation.oid = ${index_oid} AND database.datname = pg_catalog.current_database();" remove_index_lineage public.lifecycle_legacy_race_idx PGAPPNAME=lifecycle-legacy-race-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 6); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/legacy-backfill-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 6 AND granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-legacy-race-first \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.lifecycle_legacy_race_docs SELECT 1000 + document_number, pg_catalog.format( 'first writer document %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index( 'public.lifecycle_legacy_race_idx'); INSERT INTO public.lifecycle_legacy_race_docs SELECT 2000 + document_number, pg_catalog.format( 'first writer second %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index( 'public.lifecycle_legacy_race_idx'); SELECT pg_catalog.pg_advisory_xact_lock_shared(478, 6); COMMIT;" >"${first_output}" 2>&1 & first_pid=$! PGAPPNAME=lifecycle-legacy-race-second \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.lifecycle_legacy_race_docs SELECT 3000 + document_number, pg_catalog.format( 'second writer document %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index( 'public.lifecycle_legacy_race_idx'); INSERT INTO public.lifecycle_legacy_race_docs SELECT 4000 + document_number, pg_catalog.format( 'second writer second %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index( 'public.lifecycle_legacy_race_idx'); SELECT pg_catalog.pg_advisory_xact_lock_shared(478, 6); COMMIT;" >"${second_output}" 2>&1 & second_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ( 'lifecycle-legacy-race-first', 'lifecycle-legacy-race-second') AND wait_event = 'advisory';")" = "2" ]; then break fi sleep 0.1 done assert_eq "concurrent legacy writers reach the commit gate" "2" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ( 'lifecycle-legacy-race-first', 'lifecycle-legacy-race-second') AND wait_event = 'advisory';")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-legacy-race-gate';" >/dev/null wait "${gate_pid}" || true wait "${first_pid}" || first_status=$? wait "${second_pid}" || second_status=$? if [ "${first_status}" -ne 0 ] || [ "${second_status}" -ne 0 ]; then error "concurrent legacy backfill failed: first: $(cat "${first_output}") second: $(cat "${second_output}")" fi assert_eq "both concurrent legacy writers commit" "80" \ "$(sql_super -c "SELECT count(*) FROM public.lifecycle_legacy_race_docs;")" final_lineage="$(index_lineage public.lifecycle_legacy_race_idx)" assert_eq "concurrent legacy writers converge on one 128-bit lineage" \ "32" "${#final_lineage}" assert_eq "concurrent legacy writers leave one lineage workflow" "1" \ "$(sql_super -c "SELECT count(*) FROM df.instances AS instance JOIN pg_catalog.pg_class AS relation ON relation.oid = ${index_oid} JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE instance.label OPERATOR(pg_catalog.~~) pg_catalog.format( 'pg_textsearch:bg:v1:%s:%s:%s:%s:%s:%%:%s:%%', database.oid, relation.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner, '${final_lineage}') AND instance.submitted_by::pg_catalog.oid = relation.relowner AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);")" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'legacy race test complete') FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${index_oid}:%' AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_super -c "DROP TABLE public.lifecycle_legacy_race_docs;" } test_legacy_reindex_spill_lock_order() { local blocker_output blocker_pid final_lineage first_oid local reindex_output reindex_pid local reindex_status=0 spill_output spill_pid spill_status=0 target_oid local target_oid_after blocker_output="${DATA_DIR}/legacy-lock-blocker.out" reindex_output="${DATA_DIR}/legacy-lock-reindex.out" spill_output="${DATA_DIR}/legacy-lock-spill.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_legacy_lock_docs (id integer, body text); CREATE INDEX lifecycle_legacy_lock_first_idx ON public.lifecycle_legacy_lock_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE INDEX lifecycle_legacy_lock_target_idx ON public.lifecycle_legacy_lock_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL first_oid="$(sql_super -c "SELECT 'public.lifecycle_legacy_lock_first_idx'::regclass::oid;")" target_oid="$(sql_super -c "SELECT 'public.lifecycle_legacy_lock_target_idx'::regclass::oid;")" remove_index_lineage public.lifecycle_legacy_lock_first_idx remove_index_lineage public.lifecycle_legacy_lock_target_idx PGAPPNAME=lifecycle-legacy-lock-blocker sql_super -c " BEGIN; UPDATE pg_catalog.pg_class SET reloptions = reloptions WHERE oid = ${first_oid}; SELECT pg_catalog.pg_sleep(120);" \ >"${blocker_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-legacy-lock-blocker' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done assert_eq "legacy lock blocker holds the first catalog row" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-legacy-lock-blocker' AND wait_event = 'PgSleep';")" PGAPPNAME=lifecycle-legacy-lock-reindex \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "REINDEX TABLE CONCURRENTLY public.lifecycle_legacy_lock_docs;" \ >"${reindex_output}" 2>&1 & reindex_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity WHERE activity.application_name = 'lifecycle-legacy-lock-reindex' AND activity.wait_event_type = 'Lock' AND EXISTS ( SELECT 1 FROM pg_catalog.pg_locks AS relation_lock WHERE relation_lock.pid = activity.pid AND relation_lock.locktype = 'relation' AND relation_lock.relation = ${target_oid} AND relation_lock.mode = 'ShareUpdateExclusiveLock' AND relation_lock.granted);")" = "1" ]; then break fi sleep 0.1 done assert_eq "tracked REINDEX holds the target before legacy backfill" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity WHERE activity.application_name = 'lifecycle-legacy-lock-reindex' AND activity.wait_event_type = 'Lock' AND EXISTS ( SELECT 1 FROM pg_catalog.pg_locks AS relation_lock WHERE relation_lock.pid = activity.pid AND relation_lock.locktype = 'relation' AND relation_lock.relation = ${target_oid} AND relation_lock.mode = 'ShareUpdateExclusiveLock' AND relation_lock.granted);")" PGAPPNAME=lifecycle-legacy-lock-spill \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.lifecycle_legacy_lock_docs SELECT document_number, pg_catalog.format( 'legacy lock first %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index( 'public.lifecycle_legacy_lock_target_idx'); INSERT INTO public.lifecycle_legacy_lock_docs SELECT 100 + document_number, pg_catalog.format( 'legacy lock second %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index( 'public.lifecycle_legacy_lock_target_idx'); COMMIT;" >"${spill_output}" 2>&1 & spill_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-legacy-lock-spill' AND relation_lock.locktype = 'relation' AND relation_lock.relation = ${target_oid} AND relation_lock.mode = 'ShareUpdateExclusiveLock' AND NOT relation_lock.granted;")" = "1" ] || [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-legacy-lock-spill';")" = "0" ]; then break fi sleep 0.1 done assert_eq "contended legacy spill does not hold the private index lock" \ "0" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS private_lock ON private_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-legacy-lock-spill' AND private_lock.locktype = 'object' AND private_lock.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND private_lock.objid = ${target_oid} AND private_lock.objsubid = 1 AND private_lock.mode = 'ExclusiveLock' AND private_lock.granted;")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-legacy-lock-blocker';" >/dev/null wait "${blocker_pid}" || true wait "${reindex_pid}" || reindex_status=$? wait "${spill_pid}" || spill_status=$? if [ "${reindex_status}" -ne 0 ] || [ "${spill_status}" -ne 0 ]; then error "legacy REINDEX/spill lock race failed: reindex: $(cat "${reindex_output}") spill: $(cat "${spill_output}")" fi assert_eq "legacy REINDEX/spill preserves writer rows" "40" \ "$(sql_super -c "SELECT count(*) FROM public.lifecycle_legacy_lock_docs;")" target_oid_after="$(sql_super -c "SELECT 'public.lifecycle_legacy_lock_target_idx'::regclass::oid;")" final_lineage="$( index_lineage public.lifecycle_legacy_lock_target_idx )" assert_eq "legacy REINDEX/spill converges on one lineage" "32" \ "${#final_lineage}" assert_eq "legacy REINDEX/spill leaves one current workflow" "1" \ "$(current_generation_job_count "${target_oid_after}")" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'legacy lock race complete') FROM df.instances AS instance WHERE instance.submitted_by = 'durable_owner'::regrole AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]) AND (instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${target_oid_after}:%');" >/dev/null sql_super -c "DROP TABLE public.lifecycle_legacy_lock_docs;" } test_managed_lock_order() { local blocker_output blocker_pid index_oid reconcile_completed=false local reconcile_output reconcile_pid reconcile_status=0 local signal_completed=false signal_output signal_pid signal_status=0 blocker_output="${DATA_DIR}/managed-lock-blocker.out" reconcile_output="${DATA_DIR}/managed-lock-reconcile.out" signal_output="${DATA_DIR}/managed-lock-signal.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_managed_lock_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_managed_lock_leaf PARTITION OF public.lifecycle_managed_lock_docs FOR VALUES FROM (0) TO (1000); CREATE INDEX lifecycle_managed_lock_leaf_idx ON public.lifecycle_managed_lock_leaf USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); INSERT INTO public.lifecycle_managed_lock_leaf SELECT document_number, pg_catalog.format('managed lock first %s filler', document_number) FROM generate_series(1, 20) AS document_number; SELECT bm25_spill_index('public.lifecycle_managed_lock_leaf_idx'); INSERT INTO public.lifecycle_managed_lock_leaf SELECT 100 + document_number, pg_catalog.format('managed lock second %s filler', document_number) FROM generate_series(1, 20) AS document_number; SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_managed_lock_leaf_idx'::regclass::oid;")" PGAPPNAME=lifecycle-managed-lock-blocker sql_super -c " BEGIN; COMMENT ON ACCESS METHOD bm25 IS 'managed lock gate'; SELECT pg_catalog.pg_sleep(120);" \ >"${blocker_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-managed-lock-blocker' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done assert_eq "managed lock blocker holds the dependency object" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS dependency ON dependency.pid = activity.pid JOIN pg_catalog.pg_am AS access_method ON access_method.oid = dependency.objid WHERE activity.application_name = 'lifecycle-managed-lock-blocker' AND activity.wait_event = 'PgSleep' AND dependency.locktype = 'object' AND dependency.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND access_method.amname = 'bm25' AND dependency.objsubid = 0 AND dependency.mode = 'ShareUpdateExclusiveLock' AND dependency.granted;")" PGAPPNAME=lifecycle-managed-lock-signal \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "SET statement_timeout = '15s'; BEGIN; SELECT bm25_spill_index( 'public.lifecycle_managed_lock_leaf_idx'); COMMIT;" >"${signal_output}" 2>&1 & signal_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-managed-lock-signal';")" = "0" ]; then signal_completed=true break fi sleep 0.1 done if [ "${signal_completed}" = "true" ]; then wait "${signal_pid}" || signal_status=$? fi PGAPPNAME=lifecycle-managed-lock-reconcile \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "SET statement_timeout = '15s'; CREATE INDEX lifecycle_managed_lock_parent_idx ON public.lifecycle_managed_lock_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '5 4 3 2 *');" \ >"${reconcile_output}" 2>&1 & reconcile_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-managed-lock-reconcile';")" = "0" ]; then reconcile_completed=true break fi sleep 0.1 done if [ "${reconcile_completed}" = "true" ]; then wait "${reconcile_pid}" || reconcile_status=$? fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-managed-lock-blocker';" >/dev/null wait "${blocker_pid}" || true if [ "${signal_completed}" != "true" ]; then wait "${signal_pid}" || signal_status=$? fi if [ "${reconcile_completed}" != "true" ]; then wait "${reconcile_pid}" || reconcile_status=$? fi if [ "${signal_completed}" != "true" ] || [ "${reconcile_completed}" != "true" ] || [ "${reconcile_status}" -ne 0 ] || [ "${signal_status}" -ne 0 ]; then error "managed lifecycle lock ordering failed: reconcile: $(cat "${reconcile_output}") signal: $(cat "${signal_output}")" fi if grep -Fq "deadlock detected" "${reconcile_output}" || grep -Fq "deadlock detected" "${signal_output}"; then error "managed lifecycle lock ordering reported a caught deadlock: reconcile: $(cat "${reconcile_output}") signal: $(cat "${signal_output}")" fi if ! grep -Fq \ "background compaction lifecycle reconciliation was deferred" \ "${reconcile_output}"; then error "contended partition reconciliation was not reported: $(cat "${reconcile_output}")" fi log "PASS: PRECOMMIT managed paths defer instead of waiting" assert_eq "partition option reconciliation completes" "t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', reloptions @> ARRAY['compaction=background'], reloptions @> ARRAY['compaction_schedule=5 4 3 2 *']) FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" sql_as durable_owner -c " ALTER INDEX public.lifecycle_managed_lock_leaf_idx SET (compaction_schedule = '5 4 3 2 *');" >/dev/null assert_eq "deferred partition schedule is retryable" "t" \ "$(sql_super -c "SELECT EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:${index_oid}:%:' || pg_catalog.encode( pg_catalog.convert_to('5 4 3 2 *', 'UTF8'), 'hex')) AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" sql_super -c "DROP TABLE public.lifecycle_managed_lock_docs;" } test_alter_reindex_lock_order() { local alter_admission_before alter_output alter_pid alter_status=0 local alter_waited=false gate_pid index_oid reindex_inverse local reindex_output reindex_pid reindex_status=0 alter_output="${DATA_DIR}/alter-reindex-alter.out" reindex_output="${DATA_DIR}/alter-reindex-reindex.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_alter_reindex_docs (body text); CREATE INDEX lifecycle_alter_reindex_idx ON public.lifecycle_alter_reindex_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_uppercase_background_docs (body text); CREATE INDEX lifecycle_uppercase_background_idx ON public.lifecycle_uppercase_background_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE FUNCTION public.lifecycle_alter_reindex_pause() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-alter-reindex-alter' THEN PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 14); END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_alter_reindex_pause ON ddl_command_start WHEN TAG IN ('ALTER INDEX') EXECUTE FUNCTION public.lifecycle_alter_reindex_pause();" \ >/dev/null index_oid="$(sql_super -c "SELECT 'public.lifecycle_alter_reindex_idx'::regclass::oid;")" PGAPPNAME=lifecycle-alter-reindex-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 14); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/alter-reindex-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-alter-reindex-gate' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-alter-reindex-alter \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "ALTER INDEX public.lifecycle_alter_reindex_idx SET (compaction_schedule = '1 0 1 1 *');" \ >"${alter_output}" 2>&1 & alter_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-alter-reindex-alter' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "managed ALTER reaches the pre-core barrier" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-alter-reindex-alter' AND wait_event = 'advisory';")" alter_admission_before="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-alter-reindex-alter' AND activity.wait_event = 'advisory' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND admission.granted;")" if [ "${alter_admission_before}" = "0" ]; then sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-alter-reindex-gate';" >/dev/null wait "${gate_pid}" || true wait "${alter_pid}" || alter_status=$? alter_waited=true fi PGAPPNAME=lifecycle-alter-reindex-reindex \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "REINDEX INDEX CONCURRENTLY public.lifecycle_alter_reindex_idx;" \ >"${reindex_output}" 2>&1 & reindex_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-alter-reindex-reindex' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND NOT admission.granted;")" = "1" ]; then break fi if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-alter-reindex-reindex';")" = "0" ]; then break fi sleep 0.1 done reindex_inverse="$(sql_super -c "SELECT count(*) FILTER ( WHERE relation_lock.locktype = 'relation' AND relation_lock.relation = ${index_oid} AND relation_lock.mode = 'ShareUpdateExclusiveLock' AND relation_lock.granted) || ':' || count(*) FILTER ( WHERE relation_lock.locktype = 'object' AND relation_lock.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND relation_lock.objid = ${index_oid} AND relation_lock.objsubid = 1 AND relation_lock.mode = 'ExclusiveLock' AND NOT relation_lock.granted) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS relation_lock ON relation_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-alter-reindex-reindex';")" if [ "${alter_admission_before}" = "1" ]; then assert_eq "old-order REINDEX reaches the inverse lock edge" "1:1" \ "${reindex_inverse}" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-alter-reindex-gate';" >/dev/null wait "${gate_pid}" || true fi if [ "${alter_waited}" = "false" ]; then wait "${alter_pid}" || alter_status=$? fi wait "${reindex_pid}" || reindex_status=$? if [ "${alter_status}" -ne 0 ] || [ "${reindex_status}" -ne 0 ] || grep -Fq "deadlock detected" "${alter_output}" || grep -Fq "deadlock detected" "${reindex_output}"; then error "ALTER/concurrent REINDEX lock order failed: alter: $(cat "${alter_output}") reindex: $(cat "${reindex_output}")" fi assert_eq "managed ALTER takes no admission before core relation lock" \ "0" "${alter_admission_before}" assert_eq "ALTER schedule survives concurrent REINDEX" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction_schedule=1 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_alter_reindex_idx'::regclass;")" sql_as durable_owner -c " ALTER INDEX public.lifecycle_uppercase_background_idx SET (compaction = 'BACKGROUND');" >/dev/null assert_eq "accepted mixed-case background option activates" "1" \ "$(active_jobs_for_index "$(sql_super -c "SELECT 'public.lifecycle_uppercase_background_idx'::regclass::oid;")")" sql_super -c " DROP EVENT TRIGGER lifecycle_alter_reindex_pause; DROP FUNCTION public.lifecycle_alter_reindex_pause(); DROP TABLE public.lifecycle_alter_reindex_docs, public.lifecycle_uppercase_background_docs;" >/dev/null } test_cross_statement_reindex_lock_order() { local am_oid dependency_before first_output first_pid first_status=0 local gate_pid second_index_oid second_output local second_pid second_status=0 first_output="${DATA_DIR}/cross-reindex-first.out" second_output="${DATA_DIR}/cross-reindex-second.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_cross_reindex_first_docs (body text); CREATE INDEX lifecycle_cross_reindex_first_idx ON public.lifecycle_cross_reindex_first_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_cross_reindex_second_docs (body text); CREATE INDEX lifecycle_cross_reindex_second_idx ON public.lifecycle_cross_reindex_second_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL second_index_oid="$(sql_super -c "SELECT 'public.lifecycle_cross_reindex_second_idx'::regclass::oid;")" am_oid="$(sql_super -c \ "SELECT oid FROM pg_catalog.pg_am WHERE amname = 'bm25';")" PGAPPNAME=lifecycle-cross-reindex-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 15); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/cross-reindex-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-reindex-gate' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-cross-reindex-first \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; REINDEX INDEX public.lifecycle_cross_reindex_first_idx; SELECT pg_catalog.pg_advisory_xact_lock(478, 15); REINDEX INDEX public.lifecycle_cross_reindex_second_idx; COMMIT;" >"${first_output}" 2>&1 & first_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-reindex-first' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "first REINDEX transaction reaches the statement barrier" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-reindex-first' AND wait_event = 'advisory';")" dependency_before="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS dependency ON dependency.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-reindex-first' AND dependency.locktype = 'object' AND dependency.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND dependency.objid = ${am_oid} AND dependency.objsubid = 0 AND dependency.mode = 'ShareRowExclusiveLock' AND dependency.granted;")" PGAPPNAME=lifecycle-cross-reindex-second \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; REINDEX INDEX public.lifecycle_cross_reindex_second_idx; COMMIT;" >"${second_output}" 2>&1 & second_pid=$! if [ "${dependency_before}" = "1" ]; then for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS dependency ON dependency.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-reindex-second' AND dependency.locktype = 'object' AND dependency.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND dependency.objid = ${am_oid} AND dependency.objsubid = 0 AND dependency.mode = 'ShareRowExclusiveLock' AND NOT dependency.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "inverse REINDEX holds relation while waiting dependency" \ "1:1" \ "$(sql_super -c "SELECT count(*) FILTER ( WHERE managed.locktype = 'relation' AND managed.relation = ${second_index_oid} AND managed.mode = 'AccessExclusiveLock' AND managed.granted) || ':' || count(*) FILTER ( WHERE managed.locktype = 'object' AND managed.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND managed.objid = ${am_oid} AND managed.objsubid = 0 AND managed.mode = 'ShareRowExclusiveLock' AND NOT managed.granted) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS managed ON managed.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-reindex-second';")" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-reindex-gate';" >/dev/null wait "${gate_pid}" || true wait "${first_pid}" || first_status=$? wait "${second_pid}" || second_status=$? if [ "${first_status}" -ne 0 ] || [ "${second_status}" -ne 0 ] || grep -Fq "deadlock detected" "${first_output}" || grep -Fq "deadlock detected" "${second_output}"; then error "cross-statement REINDEX lock order failed: first: $(cat "${first_output}") second: $(cat "${second_output}")" fi assert_eq "REINDEX defers dependency work to the terminal batch" \ "0" "${dependency_before}" assert_eq "both REINDEX workflows remain current" "1:1" \ "$(current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_cross_reindex_first_idx'::regclass::oid;")"):$( current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_cross_reindex_second_idx'::regclass::oid;")")" sql_super -c " DROP TABLE public.lifecycle_cross_reindex_first_docs, public.lifecycle_cross_reindex_second_docs;" >/dev/null } test_cross_statement_owner_lock_order() { local am_oid dependency_before first_output first_pid first_status=0 local gate_pid second_heap_oid second_output second_waited=false local second_pid second_status=0 first_output="${DATA_DIR}/cross-owner-first.out" second_output="${DATA_DIR}/cross-owner-second.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_cross_owner_first_docs (body text); CREATE INDEX lifecycle_cross_owner_first_idx ON public.lifecycle_cross_owner_first_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_cross_owner_second_docs (body text); CREATE INDEX lifecycle_cross_owner_second_idx ON public.lifecycle_cross_owner_second_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL second_heap_oid="$(sql_super -c "SELECT 'public.lifecycle_cross_owner_second_docs'::regclass::oid;")" am_oid="$(sql_super -c \ "SELECT oid FROM pg_catalog.pg_am WHERE amname = 'bm25';")" PGAPPNAME=lifecycle-cross-owner-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 16); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/cross-owner-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-owner-gate' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-cross-owner-first \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; ALTER TABLE public.lifecycle_cross_owner_first_docs OWNER TO durable_owner_two; SELECT pg_catalog.pg_advisory_xact_lock(478, 16); ALTER TABLE public.lifecycle_cross_owner_second_docs OWNER TO durable_owner_two; COMMIT;" >"${first_output}" 2>&1 & first_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-owner-first' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "first owner transaction reaches the statement barrier" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-owner-first' AND wait_event = 'advisory';")" dependency_before="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS dependency ON dependency.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-owner-first' AND dependency.locktype = 'object' AND dependency.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND dependency.objid = ${am_oid} AND dependency.objsubid = 0 AND dependency.mode = 'ShareRowExclusiveLock' AND dependency.granted;")" PGAPPNAME=lifecycle-cross-owner-second \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; ALTER TABLE public.lifecycle_cross_owner_second_docs OWNER TO durable_owner_two; COMMIT;" >"${second_output}" 2>&1 & second_pid=$! if [ "${dependency_before}" = "1" ]; then for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS dependency ON dependency.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-owner-second' AND dependency.locktype = 'object' AND dependency.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND dependency.objid = ${am_oid} AND dependency.objsubid = 0 AND dependency.mode = 'ShareRowExclusiveLock' AND NOT dependency.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "inverse owner change holds heap while waiting dependency" \ "1:1" \ "$(sql_super -c "SELECT count(*) FILTER ( WHERE managed.locktype = 'relation' AND managed.relation = ${second_heap_oid} AND managed.mode = 'AccessExclusiveLock' AND managed.granted) || ':' || count(*) FILTER ( WHERE managed.locktype = 'object' AND managed.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND managed.objid = ${am_oid} AND managed.objsubid = 0 AND managed.mode = 'ShareRowExclusiveLock' AND NOT managed.granted) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS managed ON managed.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-owner-second';")" fi # Keep statement-level overlap, but serialize terminal reconciliation # so contention on shared extension objects cannot defer either job. if [ "${dependency_before}" = "0" ]; then wait "${second_pid}" || second_status=$? second_waited=true fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-owner-gate';" >/dev/null wait "${gate_pid}" || true wait "${first_pid}" || first_status=$? if [ "${second_waited}" = "false" ]; then wait "${second_pid}" || second_status=$? fi if [ "${first_status}" -ne 0 ] || [ "${second_status}" -ne 0 ] || grep -Fq "deadlock detected" "${first_output}" || grep -Fq "deadlock detected" "${second_output}"; then error "cross-statement owner lock order failed: first: $(cat "${first_output}") second: $(cat "${second_output}")" fi assert_eq "owner changes defer dependency work to the terminal batch" \ "0" "${dependency_before}" assert_eq "both owner changes preserve current workflows" "1:1" \ "$(current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_cross_owner_first_idx'::regclass::oid;")"):$( current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_cross_owner_second_idx'::regclass::oid;")")" assert_eq "both owner changes complete" \ "durable_owner_two:durable_owner_two" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid( (SELECT relowner FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_cross_owner_first_docs'::regclass)) || ':' || pg_catalog.pg_get_userbyid( (SELECT relowner FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_cross_owner_second_docs'::regclass));")" sql_super -c " DROP TABLE public.lifecycle_cross_owner_first_docs, public.lifecycle_cross_owner_second_docs;" >/dev/null } test_multi_family_partition_attach_batch() { local attach_output local child_a_oid child_b_oid parent_a_lineage parent_b_lineage sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_multi_family_parent ( id integer, body_a text, body_b text ) PARTITION BY RANGE (id); CREATE INDEX lifecycle_multi_family_parent_a_idx ON public.lifecycle_multi_family_parent USING bm25(body_a) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '1 0 1 1 *'); CREATE INDEX lifecycle_multi_family_parent_b_idx ON public.lifecycle_multi_family_parent USING bm25(body_b) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '2 0 1 1 *'); CREATE TABLE public.lifecycle_multi_family_child ( id integer, body_a text, body_b text ); SQL if ! attach_output="$(sql_as durable_owner -c " ALTER TABLE public.lifecycle_multi_family_parent ATTACH PARTITION public.lifecycle_multi_family_child FOR VALUES FROM (0) TO (100);" 2>&1)"; then error "two-family partition attachment failed: ${attach_output}" fi child_a_oid="$(sql_super -c "SELECT inhrelid FROM pg_catalog.pg_inherits WHERE inhparent = 'public.lifecycle_multi_family_parent_a_idx'::regclass;")" child_b_oid="$(sql_super -c "SELECT inhrelid FROM pg_catalog.pg_inherits WHERE inhparent = 'public.lifecycle_multi_family_parent_b_idx'::regclass;")" parent_a_lineage="$( index_lineage public.lifecycle_multi_family_parent_a_idx )" parent_b_lineage="$( index_lineage public.lifecycle_multi_family_parent_b_idx )" assert_eq "two-family attachment creates both physical indexes" "t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', ${child_a_oid} <> ${child_b_oid}, (SELECT count(*) FROM pg_catalog.pg_class WHERE oid IN (${child_a_oid}, ${child_b_oid})) = 2);")" assert_eq "first attached family inherits lifecycle options" \ "${parent_a_lineage}:true" \ "$(sql_super -c "SELECT pg_catalog.substr( lineage_option, pg_catalog.length('compaction_lineage=') + 1) || ':' || (relation.reloptions @> ARRAY['compaction_schedule=1 0 1 1 *'])::text FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest( relation.reloptions) AS lineage_option WHERE relation.oid = ${child_a_oid} AND lineage_option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "second attached family inherits lifecycle options" \ "${parent_b_lineage}:true" \ "$(sql_super -c "SELECT pg_catalog.substr( lineage_option, pg_catalog.length('compaction_lineage=') + 1) || ':' || (relation.reloptions @> ARRAY['compaction_schedule=2 0 1 1 *'])::text FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest( relation.reloptions) AS lineage_option WHERE relation.oid = ${child_b_oid} AND lineage_option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "two-family attachment activates both leaves" "1:1" \ "$(current_generation_job_count "${child_a_oid}"):$( current_generation_job_count "${child_b_oid}")" sql_super -c "DROP TABLE public.lifecycle_multi_family_parent;" \ >/dev/null } test_attached_index_rewrite_preserves_parent_options() { local child_oid parent_lineage tablespace_dir tablespace_dir="$( mktemp -d "${TMPDIR:-/tmp}/pg_textsearch_attach_ts.XXXXXX" )" sql_super -c "CREATE TABLESPACE lifecycle_attach_rewrite_ts LOCATION '${tablespace_dir}';" >/dev/null sql_super -c "GRANT CREATE ON TABLESPACE lifecycle_attach_rewrite_ts TO durable_owner;" >/dev/null sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_attach_reindex_parent ( id integer, body text ) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_attach_reindex_child PARTITION OF public.lifecycle_attach_reindex_parent FOR VALUES FROM (0) TO (100); CREATE INDEX lifecycle_attach_reindex_parent_idx ON ONLY public.lifecycle_attach_reindex_parent USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '1 0 1 1 *'); CREATE INDEX lifecycle_attach_reindex_child_idx ON public.lifecycle_attach_reindex_child USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '2 0 1 1 *'); BEGIN; ALTER INDEX public.lifecycle_attach_reindex_parent_idx ATTACH PARTITION public.lifecycle_attach_reindex_child_idx; ALTER TABLE public.lifecycle_attach_reindex_child SET TABLESPACE lifecycle_attach_rewrite_ts; COMMIT; SQL child_oid="$(sql_super -c "SELECT 'public.lifecycle_attach_reindex_child_idx'::regclass::oid;")" parent_lineage="$( index_lineage public.lifecycle_attach_reindex_parent_idx )" assert_eq "attached then rewritten child inherits parent options" \ "${parent_lineage}:true" \ "$(sql_super -c "SELECT pg_catalog.substr( lineage_option, pg_catalog.length('compaction_lineage=') + 1) || ':' || (relation.reloptions @> ARRAY['compaction_schedule=1 0 1 1 *'])::text FROM pg_catalog.pg_class AS relation CROSS JOIN LATERAL pg_catalog.unnest( relation.reloptions) AS lineage_option WHERE relation.oid = ${child_oid} AND lineage_option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "attached then rewritten child activates parent schedule" \ "true" \ "$(sql_super -c "SELECT (label OPERATOR(pg_catalog.~~) ('%:' || pg_catalog.encode( pg_catalog.convert_to('1 0 1 1 *', 'UTF8'), 'hex')))::text FROM df.instances WHERE id = '$(current_generation_job_id "${child_oid}")';")" sql_super -c \ "DROP TABLE public.lifecycle_attach_reindex_parent;" >/dev/null sql_super -c "DROP TABLESPACE lifecycle_attach_rewrite_ts;" >/dev/null rmdir "${tablespace_dir}" } test_lineage_lookup_drop_durable_order() { local create_output create_pid create_status=0 durable_oid local drop_output drop_pid drop_status=0 gate_create_pid gate_drop_pid local instances_oid member_before create_output="${DATA_DIR}/lineage-drop-create.out" drop_output="${DATA_DIR}/lineage-drop-extension.out" durable_oid="$(sql_super -c "SELECT oid FROM pg_catalog.pg_extension WHERE extname = 'pg_durable';")" instances_oid="$(sql_super -c "SELECT 'df.instances'::regclass::oid;")" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_lineage_drop_docs (body text); CREATE FUNCTION public.lifecycle_lineage_drop_pause() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-lineage-drop-create' THEN PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 17); END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_lineage_drop_pause ON ddl_command_start WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION public.lifecycle_lineage_drop_pause();" \ >/dev/null PGAPPNAME=lifecycle-lineage-drop-create-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 17); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/lineage-drop-create-gate.out" 2>&1 & gate_create_pid=$! PGAPPNAME=lifecycle-lineage-drop-extension-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 18); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/lineage-drop-extension-gate.out" 2>&1 & gate_drop_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ( 'lifecycle-lineage-drop-create-gate', 'lifecycle-lineage-drop-extension-gate') AND wait_event = 'PgSleep';")" = "2" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-lineage-drop-create \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "CREATE INDEX lifecycle_lineage_drop_idx ON public.lifecycle_lineage_drop_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" \ >"${create_output}" 2>&1 & create_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-create' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "lineage CREATE reaches its pre-core barrier" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-create' AND wait_event = 'advisory';")" member_before="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS member_lock ON member_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-lineage-drop-create' AND member_lock.locktype = 'relation' AND member_lock.relation = ${instances_oid} AND member_lock.mode = 'RowExclusiveLock' AND member_lock.granted;")" PGAPPNAME=lifecycle-lineage-drop-extension \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; ALTER EXTENSION pg_durable DROP TABLE df.instances; LOCK TABLE df.instances IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_advisory_xact_lock(478, 18); ROLLBACK;" >"${drop_output}" 2>&1 & drop_pid=$! if [ "${member_before}" = "1" ]; then for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS member_lock ON member_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-lineage-drop-extension' AND member_lock.locktype = 'relation' AND member_lock.relation = ${instances_oid} AND member_lock.mode = 'AccessExclusiveLock' AND NOT member_lock.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "pg_durable member change waits behind lineage SPI" "1:1" \ "$(sql_super -c "SELECT count(*) FILTER ( WHERE extension_lock.locktype = 'object' AND extension_lock.classid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND extension_lock.objid = ${durable_oid} AND extension_lock.mode = 'AccessExclusiveLock' AND extension_lock.granted) || ':' || count(*) FILTER ( WHERE extension_lock.locktype = 'relation' AND extension_lock.relation = ${instances_oid} AND extension_lock.mode = 'AccessExclusiveLock' AND NOT extension_lock.granted) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS extension_lock ON extension_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-lineage-drop-extension';")" else for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-extension' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-extension' AND wait_event = 'advisory';")" != "1" ]; then error "pg_durable member change did not reach its rollback barrier: activity: $(sql_super -c "SELECT pg_catalog.concat_ws(':', state, wait_event_type, wait_event, query) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-extension';") output: $(cat "${drop_output}")" fi log "PASS: pg_durable member change reaches its rollback barrier" fi if [ "${member_before}" = "0" ]; then sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-extension-gate';" >/dev/null wait "${gate_drop_pid}" || true wait "${drop_pid}" || drop_status=$? fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-create-gate';" >/dev/null wait "${gate_create_pid}" || true wait "${create_pid}" || create_status=$? if [ "${member_before}" = "1" ]; then sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-drop-extension-gate';" >/dev/null wait "${gate_drop_pid}" || true wait "${drop_pid}" || drop_status=$? fi if [ "${create_status}" -ne 0 ] || [ "${drop_status}" -ne 0 ] || grep -Fq "deadlock detected" "${create_output}" || grep -Fq "deadlock detected" "${drop_output}"; then error "lineage lookup and DROP pg_durable lock order failed: create: $(cat "${create_output}") drop: $(cat "${drop_output}")" fi assert_eq "lineage collection takes no pg_durable member lock" \ "0" "${member_before}" assert_eq "rolled-back pg_durable drop preserves both extensions" \ "1:1" \ "$(sql_super -c "SELECT count(*) FILTER (WHERE extname = 'pg_durable') || ':' || count(*) FILTER (WHERE extname = 'pg_textsearch') FROM pg_catalog.pg_extension;")" assert_eq "lineage CREATE activates after the drop rollback" "1" \ "$(current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_lineage_drop_idx'::regclass::oid;")")" sql_super -c " DROP EVENT TRIGGER lifecycle_lineage_drop_pause; DROP FUNCTION public.lifecycle_lineage_drop_pause(); DROP TABLE public.lifecycle_lineage_drop_docs;" >/dev/null } test_textsearch_extension_dependency_order() { local alter_completed=false alter_output alter_pid alter_status=0 local create_output local extension_oid extension_output extension_pid extension_status=0 local gate_pid index_oid step_function_oid alter_output="${DATA_DIR}/textsearch-extension-alter.out" extension_output="${DATA_DIR}/textsearch-extension-holder.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_textsearch_extension_docs (body text); CREATE INDEX lifecycle_textsearch_extension_idx ON public.lifecycle_textsearch_extension_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_textsearch_lineage_docs (body text); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_textsearch_extension_idx'::regclass::oid;")" extension_oid="$(sql_super -c "SELECT oid FROM pg_catalog.pg_extension WHERE extname = 'pg_textsearch';")" step_function_oid="$(sql_super -c "SELECT 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)'::regprocedure::oid;")" PGAPPNAME=lifecycle-textsearch-extension-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 19); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/textsearch-extension-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-textsearch-extension-gate' AND wait_event = 'PgSleep';")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-textsearch-extension-holder \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=30s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; ALTER EXTENSION pg_textsearch DROP FUNCTION bm25_compact_step_if_current(oid, oid, oid, oid, oid); DROP FUNCTION bm25_compact_step_if_current(oid, oid, oid, oid, oid); SELECT pg_catalog.pg_advisory_xact_lock_shared(478, 19); DROP EXTENSION pg_textsearch CASCADE; ROLLBACK;" >"${extension_output}" 2>&1 & extension_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS member_lock ON member_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-textsearch-extension-holder' AND activity.wait_event = 'advisory' AND member_lock.locktype = 'object' AND member_lock.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass AND member_lock.objid = ${step_function_oid} AND member_lock.mode = 'AccessExclusiveLock' AND member_lock.granted;")" = "1" ]; then break fi sleep 0.1 done if [ "$(sql_super -c "SELECT count(*) FILTER ( WHERE object_lock.classid = 'pg_catalog.pg_extension'::pg_catalog.regclass AND object_lock.objid = ${extension_oid} AND object_lock.mode = 'AccessShareLock') || ':' || count(*) FILTER ( WHERE object_lock.classid = 'pg_catalog.pg_proc'::pg_catalog.regclass AND object_lock.objid = ${step_function_oid} AND object_lock.mode = 'AccessExclusiveLock') FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS object_lock ON object_lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-textsearch-extension-holder' AND activity.wait_event = 'advisory' AND object_lock.locktype = 'object' AND object_lock.granted;")" != "1:1" ]; then error "pg_textsearch drop did not hold its extension and member: activity: $(sql_super -c "SELECT pg_catalog.concat_ws(':', state, wait_event_type, wait_event, query) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-textsearch-extension-holder';") locks: $(sql_super -c "SELECT pg_catalog.string_agg( lock.locktype || ':' || coalesce(lock.classid::text, '') || ':' || coalesce(lock.objid::text, '') || ':' || lock.mode || ':' || lock.granted::text, ',') FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS lock ON lock.pid = activity.pid WHERE activity.application_name = 'lifecycle-textsearch-extension-holder';") output: $(cat "${extension_output}")" fi log "PASS: pg_textsearch drop holds its extension and member" if create_output="$(PGOPTIONS="-c statement_timeout=2s" \ sql_as durable_owner -c " CREATE INDEX lifecycle_textsearch_lineage_idx ON public.lifecycle_textsearch_lineage_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_lineage = 'bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb');" 2>&1)"; then error "supplied lineage bypassed extension lifecycle serialization: ${create_output}" fi if ! grep -Fq "canceling statement due to statement timeout" \ <<<"${create_output}"; then error "supplied lineage lifecycle wait failed unexpectedly: ${create_output}" fi assert_eq "serialized supplied lineage leaves no index" "" \ "$(sql_super -c "SELECT pg_catalog.to_regclass( 'public.lifecycle_textsearch_lineage_idx');")" log "PASS: extension lifecycle serializes supplied lineage CREATE" PGAPPNAME=lifecycle-textsearch-extension-alter \ PGOPTIONS="-c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "ALTER INDEX public.lifecycle_textsearch_extension_idx SET (compaction_schedule = '7 0 1 1 *');" \ >"${alter_output}" 2>&1 & alter_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-textsearch-extension-alter';")" = "0" ]; then alter_completed=true break fi sleep 0.1 done if [ "${alter_completed}" = "true" ]; then wait "${alter_pid}" || alter_status=$? fi assert_eq "contended extension root defers terminal activation" "f" \ "$(sql_super -c "SELECT EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:${index_oid}:%:' || pg_catalog.encode( pg_catalog.convert_to('7 0 1 1 *', 'UTF8'), 'hex')) AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-textsearch-extension-gate';" >/dev/null wait "${gate_pid}" || true wait "${extension_pid}" || extension_status=$? if [ "${alter_completed}" != "true" ]; then wait "${alter_pid}" || alter_status=$? fi if [ "${alter_completed}" != "true" ] || [ "${alter_status}" -ne 0 ] || [ "${extension_status}" -ne 0 ] || ! grep -Fq \ "background compaction lifecycle reconciliation was deferred" \ "${alter_output}" || grep -Fq "deadlock detected" "${alter_output}" || grep -Fq "deadlock detected" "${extension_output}"; then error "pg_textsearch extension/dependency ordering failed: alter: $(cat "${alter_output}") extension: $(cat "${extension_output}")" fi log "PASS: contended extension root is checked before dependency work" sql_as durable_owner -c " ALTER INDEX public.lifecycle_textsearch_extension_idx SET (compaction_schedule = '7 0 1 1 *');" >/dev/null assert_eq "explicit retry publishes the deferred schedule" "t" \ "$(sql_super -c "SELECT EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:${index_oid}:%:' || pg_catalog.encode( pg_catalog.convert_to('7 0 1 1 *', 'UTF8'), 'hex')) AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" sql_super -c " DROP TABLE public.lifecycle_textsearch_extension_docs, public.lifecycle_textsearch_lineage_docs;" >/dev/null } test_precommit_request_admission_nowait() { local blocker_output blocker_pid blocker_status=0 gate_fifo gate_pid local index_oid local writer_output writer_pid writer_status=0 writer_waiting blocker_output="${DATA_DIR}/precommit-admission-blocker.out" writer_output="${DATA_DIR}/precommit-admission-writer.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_precommit_admission_docs ( id integer, body text ); CREATE INDEX lifecycle_precommit_admission_idx ON public.lifecycle_precommit_admission_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_precommit_admission_idx'::regclass::oid;")" install_signal_probe reset_signal_probe sql_super -c "INSERT INTO public.compaction_signal_fault VALUES ('*', 'gate');" gate_fifo="${DATA_DIR}/precommit-admission-gate.fifo" mkfifo "${gate_fifo}" exec 7<>"${gate_fifo}" PGAPPNAME=lifecycle-precommit-admission-gate \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ <"${gate_fifo}" >"${DATA_DIR}/precommit-admission-gate.out" 2>&1 & gate_pid=$! printf '%s\n' "SELECT pg_catalog.pg_advisory_lock(478, 11);" >&7 for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS gate ON gate.pid = activity.pid WHERE activity.application_name = 'lifecycle-precommit-admission-gate' AND activity.state = 'idle' AND gate.locktype = 'advisory' AND gate.classid = 478 AND gate.objid = 11 AND gate.mode = 'ExclusiveLock' AND gate.granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-precommit-admission-blocker \ PGOPTIONS="-c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.lifecycle_precommit_admission_docs SELECT value, pg_catalog.format('blocker %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index( 'public.lifecycle_precommit_admission_idx'); INSERT INTO public.lifecycle_precommit_admission_docs SELECT 50 + value, pg_catalog.format('blocker second %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index( 'public.lifecycle_precommit_admission_idx'); COMMIT;" \ >"${blocker_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-precommit-admission-blocker' AND activity.wait_event = 'advisory' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND admission.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "request blocker holds the target admission" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-precommit-admission-blocker' AND activity.wait_event = 'advisory' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND admission.granted;")" PGAPPNAME=lifecycle-precommit-admission-writer \ PGOPTIONS="-c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.lifecycle_precommit_admission_docs SELECT 100 + value, pg_catalog.format('writer %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index( 'public.lifecycle_precommit_admission_idx'); COMMIT;" >"${writer_output}" 2>&1 & writer_pid=$! writer_waiting=0 for _ in $(seq 1 100); do writer_waiting="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-precommit-admission-writer' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND NOT admission.granted;")" if [ "${writer_waiting}" = "1" ] || [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-precommit-admission-writer';")" = "0" ]; then break fi sleep 0.1 done if [ "${writer_waiting}" = "0" ]; then wait "${writer_pid}" || writer_status=$? fi printf '%s\n' \ "SELECT pg_catalog.pg_advisory_unlock(478, 11);" \ '\q' >&7 exec 7>&- wait "${gate_pid}" || true if [ "${writer_waiting}" = "1" ]; then wait "${writer_pid}" || writer_status=$? fi wait "${blocker_pid}" || blocker_status=$? if [ "${writer_status}" -ne 0 ] || [ "${blocker_status}" -ne 0 ] || grep -Fq "deadlock detected" "${writer_output}" || grep -Fq "deadlock detected" "${blocker_output}"; then error "PRE_COMMIT admission deferral failed: writer: $(cat "${writer_output}") blocker: $(cat "${blocker_output}")" fi assert_eq "PRE_COMMIT request never waits on admission" \ "0" "${writer_waiting}" assert_eq "deferred request preserves writer rows" "60" \ "$(sql_super -c "SELECT count(*) FROM public.lifecycle_precommit_admission_docs;")" restore_signal_probe sql_super -c "DROP TABLE public.lifecycle_precommit_admission_docs;" >/dev/null } test_terminal_grant_reentry_is_rejected() { local alter_output sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_grant_reentry_docs ( body_a text, body_b text ); CREATE INDEX lifecycle_grant_reentry_outer_idx ON public.lifecycle_grant_reentry_docs USING bm25(body_a) WITH (text_config = 'english', compaction = 'manual'); CREATE FUNCTION public.lifecycle_grant_reenter() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-grant-reentry' AND pg_catalog.to_regclass( 'public.lifecycle_grant_reentry_nested_idx') IS NULL THEN EXECUTE 'CREATE INDEX lifecycle_grant_reentry_nested_idx ' 'ON public.lifecycle_grant_reentry_docs USING bm25(body_b) ' 'WITH (text_config = ''english'', ' 'compaction = ''background'')'; END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_grant_reenter ON ddl_command_end WHEN TAG IN ('GRANT') EXECUTE FUNCTION public.lifecycle_grant_reenter(); REVOKE EXECUTE ON FUNCTION bm25_compact_step_if_current(oid, oid, oid, oid, oid), bm25_background_target_is_current(oid, oid, oid, oid, oid) FROM durable_owner;" >/dev/null if alter_output="$(PGAPPNAME=lifecycle-grant-reentry \ sql_as durable_owner -c " ALTER INDEX public.lifecycle_grant_reentry_outer_idx SET (compaction = 'background');" 2>&1)"; then error "managed DDL reentered terminal reconciliation: ${alter_output}" fi if ! grep -Fq \ "cannot execute DDL during background compaction reconciliation" \ <<<"${alter_output}"; then error "terminal reentry failed for the wrong reason: ${alter_output}" fi assert_eq "terminal reentry rolls back nested managed DDL" "" \ "$(sql_super -c "SELECT pg_catalog.to_regclass( 'public.lifecycle_grant_reentry_nested_idx');")" sql_super -c " DROP EVENT TRIGGER lifecycle_grant_reenter; DROP FUNCTION public.lifecycle_grant_reenter(); DROP TABLE public.lifecycle_grant_reentry_docs; SELECT df.grant_usage('durable_owner');" >/dev/null } test_post_publication_reindex_defers() { local blocker_output blocker_pid blocker_status=0 gate_fifo gate_pid local index_oid_after index_oid_before outer_output outer_pid local outer_status=0 outer_waiting relfilenumber_before signal_gate_fifo local renamed_output signal_gate_pid blocker_output="${DATA_DIR}/post-publication-blocker.out" outer_output="${DATA_DIR}/post-publication-reindex.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_post_publication_docs ( id integer, body text ); INSERT INTO public.lifecycle_post_publication_docs VALUES (1, 'before'); CREATE INDEX lifecycle_post_publication_idx ON public.lifecycle_post_publication_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_post_publication_nested_docs (body text); CREATE INDEX lifecycle_post_publication_nested_idx ON public.lifecycle_post_publication_nested_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE FUNCTION public.lifecycle_post_publication_pause() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-post-publication-reindex' AND COALESCE( pg_catalog.current_setting( 'lifecycle.post_publication_reentry', true), '') OPERATOR(pg_catalog.<>) 'done' THEN PERFORM pg_catalog.set_config( 'lifecycle.post_publication_reentry', 'done', false); EXECUTE 'ALTER INDEX public.lifecycle_post_publication_nested_idx ' 'SET (compaction_schedule = ''1 0 1 1 *'')'; PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 20); END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_post_publication_pause ON ddl_command_end WHEN TAG IN ('REINDEX') EXECUTE FUNCTION public.lifecycle_post_publication_pause();" \ >/dev/null index_oid_before="$(sql_super -c "SELECT 'public.lifecycle_post_publication_idx'::regclass::oid;")" relfilenumber_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode( 'public.lifecycle_post_publication_idx'::regclass);")" install_signal_probe reset_signal_probe sql_super -c "INSERT INTO public.compaction_signal_fault VALUES ('*', 'gate');" gate_fifo="${DATA_DIR}/post-publication-gate.fifo" mkfifo "${gate_fifo}" exec 7<>"${gate_fifo}" PGAPPNAME=lifecycle-post-publication-gate \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ <"${gate_fifo}" >"${DATA_DIR}/post-publication-gate.out" 2>&1 & gate_pid=$! printf '%s\n' \ "SELECT pg_catalog.pg_advisory_lock(478, 20);" >&7 for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS gate ON gate.pid = activity.pid WHERE activity.application_name = 'lifecycle-post-publication-gate' AND activity.state = 'idle' AND gate.locktype = 'advisory' AND gate.classid = 478 AND gate.objid = 20 AND gate.mode = 'ExclusiveLock' AND gate.granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-post-publication-reindex \ PGOPTIONS="-c deadlock_timeout=100ms -c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "REINDEX INDEX CONCURRENTLY public.lifecycle_post_publication_idx;" \ >"${outer_output}" 2>&1 & outer_pid=$! for _ in $(seq 1 200); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-post-publication-reindex' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "concurrent REINDEX reaches post-publication reentry" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-post-publication-reindex' AND wait_event = 'advisory';")" index_oid_after="$(sql_super -c "SELECT 'public.lifecycle_post_publication_idx'::regclass::oid;")" if [ "$(sql_super -c "SELECT pg_catalog.pg_relation_filenode( 'public.lifecycle_post_publication_idx'::regclass);")" = \ "${relfilenumber_before}" ]; then error "concurrent REINDEX paused before publishing replacement storage" fi signal_gate_fifo="${DATA_DIR}/post-publication-signal-gate.fifo" mkfifo "${signal_gate_fifo}" exec 8<>"${signal_gate_fifo}" PGAPPNAME=lifecycle-post-publication-signal-gate \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ <"${signal_gate_fifo}" \ >"${DATA_DIR}/post-publication-signal-gate.out" 2>&1 & signal_gate_pid=$! printf '%s\n' \ "SELECT pg_catalog.pg_advisory_lock(478, 11);" >&8 for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS gate ON gate.pid = activity.pid WHERE activity.application_name = 'lifecycle-post-publication-signal-gate' AND activity.state = 'idle' AND gate.locktype = 'advisory' AND gate.classid = 478 AND gate.objid = 11 AND gate.mode = 'ExclusiveLock' AND gate.granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-post-publication-blocker \ PGOPTIONS="-c statement_timeout=15s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.lifecycle_post_publication_docs SELECT 100 + value, pg_catalog.format( 'post publication first %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index( 'public.lifecycle_post_publication_idx'); INSERT INTO public.lifecycle_post_publication_docs SELECT 200 + value, pg_catalog.format( 'post publication second %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index( 'public.lifecycle_post_publication_idx'); COMMIT;" >"${blocker_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 200); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-post-publication-blocker' AND activity.wait_event = 'advisory' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid_after} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND admission.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "signal blocker holds the published target admission" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-post-publication-blocker' AND activity.wait_event = 'advisory' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid_after} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND admission.granted;")" printf '%s\n' \ "SELECT pg_catalog.pg_advisory_unlock(478, 20);" \ '\q' >&7 exec 7>&- wait "${gate_pid}" || true outer_waiting=0 for _ in $(seq 1 50); do outer_waiting="$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS admission ON admission.pid = activity.pid WHERE activity.application_name = 'lifecycle-post-publication-reindex' AND admission.locktype = 'object' AND admission.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND admission.objid = ${index_oid_after} AND admission.objsubid = 1 AND admission.mode = 'ExclusiveLock' AND NOT admission.granted;")" if [ "${outer_waiting}" = "1" ] || [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-post-publication-reindex';")" = "0" ]; then break fi sleep 0.1 done if [ "${outer_waiting}" = "0" ]; then wait "${outer_pid}" || outer_status=$? fi printf '%s\n' \ "SELECT pg_catalog.pg_advisory_unlock(478, 11);" \ '\q' >&8 exec 8>&- wait "${signal_gate_pid}" || true wait "${blocker_pid}" || blocker_status=$? if [ "${outer_waiting}" = "1" ]; then wait "${outer_pid}" || outer_status=$? fi if [ "${outer_status}" -ne 0 ] || [ "${blocker_status}" -ne 0 ] || grep -Fq "deadlock detected" "${outer_output}" || grep -Fq "deadlock detected" "${blocker_output}"; then error "post-publication reconciliation failed: reindex: $(cat "${outer_output}") blocker: $(cat "${blocker_output}")" fi assert_eq "post-publication reconciliation never waits on admission" \ "0" "${outer_waiting}" assert_eq "published concurrent REINDEX remains visible" \ "${index_oid_after}" \ "$(sql_super -c "SELECT 'public.lifecycle_post_publication_idx'::regclass::oid;")" assert_eq "nested reentry option change commits" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction_schedule=1 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_post_publication_nested_idx'::regclass;")" restore_signal_probe renamed_output="${DATA_DIR}/post-publication-renamed-object.out" sql_super -c "ALTER FUNCTION df.explain(text) RENAME TO explain_temporarily_unavailable;" if ! sql_as durable_owner -c " REINDEX INDEX CONCURRENTLY public.lifecycle_post_publication_idx;" \ >"${renamed_output}" 2>&1; then error "published REINDEX reported an object-discovery failure: $(cat "${renamed_output}")" fi if ! grep -Fq \ "background compaction lifecycle reconciliation was deferred" \ "${renamed_output}"; then error "published REINDEX did not report deferred object discovery: $(cat "${renamed_output}")" fi sql_super -c "ALTER FUNCTION df.explain_temporarily_unavailable(text) RENAME TO explain;" sql_as durable_owner -c " ALTER INDEX public.lifecycle_post_publication_idx SET (compaction_schedule = '0 0 1 1 *');" >/dev/null assert_eq "object-discovery deferral remains explicitly retryable" "1" \ "$(current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_post_publication_idx'::regclass::oid;")")" sql_super -c " DROP EVENT TRIGGER lifecycle_post_publication_pause; DROP FUNCTION public.lifecycle_post_publication_pause(); DROP TABLE public.lifecycle_post_publication_docs, public.lifecycle_post_publication_nested_docs;" >/dev/null } test_post_publication_schedule_override() { local index_oid job_after job_label reindex_output schedule_matches sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_post_publication_schedule_docs (body text); INSERT INTO public.lifecycle_post_publication_schedule_docs VALUES ('before'); CREATE INDEX lifecycle_post_publication_schedule_idx ON public.lifecycle_post_publication_schedule_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE FUNCTION public.lifecycle_post_publication_schedule_change() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-post-publication-schedule' THEN ALTER INDEX public.lifecycle_post_publication_schedule_idx SET (compaction_schedule = '2 0 1 1 *'); END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_post_publication_schedule_change ON ddl_command_end WHEN TAG IN ('REINDEX') EXECUTE FUNCTION public.lifecycle_post_publication_schedule_change();" >/dev/null index_oid="$(sql_super -c "SELECT 'public.lifecycle_post_publication_schedule_idx'::regclass::oid;")" reindex_output="$(PGAPPNAME=lifecycle-post-publication-schedule \ sql_as durable_owner -c " REINDEX INDEX CONCURRENTLY public.lifecycle_post_publication_schedule_idx;" 2>&1)" index_oid="$(sql_super -c "SELECT 'public.lifecycle_post_publication_schedule_idx'::regclass::oid;")" assert_eq "post-publication schedule option change commits" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction_schedule=2 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" schedule_matches=0 for _ in $(seq 1 100); do schedule_matches="$( current_generation_schedule_job_count \ "${index_oid}" "2 0 1 1 *" )" if [ "${schedule_matches}" = "1" ]; then break fi sleep 0.1 done if [ "${schedule_matches}" != "1" ]; then job_after="$(current_generation_job_id "${index_oid}")" job_label="$(sql_super -c "SELECT label FROM df.instances WHERE id = '${job_after}';")" error "post-publication schedule change was not authoritative: workflow ${job_after} has label ${job_label} REINDEX output: ${reindex_output}" fi log "PASS: post-publication schedule change remains authoritative" sql_super -c " DROP EVENT TRIGGER lifecycle_post_publication_schedule_change; DROP FUNCTION public.lifecycle_post_publication_schedule_change(); DROP TABLE public.lifecycle_post_publication_schedule_docs;" \ >/dev/null } test_post_publication_background_activation() { local index_oid schedule_matches sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_post_publication_activation_docs (body text); INSERT INTO public.lifecycle_post_publication_activation_docs VALUES ('before'); CREATE INDEX lifecycle_post_publication_activation_idx ON public.lifecycle_post_publication_activation_docs USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); CREATE FUNCTION public.lifecycle_post_publication_enable_background() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-post-publication-activation' THEN ALTER INDEX public.lifecycle_post_publication_activation_idx SET (compaction = 'background', compaction_schedule = '3 0 1 1 *'); END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_post_publication_enable_background ON ddl_command_end WHEN TAG IN ('REINDEX') EXECUTE FUNCTION public.lifecycle_post_publication_enable_background();" \ >/dev/null PGAPPNAME=lifecycle-post-publication-activation \ sql_as durable_owner -c " REINDEX INDEX CONCURRENTLY public.lifecycle_post_publication_activation_idx;" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_post_publication_activation_idx'::regclass::oid;")" assert_eq "post-publication background activation commits" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction=background', 'compaction_schedule=3 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = ${index_oid};")" schedule_matches=0 for _ in $(seq 1 100); do schedule_matches="$( current_generation_schedule_job_count \ "${index_oid}" "3 0 1 1 *" )" if [ "${schedule_matches}" = "1" ]; then break fi sleep 0.1 done assert_eq "post-publication activation creates the current workflow" \ "1" "${schedule_matches}" sql_super -c " DROP EVENT TRIGGER lifecycle_post_publication_enable_background; DROP FUNCTION public.lifecycle_post_publication_enable_background(); DROP TABLE public.lifecycle_post_publication_activation_docs;" \ >/dev/null } test_post_publication_activation_error_defers() { local index_oid output relfilenumber_before sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_post_publication_error_docs (body text); INSERT INTO public.lifecycle_post_publication_error_docs VALUES ('before'); CREATE INDEX lifecycle_post_publication_error_idx ON public.lifecycle_post_publication_error_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE FUNCTION public.lifecycle_post_publication_revoke() RETURNS event_trigger LANGUAGE plpgsql SECURITY DEFINER SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-post-publication-error' THEN ALTER INDEX public.lifecycle_post_publication_error_idx RENAME TO lifecycle_post_publication_error_renamed_idx; REVOKE USAGE ON SCHEMA df FROM durable_owner; END IF; END $body$; SQL sql_super -c " ALTER FUNCTION public.lifecycle_post_publication_revoke() OWNER TO postgres; CREATE EVENT TRIGGER lifecycle_post_publication_revoke ON ddl_command_end WHEN TAG IN ('REINDEX') EXECUTE FUNCTION public.lifecycle_post_publication_revoke();" \ >/dev/null index_oid="$(sql_super -c "SELECT 'public.lifecycle_post_publication_error_idx'::regclass::oid;")" relfilenumber_before="$(sql_super -c "SELECT pg_catalog.pg_relation_filenode(${index_oid});")" if ! output="$(PGAPPNAME=lifecycle-post-publication-error \ sql_as durable_owner -c " REINDEX INDEX CONCURRENTLY public.lifecycle_post_publication_error_idx;" 2>&1)"; then error "published REINDEX reported an activation error: ${output}" fi if ! grep -Fq \ "background compaction lifecycle reconciliation was deferred" \ <<<"${output}"; then error "published REINDEX did not report activation deferral: ${output}" fi if [ "$(sql_super -c "SELECT pg_catalog.pg_relation_filenode( 'public.lifecycle_post_publication_error_renamed_idx' ::regclass);")" = \ "${relfilenumber_before}" ]; then error "activation-error test did not publish replacement storage" fi sql_super -c "SELECT df.grant_usage('durable_owner');" >/dev/null sql_as durable_owner -c " ALTER INDEX public.lifecycle_post_publication_error_renamed_idx SET (compaction_schedule = '0 0 1 1 *');" >/dev/null assert_eq "activation-error deferral remains explicitly retryable" "1" \ "$(current_generation_job_count "$(sql_super -c "SELECT 'public.lifecycle_post_publication_error_renamed_idx' ::regclass::oid;")")" sql_super -c " DROP EVENT TRIGGER lifecycle_post_publication_revoke; DROP FUNCTION public.lifecycle_post_publication_revoke(); DROP TABLE public.lifecycle_post_publication_error_docs;" >/dev/null } test_cross_statement_managed_lock_order() { local am_oid first_index_oid first_output first_pid first_status=0 local gate_pid second_index_oid second_output second_pid second_status=0 first_output="${DATA_DIR}/cross-statement-lock-first.out" second_output="${DATA_DIR}/cross-statement-lock-second.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_cross_statement_first_docs (body text); CREATE INDEX lifecycle_cross_statement_first_idx ON public.lifecycle_cross_statement_first_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_cross_statement_second_docs (body text); CREATE INDEX lifecycle_cross_statement_second_idx ON public.lifecycle_cross_statement_second_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL first_index_oid="$(sql_super -c "SELECT 'public.lifecycle_cross_statement_first_idx'::regclass::oid;")" second_index_oid="$(sql_super -c "SELECT 'public.lifecycle_cross_statement_second_idx'::regclass::oid;")" am_oid="$(sql_super -c \ "SELECT oid FROM pg_catalog.pg_am WHERE amname = 'bm25';")" PGAPPNAME=lifecycle-cross-statement-gate sql_super -c " SELECT pg_catalog.pg_advisory_lock(478, 13); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/cross-statement-lock-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS gate ON gate.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-statement-gate' AND activity.wait_event = 'PgSleep' AND gate.locktype = 'advisory' AND gate.classid = 478 AND gate.objid = 13 AND gate.mode = 'ExclusiveLock' AND gate.granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "cross-statement gate is held" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS gate ON gate.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-statement-gate' AND activity.wait_event = 'PgSleep' AND gate.locktype = 'advisory' AND gate.classid = 478 AND gate.objid = 13 AND gate.mode = 'ExclusiveLock' AND gate.granted;")" PGAPPNAME=lifecycle-cross-statement-first \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "SET deadlock_timeout = '100ms'; SET statement_timeout = '15s'; BEGIN; ALTER INDEX public.lifecycle_cross_statement_first_idx SET (compaction_schedule = '1 0 1 1 *'); SELECT pg_catalog.pg_advisory_xact_lock(478, 13); ALTER INDEX public.lifecycle_cross_statement_second_idx SET (compaction_schedule = '2 0 1 1 *'); COMMIT;" >"${first_output}" 2>&1 & first_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-statement-first' AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done assert_eq "first transaction reaches the statement barrier" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-statement-first' AND wait_event_type = 'Lock';")" assert_eq "collecting transaction holds no managed locks" "0" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity AS activity JOIN pg_catalog.pg_locks AS managed ON managed.pid = activity.pid WHERE activity.application_name = 'lifecycle-cross-statement-first' AND managed.locktype = 'object' AND managed.classid = 'pg_catalog.pg_am'::pg_catalog.regclass AND managed.granted AND ((managed.objid = ${first_index_oid} AND managed.objsubid = 1 AND managed.mode = 'ExclusiveLock') OR (managed.objid = ${am_oid} AND managed.objsubid = 0 AND managed.mode = 'ShareRowExclusiveLock'));")" PGAPPNAME=lifecycle-cross-statement-second \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "SET deadlock_timeout = '10s'; SET statement_timeout = '15s'; BEGIN; ALTER INDEX public.lifecycle_cross_statement_second_idx SET (compaction_schedule = '3 0 1 1 *'); COMMIT;" >"${second_output}" 2>&1 & second_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-statement-second';")" = "0" ]; then break fi sleep 0.1 done wait "${second_pid}" || second_status=$? if [ "${second_status}" -ne 0 ]; then error "independent managed transaction was blocked: $(cat "${second_output}")" fi log "PASS: independent managed transaction completes during collection" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-cross-statement-gate';" >/dev/null wait "${gate_pid}" || true wait "${first_pid}" || first_status=$? if [ "${first_status}" -ne 0 ]; then error "cross-statement managed lock test failed: first: $(cat "${first_output}") second: $(cat "${second_output}")" fi if grep -Fq "deadlock detected" "${first_output}" || grep -Fq "deadlock detected" "${second_output}"; then error "cross-statement managed operations deadlocked: first: $(cat "${first_output}") second: $(cat "${second_output}")" fi assert_eq "later statement wins after collected transaction resumes" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction_schedule=2 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = ${second_index_oid};")" assert_eq "cross-statement batch publishes the first schedule" "t" \ "$(sql_super -c "SELECT EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:${first_index_oid}:%:' || pg_catalog.encode( pg_catalog.convert_to('1 0 1 1 *', 'UTF8'), 'hex')) AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" assert_eq "cross-statement batch publishes the second schedule" "t" \ "$(sql_super -c "SELECT EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:${second_index_oid}:%:' || pg_catalog.encode( pg_catalog.convert_to('2 0 1 1 *', 'UTF8'), 'hex')) AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" sql_super -c " DROP TABLE public.lifecycle_cross_statement_first_docs; DROP TABLE public.lifecycle_cross_statement_second_docs;" } test_managed_intent_savepoint_recovery() { local fifo output pid status=0 fifo="${DATA_DIR}/managed-intent-savepoint.fifo" output="${DATA_DIR}/managed-intent-savepoint.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_savepoint_first_docs (body text); CREATE INDEX lifecycle_savepoint_first_idx ON public.lifecycle_savepoint_first_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_savepoint_second_docs (body text); CREATE INDEX lifecycle_savepoint_second_idx ON public.lifecycle_savepoint_second_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_savepoint_created_docs (body text); CREATE TABLE public.lifecycle_intent_disabled_docs (body text); CREATE TABLE public.lifecycle_intent_rollback_docs (body text); CREATE TABLE public.lifecycle_intent_timestamp_docs (body text); CREATE INDEX lifecycle_intent_timestamp_idx ON public.lifecycle_intent_timestamp_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_intent_order_a_docs (body text); CREATE INDEX lifecycle_intent_order_a_idx ON public.lifecycle_intent_order_a_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_intent_order_b_docs (body text); CREATE INDEX lifecycle_intent_order_b_idx ON public.lifecycle_intent_order_b_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL mkfifo "${fifo}" exec 9<>"${fifo}" PGAPPNAME=lifecycle-managed-intent-savepoint \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U postgres -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=0 \ <"${fifo}" >"${output}" 2>&1 & pid=$! printf '%s\n' 'BEGIN;' '\echo begin_done' >&9 for _ in $(seq 1 100); do grep -Fq "begin_done" "${output}" && break sleep 0.1 done sql_as durable_owner -c " ALTER INDEX public.lifecycle_intent_timestamp_idx SET (compaction_schedule = '14 0 1 1 *');" >/dev/null printf '%s\n' \ "ALTER INDEX public.lifecycle_intent_timestamp_idx SET (compaction_schedule = '15 0 1 1 *');" \ '\echo timestamp_intent_done' >&9 for _ in $(seq 1 100); do grep -Fq "timestamp_intent_done" "${output}" && break sleep 0.1 done printf '%s\n' \ "ALTER INDEX public.lifecycle_savepoint_first_idx SET (compaction_schedule = '4 0 1 1 *');" \ '\echo first_done' >&9 for _ in $(seq 1 100); do grep -Fq "first_done" "${output}" && break sleep 0.1 done printf '%s\n' \ "CREATE INDEX lifecycle_savepoint_created_idx ON public.lifecycle_savepoint_created_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '3 0 1 1 *');" \ "ALTER INDEX public.lifecycle_savepoint_created_idx SET (compaction_schedule = '9 0 1 1 *');" \ '\echo create_alter_done' >&9 for _ in $(seq 1 100); do grep -Fq "create_alter_done" "${output}" && break sleep 0.1 done printf '%s\n' \ "CREATE INDEX lifecycle_intent_disabled_idx ON public.lifecycle_intent_disabled_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '12 0 1 1 *');" \ "ALTER INDEX public.lifecycle_intent_disabled_idx SET (compaction = 'manual');" \ "CREATE INDEX lifecycle_intent_rollback_idx ON public.lifecycle_intent_rollback_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '13 0 1 1 *');" \ 'SAVEPOINT disable_intent;' \ "ALTER INDEX public.lifecycle_intent_rollback_idx SET (compaction = 'manual');" \ 'ROLLBACK TO SAVEPOINT disable_intent;' \ '\echo mode_intents_done' >&9 for _ in $(seq 1 100); do grep -Fq "mode_intents_done" "${output}" && break sleep 0.1 done printf '%s\n' \ "ALTER INDEX public.lifecycle_intent_order_a_idx SET (compaction_schedule = '10 0 1 1 *');" \ "ALTER TABLE public.lifecycle_intent_order_a_docs OWNER TO durable_owner_two;" \ "ALTER TABLE public.lifecycle_intent_order_b_docs OWNER TO durable_owner_two;" \ "ALTER INDEX public.lifecycle_intent_order_b_idx SET (compaction_schedule = '11 0 1 1 *');" \ '\echo intent_order_done' >&9 for _ in $(seq 1 100); do grep -Fq "intent_order_done" "${output}" && break sleep 0.1 done printf '%s\n' 'SAVEPOINT managed_intent;' '\echo savepoint_done' >&9 for _ in $(seq 1 100); do grep -Fq "savepoint_done" "${output}" && break sleep 0.1 done printf '%s\n' \ "ALTER INDEX public.lifecycle_savepoint_second_idx SET (compaction_schedule = '5 0 1 1 *');" \ '\echo rolled_back_intent_done' >&9 for _ in $(seq 1 100); do grep -Fq "rolled_back_intent_done" "${output}" && break sleep 0.1 done printf '%s\n' 'SELECT 1 / 0;' '\echo expected_error_done' >&9 for _ in $(seq 1 100); do grep -Fq "expected_error_done" "${output}" && break sleep 0.1 done printf '%s\n' \ 'ROLLBACK TO SAVEPOINT managed_intent;' \ '\echo rollback_done' >&9 for _ in $(seq 1 100); do grep -Fq "rollback_done" "${output}" && break sleep 0.1 done printf '%s\n' \ 'SAVEPOINT merged_intent;' \ "ALTER INDEX public.lifecycle_savepoint_first_idx SET (compaction_schedule = '8 0 1 1 *');" \ 'RELEASE SAVEPOINT merged_intent;' \ '\echo release_done' >&9 for _ in $(seq 1 100); do grep -Fq "release_done" "${output}" && break sleep 0.1 done printf '%s\n' \ "ALTER INDEX public.lifecycle_savepoint_second_idx SET (compaction_schedule = '6 0 1 1 *');" \ '\echo replacement_intent_done' >&9 for _ in $(seq 1 100); do grep -Fq "replacement_intent_done" "${output}" && break sleep 0.1 done printf '%s\n' 'COMMIT;' '\echo commit_done' '\q' >&9 exec 9>&- wait "${pid}" || status=$? if [ "${status}" -ne 0 ] || ! grep -Fq "division by zero" "${output}" || ! grep -Fq "rollback_done" "${output}" || ! grep -Fq "commit_done" "${output}" || grep -Fq "current transaction is aborted" "${output}"; then error "managed intent savepoint recovery failed: $(cat "${output}")" fi assert_eq "savepoint recovery commits the parent intent" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction_schedule=8 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_savepoint_first_idx'::regclass;")" assert_eq "savepoint recovery discards and replaces the child intent" \ "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction_schedule=6 0 1 1 *'] FROM pg_catalog.pg_class WHERE oid = 'public.lifecycle_savepoint_second_idx'::regclass;")" assert_eq "later ALTER overrides a CREATE intent schedule" "t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction_schedule=9 0 1 1 *'], EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%:' || pg_catalog.encode( pg_catalog.convert_to('9 0 1 1 *', 'UTF8'), 'hex')) AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]))) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'public.lifecycle_savepoint_created_idx'::regclass;")" assert_eq "later manual mode cancels a CREATE activation" "t:0" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=manual'], (SELECT count(*) FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%') AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]))) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'public.lifecycle_intent_disabled_idx'::regclass;")" assert_eq "rolled-back manual mode preserves CREATE activation" "t:1" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.reloptions @> ARRAY['compaction=background'], (SELECT count(*) FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%') AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]))) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'public.lifecycle_intent_rollback_idx'::regclass;")" assert_eq "workflow ordering follows activation rather than xact start" \ "t" \ "$(sql_super -c "SELECT ( SELECT created_at FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%:' || pg_catalog.encode( pg_catalog.convert_to('15 0 1 1 *', 'UTF8'), 'hex')) ORDER BY created_at DESC, id DESC LIMIT 1) > (SELECT created_at FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%:' || pg_catalog.encode( pg_catalog.convert_to('14 0 1 1 *', 'UTF8'), 'hex')) ORDER BY created_at DESC, id DESC LIMIT 1) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'public.lifecycle_intent_timestamp_idx'::regclass;")" assert_eq "ALTER then owner change keeps the altered schedule" "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.relowner = 'durable_owner_two'::regrole, relation.reloptions @> ARRAY['compaction_schedule=10 0 1 1 *'], EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%:' || relation.relowner || ':%:' || pg_catalog.encode( pg_catalog.convert_to('10 0 1 1 *', 'UTF8'), 'hex')) AND submitted_by::pg_catalog.oid = relation.relowner AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]))) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'public.lifecycle_intent_order_a_idx'::regclass;")" assert_eq "owner change then ALTER uses the final schedule" "t:t:t" \ "$(sql_super -c "SELECT pg_catalog.concat_ws( ':', relation.relowner = 'durable_owner_two'::regrole, relation.reloptions @> ARRAY['compaction_schedule=11 0 1 1 *'], EXISTS ( SELECT 1 FROM df.instances WHERE label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid || ':%:' || relation.relowner || ':%:' || pg_catalog.encode( pg_catalog.convert_to('11 0 1 1 *', 'UTF8'), 'hex')) AND submitted_by::pg_catalog.oid = relation.relowner AND status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]))) FROM pg_catalog.pg_class AS relation WHERE relation.oid = 'public.lifecycle_intent_order_b_idx'::regclass;")" sql_super -c " DROP TABLE public.lifecycle_savepoint_first_docs; DROP TABLE public.lifecycle_savepoint_second_docs; DROP TABLE public.lifecycle_savepoint_created_docs; DROP TABLE public.lifecycle_intent_disabled_docs; DROP TABLE public.lifecycle_intent_rollback_docs; DROP TABLE public.lifecycle_intent_timestamp_docs; DROP TABLE public.lifecycle_intent_order_a_docs; DROP TABLE public.lifecycle_intent_order_b_docs;" } test_internal_lock_namespace() { local admission_error database_oid gate_pid index_oid lock_key owner_error sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_private_lock_docs (body text); CREATE INDEX lifecycle_private_lock_idx ON public.lifecycle_private_lock_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL index_oid="$(sql_super -c "SELECT 'public.lifecycle_private_lock_idx'::regclass::oid;")" remove_index_lineage public.lifecycle_private_lock_idx lock_key="$((1885828211 * 4294967296 + index_oid))" PGAPPNAME=lifecycle-public-advisory-lock sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(${lock_key}); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/public-advisory-lock.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 1885828211 AND objid = ${index_oid} AND granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "public advisory collision gate is held" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 1885828211 AND objid = ${index_oid} AND granted;")" if ! owner_error="$(sql_super -c " SET statement_timeout = '2s'; ALTER TABLE public.lifecycle_private_lock_docs OWNER TO durable_owner_two;" 2>&1)"; then error "public advisory lock blocked internal lineage serialization: \ ${owner_error}" fi assert_eq "public advisory lock does not block internal lineage lock" \ "durable_owner_two:32" \ "$(sql_super -c "SELECT pg_catalog.pg_get_userbyid(relation.relowner) || ':' || pg_catalog.length(pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1)) FROM pg_catalog.pg_class AS relation, LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-public-advisory-lock';" >/dev/null wait "${gate_pid}" || true database_oid="$(sql_super -c \ "SELECT oid FROM pg_catalog.pg_database WHERE datname = pg_catalog.current_database();")" PGAPPNAME=lifecycle-admission-advisory-lock \ sql_as durable_writer -c \ "SELECT pg_catalog.pg_advisory_lock( ${database_oid}::integer, ${index_oid}::integer); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/admission-advisory-lock.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = ${database_oid} AND objid = ${index_oid} AND granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "public admission collision gate is held" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = ${database_oid} AND objid = ${index_oid} AND granted;")" if ! admission_error="$(sql_as durable_owner_two -c " SET statement_timeout = '2s'; ALTER INDEX public.lifecycle_private_lock_idx SET ( compaction_schedule = '1 0 1 1 *');" 2>&1)"; then error "public advisory lock blocked workflow admission: \ ${admission_error}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-admission-advisory-lock';" >/dev/null wait "${gate_pid}" || true sql_super -c "DROP TABLE public.lifecycle_private_lock_docs;" } test_lineage_ddl_guards() { local cic_error duplicate_error lineage replay_error reset_error set_error sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_lineage_source_docs (body text); CREATE INDEX lifecycle_lineage_source_idx ON public.lifecycle_lineage_source_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_lineage_duplicate_docs (body text); SQL lineage="$(index_lineage public.lifecycle_lineage_source_idx)" if set_error="$(sql_as durable_owner -c " ALTER INDEX public.lifecycle_lineage_source_idx SET ( compaction_lineage = '11111111111111111111111111111111');" 2>&1)"; then error "ALTER INDEX SET changed managed lineage" fi if ! grep -Fq "cannot alter internal background compaction lineage" \ <<<"${set_error}"; then error "lineage SET failed for the wrong reason: ${set_error}" fi assert_eq "rejected lineage SET preserves identity" "${lineage}" \ "$(index_lineage public.lifecycle_lineage_source_idx)" if reset_error="$(sql_as durable_owner -c " ALTER INDEX public.lifecycle_lineage_source_idx RESET (compaction_lineage);" 2>&1)"; then error "ALTER INDEX RESET removed managed lineage" fi if ! grep -Fq "cannot alter internal background compaction lineage" \ <<<"${reset_error}"; then error "lineage RESET failed for the wrong reason: ${reset_error}" fi assert_eq "rejected lineage RESET preserves identity" "${lineage}" \ "$(index_lineage public.lifecycle_lineage_source_idx)" if duplicate_error="$(sql_as durable_owner -c " CREATE INDEX lifecycle_lineage_duplicate_options_idx ON public.lifecycle_lineage_duplicate_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_lineage = '${lineage}', compaction_lineage = '${lineage}');" 2>&1)"; then error "CREATE accepted duplicate lineage options" fi if ! grep -Fq "compaction_lineage" <<<"${duplicate_error}"; then error "duplicate lineage options failed unexpectedly: \ ${duplicate_error}" fi if duplicate_error="$(sql_as durable_owner -c " CREATE INDEX lifecycle_lineage_duplicate_idx ON public.lifecycle_lineage_duplicate_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_lineage = '${lineage}');" 2>&1)"; then error "CREATE reused a live background compaction lineage" fi if ! grep -Fq "background compaction lineage is already in use" \ <<<"${duplicate_error}"; then error "live lineage reuse failed unexpectedly: ${duplicate_error}" fi assert_eq "rejected live lineage reuse creates no index" "" \ "$(sql_super -c "SELECT pg_catalog.to_regclass( 'public.lifecycle_lineage_duplicate_idx');")" if cic_error="$(sql_as durable_owner -c " CREATE INDEX CONCURRENTLY lifecycle_lineage_cic_idx ON public.lifecycle_lineage_duplicate_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_lineage = 'cccccccccccccccccccccccccccccccc');" 2>&1)"; then error "concurrent CREATE accepted an explicit lineage" fi if ! grep -Fq \ "explicit background compaction lineage is not supported" \ <<<"${cic_error}"; then error "concurrent explicit lineage failed unexpectedly: ${cic_error}" fi assert_eq "rejected concurrent explicit lineage leaves no index" "" \ "$(sql_super -c "SELECT pg_catalog.to_regclass( 'public.lifecycle_lineage_cic_idx');")" sql_as durable_owner -c \ "DROP INDEX public.lifecycle_lineage_source_idx;" >/dev/null if replay_error="$(sql_as durable_owner -c " CREATE INDEX lifecycle_lineage_source_idx ON public.lifecycle_lineage_source_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_lineage = '${lineage}');" 2>&1)"; then error "drop/recreate replay reused retained lineage history" fi if ! grep -Fq "background compaction lineage is already in use" \ <<<"${replay_error}"; then error "retained lineage replay failed unexpectedly: ${replay_error}" fi sql_super -c "DROP TABLE public.lifecycle_lineage_source_docs, public.lifecycle_lineage_duplicate_docs;" } test_partitioned_lineage_history() { local lineage replay_error sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_partition_history_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_partition_history_low PARTITION OF public.lifecycle_partition_history_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_partition_history_high PARTITION OF public.lifecycle_partition_history_docs FOR VALUES FROM (100) TO (200); SQL sql_super -c "ALTER TABLE public.lifecycle_partition_history_high OWNER TO durable_owner_two;" sql_as durable_owner -c " CREATE INDEX lifecycle_partition_history_idx ON public.lifecycle_partition_history_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); " lineage="$(index_lineage public.lifecycle_partition_history_idx)" assert_eq "partition history has mixed physical leaf owners" "2" \ "$(sql_super -c "SELECT count(DISTINCT heap.relowner) FROM pg_catalog.pg_partition_tree( 'public.lifecycle_partition_history_idx'::regclass) AS tree JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = tree.relid JOIN pg_catalog.pg_class AS heap ON heap.oid = index_catalog.indrelid WHERE tree.isleaf;")" sql_as durable_owner -c "SELECT df.cancel( instance.id, 'partition lineage history test') FROM df.instances AS instance JOIN pg_catalog.pg_inherits AS inheritance ON instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || inheritance.inhrelid::pg_catalog.text || ':%') WHERE inheritance.inhparent = 'public.lifecycle_partition_history_idx'::regclass AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null sql_super -c "UPDATE df.instances SET label = 'retired-partition-owner-' || id WHERE submitted_by = 'durable_owner'::regrole AND label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:%:%:%:%:%:${lineage}:%';" assert_eq "mixed-owner retained leaf history remains" "1" \ "$(sql_super -c "SELECT count(*) FROM df.instances WHERE submitted_by = 'durable_owner_two'::regrole AND label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:%:%:%:%:%:${lineage}:%';")" sql_as durable_owner -c \ "DROP INDEX public.lifecycle_partition_history_idx;" >/dev/null if replay_error="$(sql_as durable_owner -c " CREATE INDEX lifecycle_partition_history_idx ON public.lifecycle_partition_history_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *', compaction_lineage = '${lineage}');" 2>&1)"; then error "partitioned drop/recreate reused retained leaf lineage" fi if ! grep -Fq "background compaction lineage is already in use" \ <<<"${replay_error}"; then error "partitioned retained history failed unexpectedly: \ ${replay_error}" fi log "PASS: partition root rejects retained physical-leaf lineage" sql_super -c \ "DROP TABLE public.lifecycle_partition_history_docs;" } test_forged_lineage_submitter() { local create_error foreign_instance forged_label lineage new_index_oid local new_instance original_instance sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_forged_history_docs (body text); CREATE INDEX lifecycle_forged_history_idx ON public.lifecycle_forged_history_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL new_index_oid="$(sql_super -c "SELECT 'public.lifecycle_forged_history_idx'::regclass::oid;")" lineage="$(index_lineage public.lifecycle_forged_history_idx)" original_instance="$(current_generation_job_id "${new_index_oid}")" wait_for_signal_node "${original_instance}" 30 forged_label="$(sql_super -c "SELECT label FROM df.instances WHERE id = '${original_instance}';")" sql_as durable_owner -c "SELECT df.cancel( '${original_instance}', 'forge submitter test');" >/dev/null wait_for_terminal "${original_instance}" 30 sql_super -c "UPDATE df.instances SET label = 'retired-forged-source-' || id WHERE id = '${original_instance}';" sql_as durable_owner -c \ "DROP INDEX public.lifecycle_forged_history_idx;" >/dev/null foreign_instance="$(sql_as durable_owner_two -c " SELECT df.start( df.wait_for_signal('hold', 300), '${forged_label}', pg_catalog.current_database(), 'caller');")" wait_for_signal_node "${foreign_instance}" 30 if ! create_error="$(sql_as durable_owner -c " CREATE INDEX lifecycle_forged_history_idx ON public.lifecycle_forged_history_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *', compaction_lineage = '${lineage}');" 2>&1)"; then error "foreign-submitter forged label blocked legitimate lineage \ reuse: ${create_error}" fi new_index_oid="$(sql_super -c "SELECT 'public.lifecycle_forged_history_idx'::regclass::oid;")" new_instance="$(current_generation_job_id "${new_index_oid}")" if [ -z "${new_instance}" ]; then error "foreign-submitter forged label prevented owner activation" fi assert_eq "forged label remains owned by its foreign submitter" \ "durable_owner_two" \ "$(sql_super -c "SELECT submitted_by::pg_catalog.text FROM df.instances WHERE id = '${foreign_instance}';")" assert_eq "owner may reuse lineage present only in forged history" \ "${lineage}" \ "$(index_lineage public.lifecycle_forged_history_idx)" sql_as durable_owner_two -c "SELECT df.cancel( '${foreign_instance}', 'forge submitter test complete');" >/dev/null sql_as durable_owner -c "SELECT df.cancel( '${new_instance}', 'forge submitter test complete');" >/dev/null sql_super -c "DROP TABLE public.lifecycle_forged_history_docs;" } test_concurrent_supplied_lineage_create() { local first_output first_pid first_status=0 gate_pid local second_output second_pid second_status=0 successes local supplied_lineage=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa first_output="${DATA_DIR}/lineage-create-first.out" second_output="${DATA_DIR}/lineage-create-second.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE FUNCTION public.lifecycle_lineage_create_pause(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $body$ BEGIN PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 7); RETURN value; END $body$; CREATE TABLE public.lifecycle_lineage_create_docs (body text); INSERT INTO public.lifecycle_lineage_create_docs VALUES ('one'), ('two'); SQL PGAPPNAME=lifecycle-lineage-create-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 7); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/lineage-create-gate.out" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 7 AND granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-lineage-create-first \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "CREATE INDEX lifecycle_lineage_create_first_idx ON public.lifecycle_lineage_create_docs USING bm25 ( public.lifecycle_lineage_create_pause(body)) WITH (text_config = 'english', compaction = 'background', compaction_lineage = '${supplied_lineage}');" \ >"${first_output}" 2>&1 & first_pid=$! PGAPPNAME=lifecycle-lineage-create-second \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "CREATE INDEX lifecycle_lineage_create_second_idx ON public.lifecycle_lineage_create_docs USING bm25 ( public.lifecycle_lineage_create_pause(body)) WITH (text_config = 'english', compaction = 'background', compaction_lineage = '${supplied_lineage}');" \ >"${second_output}" 2>&1 & second_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ( 'lifecycle-lineage-create-first', 'lifecycle-lineage-create-second') AND wait_event_type = 'Lock';")" = "2" ]; then break fi sleep 0.1 done assert_eq "concurrent lineage CREATE reaches serialization gates" "2" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ( 'lifecycle-lineage-create-first', 'lifecycle-lineage-create-second') AND wait_event_type = 'Lock';")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-create-gate';" >/dev/null wait "${gate_pid}" || true wait "${first_pid}" || first_status=$? wait "${second_pid}" || second_status=$? successes=0 if [ "${first_status}" -eq 0 ]; then successes=$((successes + 1)) elif ! grep -Eq \ "background compaction lineage is already in use|could not validate background compaction lineage" \ "${first_output}"; then error "first concurrent CREATE failed unexpectedly: \ $(cat "${first_output}")" fi if [ "${second_status}" -eq 0 ]; then successes=$((successes + 1)) elif ! grep -Eq \ "background compaction lineage is already in use|could not validate background compaction lineage" \ "${second_output}"; then error "second concurrent CREATE failed unexpectedly: \ $(cat "${second_output}")" fi assert_eq "one concurrent same-heap lineage CREATE succeeds" "1" \ "${successes}" assert_eq "same-heap lineage remains unique after concurrent CREATE" \ "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = relation.oid CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE index_catalog.indrelid = 'public.lifecycle_lineage_create_docs'::regclass AND option OPERATOR(pg_catalog.=) 'compaction_lineage=${supplied_lineage}';")" assert_eq "concurrent same-heap CREATE leaves one workflow" "1" \ "$(sql_super -c "SELECT count(*) FROM df.instances AS instance JOIN pg_catalog.pg_class AS relation ON instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid::pg_catalog.text || ':%') JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = relation.oid WHERE index_catalog.indrelid = 'public.lifecycle_lineage_create_docs'::regclass AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);")" sql_super -c "DROP TABLE public.lifecycle_lineage_create_docs; DROP FUNCTION public.lifecycle_lineage_create_pause(text);" } test_reindex_lineage_replacement_isolation() { local blocker_pid old_index_name old_index_oid old_lineage local reindex_pid reindex_output replacement_instance replacement_oid local replacement_lineage local lock_output="${DATA_DIR}/reindex-lineage-lock.out" reindex_output="${DATA_DIR}/reindex-lineage-replacement.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE FUNCTION public.lifecycle_lineage_pause(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF value OPERATOR(pg_catalog.=) 'pause' THEN PERFORM pg_catalog.pg_advisory_xact_lock(478, 4); END IF; RETURN value; END $body$; CREATE TABLE public.lifecycle_lineage_reindex_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_lineage_reindex_low PARTITION OF public.lifecycle_lineage_reindex_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_lineage_reindex_high PARTITION OF public.lifecycle_lineage_reindex_docs FOR VALUES FROM (100) TO (200); INSERT INTO public.lifecycle_lineage_reindex_docs VALUES (1, 'pause'), (101, 'continue'); CREATE INDEX lifecycle_lineage_reindex_idx ON public.lifecycle_lineage_reindex_docs USING bm25 (public.lifecycle_lineage_pause(body)) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL old_index_oid="$(sql_super -c "SELECT index_class.oid FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS index_class ON index_class.oid = inheritance.inhrelid JOIN pg_catalog.pg_index AS index_catalog ON index_catalog.indexrelid = index_class.oid WHERE inheritance.inhparent = 'public.lifecycle_lineage_reindex_idx'::regclass AND index_catalog.indrelid = 'public.lifecycle_lineage_reindex_high'::regclass;")" old_index_name="$(sql_super -c "SELECT relname FROM pg_catalog.pg_class WHERE oid = ${old_index_oid};")" old_lineage="$(index_lineage "public.${old_index_name}")" sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 4); SELECT pg_catalog.pg_sleep(120);" \ >"${lock_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 4 AND granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-lineage-replacement \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "REINDEX INDEX public.lifecycle_lineage_reindex_idx;" \ >"${reindex_output}" 2>&1 & reindex_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-replacement' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "lineage isolation pauses before the pending leaf" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-replacement' AND wait_event = 'advisory';")" sql_as durable_owner -c " ALTER TABLE public.lifecycle_lineage_reindex_docs DETACH PARTITION public.lifecycle_lineage_reindex_high; DROP INDEX public.${old_index_name}; CREATE INDEX ${old_index_name} ON public.lifecycle_lineage_reindex_high USING bm25 (public.lifecycle_lineage_pause(body)) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '5 4 3 2 *');" >/dev/null 2>&1 replacement_oid="$(sql_super -c \ "SELECT 'public.${old_index_name}'::regclass::oid;")" replacement_lineage="$(index_lineage "public.${old_index_name}")" if [ "${replacement_oid}" = "${old_index_oid}" ] || [ "${replacement_lineage}" = "${old_lineage}" ]; then error "lineage replacement setup reused the original identity" fi replacement_instance="$(current_generation_job_id "${replacement_oid}")" sql_as durable_owner -c "SELECT df.cancel( '${replacement_instance}', 'hide unrelated replacement');" >/dev/null wait_for_terminal "${replacement_instance}" 30 sql_super -c "UPDATE df.instances SET label = 'retired-lineage-replacement-' || id WHERE id = '${replacement_instance}';" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE pid <> pg_catalog.pg_backend_pid() AND query OPERATOR(pg_catalog.~~) 'SELECT pg_catalog.pg_advisory_lock(478, 4)%';" >/dev/null wait "${blocker_pid}" || true if ! wait "${reindex_pid}"; then error "partitioned REINDEX replacement test failed: \ $(cat "${reindex_output}")" fi assert_eq "outer REINDEX ignores same-name different-lineage index" "" \ "$(current_generation_job_id "${replacement_oid}")" sql_as durable_owner -c " DROP TABLE public.lifecycle_lineage_reindex_docs, public.lifecycle_lineage_reindex_high; DROP FUNCTION public.lifecycle_lineage_pause(text);" } test_reindex_authorization_ordering() { local blocker_pid index_error index_oid jobs_before lineage_before local lock_output="${DATA_DIR}/reindex-auth-lock.out" local partition_child_index partition_error partition_jobs_before local partition_parent_oid local tablespace_dir tablespace_error transaction_error sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_auth_docs (body text); CREATE INDEX lifecycle_auth_idx ON public.lifecycle_auth_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_auth_partitioned_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_auth_partitioned_low PARTITION OF public.lifecycle_auth_partitioned_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_auth_partitioned_high PARTITION OF public.lifecycle_auth_partitioned_docs FOR VALUES FROM (100) TO (200); CREATE INDEX lifecycle_auth_partitioned_idx ON public.lifecycle_auth_partitioned_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); SQL tablespace_dir="${DATA_DIR}-lifecycle-auth-tablespace" mkdir -p "${tablespace_dir}" sql_super -c " CREATE TABLESPACE lifecycle_auth_tablespace OWNER durable_writer LOCATION '${tablespace_dir}';" >/dev/null index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_auth_idx'::regclass::oid;")" lineage_before="$(index_lineage public.lifecycle_auth_idx)" jobs_before="$(managed_job_count)" PGAPPNAME=lifecycle-auth-index-lock sql_as durable_owner -c \ "BEGIN; LOCK TABLE public.lifecycle_auth_docs IN SHARE MODE; SELECT pg_catalog.pg_sleep(120);" >"${lock_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_auth_docs'::regclass AND mode = 'ShareLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if transaction_error="$(sql_as durable_owner -c " BEGIN; SET LOCAL statement_timeout = '1s'; REINDEX INDEX CONCURRENTLY public.lifecycle_auth_idx;" 2>&1)"; then error "concurrent REINDEX unexpectedly ran inside a transaction" fi if ! grep -Fq "cannot run inside a transaction block" \ <<<"${transaction_error}"; then error "concurrent REINDEX waited before its transaction check: \ ${transaction_error}" fi if tablespace_error="$(sql_as durable_owner -c " SET statement_timeout = '1s'; REINDEX (TABLESPACE lifecycle_auth_missing) INDEX public.lifecycle_auth_idx;" 2>&1)"; then error "REINDEX unexpectedly accepted a missing tablespace" fi if ! grep -Fq \ 'tablespace "lifecycle_auth_missing" does not exist' \ <<<"${tablespace_error}"; then error "REINDEX waited before destination tablespace lookup: \ ${tablespace_error}" fi if tablespace_error="$(sql_as durable_owner -c " SET statement_timeout = '1s'; REINDEX (TABLESPACE lifecycle_auth_tablespace) INDEX public.lifecycle_auth_idx;" 2>&1)"; then error "REINDEX unexpectedly bypassed tablespace CREATE privilege" fi if ! grep -Fq \ "permission denied for tablespace lifecycle_auth_tablespace" \ <<<"${tablespace_error}"; then error "REINDEX waited before destination tablespace ACL checks: \ ${tablespace_error}" fi if index_error="$(PGOPTIONS='-c statement_timeout=1s' \ sql_as durable_writer -c " REINDEX INDEX CONCURRENTLY public.lifecycle_auth_idx;" 2>&1)"; then error "unauthorized concurrent REINDEX unexpectedly succeeded" fi if ! grep -Fq "permission denied for index lifecycle_auth_idx" \ <<<"${index_error}"; then error "unauthorized concurrent REINDEX did not fail before locking: \ ${index_error}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE pid <> pg_catalog.pg_backend_pid() AND application_name = 'lifecycle-auth-index-lock';" >/dev/null wait "${blocker_pid}" || true assert_eq "unauthorized concurrent REINDEX preserves lineage" \ "${lineage_before}" "$(index_lineage public.lifecycle_auth_idx)" assert_eq "unauthorized concurrent REINDEX preserves index OID" \ "${index_oid}" \ "$(sql_super -c "SELECT 'public.lifecycle_auth_idx'::regclass::oid;")" assert_eq "unauthorized concurrent REINDEX creates no workflow" \ "${jobs_before}" "$(managed_job_count)" partition_parent_oid="$(sql_super -c "SELECT 'public.lifecycle_auth_partitioned_idx'::regclass::oid;")" partition_jobs_before="$(managed_job_count)" partition_child_index="$(sql_super -c " SELECT child_index.oid::regclass::text FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS child_index ON child_index.oid = inheritance.inhrelid WHERE inheritance.inhparent = ${partition_parent_oid} ORDER BY child_index.oid LIMIT 1;")" PGAPPNAME=lifecycle-auth-partition-child-lock \ sql_as durable_owner -c " BEGIN; ALTER INDEX ${partition_child_index} SET (compaction_schedule = '1 2 3 4 *'); SELECT pg_catalog.pg_sleep(120);" >"${lock_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = '${partition_child_index}'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if transaction_error="$(sql_as durable_owner -c " BEGIN; SET LOCAL statement_timeout = '1s'; REINDEX TABLE public.lifecycle_auth_partitioned_docs;" 2>&1)"; then error "partitioned REINDEX unexpectedly ran inside a transaction" fi if ! grep -Fq "cannot run inside a transaction block" \ <<<"${transaction_error}"; then error "partitioned REINDEX prelocked descendants before preflight: \ ${transaction_error}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-auth-partition-child-lock';" >/dev/null wait "${blocker_pid}" || true PGAPPNAME=lifecycle-auth-partition-lock sql_as durable_owner -c \ "BEGIN; LOCK TABLE public.lifecycle_auth_partitioned_docs IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_sleep(120);" >"${lock_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_auth_partitioned_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done if transaction_error="$(sql_as durable_owner -c " BEGIN; SET LOCAL statement_timeout = '1s'; REINDEX TABLE CONCURRENTLY public.lifecycle_auth_partitioned_docs;" 2>&1)"; then error "partitioned concurrent REINDEX ran inside a transaction" fi if ! grep -Fq "cannot run inside a transaction block" \ <<<"${transaction_error}"; then error "partitioned REINDEX enumerated descendants before preflight: \ ${transaction_error}" fi if partition_error="$(sql_as durable_writer -c " SET statement_timeout = '1s'; REINDEX TABLE public.lifecycle_auth_partitioned_docs;" 2>&1)"; then error "unauthorized partitioned REINDEX unexpectedly succeeded" fi if ! grep -Fq \ "permission denied for table lifecycle_auth_partitioned_docs" \ <<<"${partition_error}"; then error "unauthorized partitioned REINDEX did not fail before locking: \ ${partition_error}" fi sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE pid <> pg_catalog.pg_backend_pid() AND application_name = 'lifecycle-auth-partition-lock';" >/dev/null wait "${blocker_pid}" || true assert_eq "unauthorized partitioned REINDEX preserves parent OID" \ "${partition_parent_oid}" \ "$(sql_super -c "SELECT 'public.lifecycle_auth_partitioned_idx'::regclass::oid;")" assert_eq "unauthorized partitioned REINDEX creates no workflow" \ "${partition_jobs_before}" "$(managed_job_count)" sql_super -c "GRANT MAINTAIN ON public.lifecycle_auth_docs TO durable_writer;" sql_as durable_writer -c "REINDEX INDEX CONCURRENTLY public.lifecycle_auth_idx;" >/dev/null 2>&1 log "PASS: MAINTAIN permits tracked concurrent REINDEX" sql_super -c "DROP TABLE public.lifecycle_auth_docs, public.lifecycle_auth_partitioned_docs;" \ >/dev/null sql_super -c "DROP TABLESPACE lifecycle_auth_tablespace;" >/dev/null } test_create_authorization_ordering() { local blocker_pid create_error jobs_before malformed_error local lock_output="${DATA_DIR}/create-auth-lock.out" sql_as durable_owner -c " CREATE TABLE public.lifecycle_create_auth_docs (body text);" \ >/dev/null jobs_before="$(managed_job_count)" PGAPPNAME=lifecycle-create-auth-lock sql_as durable_owner -c \ "BEGIN; LOCK TABLE public.lifecycle_create_auth_docs IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_sleep(120);" >"${lock_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_create_auth_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done assert_eq "CREATE authorization blocker holds the table lock" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_create_auth_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" if create_error="$(sql_as durable_writer -c " SET statement_timeout = '1s'; CREATE INDEX lifecycle_create_auth_idx ON public.lifecycle_create_auth_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" 2>&1)"; then error "unauthorized CREATE INDEX unexpectedly succeeded" fi if ! grep -Fq "must be owner of table lifecycle_create_auth_docs" \ <<<"${create_error}"; then error "unauthorized CREATE waited on the custom strong lock: \ ${create_error}" fi log "PASS: unauthorized CREATE fails before custom strong locking" if malformed_error="$(sql_as durable_writer -c " SET statement_timeout = '1s'; CREATE INDEX lifecycle_create_auth_malformed_idx ON public.lifecycle_create_auth_docs USING bm25(body) WITH (text_config = 'english', compaction);" 2>&1)"; then error "unauthorized malformed CREATE INDEX unexpectedly succeeded" fi if ! grep -Fq "must be owner of table lifecycle_create_auth_docs" \ <<<"${malformed_error}"; then error "malformed compaction option preceded core permission error: \ ${malformed_error}" fi log "PASS: CREATE ownership check precedes malformed option extraction" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-create-auth-lock';" >/dev/null wait "${blocker_pid}" || true assert_eq "unauthorized CREATE leaves no index" "" \ "$(sql_super -c "SELECT pg_catalog.to_regclass( 'public.lifecycle_create_auth_idx');")" assert_eq "unauthorized CREATE creates no workflow" \ "${jobs_before}" "$(managed_job_count)" sql_super -c "DROP TABLE public.lifecycle_create_auth_docs;" } test_lineage_guard_name_race() { local alter_output alter_pid alter_status=0 blocker_pid local lineage rename_output rename_pid rename_status=0 replacement_oid sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_lineage_guard_old_docs (body text); CREATE INDEX lifecycle_lineage_guard_idx ON public.lifecycle_lineage_guard_old_docs USING btree(body); CREATE TABLE public.lifecycle_lineage_guard_new_docs (body text); CREATE INDEX lifecycle_lineage_guard_replacement_idx ON public.lifecycle_lineage_guard_new_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE FUNCTION public.lifecycle_lineage_guard_pause() RETURNS event_trigger LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $body$ BEGIN IF pg_catalog.current_setting('application_name') OPERATOR(pg_catalog.=) 'lifecycle-lineage-guard-alter' THEN PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 10); END IF; END $body$; SQL sql_super -c " CREATE EVENT TRIGGER lifecycle_lineage_guard_pause ON ddl_command_start WHEN TAG IN ('ALTER INDEX') EXECUTE FUNCTION public.lifecycle_lineage_guard_pause();" \ >/dev/null lineage="$( index_lineage public.lifecycle_lineage_guard_replacement_idx )" replacement_oid="$(sql_super -c "SELECT 'public.lifecycle_lineage_guard_replacement_idx'::regclass::oid;")" PGAPPNAME=lifecycle-lineage-guard-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 10); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/lineage-guard-gate.out" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 10 AND granted;")" = "1" ]; then break fi sleep 0.1 done alter_output="${DATA_DIR}/lineage-guard-alter.out" PGAPPNAME=lifecycle-lineage-guard-alter \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "ALTER INDEX public.lifecycle_lineage_guard_idx RESET (compaction_lineage);" >"${alter_output}" 2>&1 & alter_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-guard-alter' AND wait_event = 'advisory';")" = "1" ]; then break fi sleep 0.1 done assert_eq "lineage guard pauses after inspecting its target" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-guard-alter' AND wait_event = 'advisory';")" rename_output="${DATA_DIR}/lineage-guard-rename.out" PGAPPNAME=lifecycle-lineage-guard-rename \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; ALTER INDEX public.lifecycle_lineage_guard_idx RENAME TO lifecycle_lineage_guard_retired_idx; ALTER INDEX public.lifecycle_lineage_guard_replacement_idx RENAME TO lifecycle_lineage_guard_idx; COMMIT;" >"${rename_output}" 2>&1 & rename_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT CASE WHEN 'public.lifecycle_lineage_guard_idx'::regclass::oid = ${replacement_oid} THEN 1 ELSE 0 END;" 2>/dev/null || printf '0')" = "1" ] || [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-guard-rename' AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-lineage-guard-gate';" >/dev/null wait "${blocker_pid}" || true wait "${alter_pid}" || alter_status=$? wait "${rename_pid}" || rename_status=$? if [ "${rename_status}" -ne 0 ]; then error "lineage guard name swap failed: $(cat "${rename_output}")" fi if [ "${alter_status}" -ne 0 ] && ! grep -Fq "compaction_lineage" "${alter_output}"; then error "lineage guard failed for an unexpected reason: \ $(cat "${alter_output}")" fi assert_eq "name-swapped BM25 replacement keeps its lineage" \ "${lineage}" \ "$(index_lineage public.lifecycle_lineage_guard_idx)" sql_super -c " DROP EVENT TRIGGER lifecycle_lineage_guard_pause; DROP FUNCTION public.lifecycle_lineage_guard_pause(); DROP TABLE public.lifecycle_lineage_guard_old_docs, public.lifecycle_lineage_guard_new_docs;" >/dev/null } test_reindex_authorization_resolution_race() { local gate_pid lock_output locker_pid rename_output rename_pid local reindex_error reindex_pid reindex_waiting lock_output="${DATA_DIR}/reindex-auth-race-lock.out" rename_output="${DATA_DIR}/reindex-auth-race-rename.out" sql_as durable_owner <<'SQL' >/dev/null 2>&1 CREATE TABLE public.lifecycle_auth_race_old_docs (body text); CREATE INDEX lifecycle_auth_race_idx ON public.lifecycle_auth_race_old_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.lifecycle_auth_race_new_docs (body text); CREATE INDEX lifecycle_auth_race_replacement_idx ON public.lifecycle_auth_race_new_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); GRANT MAINTAIN ON public.lifecycle_auth_race_old_docs TO durable_writer; SQL PGAPPNAME=lifecycle-auth-race-new-lock sql_as durable_owner -c \ "BEGIN; LOCK TABLE public.lifecycle_auth_race_new_docs IN SHARE MODE; SELECT pg_catalog.pg_sleep(120);" >"${lock_output}" 2>&1 & locker_pid=$! PGAPPNAME=lifecycle-auth-race-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 5); SELECT pg_catalog.pg_sleep(120);" >"${lock_output}.gate" 2>&1 & gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 5 AND granted;")" = "1" ] && [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_auth_race_new_docs'::regclass AND mode = 'ShareLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-auth-race-rename sql_as durable_owner -c \ "BEGIN; LOCK TABLE public.lifecycle_auth_race_old_docs IN ACCESS EXCLUSIVE MODE; SELECT pg_catalog.pg_advisory_lock(478, 5); ALTER INDEX public.lifecycle_auth_race_idx RENAME TO lifecycle_auth_race_retired_idx; ALTER INDEX public.lifecycle_auth_race_replacement_idx RENAME TO lifecycle_auth_race_idx; COMMIT;" >"${rename_output}" 2>&1 & rename_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE relation = 'public.lifecycle_auth_race_old_docs'::regclass AND mode = 'AccessExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done PGAPPNAME=lifecycle-auth-race-reindex \ PGOPTIONS="-c statement_timeout=5s" \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_writer -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "REINDEX INDEX CONCURRENTLY public.lifecycle_auth_race_idx;" \ >"${lock_output}.reindex" 2>&1 & reindex_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-auth-race-reindex' AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done reindex_waiting="$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-auth-race-reindex' AND wait_event_type = 'Lock';")" if [ "${reindex_waiting}" != "1" ]; then error "authorization race did not reach the original heap lock: $(cat "${lock_output}.reindex")" fi log "PASS: authorization race reaches the original heap lock" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-auth-race-gate';" >/dev/null wait "${gate_pid}" || true if ! wait "${rename_pid}"; then error "authorization race rename failed: $(cat "${rename_output}")" fi if wait "${reindex_pid}"; then error "authorization race REINDEX unexpectedly succeeded" fi reindex_error="$(cat "${lock_output}.reindex")" if ! grep -Fq "permission denied for index lifecycle_auth_race_idx" \ <<<"${reindex_error}"; then error "authorization followed a stale relation after rename: \ ${reindex_error}" fi log "PASS: REINDEX authorizes the relation resolved under lock" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-auth-race-new-lock';" >/dev/null wait "${locker_pid}" || true sql_super -c "DROP TABLE public.lifecycle_auth_race_old_docs, public.lifecycle_auth_race_new_docs;" } test_prior_generation_spill_adoption() { local index_oid index_oid_before old_instance current_instance local invalid_lineage_error lineage_after lineage_before local replacement_instance reuse_output reused_instance reused_lineage local threshold_before install_signal_probe sql_as durable_owner -c " CREATE TABLE public.lifecycle_adopt_docs (id integer, body text); INSERT INTO public.lifecycle_adopt_docs SELECT i, pg_catalog.format('seed document %s filler', i) FROM generate_series(1, 100) AS i;" >/dev/null if invalid_lineage_error="$(sql_as durable_owner -c " CREATE INDEX lifecycle_adopt_invalid_idx ON public.lifecycle_adopt_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_lineage = 'bad');" 2>&1)"; then error "background CREATE accepted an invalid lineage" fi if ! grep -Fq "invalid background compaction lineage" \ <<<"${invalid_lineage_error}"; then error "invalid lineage failed for the wrong reason: \ ${invalid_lineage_error}" fi sql_as durable_owner -c " CREATE INDEX lifecycle_adopt_idx ON public.lifecycle_adopt_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); GRANT INSERT ON public.lifecycle_adopt_docs TO durable_writer;" \ >/dev/null 2>&1 index_oid_before="$(sql_super -c \ "SELECT 'public.lifecycle_adopt_idx'::regclass::oid;")" lineage_before="$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation, LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${index_oid_before} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" old_instance="$(current_generation_job_id "${index_oid_before}")" wait_for_signal_node "${old_instance}" 30 sql_as durable_owner -c \ "REINDEX INDEX CONCURRENTLY public.lifecycle_adopt_idx;" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_adopt_idx'::regclass::oid;")" if [ "${index_oid}" = "${index_oid_before}" ]; then error "spill adoption setup did not replace the index OID" fi lineage_after="$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation, LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "concurrent REINDEX preserves logical lineage" \ "${lineage_before}" "${lineage_after}" current_instance="$(current_generation_job_id "${index_oid}")" if [ -n "${current_instance}" ] && [ "${current_instance}" != "${old_instance}" ]; then sql_as durable_owner -c "SELECT df.cancel( '${current_instance}', 'force spill-time adoption');" >/dev/null wait_for_terminal "${current_instance}" 30 sql_super -c "UPDATE df.instances SET label = 'retired-current-generation-' || id WHERE id = '${current_instance}';" fi sql_super -c "REVOKE EXECUTE ON FUNCTION bm25_compact_step_if_current(oid, oid, oid, oid, oid), bm25_background_target_is_current(oid, oid, oid, oid, oid) FROM durable_owner;" threshold_before="$(sql_super -c \ "SHOW pg_textsearch.memtable_pages_threshold;")" sql_super -c "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" >/dev/null sql_super -c "SELECT pg_catalog.pg_reload_conf();" >/dev/null reset_signal_probe sql_as durable_writer -c "INSERT INTO public.lifecycle_adopt_docs SELECT 1000 + document_number, (SELECT pg_catalog.string_agg( pg_catalog.format( 'adopt%sterm%s', document_number, term_number), ' ') FROM generate_series(1, 200) AS term_number) FROM generate_series(1, 6) AS document_number;" >/dev/null replacement_instance="$(current_generation_job_id "${index_oid}")" if [ -z "${replacement_instance}" ] || [ "${replacement_instance}" = "${old_instance}" ] || [ "${replacement_instance}" = "${current_instance}" ]; then error "spill did not adopt the prior generation schedule" fi assert_eq "spill adoption submits as the physical index owner" \ "durable_owner" \ "$(sql_super -c "SELECT submitted_by::pg_catalog.text FROM df.instances WHERE id = '${replacement_instance}';")" assert_eq "spill adoption signals the replacement once as owner" \ "${replacement_instance}:durable_owner" \ "$(sql_super -c "SELECT instance_id || ':' || role_name FROM public.compaction_signal_audit;")" assert_eq "spill adoption attempts exactly one signal" "1" \ "$(signal_attempt_count)" assert_eq "spill adoption preserves the prior generation schedule" "t" \ "$(sql_super -c "SELECT label LIKE '%:' || pg_catalog.encode( pg_catalog.convert_to('0 0 1 1 *', 'UTF8'), 'hex') FROM df.instances WHERE id = '${replacement_instance}';")" assert_eq "spill adoption restores private helper access" "t:t" \ "$(helper_privileges_for_role durable_owner)" assert_eq "spill adoption preserves the durable dependency" "1" \ "$(dependency_count)" sql_as durable_owner -c "SELECT df.cancel( '${replacement_instance}', 'lineage reuse test');" >/dev/null wait_for_terminal "${replacement_instance}" 30 sql_as durable_owner -c " DROP INDEX public.lifecycle_adopt_idx; CREATE INDEX lifecycle_adopt_idx ON public.lifecycle_adopt_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '5 4 3 2 *');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'public.lifecycle_adopt_idx'::regclass::oid;")" reused_lineage="$(sql_super -c "SELECT pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) FROM pg_catalog.pg_class AS relation, LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE relation.oid = ${index_oid} AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" if [ "${reused_lineage}" = "${lineage_before}" ]; then error "recreated index reused the dropped index lineage" fi reused_instance="$(current_generation_job_id "${index_oid}")" wait_for_signal_node "${reused_instance}" 30 sql_as durable_owner -c "SELECT df.cancel( '${reused_instance}', 'hide replacement lineage history');" \ >/dev/null wait_for_terminal "${reused_instance}" 30 sql_super -c "UPDATE df.instances SET label = 'retired-reused-index-' || id WHERE id = '${reused_instance}';" reset_signal_probe reuse_output="$(sql_as durable_writer -c " INSERT INTO public.lifecycle_adopt_docs SELECT 2000 + document_number, (SELECT pg_catalog.string_agg( pg_catalog.format( 'reuse%sterm%s', document_number, term_number), ' ') FROM generate_series(1, 200) AS term_number) FROM generate_series(1, 6) AS document_number;" 2>&1)" if ! grep -Fq "requires explicit adoption" <<<"${reuse_output}"; then error "recreated index did not reject unrelated lineage history: \ ${reuse_output}" fi assert_eq "recreated index does not signal an unrelated workflow" \ "0" "$(signal_attempt_count)" assert_eq "recreated index does not adopt an unrelated schedule" \ "" "$(current_generation_job_id "${index_oid}")" sql_super -c "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = '${threshold_before}';" \ >/dev/null sql_super -c "SELECT pg_catalog.pg_reload_conf();" >/dev/null sql_as durable_owner -c "SELECT df.cancel( instance.id, 'spill adoption test complete') FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) 'pg_textsearch:bg:v1:%:${index_oid}:%' AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]);" \ >/dev/null restore_signal_probe sql_as durable_owner -c "DROP TABLE public.lifecycle_adopt_docs;" } test_background_vacuum_rewrites_sparse_singleton() { local after_docs after_root after_state before_docs before_root before_state local index_oid instance_id signal_events_after signal_events_before local rewritten=false sql_as durable_owner -c " CREATE TABLE public.background_vacuum_sparse_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO public.background_vacuum_sparse_docs SELECT id, pg_catalog.format( 'sparsevacuum document %s stable filler', id) FROM pg_catalog.generate_series(1, 100) AS id; CREATE INDEX background_vacuum_sparse_idx ON public.background_vacuum_sparse_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.background_vacuum_sparse_idx'::regclass::oid;")" instance_id="$(current_generation_job_id "${index_oid}")" [ -n "${instance_id}" ] || error "sparse VACUUM fixture created no managed workflow" wait_for_signal_node "${instance_id}" 30 before_state="$(sql_super -c " WITH summary AS ( SELECT bm25_summarize_index( 'public.background_vacuum_sparse_idx') AS value ) SELECT pg_catalog.substring( value, 'Segment 1: block=([0-9]+)') || ':' || pg_catalog.substring(value, 'docs=([0-9]+)') FROM summary;")" before_root="${before_state%%:*}" before_docs="${before_state##*:}" assert_eq "sparse VACUUM fixture starts with 100 persisted docs" \ "100" "${before_docs}" signal_events_before="$(sql_super -c "SELECT count(*) FROM _duroxide.history WHERE instance_id = '${instance_id}' AND event_data::jsonb->>'type' = 'ExternalEvent' AND event_data::jsonb->>'name' = 'compact';")" sql_as durable_owner -c "DELETE FROM public.background_vacuum_sparse_docs WHERE id <= 50;" >/dev/null sql_as durable_owner -c \ "VACUUM public.background_vacuum_sparse_docs;" >/dev/null after_state="${before_state}" signal_events_after="${signal_events_before}" for _ in $(seq 1 60); do signal_events_after="$(sql_super -c "SELECT count(*) FROM _duroxide.history WHERE instance_id = '${instance_id}' AND event_data::jsonb->>'type' = 'ExternalEvent' AND event_data::jsonb->>'name' = 'compact';")" after_state="$(sql_super -c " WITH summary AS ( SELECT bm25_summarize_index( 'public.background_vacuum_sparse_idx') AS value ) SELECT pg_catalog.substring( value, 'Segment 1: block=([0-9]+)') || ':' || pg_catalog.substring(value, 'docs=([0-9]+)') FROM summary;")" after_root="${after_state%%:*}" after_docs="${after_state##*:}" if [ "${signal_events_after}" -gt "${signal_events_before}" ]; then if [ "${after_root}" != "${before_root}" ]; then rewritten=true break fi fi sleep 1 done if ! "${rewritten}"; then error "background VACUUM did not dispatch and rewrite the sparse \ singleton: before=${before_state}, after=${after_state}, \ signals=${signal_events_before}:${signal_events_after}" fi wait_for_signal_node "${instance_id}" 30 assert_eq "background VACUUM rewrites exactly the 50 live documents" \ "50" "${after_docs}" assert_eq "background VACUUM keeps the managed workflow active" \ "${instance_id}" "$(current_generation_job_id "${index_oid}")" assert_eq "background VACUUM preserves the exact surviving result IDs" \ "50:true" \ "$(sql_as durable_owner -c " SET enable_seqscan = off; WITH ranked AS ( SELECT id FROM public.background_vacuum_sparse_docs ORDER BY body <@> to_bm25query( 'sparsevacuum', 'background_vacuum_sparse_idx') LIMIT 100 ) SELECT count(*) || ':' || (pg_catalog.array_agg(id ORDER BY id) = ARRAY(SELECT pg_catalog.generate_series(51, 100))) FROM ranked;")" sql_as durable_owner -c "SELECT df.cancel( '${instance_id}', 'sparse VACUUM test complete');" >/dev/null wait_for_terminal "${instance_id}" 30 sql_as durable_owner -c \ "DROP TABLE public.background_vacuum_sparse_docs;" } install_signal_probe() { sql_super <<'SQL' ALTER FUNCTION df.signal(text, text, text) RENAME TO signal_v028; CREATE SEQUENCE public.compaction_signal_attempt_seq; CREATE TABLE public.compaction_signal_audit ( attempt bigint PRIMARY KEY, instance_id text NOT NULL, role_name name NOT NULL ); CREATE TABLE public.compaction_signal_fault ( instance_id text PRIMARY KEY, fault text NOT NULL ); GRANT USAGE, SELECT ON SEQUENCE public.compaction_signal_attempt_seq TO durable_owner, durable_owner_two; GRANT INSERT ON public.compaction_signal_audit TO durable_owner, durable_owner_two; GRANT SELECT ON public.compaction_signal_fault TO durable_owner, durable_owner_two; CREATE FUNCTION df.signal( instance_id text, signal_name text, signal_data text DEFAULT '{}') RETURNS text LANGUAGE plpgsql STRICT SET search_path = pg_catalog, pg_temp AS $body$ DECLARE attempt_number bigint; injected_fault text; BEGIN attempt_number := nextval('public.compaction_signal_attempt_seq'::regclass); SELECT fault INTO injected_fault FROM public.compaction_signal_fault AS control WHERE control.instance_id OPERATOR(pg_catalog.=) $1 OR control.instance_id OPERATOR(pg_catalog.=) '*' ORDER BY control.instance_id OPERATOR(pg_catalog.=) $1 DESC LIMIT 1; IF injected_fault OPERATOR(pg_catalog.=) 'error' THEN RAISE EXCEPTION 'probe ordinary signal failure'; ELSIF injected_fault OPERATOR(pg_catalog.=) 'cancel' THEN RAISE EXCEPTION 'probe query cancellation' USING ERRCODE = '57014'; ELSIF injected_fault OPERATOR(pg_catalog.=) 'gate' THEN PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 11); END IF; INSERT INTO public.compaction_signal_audit( attempt, instance_id, role_name) VALUES (attempt_number, $1, current_user); RETURN pg_catalog.format('probe-%s', attempt_number); END $body$; ALTER EXTENSION pg_durable ADD FUNCTION df.signal(text, text, text); REVOKE ALL ON FUNCTION df.signal(text, text, text) FROM PUBLIC; GRANT EXECUTE ON FUNCTION df.signal(text, text, text) TO durable_owner, durable_owner_two; CREATE FUNCTION public.queue_force_spills( target_table regclass, target_index regclass, first_id integer, rounds integer) RETURNS void LANGUAGE plpgsql SET search_path = pg_catalog, pg_temp AS $body$ BEGIN FOR round_number IN 1..rounds LOOP EXECUTE pg_catalog.format( 'INSERT INTO %s(id, body) ' 'SELECT $1 + $2 * 100 + value, ' 'pg_catalog.format(''queue round %%s row %%s filler'', ' '$2, value) FROM pg_catalog.generate_series(1, 20) AS value', target_table) USING first_id, round_number; PERFORM public.bm25_spill_index(target_index::text); END LOOP; END $body$; SQL } restore_signal_probe() { sql_super <<'SQL' ALTER EXTENSION pg_durable DROP FUNCTION df.signal(text, text, text); DROP FUNCTION df.signal(text, text, text); ALTER FUNCTION df.signal_v028(text, text, text) RENAME TO signal; DROP FUNCTION public.queue_force_spills(regclass, regclass, integer, integer); DROP TABLE public.compaction_signal_fault; DROP TABLE public.compaction_signal_audit; DROP SEQUENCE public.compaction_signal_attempt_seq; SQL } reset_signal_probe() { sql_super -c "TRUNCATE public.compaction_signal_audit, public.compaction_signal_fault; ALTER SEQUENCE public.compaction_signal_attempt_seq RESTART WITH 1;" } signal_attempt_count() { sql_super -c "SELECT CASE WHEN is_called THEN last_value ELSE 0 END FROM public.compaction_signal_attempt_seq;" } test_request_queue_runtime() { local abort_instance cancel_error cancel_instance dedup_a_instance local dedup_b_instance drop_instance failure_a_instance failure_b_instance local failure_output lock_a_instance lock_b_instance lock_a_output local lock_a_pid lock_a_status=0 lock_b_output lock_b_pid lock_b_status=0 local lock_gate_pid savepoint_instance sql_as durable_owner -c " CREATE TABLE public.queue_below_docs (id integer, body text); CREATE INDEX queue_below_idx ON public.queue_below_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_abort_docs (id integer, body text); CREATE INDEX queue_abort_idx ON public.queue_abort_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_dedup_a_docs (id integer, body text); CREATE INDEX queue_dedup_a_idx ON public.queue_dedup_a_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_dedup_b_docs (id integer, body text); CREATE INDEX queue_dedup_b_idx ON public.queue_dedup_b_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_savepoint_docs (id integer, body text); CREATE INDEX queue_savepoint_idx ON public.queue_savepoint_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_drop_docs (id integer, body text); CREATE INDEX queue_drop_idx ON public.queue_drop_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_failure_a_docs (id integer, body text); CREATE INDEX queue_failure_a_idx ON public.queue_failure_a_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_failure_b_docs (id integer, body text); CREATE INDEX queue_failure_b_idx ON public.queue_failure_b_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_cancel_docs (id integer, body text); CREATE INDEX queue_cancel_idx ON public.queue_cancel_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_lock_a_docs (id integer, body text); CREATE INDEX queue_lock_a_idx ON public.queue_lock_a_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *'); CREATE TABLE public.queue_lock_b_docs (id integer, body text); CREATE INDEX queue_lock_b_idx ON public.queue_lock_b_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 abort_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_abort_idx'::regclass::oid;")")" dedup_a_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_dedup_a_idx'::regclass::oid;")")" dedup_b_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_dedup_b_idx'::regclass::oid;")")" savepoint_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_savepoint_idx'::regclass::oid;")")" drop_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_drop_idx'::regclass::oid;")")" failure_a_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_failure_a_idx'::regclass::oid;")")" failure_b_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_failure_b_idx'::regclass::oid;")")" cancel_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_cancel_idx'::regclass::oid;")")" lock_a_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_lock_a_idx'::regclass::oid;")")" lock_b_instance="$(current_generation_job_id \ "$(sql_super -c \ "SELECT 'public.queue_lock_b_idx'::regclass::oid;")")" install_signal_probe reset_signal_probe sql_as durable_owner -c "SELECT public.queue_force_spills( 'public.queue_below_docs'::regclass, 'public.queue_below_idx'::regclass, 0, 1);" >/dev/null assert_eq "below-threshold spill signals without writer-side planning" "1" \ "$(signal_attempt_count)" reset_signal_probe sql_as durable_owner <<'SQL' >/dev/null BEGIN; SELECT public.queue_force_spills( 'public.queue_abort_docs'::regclass, 'public.queue_abort_idx'::regclass, 1000, 2); ROLLBACK; BEGIN; SELECT 1; COMMIT; SQL assert_eq "top-level abort does not leak a pending request" "0" \ "$(signal_attempt_count)" assert_eq "aborted queue test targeted a live workflow" "1" \ "$(sql_super -c "SELECT count(*) FROM df.instances WHERE id = '${abort_instance}';")" reset_signal_probe sql_as durable_owner <<'SQL' >/dev/null BEGIN; SELECT public.queue_force_spills( 'public.queue_dedup_a_docs'::regclass, 'public.queue_dedup_a_idx'::regclass, 2000, 3); SELECT public.queue_force_spills( 'public.queue_dedup_b_docs'::regclass, 'public.queue_dedup_b_idx'::regclass, 3000, 2); COMMIT; SQL assert_eq "request queue signals two indexes once each" "2" \ "$(signal_attempt_count)" assert_eq "multiple request hits deduplicate per index" "1:1" \ "$(sql_super -c "SELECT count(*) FILTER (WHERE instance_id = '${dedup_a_instance}') || ':' || count(*) FILTER (WHERE instance_id = '${dedup_b_instance}') FROM public.compaction_signal_audit;")" reset_signal_probe sql_as durable_owner <<'SQL' >/dev/null BEGIN; SAVEPOINT queue_spill; SELECT public.queue_force_spills( 'public.queue_savepoint_docs'::regclass, 'public.queue_savepoint_idx'::regclass, 4000, 2); DROP INDEX public.queue_savepoint_idx; ROLLBACK TO SAVEPOINT queue_spill; COMMIT; SQL assert_eq "savepoint-surviving spill request dispatches" "1" \ "$(signal_attempt_count)" assert_eq "rolled-back DROP retains the original request target" \ "${savepoint_instance}" \ "$(sql_super -c "SELECT instance_id FROM public.compaction_signal_audit;")" reset_signal_probe sql_as durable_owner <<'SQL' >/dev/null BEGIN; SELECT public.queue_force_spills( 'public.queue_drop_docs'::regclass, 'public.queue_drop_idx'::regclass, 5000, 2); DROP INDEX public.queue_drop_idx; COMMIT; SQL assert_eq "committed DROP suppresses stale-OID dispatch" "0" \ "$(signal_attempt_count)" assert_eq "dropped request targeted an existing workflow" "1" \ "$(sql_super -c "SELECT count(*) FROM df.instances WHERE id = '${drop_instance}';")" reset_signal_probe sql_super -c "INSERT INTO public.compaction_signal_fault VALUES ('${failure_a_instance}', 'error');" if ! failure_output="$(sql_as durable_owner <<'SQL' 2>&1 BEGIN; SELECT public.queue_force_spills( 'public.queue_failure_a_docs'::regclass, 'public.queue_failure_a_idx'::regclass, 6000, 2); SELECT public.queue_force_spills( 'public.queue_failure_b_docs'::regclass, 'public.queue_failure_b_idx'::regclass, 7000, 2); COMMIT; SQL )"; then error "ordinary PRE_COMMIT signal failure aborted writer data: \ ${failure_output}" fi if ! grep -Fq "probe ordinary signal failure" \ <<<"${failure_output}"; then error "ordinary PRE_COMMIT signal failure emitted no warning" fi assert_eq "ordinary failure continues later dispatch" "2" \ "$(signal_attempt_count)" assert_eq "ordinary failure preserves both writers' rows" "40:40" \ "$(sql_super -c "SELECT (SELECT count(*) FROM public.queue_failure_a_docs) || ':' || (SELECT count(*) FROM public.queue_failure_b_docs);")" assert_eq "ordinary failure continues to the second index" \ "${failure_b_instance}:durable_owner" \ "$(sql_super -c "SELECT instance_id || ':' || role_name FROM public.compaction_signal_audit;")" reset_signal_probe sql_super -c "INSERT INTO public.compaction_signal_fault VALUES ('${cancel_instance}', 'cancel');" if cancel_error="$(sql_as durable_owner <<'SQL' 2>&1 BEGIN; SELECT public.queue_force_spills( 'public.queue_cancel_docs'::regclass, 'public.queue_cancel_idx'::regclass, 8000, 2); COMMIT; SQL )"; then error "query cancellation during PRE_COMMIT unexpectedly committed" fi if ! grep -Fq "probe query cancellation" <<<"${cancel_error}"; then error "query cancellation did not propagate: ${cancel_error}" fi assert_eq "query cancellation attempted one signal" "1" \ "$(signal_attempt_count)" assert_eq "query cancellation rolls back writer data" "0" \ "$(sql_super -c \ "SELECT count(*) FROM public.queue_cancel_docs;")" reset_signal_probe sql_super -c "INSERT INTO public.compaction_signal_fault VALUES ('${lock_a_instance}', 'gate'), ('${lock_b_instance}', 'gate');" wait_for_signal_node "${lock_a_instance}" 30 wait_for_signal_node "${lock_b_instance}" 30 PGAPPNAME=queue-lock-gate sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 11); SELECT pg_catalog.pg_sleep(120);" \ >"${DATA_DIR}/queue-lock-gate.out" 2>&1 & lock_gate_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 11 AND granted;")" = "1" ]; then break fi sleep 0.1 done lock_a_output="${DATA_DIR}/queue-lock-a.out" lock_b_output="${DATA_DIR}/queue-lock-b.out" PGAPPNAME=queue-lock-a \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; SELECT public.queue_force_spills( 'public.queue_lock_a_docs'::regclass, 'public.queue_lock_a_idx'::regclass, 9000, 2); SELECT public.queue_force_spills( 'public.queue_lock_b_docs'::regclass, 'public.queue_lock_b_idx'::regclass, 10000, 2); COMMIT;" >"${lock_a_output}" 2>&1 & lock_a_pid=$! PGAPPNAME=queue-lock-b \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; SELECT public.queue_force_spills( 'public.queue_lock_b_docs'::regclass, 'public.queue_lock_b_idx'::regclass, 11000, 2); SELECT public.queue_force_spills( 'public.queue_lock_a_docs'::regclass, 'public.queue_lock_a_idx'::regclass, 12000, 2); COMMIT;" >"${lock_b_output}" 2>&1 & lock_b_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ('queue-lock-a', 'queue-lock-b') AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done assert_eq "reverse-order request flush has one admitted writer" "1" \ "$(sql_super -c "SELECT pg_catalog.count(*) FROM pg_catalog.pg_stat_activity WHERE application_name IN ('queue-lock-a', 'queue-lock-b') AND wait_event_type = 'Lock';")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'queue-lock-gate';" >/dev/null wait "${lock_gate_pid}" || true wait "${lock_a_pid}" || lock_a_status=$? wait "${lock_b_pid}" || lock_b_status=$? if [ "${lock_a_status}" -ne 0 ] || [ "${lock_b_status}" -ne 0 ]; then error "reverse-order request flush deadlocked: first: $(cat "${lock_a_output}") second: $(cat "${lock_b_output}")" fi if grep -Fq "deadlock detected" "${lock_a_output}" || grep -Fq "deadlock detected" "${lock_b_output}"; then error "reverse-order request flush reported a caught deadlock" fi assert_eq "reverse-order request flush commits both writers" "80:80" \ "$(sql_super -c "SELECT (SELECT count(*) FROM public.queue_lock_a_docs) || ':' || (SELECT count(*) FROM public.queue_lock_b_docs);")" assert_eq "contended request flush signals admitted work once" "t" \ "$(sql_super -c "SELECT count(*) OPERATOR(pg_catalog.>=) 1 AND count(*) OPERATOR(pg_catalog.<=) 2 AND count(*) FILTER ( WHERE instance_id = '${lock_a_instance}') OPERATOR(pg_catalog.<=) 1 AND count(*) FILTER ( WHERE instance_id = '${lock_b_instance}') OPERATOR(pg_catalog.<=) 1 FROM public.compaction_signal_audit;")" restore_signal_probe sql_super -c "DROP TABLE public.queue_below_docs, public.queue_abort_docs, public.queue_dedup_a_docs, public.queue_dedup_b_docs, public.queue_savepoint_docs, public.queue_drop_docs, public.queue_failure_a_docs, public.queue_failure_b_docs, public.queue_cancel_docs, public.queue_lock_a_docs, public.queue_lock_b_docs;" } test_precommit_signal_observes_published_spill() { local blocker_pid blocker_output index_oid instance_id local writer_pid writer_output writer_status=0 sql_as durable_owner -c " CREATE TABLE public.precommit_signal_docs (id integer, body text); CREATE INDEX precommit_signal_idx ON public.precommit_signal_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'public.precommit_signal_idx'::regclass::oid;")" instance_id="$(current_generation_job_id "${index_oid}")" wait_for_signal_node "${instance_id}" 90 sql_super <<'SQL' CREATE TABLE public.precommit_compaction_audit ( role_name name NOT NULL ); ALTER TABLE public.precommit_compaction_audit OWNER TO durable_owner; REVOKE ALL ON public.precommit_compaction_audit FROM PUBLIC; GRANT INSERT ON public.precommit_compaction_audit TO durable_owner; CREATE FUNCTION bm25_compact_step_if_current_precommit_test_c( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; REVOKE ALL ON FUNCTION bm25_compact_step_if_current_precommit_test_c(oid, oid, oid, oid, oid) FROM PUBLIC; GRANT EXECUTE ON FUNCTION bm25_compact_step_if_current_precommit_test_c(oid, oid, oid, oid, oid) TO durable_owner; CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path = pg_catalog, pg_temp AS $body$ BEGIN INSERT INTO public.precommit_compaction_audit(role_name) VALUES (current_user); RETURN public.bm25_compact_step_if_current_precommit_test_c( index_oid, database_oid, tablespace_oid, relfilenumber, owner_oid); END $body$; ALTER FUNCTION df.signal(text, text, text) RENAME TO signal_v028; CREATE FUNCTION df.signal( instance_id text, signal_name text, signal_data text DEFAULT '{}') RETURNS text LANGUAGE plpgsql STRICT SET search_path = pg_catalog, pg_temp AS $body$ DECLARE result text; BEGIN result := df.signal_v028($1, $2, $3); PERFORM pg_catalog.pg_advisory_xact_lock_shared(478, 12); RETURN result; END $body$; ALTER EXTENSION pg_durable ADD FUNCTION df.signal(text, text, text); REVOKE ALL ON FUNCTION df.signal(text, text, text) FROM PUBLIC; GRANT EXECUTE ON FUNCTION df.signal(text, text, text) TO durable_owner; SQL blocker_output="${DATA_DIR}/precommit-signal-blocker.out" PGAPPNAME=precommit-signal-blocker sql_super -c \ "SELECT pg_catalog.pg_advisory_lock(478, 12); SELECT pg_catalog.pg_sleep(120);" \ >"${blocker_output}" 2>&1 & blocker_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_locks WHERE locktype = 'advisory' AND classid = 478 AND objid = 12 AND mode = 'ExclusiveLock' AND granted;")" = "1" ]; then break fi sleep 0.1 done writer_output="${DATA_DIR}/precommit-signal-writer.out" PGAPPNAME=precommit-signal-writer \ "${PGBINDIR}/psql" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 \ -c "BEGIN; INSERT INTO public.precommit_signal_docs SELECT value, pg_catalog.format( 'precommit first row %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.precommit_signal_idx'); INSERT INTO public.precommit_signal_docs SELECT value + 20, pg_catalog.format( 'precommit second row %s filler', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.precommit_signal_idx'); COMMIT;" >"${writer_output}" 2>&1 & writer_pid=$! for _ in $(seq 1 300); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'precommit-signal-writer' AND wait_event_type = 'Lock';")" = "1" ]; then break fi sleep 0.1 done assert_eq "writer waits after sending the PRE_COMMIT signal" "1" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'precommit-signal-writer' AND wait_event_type = 'Lock';")" assert_eq "writer rows remain uncommitted while the signal is live" "0" \ "$(sql_super -c \ "SELECT count(*) FROM public.precommit_signal_docs;")" for _ in $(seq 1 300); do if [ "$(sql_super -c \ "SELECT count(*) FROM public.precommit_compaction_audit;")" \ -gt 0 ]; then break fi sleep 0.1 done assert_eq "worker runs before the spilling writer commits" "t" \ "$(sql_super -c "SELECT count(*) > 0 AND pg_catalog.bool_and( role_name = 'durable_owner') FROM public.precommit_compaction_audit;")" assert_eq "pre-commit worker consumes the published compaction debt" "f" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'public.precommit_signal_idx'::regclass);")" sql_super -c "SELECT pg_catalog.pg_terminate_backend(pid) FROM pg_catalog.pg_stat_activity WHERE application_name = 'precommit-signal-blocker';" >/dev/null wait "${blocker_pid}" || true wait "${writer_pid}" || writer_status=$? if [ "${writer_status}" -ne 0 ]; then error "writer failed after PRE_COMMIT signal gate: $(cat "${writer_output}")" fi assert_eq "writer commits after the worker has compacted" "40" \ "$(sql_super -c \ "SELECT count(*) FROM public.precommit_signal_docs;")" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; ALTER EXTENSION pg_durable DROP FUNCTION df.signal(text, text, text); DROP FUNCTION df.signal(text, text, text); ALTER FUNCTION df.signal_v028(text, text, text) RENAME TO signal; DROP FUNCTION bm25_compact_step_if_current_precommit_test_c(oid, oid, oid, oid, oid); DROP TABLE public.precommit_compaction_audit; SQL sql_as durable_owner -c \ "SELECT df.cancel('${instance_id}', 'precommit test complete');" \ >/dev/null wait_for_terminal "${instance_id}" 30 sql_as durable_owner -c "DROP TABLE public.precommit_signal_docs;" } test_repeatable_read_admission_snapshot() { local index_oid initial_job reader_output reader_pid sql_as durable_owner -c " CREATE TABLE public.lifecycle_rr_docs (id integer, body text); CREATE INDEX lifecycle_rr_idx ON public.lifecycle_rr_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_oid="$(sql_super -c "SELECT 'public.lifecycle_rr_idx'::regclass::oid;")" initial_job="$(current_generation_job_id "${index_oid}")" sql_as durable_owner -c \ "SELECT df.cancel('${initial_job}', 'repeatable-read setup');" \ >/dev/null wait_for_terminal "${initial_job}" 30 reader_output="${DATA_DIR}/repeatable-read-admission.out" PGAPPNAME=lifecycle-repeatable-read sql_as durable_owner <<'SQL' \ >"${reader_output}" 2>&1 & BEGIN ISOLATION LEVEL REPEATABLE READ; SELECT count(*) FROM df.instances; SELECT pg_catalog.pg_sleep(5); INSERT INTO public.lifecycle_rr_docs SELECT value, pg_catalog.format('reader document %s alpha beta gamma', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.lifecycle_rr_idx'); INSERT INTO public.lifecycle_rr_docs SELECT 100 + value, pg_catalog.format('reader second document %s delta epsilon', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.lifecycle_rr_idx'); COMMIT; SQL reader_pid=$! for _ in $(seq 1 100); do if [ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_stat_activity WHERE application_name = 'lifecycle-repeatable-read' AND query OPERATOR(pg_catalog.~~) '%pg_sleep%';")" = "1" ]; then break fi sleep 0.1 done sql_as durable_owner -c " INSERT INTO public.lifecycle_rr_docs SELECT 1000 + value, pg_catalog.format('writer document %s one two three', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.lifecycle_rr_idx'); INSERT INTO public.lifecycle_rr_docs SELECT 2000 + value, pg_catalog.format('writer second %s four five six', value) FROM pg_catalog.generate_series(1, 20) AS value; SELECT bm25_spill_index('public.lifecycle_rr_idx');" >/dev/null 2>&1 wait "${reader_pid}" || { cat "${reader_output}" >&2 error "repeatable-read writer failed" } assert_eq "repeatable-read admission keeps one current workflow" "1" \ "$(current_generation_job_count "${index_oid}")" sql_super -c "DROP TABLE public.lifecycle_rr_docs;" } test_actor_writer_worker_identity() { local activation_audit_id activation_audit_rows index_oid instance_id local memtable_threshold_before writer_audit_id writer_audit_rows sql_super -c "REVOKE durable_no_textsearch_schema FROM durable_writer; GRANT durable_owner TO durable_actor; REVOKE CREATE ON SCHEMA public FROM durable_writer; CREATE TABLE identity_docs (id integer, body text); ALTER TABLE identity_docs OWNER TO durable_owner; GRANT INSERT ON identity_docs TO durable_writer; CREATE INDEX identity_docs_idx ON identity_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual', compaction_schedule = '0 0 1 1 *'); CREATE TABLE worker_identity_audit ( audit_id bigint GENERATED ALWAYS AS IDENTITY, role_name name); ALTER TABLE worker_identity_audit OWNER TO durable_owner; REVOKE ALL ON worker_identity_audit FROM PUBLIC; GRANT INSERT ON worker_identity_audit TO durable_owner;" sql_super <<'SQL' CREATE FUNCTION bm25_compact_step_if_current_test_c( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; REVOKE ALL ON FUNCTION bm25_compact_step_if_current_test_c(oid, oid, oid, oid, oid) FROM PUBLIC; GRANT EXECUTE ON FUNCTION bm25_compact_step_if_current_test_c(oid, oid, oid, oid, oid) TO durable_owner; CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path = pg_catalog, pg_temp AS $body$ BEGIN INSERT INTO public.worker_identity_audit(role_name) VALUES (current_user); RETURN false; END $body$; SQL assert_eq "writer is not a member of a pg_durable-enabled role" "f" \ "$(sql_super -c "SELECT pg_catalog.pg_has_role( 'durable_writer', 'durable_no_textsearch_schema', 'MEMBER');")" assert_eq "writer lacks effective df schema access" "f" \ "$(sql_super -c "SELECT pg_catalog.has_schema_privilege( 'durable_writer', 'df', 'USAGE');")" assert_eq "writer lacks effective df.start access" "t" \ "$(sql_super -c "SELECT NOT ( pg_catalog.has_schema_privilege( 'durable_writer', 'df', 'USAGE') AND pg_catalog.has_function_privilege( 'durable_writer', procedure.oid, 'EXECUTE')) FROM pg_catalog.pg_proc AS procedure JOIN pg_catalog.pg_namespace AS namespace ON namespace.oid = procedure.pronamespace JOIN pg_catalog.pg_depend AS dependency ON dependency.classid = 'pg_catalog.pg_proc'::regclass AND dependency.objid = procedure.oid AND dependency.deptype = 'e' JOIN pg_catalog.pg_extension AS extension ON extension.oid = dependency.refobjid WHERE namespace.nspname = 'df' AND procedure.proname = 'start' AND extension.extname = 'pg_durable';")" assert_eq "writer lacks privileges on required df tables" "t" \ "$(sql_super -c "SELECT NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'df.instances', 'SELECT') AND NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'df.instances', 'INSERT') AND NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'df.nodes', 'INSERT') AND NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'df.vars', 'SELECT');")" assert_eq "writer lacks private helper EXECUTE" "t" \ "$(sql_super -c "SELECT NOT pg_catalog.has_function_privilege( 'durable_writer', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') AND NOT pg_catalog.has_function_privilege( 'durable_writer', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE');")" assert_eq "writer has only INSERT table privilege" "t" \ "$(sql_super -c "SELECT pg_catalog.has_table_privilege( 'durable_writer', 'identity_docs', 'INSERT') AND NOT pg_catalog.has_table_privilege( 'durable_writer', 'identity_docs', 'SELECT, UPDATE, DELETE, TRUNCATE, REFERENCES, TRIGGER, ' || 'MAINTAIN') AND NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'identity_docs', 'SELECT') AND NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'identity_docs', 'UPDATE') AND NOT pg_catalog.has_any_column_privilege( 'durable_writer', 'identity_docs', 'REFERENCES');")" assert_eq "actor command starts without SET ROLE" \ "durable_actor:durable_actor" \ "$(sql_as durable_actor -c \ "SELECT current_user || ':' || session_user;")" sql_as durable_actor -c " ALTER INDEX public.identity_docs_idx SET (compaction = 'background');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'identity_docs_idx'::regclass::oid;")" instance_id="$(active_job_id_for_owner \ "${index_oid}" durable_owner)" [ -n "${instance_id}" ] || error "actor-enabled index has no durable_owner workflow" assert_eq "actor activation submits as the index owner" \ "durable_owner" \ "$(sql_super -c "SELECT submitted_by::text FROM df.instances WHERE id = '${instance_id}';")" wait_for_audit_rows 1 30 wait_for_signal_node "${instance_id}" 90 activation_audit_rows="$(sql_super -c \ "SELECT count(*) FROM worker_identity_audit;")" activation_audit_id="$(sql_super -c \ "SELECT max(audit_id) FROM worker_identity_audit;")" assert_eq "initial activation cascade executes as the index owner" "t" \ "$(sql_super -c "SELECT count(*) > 0 AND pg_catalog.bool_and( role_name = 'durable_owner') FROM worker_identity_audit WHERE audit_id <= ${activation_audit_id};")" memtable_threshold_before="$(sql_super -c \ "SHOW pg_textsearch.memtable_pages_threshold;")" sql_super -c "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" >/dev/null sql_super -c "SELECT pg_catalog.pg_reload_conf();" >/dev/null assert_eq "writer spill threshold is active" "1" \ "$(sql_super -c \ "SHOW pg_textsearch.memtable_pages_threshold;")" sql_as durable_writer -c "INSERT INTO public.identity_docs SELECT document_number, (SELECT pg_catalog.string_agg( pg_catalog.format( 'writer%sterm%s', document_number, term_number), ' ') FROM generate_series(1, 200) AS term_number) FROM generate_series(1, 6) AS document_number;" >/dev/null wait_for_audit_rows "$((activation_audit_rows + 1))" 30 assert_eq "writer-triggered signal executes as the index owner" "t" \ "$(sql_super -c "SELECT count(*) > 0 AND pg_catalog.bool_and( role_name = 'durable_owner') FROM worker_identity_audit WHERE audit_id > ${activation_audit_id};")" wait_for_signal_node "${instance_id}" 30 writer_audit_rows="$(sql_super -c \ "SELECT count(*) FROM worker_identity_audit;")" writer_audit_id="$(sql_super -c \ "SELECT max(audit_id) FROM worker_identity_audit;")" sql_super -c "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" >/dev/null sql_super -c "SELECT pg_catalog.pg_reload_conf();" >/dev/null assert_eq "writer spill threshold is restored" \ "${memtable_threshold_before}" \ "$(sql_super -c \ "SHOW pg_textsearch.memtable_pages_threshold;")" assert_eq "insert-only writer creates compaction debt" "t" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'identity_docs_idx'::regclass);")" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT SET search_path = pg_catalog, pg_temp AS $body$ BEGIN INSERT INTO public.worker_identity_audit(role_name) VALUES (current_user); RETURN public.bm25_compact_step_if_current_test_c( index_oid, database_oid, tablespace_oid, relfilenumber, owner_oid); END $body$; SQL sql_as durable_owner -c \ "SELECT df.signal('${instance_id}', 'compact', '{}');" >/dev/null wait_for_audit_rows "$((writer_audit_rows + 1))" 30 assert_eq "controlled consuming signal executes as the index owner" "t" \ "$(sql_super -c "SELECT count(*) > 0 AND pg_catalog.bool_and( role_name = 'durable_owner') FROM worker_identity_audit WHERE audit_id > ${writer_audit_id};")" wait_for_no_debt identity_docs_idx 60 assert_eq "owner signal consumes writer-created debt" "f" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'identity_docs_idx'::regclass);")" assert_eq "every worker step executes as the index owner" "t" \ "$(sql_super -c "SELECT count(*) >= 3 AND pg_catalog.bool_and( role_name = 'durable_owner') FROM worker_identity_audit;")" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; SQL assert_eq "worker helper definition is restored" \ "c:tp_compact_index_step_if_current:true" \ "$(sql_super -c "SELECT language.lanname || ':' || procedure.prosrc || ':' || (procedure.proconfig IS NULL) FROM pg_catalog.pg_proc AS procedure JOIN pg_catalog.pg_language AS language ON language.oid = procedure.prolang WHERE procedure.oid = pg_catalog.to_regprocedure( 'bm25_compact_step_if_current' || '(oid,oid,oid,oid,oid)');")" sql_as durable_owner -c \ "SELECT df.cancel('${instance_id}', 'identity test complete');" \ >/dev/null wait_for_terminal "${instance_id}" 30 sql_super -c "DROP FUNCTION bm25_compact_step_if_current_test_c( oid, oid, oid, oid, oid); DROP TABLE identity_docs, worker_identity_audit; GRANT CREATE ON SCHEMA public TO durable_writer; REVOKE durable_owner FROM durable_actor;" } test_scheduled_failure_continuation() { local index_oid instance_id failure_message index_oid="$(sql_super -c \ "SELECT 'documents_idx'::regclass::oid;")" instance_id="$(active_job_id "${index_oid}")" [ -n "${instance_id}" ] || error "failure-continuation test found no managed workflow" wait_for_signal_node "${instance_id}" 30 assert_eq "scheduled loop continues after body failures" "t" \ "$(sql_super -c "SELECT df.explain('${instance_id}') LIKE '%LOOP (infinite, continue on failure)%';")" failure_message="pg_textsearch transient compaction test failure" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT AS $body$ BEGIN RAISE EXCEPTION 'pg_textsearch transient compaction test failure'; END $body$; SQL create_compaction_debt wait_for_log_message "${failure_message}" 30 assert_eq "failed scheduled iteration leaves workflow running" "running" \ "$(sql_super -c "SELECT status FROM df.instances WHERE id = '${instance_id}';")" assert_eq "failed scheduled iteration preserves compaction debt" "t" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'documents_idx'::regclass);")" wait_for_signal_node "${instance_id}" 30 sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; SQL sql_as durable_owner -c \ "SELECT df.signal('${instance_id}', 'compact', '{}');" >/dev/null wait_for_no_debt documents_idx 30 assert_eq "same workflow recovers after the transient failure" \ "${instance_id}" "$(active_job_id "${index_oid}")" } test_initial_failure_continuation() { local index_oid instance_id failure_message sql_super -c "CREATE TABLE initial_failure_docs (id integer, body text); ALTER TABLE initial_failure_docs OWNER TO durable_owner;" sql_as durable_owner -c " CREATE INDEX initial_failure_docs_idx ON initial_failure_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual');" sql_as durable_owner <<'SQL' >/dev/null DO $body$ BEGIN FOR n IN 1..2 LOOP INSERT INTO initial_failure_docs (id, body) SELECT n * 100 + i, format('initial failure round %s document %s', n, i) FROM generate_series(1, 20) AS i; PERFORM bm25_spill_index('initial_failure_docs_idx'); END LOOP; END $body$; SQL assert_eq "manual index has startup compaction debt" "t" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'initial_failure_docs_idx'::regclass);")" failure_message="pg_textsearch initial compaction test failure" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE plpgsql VOLATILE STRICT AS $body$ BEGIN RAISE EXCEPTION 'pg_textsearch initial compaction test failure'; END $body$; SQL sql_as durable_owner -c " ALTER INDEX initial_failure_docs_idx SET (compaction = 'background');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'initial_failure_docs_idx'::regclass::oid;")" instance_id="$(latest_job_id "${index_oid}")" [ -n "${instance_id}" ] || error "startup failure test found no managed workflow" wait_for_log_message "${failure_message}" 30 assert_eq "failed startup cascade leaves workflow running" "running" \ "$(sql_super -c "SELECT status FROM df.instances WHERE id = '${instance_id}';")" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; SQL wait_for_no_debt initial_failure_docs_idx 30 assert_eq "same workflow recovers its startup cascade" \ "${instance_id}" "$(active_job_id "${index_oid}")" sql_as durable_owner -c \ "SELECT df.cancel('${instance_id}', 'startup test complete');" \ >/dev/null wait_for_terminal "${instance_id}" 30 sql_super -c "DROP TABLE initial_failure_docs;" } test_cross_owner_helper_isolation() { local current_error index_one_oid index_two_oid instance_one instance_two local debt_before level_counts_before step_error workflow_state_before local database_oid tablespace_oid relfilenumber owner_oid sql_super -c "CREATE TABLE isolation_one_docs (body text); ALTER TABLE isolation_one_docs OWNER TO durable_owner; CREATE TABLE isolation_two_docs (body text); ALTER TABLE isolation_two_docs OWNER TO durable_owner_two;" sql_as durable_owner -c " CREATE INDEX isolation_one_idx ON public.isolation_one_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual', compaction_schedule = '0 0 1 1 *'); ALTER INDEX public.isolation_one_idx SET (compaction = 'background');" >/dev/null 2>&1 sql_as durable_owner_two -c " CREATE INDEX isolation_two_idx ON public.isolation_two_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 index_one_oid="$(sql_super -c \ "SELECT 'isolation_one_idx'::regclass::oid;")" instance_one="$(active_job_id_for_owner \ "${index_one_oid}" durable_owner)" [ -n "${instance_one}" ] || error "first isolation owner has no managed workflow" wait_for_signal_node "${instance_one}" 30 sql_as durable_owner_two <<'SQL' >/dev/null BEGIN; DO $body$ BEGIN FOR n IN 1..2 LOOP INSERT INTO isolation_two_docs (body) SELECT format( 'second owner round %s document %s filler', n, i) FROM generate_series(1, 20) AS i; PERFORM bm25_spill_index('isolation_two_idx'); END LOOP; END $body$; COMMIT; SQL assert_eq "manual second-owner target starts with compaction debt" "t" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'isolation_two_idx'::regclass);")" sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean LANGUAGE sql VOLATILE STRICT SET search_path = pg_catalog, pg_temp AS $body$ SELECT false $body$; SQL sql_as durable_owner_two -c " ALTER INDEX public.isolation_two_idx SET (compaction = 'background');" >/dev/null 2>&1 index_two_oid="$(sql_super -c \ "SELECT 'isolation_two_idx'::regclass::oid;")" instance_two="$(active_job_id_for_owner \ "${index_two_oid}" durable_owner_two)" [ -n "${instance_two}" ] || error "second isolation owner has no managed workflow" wait_for_signal_node "${instance_two}" 30 sql_as durable_owner_two -c \ "SELECT df.cancel('${instance_two}', 'hold isolation debt');" \ >/dev/null wait_for_terminal "${instance_two}" 30 sql_super <<'SQL' CREATE OR REPLACE FUNCTION bm25_compact_step_if_current( index_oid oid, database_oid oid, tablespace_oid oid, relfilenumber oid, owner_oid oid) RETURNS boolean AS '$libdir/pg_textsearch', 'tp_compact_index_step_if_current' LANGUAGE C VOLATILE STRICT; SQL assert_eq "cross-owner step helper is restored before rejection" \ "c:tp_compact_index_step_if_current:true" \ "$(sql_super -c "SELECT language.lanname || ':' || procedure.prosrc || ':' || (procedure.proconfig IS NULL) FROM pg_catalog.pg_proc AS procedure JOIN pg_catalog.pg_language AS language ON language.oid = procedure.prolang WHERE procedure.oid = pg_catalog.to_regprocedure( 'bm25_compact_step_if_current' || '(oid,oid,oid,oid,oid)');")" assert_eq "both owners receive private helper EXECUTE" "t" \ "$(sql_super -c "SELECT pg_catalog.has_function_privilege( 'durable_owner', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') AND pg_catalog.has_function_privilege( 'durable_owner', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE') AND pg_catalog.has_function_privilege( 'durable_owner_two', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') AND pg_catalog.has_function_privilege( 'durable_owner_two', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE');")" IFS='|' read -r database_oid tablespace_oid relfilenumber owner_oid < <( sql_super -c "SELECT database.oid, coalesce(nullif(relation.reltablespace, 0), database.dattablespace), pg_catalog.pg_relation_filenode(relation.oid), relation.relowner FROM pg_catalog.pg_class AS relation JOIN pg_catalog.pg_database AS database ON database.datname = pg_catalog.current_database() WHERE relation.oid = 'isolation_two_idx'::regclass;" ) assert_eq "second owner's physical target remains current" "t" \ "$(sql_as durable_owner_two -c " SELECT public.bm25_background_target_is_current( ${index_two_oid}, ${database_oid}, ${tablespace_oid}, ${relfilenumber}, ${owner_oid});")" debt_before="$(sql_super -c "SELECT bm25_needs_compaction( 'isolation_two_idx'::regclass);")" assert_eq "second owner's target has compaction debt" "t" \ "${debt_before}" level_counts_before="$(sql_super -c \ "SELECT bm25_level_counts('isolation_two_idx'::regclass);")" workflow_state_before="$(sql_super -c \ "SELECT id || ':' || status FROM df.instances WHERE id = '${instance_two}';")" if step_error="$(sql_as durable_owner -c " SELECT public.bm25_compact_step_if_current( ${index_two_oid}, ${database_oid}, ${tablespace_oid}, ${relfilenumber}, ${owner_oid});" 2>&1)"; then error "first owner compacted the second owner's captured target" fi if ! grep -Fq "must be owner" <<<"${step_error}"; then error "cross-owner compaction used an unexpected error: ${step_error}" fi assert_eq "rejected cross-owner step preserves level counts" \ "${level_counts_before}" \ "$(sql_super -c \ "SELECT bm25_level_counts('isolation_two_idx'::regclass);")" assert_eq "rejected cross-owner step preserves compaction debt" \ "${debt_before}" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'isolation_two_idx'::regclass);")" assert_eq "rejected cross-owner step preserves workflow state" \ "${workflow_state_before}" \ "$(sql_super -c "SELECT id || ':' || status FROM df.instances WHERE id = '${instance_two}';")" if current_error="$(sql_as durable_owner -c " SELECT public.bm25_background_target_is_current( ${index_two_oid}, ${database_oid}, ${tablespace_oid}, ${relfilenumber}, ${owner_oid});" 2>&1)"; then error "first owner inspected the second owner's captured target" fi if ! grep -Fq "must be owner" <<<"${current_error}"; then error "cross-owner current check used an unexpected error: \ ${current_error}" fi assert_eq "rejected cross-owner current check preserves level counts" \ "${level_counts_before}" \ "$(sql_super -c \ "SELECT bm25_level_counts('isolation_two_idx'::regclass);")" assert_eq "rejected cross-owner current check preserves debt" \ "${debt_before}" \ "$(sql_super -c "SELECT bm25_needs_compaction( 'isolation_two_idx'::regclass);")" assert_eq "rejected cross-owner current check preserves workflow state" \ "${workflow_state_before}" \ "$(sql_super -c "SELECT id || ':' || status FROM df.instances WHERE id = '${instance_two}';")" sql_as durable_owner -c \ "SELECT df.cancel('${instance_one}', 'isolation test complete');" \ >/dev/null wait_for_terminal "${instance_one}" 30 wait_for_terminal "${instance_two}" 30 sql_super -c "DROP TABLE isolation_one_docs, isolation_two_docs;" } test_bypassrls_owner_isolation() { local index_oid label owner_instance foreign_instance replacement_instance sql_super -c "CREATE TABLE bypass_documents (id integer, body text); ALTER TABLE bypass_documents OWNER TO durable_bypass;" sql_as durable_bypass -c "INSERT INTO bypass_documents SELECT i, 'bypass document ' || i || ' filler filler filler' FROM generate_series(1, 10000) AS i;" sql_as durable_bypass -c " CREATE INDEX CONCURRENTLY bypass_documents_idx ON bypass_documents USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'bypass_documents_idx'::regclass::oid;")" owner_instance="$(active_job_id_for_owner \ "${index_oid}" durable_bypass)" [ -n "${owner_instance}" ] || error "BYPASSRLS index owner has no managed workflow" wait_for_signal_node "${owner_instance}" 30 label="$(sql_super -c "SELECT label FROM df.instances WHERE id = '${owner_instance}';")" foreign_instance="$(sql_as durable_owner -c " SELECT df.start( df.wait_for_signal('compact', 300), '${label}', current_database(), 'caller');")" wait_for_signal_node "${foreign_instance}" 30 assert_eq "lookalike workflow has a foreign submitter" \ "durable_owner" \ "$(sql_super -c "SELECT submitted_by::text FROM df.instances WHERE id = '${foreign_instance}';")" sql_as durable_bypass -c \ "SELECT df.signal('${owner_instance}', 'compact', '{}');" \ >/dev/null wait_for_signal_node "${owner_instance}" 30 assert_eq "foreign lookalike cannot retire the owner workflow" \ "${owner_instance}" \ "$(active_job_id_for_owner "${index_oid}" durable_bypass)" sql_as durable_bypass -c \ "SELECT df.cancel('${owner_instance}', 'owner isolation recovery');" \ >/dev/null wait_for_terminal "${owner_instance}" 30 create_bypass_compaction_debt wait_for_no_debt bypass_documents_idx 30 replacement_instance="$(active_job_id_for_owner \ "${index_oid}" durable_bypass)" if [ -z "${replacement_instance}" ] || [ "${replacement_instance}" = "${owner_instance}" ]; then error "owner workflow was not recovered past a foreign lookalike" fi assert_eq "foreign lookalike cannot be selected or signaled" "t" \ "$(sql_super -c "SELECT status IN ('pending', 'running') FROM df.instances WHERE id = '${foreign_instance}';")" assert_eq "recovered workflow retains the index owner" \ "durable_bypass" \ "$(sql_super -c "SELECT submitted_by::text FROM df.instances WHERE id = '${replacement_instance}';")" } test_superuser_policy_success() { local index_oid instance_id setting_before setting_before="$(sql_super -c \ "SHOW pg_durable.enable_superuser_instances;")" assert_eq "superuser workflow policy starts disabled" "off" \ "${setting_before}" sql_super -c "ALTER SYSTEM SET pg_durable.enable_superuser_instances = on;" >/dev/null PGHOST="${SOCKET_DIR}" "${PGBINDIR}/pg_ctl" restart -D "${DATA_DIR}" \ -l "${LOGFILE}" -w >/dev/null wait_for_durable_worker assert_eq "superuser workflow policy is enabled" "on" \ "$(sql_super -c \ "SHOW pg_durable.enable_superuser_instances;")" sql_super -c "CREATE TABLE superuser_policy_docs (id integer, body text); CREATE INDEX superuser_policy_docs_idx ON superuser_policy_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual', compaction_schedule = '0 0 1 1 *'); ALTER INDEX superuser_policy_docs_idx SET (compaction = 'background');" >/dev/null 2>&1 index_oid="$(sql_super -c \ "SELECT 'superuser_policy_docs_idx'::regclass::oid;")" instance_id="$(active_job_id_for_owner "${index_oid}" postgres)" [ -n "${instance_id}" ] || error "enabled superuser policy created no postgres workflow" wait_for_signal_node "${instance_id}" 30 assert_eq "superuser workflow is submitted as postgres" "postgres" \ "$(sql_super -c "SELECT submitted_by::text FROM df.instances WHERE id = '${instance_id}';")" sql_super <<'SQL' >/dev/null BEGIN; DO $body$ BEGIN FOR n IN 1..2 LOOP INSERT INTO superuser_policy_docs (id, body) SELECT n * 100 + i, format('superuser round %s document %s filler', n, i) FROM generate_series(1, 20) AS i; PERFORM bm25_spill_index('superuser_policy_docs_idx'); END LOOP; END $body$; COMMIT; SQL wait_for_no_debt superuser_policy_docs_idx 30 assert_eq "postgres workflow executes after a spill signal" \ "${instance_id}" "$(active_job_id_for_owner "${index_oid}" postgres)" sql_super -c \ "SELECT df.cancel('${instance_id}', 'superuser test complete');" \ >/dev/null wait_for_terminal "${instance_id}" 30 sql_super -c "DROP TABLE superuser_policy_docs;" sql_super -c "ALTER SYSTEM RESET pg_durable.enable_superuser_instances;" >/dev/null PGHOST="${SOCKET_DIR}" "${PGBINDIR}/pg_ctl" restart -D "${DATA_DIR}" \ -l "${LOGFILE}" -w >/dev/null wait_for_durable_worker assert_eq "superuser workflow policy is restored" "${setting_before}" \ "$(sql_super -c \ "SHOW pg_durable.enable_superuser_instances;")" } test_remote_database_rejection() { local alter_error create_error dependency_snapshot jobs_before sql_super -c "CREATE DATABASE ${REMOTE_DB};" sql_remote_super -c "CREATE EXTENSION pg_textsearch; GRANT CREATE ON SCHEMA public TO durable_owner; ALTER ROLE durable_owner IN DATABASE ${REMOTE_DB} SET maintenance_work_mem = '1MB';" assert_eq "pg_durable exists in the control database" "t" \ "$(sql_super -c "SELECT EXISTS ( SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_durable');")" assert_eq "pg_durable is absent from the remote database" "f" \ "$(sql_remote_super -c "SELECT EXISTS ( SELECT 1 FROM pg_catalog.pg_extension WHERE extname = 'pg_durable');")" dependency_snapshot="$(remote_dependency_snapshot)" jobs_before="$(managed_job_count)" sql_remote_super -c "CREATE TABLE remote_cic_docs (body text); ALTER TABLE remote_cic_docs OWNER TO durable_owner;" if create_error="$(sql_remote_as durable_owner -c " CREATE INDEX CONCURRENTLY remote_cic_docs_idx ON remote_cic_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background');" 2>&1)"; then error "remote background CIC unexpectedly succeeded" fi if ! grep -Fq \ "pg_durable is not initialized for this database" \ <<<"${create_error}" || ! grep -Fq \ "pg_durable.database does not name the current database" \ <<<"${create_error}"; then error "remote CIC did not report the configured-database mismatch: \ ${create_error}" fi assert_eq "remote CIC leaves no relation" "t" \ "$(sql_remote_super -c "SELECT pg_catalog.to_regclass('remote_cic_docs_idx') IS NULL;")" assert_eq "remote CIC creates no control workflow" "${jobs_before}" \ "$(managed_job_count)" assert_eq "remote CIC preserves extension dependency snapshot" \ "${dependency_snapshot}" "$(remote_dependency_snapshot)" sql_remote_super -c " CREATE TABLE remote_manual_docs (id integer, body text); ALTER TABLE remote_manual_docs OWNER TO durable_owner;" sql_remote_as durable_owner <<'SQL' >/dev/null CREATE INDEX remote_manual_docs_idx ON remote_manual_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); DO $body$ BEGIN FOR n IN 1..2 LOOP INSERT INTO remote_manual_docs SELECT n * 100 + i, format('remote round %s document %s', n, i) FROM generate_series(1, 20) AS i; PERFORM bm25_spill_index('remote_manual_docs_idx'); END LOOP; END $body$; SQL assert_eq "remote manual index has compaction debt" "t" \ "$(sql_remote_as durable_owner -c "SELECT bm25_needs_compaction('remote_manual_docs_idx'::regclass);")" if alter_error="$(sql_remote_as durable_owner -c " ALTER INDEX remote_manual_docs_idx SET (compaction = 'background');" 2>&1)"; then error "remote background ALTER unexpectedly succeeded" fi if ! grep -Fq \ "pg_durable is not initialized for this database" \ <<<"${alter_error}" || ! grep -Fq \ "pg_durable.database does not name the current database" \ <<<"${alter_error}"; then error "remote ALTER did not report the configured-database mismatch: \ ${alter_error}" fi assert_eq "remote ALTER preserves manual mode" "t" \ "$(sql_remote_super -c "SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_catalog.pg_class WHERE oid = 'remote_manual_docs_idx'::regclass;")" assert_eq "remote ALTER creates no control workflow" "${jobs_before}" \ "$(managed_job_count)" assert_eq "remote ALTER creates no helper grant" "f" \ "$(remote_helper_grant)" assert_eq "remote ALTER preserves extension dependency snapshot" \ "${dependency_snapshot}" "$(remote_dependency_snapshot)" sql_remote_as durable_owner -c \ "SELECT bm25_compact('remote_manual_docs_idx'::regclass);" \ >/dev/null assert_eq "remote manual compaction clears debt" "t" \ "$(sql_remote_as durable_owner -c "SELECT NOT bm25_needs_compaction('remote_manual_docs_idx'::regclass);")" sql_super -c "DROP DATABASE ${REMOTE_DB};" } test_rollback_in_fresh_database() { local alter_error dump_file dump_lineage nologin_error local partition_lineage restore_output restored_lineage dump_file="${DATA_DIR}/lineage-dump.sql" sql_as durable_owner -c " CREATE TABLE public.lifecycle_dump_docs (body text);" >/dev/null sql_as durable_owner -c " CREATE INDEX lifecycle_dump_idx ON public.lifecycle_dump_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 sql_as durable_owner -c " CREATE TABLE public.lifecycle_dump_partitioned_docs (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE public.lifecycle_dump_partitioned_low PARTITION OF public.lifecycle_dump_partitioned_docs FOR VALUES FROM (0) TO (100); CREATE TABLE public.lifecycle_dump_partitioned_high PARTITION OF public.lifecycle_dump_partitioned_docs FOR VALUES FROM (100) TO (200); CREATE INDEX lifecycle_dump_partitioned_idx ON public.lifecycle_dump_partitioned_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background', compaction_schedule = '0 0 1 1 *');" >/dev/null 2>&1 dump_lineage="$(index_lineage public.lifecycle_dump_idx)" partition_lineage="$( index_lineage public.lifecycle_dump_partitioned_idx )" assert_eq "partitioned source parent has a 128-bit lineage" "32" \ "${#partition_lineage}" assert_eq "every partitioned source leaf matches parent lineage" "2:2" \ "$(sql_super -c "SELECT count(*) || ':' || count(*) FILTER ( WHERE pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) OPERATOR(pg_catalog.=) '${partition_lineage}') FROM pg_catalog.pg_partition_tree( 'public.lifecycle_dump_partitioned_idx'::regclass) AS tree JOIN pg_catalog.pg_class AS relation ON relation.oid = tree.relid CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE tree.isleaf AND relation.relkind = 'i' AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" "${PGBINDIR}/pg_dump" -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -U durable_owner -d "${TEST_DB}" --schema-only --no-owner \ --table=public.lifecycle_dump_docs \ --table=public.lifecycle_dump_partitioned_docs \ --table=public.lifecycle_dump_partitioned_low \ --table=public.lifecycle_dump_partitioned_high >"${dump_file}" if ! grep -Fq "compaction_lineage" "${dump_file}"; then error "pg_dump omitted the managed lineage reloption" fi sql_super -c "CREATE DATABASE ${ROLLBACK_DB};" sql_super -c "ALTER SYSTEM SET pg_durable.database = '${ROLLBACK_DB}';" PGHOST="${SOCKET_DIR}" "${PGBINDIR}/pg_ctl" restart -D "${DATA_DIR}" \ -l "${LOGFILE}" -w >/dev/null TEST_DB="${ROLLBACK_DB}" initialize_database sql_super -c "GRANT CREATE ON SCHEMA public TO durable_nologin; CREATE TABLE nologin_documents (id integer, body text); ALTER TABLE nologin_documents OWNER TO durable_nologin;" if nologin_error="$(sql_super \ -c "SET ROLE durable_nologin;" \ -c " CREATE INDEX CONCURRENTLY nologin_documents_idx ON nologin_documents USING bm25(body) WITH (text_config = 'english', compaction = 'background');" \ 2>&1)"; then error "background CREATE accepted a NOLOGIN index owner" fi if ! grep -Fq \ "index owner must have LOGIN for background compaction" \ <<<"${nologin_error}"; then error "NOLOGIN rejection did not use the stable admission message" fi log "PASS: background admission rejects a NOLOGIN index owner" assert_eq "rejected NOLOGIN CIC leaves no relation" "t" \ "$(sql_super -c "SELECT pg_catalog.to_regclass('nologin_documents_idx') IS NULL;")" assert_eq "failed NOLOGIN admission leaves no sticky dependency" "0" \ "$(dependency_count)" sql_super -c "CREATE TABLE rollback_documents (id integer, body text); ALTER TABLE rollback_documents OWNER TO durable_owner;" sql_as durable_owner <<'SQL' >/dev/null 2>&1 BEGIN; CREATE INDEX rollback_documents_idx ON rollback_documents USING bm25(body) WITH (text_config = 'english', compaction = 'background'); ROLLBACK; SQL assert_eq "rolled-back CREATE leaves no managed job" "0" \ "$(sql_super -c "SELECT count(*) FROM df.instances WHERE label LIKE 'pg_textsearch:bg:v1:%';")" assert_eq "rolled-back CREATE leaves no sticky dependency" "0" \ "$(dependency_count)" assert_eq "rolled-back CREATE leaves no private-helper grant" "f" \ "$(sql_super -c "SELECT pg_catalog.has_function_privilege( 'durable_owner', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') OR pg_catalog.has_function_privilege( 'durable_owner', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE');")" sql_super -c "CREATE TABLE alter_rollback_documents ( id integer, body text); ALTER TABLE alter_rollback_documents OWNER TO durable_owner;" sql_as durable_owner -c " CREATE INDEX alter_rollback_documents_idx ON alter_rollback_documents USING bm25(body) WITH (text_config = 'english', compaction = 'manual');" >/dev/null if alter_error="$(sql_as durable_owner <<'SQL' 2>&1 BEGIN; ALTER INDEX alter_rollback_documents_idx SET (compaction = 'background'); DO $body$ BEGIN RAISE EXCEPTION 'force background ALTER rollback'; END $body$; COMMIT; SQL )"; then error "background ALTER rollback transaction unexpectedly committed" fi if ! grep -Fq "force background ALTER rollback" \ <<<"${alter_error}"; then error "background ALTER rollback did not reach forced failure: \ ${alter_error}" fi assert_eq "rolled-back ALTER preserves manual reloption" "t" \ "$(sql_super -c "SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_catalog.pg_class WHERE oid = 'alter_rollback_documents_idx'::regclass;")" assert_eq "rolled-back ALTER leaves no managed job" "0" \ "$(managed_job_count)" assert_eq "rolled-back ALTER leaves no sticky dependency" "0" \ "$(dependency_count)" assert_eq "rolled-back ALTER leaves no private-helper grant" "f" \ "$(sql_super -c "SELECT pg_catalog.has_function_privilege( 'durable_owner', 'bm25_compact_step_if_current(oid,oid,oid,oid,oid)', 'EXECUTE') OR pg_catalog.has_function_privilege( 'durable_owner', 'bm25_background_target_is_current(oid,oid,oid,oid,oid)', 'EXECUTE');")" if ! restore_output="$(sql_as durable_owner -f "${dump_file}" 2>&1)"; then error "normal pg_dump restore rejected a fresh-database lineage: \ ${restore_output}" fi restored_lineage="$(index_lineage public.lifecycle_dump_idx)" assert_eq "pg_dump restore preserves a fresh-database lineage" \ "${dump_lineage}" "${restored_lineage}" assert_eq "partitioned pg_dump restore preserves shared lineage" \ "${partition_lineage}" \ "$(index_lineage public.lifecycle_dump_partitioned_idx)" restored_lineage="$( index_lineage public.lifecycle_dump_partitioned_idx )" assert_eq "partitioned restored parent has a 128-bit lineage" "32" \ "${#restored_lineage}" assert_eq "every restored physical leaf matches parent lineage" "2:2" \ "$(sql_super -c "SELECT count(*) || ':' || count(*) FILTER ( WHERE pg_catalog.substr( option, pg_catalog.length('compaction_lineage=') + 1) OPERATOR(pg_catalog.=) '${restored_lineage}') FROM pg_catalog.pg_partition_tree( 'public.lifecycle_dump_partitioned_idx'::regclass) AS tree JOIN pg_catalog.pg_class AS relation ON relation.oid = tree.relid CROSS JOIN LATERAL pg_catalog.unnest(relation.reloptions) AS option WHERE tree.isleaf AND relation.relkind = 'i' AND option OPERATOR(pg_catalog.~~) 'compaction_lineage=%';")" assert_eq "partitioned restore activates every physical leaf" "2" \ "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_inherits AS inheritance JOIN pg_catalog.pg_class AS relation ON relation.oid = inheritance.inhrelid WHERE inheritance.inhparent = 'public.lifecycle_dump_partitioned_idx'::regclass AND relation.relkind = 'i' AND EXISTS ( SELECT 1 FROM df.instances AS instance WHERE instance.label OPERATOR(pg_catalog.~~) ('pg_textsearch:bg:v1:%:' || relation.oid::pg_catalog.text || ':%') AND instance.status OPERATOR(pg_catalog.=) ANY (ARRAY['pending', 'running']::pg_catalog.text[]));")" sql_as durable_owner -c \ "DROP TABLE public.lifecycle_dump_docs, public.lifecycle_dump_partitioned_docs;" >/dev/null } test_sticky_dependency() { quiesce_durable_worker if sql_super -c "DROP EXTENSION pg_durable;" >/dev/null 2>&1; then error "sticky dependency allowed DROP EXTENSION pg_durable" fi log "PASS: sticky dependency blocks DROP EXTENSION pg_durable" sql_super -c "DROP EXTENSION pg_durable CASCADE;" >/dev/null assert_eq "CASCADE follows the access-method dependency to pg_textsearch" \ "0" "$(sql_super -c "SELECT count(*) FROM pg_catalog.pg_extension WHERE extname = 'pg_textsearch';")" } run_test() { local test_name=$1 local selected=",${DURABLE_TEST_FILTER:-}," if [ -z "${DURABLE_TEST_FILTER:-}" ] || [[ "${selected}" == *",${test_name},"* ]]; then "${test_name}" fi } test_signal_wait_readiness() { local instance_id waiting instance_id="$(sql_as durable_owner -c " SELECT df.start( df.seq( df.race(df.wait_for_signal('compact'), df.wait_for_signal('advance')), df.sleep(3600)), 'signal-readiness', current_database(), 'caller');")" for _ in $(seq 1 30); do waiting="$(sql_super -c "SELECT count(*) FROM df.instance_nodes('${instance_id}') WHERE node_type = 'SIGNAL' AND inferred_status = 'running';")" [ "${waiting}" = "2" ] && break sleep 1 done assert_eq "signal-readiness fixture reaches both waits" "2" "${waiting}" wait_for_signal_node "${instance_id}" 30 assert_eq "active signal wait is ready" "t" \ "$(signal_node_is_waiting "${instance_id}")" wait_for_signal_node "${instance_id}" 30 advance sql_as durable_owner -c \ "SELECT df.signal('${instance_id}', 'advance', '{}');" >/dev/null for _ in $(seq 1 30); do waiting="$(sql_super -c "SELECT count(*) FROM df.instance_nodes('${instance_id}') WHERE node_type = 'SIGNAL' AND inferred_status = 'skipped';")" [ "${waiting}" = "1" ] && break sleep 1 done assert_eq "signal-readiness fixture retires the losing wait" "1" \ "${waiting}" assert_eq "retired signal wait is not ready" "f" \ "$(signal_node_is_waiting "${instance_id}")" sql_as durable_owner -c \ "SELECT df.cancel('${instance_id}', 'readiness test complete');" \ >/dev/null wait_for_terminal "${instance_id}" 30 } stage_durable_package setup_cluster run_test test_missing_durable_cic initialize_database run_test test_signal_wait_readiness run_test test_cic_preflight_rejections run_test test_cic_owner_privilege_preflight run_test test_alter_preflight_rejections run_test test_defaulted_start_arity run_test test_reindex_nonrelation_passthrough run_test test_late_bulk_reindex_target run_test test_bulk_reindex_concurrent_mode_change run_test test_bulk_reindex_concurrent_schedule_change run_test test_failed_bulk_reindex_reconciliation run_test test_partitioned_create_activation run_test test_direct_index_partition_attach run_test test_reused_intermediate_partition_options run_test test_partition_detach_lineage run_test test_partitioned_existing_leaf_reconciliation run_test test_create_tracking_reentry run_test test_cached_create_uses_fresh_lineage run_test test_create_like_regenerates_lineage run_test test_create_tracking_concurrent run_test test_create_tracking_table_rename run_test test_owner_reconciliation run_test test_owner_change_preserves_captured_schedule run_test test_reassign_owned_reconciliation run_test test_reindex_reconciliation run_test test_plain_reindex_preserves_captured_schedule run_test test_physical_rewrite_reconciliation run_test test_database_owner_vacuum_full run_test test_vacuum_full_skip_locked run_test test_partition_vacuum_child_authorization run_test test_global_cluster_scope run_test test_global_cluster_concurrent_mode_change run_test test_inheritance_vacuum_full_scope run_test test_inheritance_alter_rewrite_scope run_test test_additional_rewrite_reconciliation run_test test_truncate_cascade_reconciliation run_test test_maintenance_privilege_compatibility run_test test_tablespace_move_without_owned_by run_test test_rewrite_preflight_ordering run_test test_concurrent_reindex_reconciliation run_test test_partitioned_reindex_reconciliation run_test test_partitioned_reindex_failure_reconciliation run_test test_partitioned_reindex_rename_reconciliation run_test test_reindex_tracking_reentry run_test test_ordinary_inheritance_reindex_scope run_test test_legacy_lineage_backfill run_test test_legacy_reconciliation_edges run_test test_refresh_selects_requested_workflow run_test test_concurrent_legacy_lineage_backfill run_test test_legacy_reindex_spill_lock_order run_test test_managed_lock_order run_test test_alter_reindex_lock_order run_test test_cross_statement_reindex_lock_order run_test test_cross_statement_owner_lock_order run_test test_multi_family_partition_attach_batch run_test test_attached_index_rewrite_preserves_parent_options run_test test_lineage_lookup_drop_durable_order run_test test_textsearch_extension_dependency_order run_test test_precommit_request_admission_nowait run_test test_terminal_grant_reentry_is_rejected run_test test_post_publication_reindex_defers run_test test_post_publication_schedule_override run_test test_post_publication_background_activation run_test test_post_publication_activation_error_defers run_test test_cross_statement_managed_lock_order run_test test_managed_intent_savepoint_recovery run_test test_internal_lock_namespace run_test test_lineage_ddl_guards run_test test_partitioned_lineage_history run_test test_forged_lineage_submitter run_test test_concurrent_supplied_lineage_create run_test test_reindex_lineage_replacement_isolation run_test test_reindex_authorization_ordering run_test test_create_authorization_ordering run_test test_lineage_guard_name_race run_test test_reindex_authorization_resolution_race run_test test_background_vacuum_rewrites_sparse_singleton run_test test_prior_generation_spill_adoption run_test test_request_queue_runtime run_test test_precommit_signal_observes_published_spill run_test test_repeatable_read_admission_snapshot run_test test_repeatable_read_direct_activation_snapshot run_test test_owner_repair_without_old_durable_acl run_test test_writer_search_path_isolation run_test test_persisted_helper_oid_guard run_test test_reassign_owned_heap_lock_order run_test test_actor_writer_worker_identity run_test test_create_activation run_test test_scheduled_failure_continuation run_test test_initial_failure_continuation run_test test_cross_owner_helper_isolation run_test test_bypassrls_owner_isolation run_test test_superuser_policy_success run_test test_remote_database_rejection run_test test_rollback_in_fresh_database run_test test_sticky_dependency log "Managed pg_durable compaction tests passed" timescale-pg_textsearch-b63fe58/test/scripts/filtered_seed_benchmark_test.sh000066400000000000000000000024471525762047700276410ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" TEST_DIR="$(mktemp -d)" trap 'rm -rf "$TEST_DIR"' EXIT cp "$ROOT_DIR/benchmarks/run_filtered_seed.sh" "$TEST_DIR/run.sh" fail() { echo "filtered-seed benchmark test failed: $*" >&2 exit 1 } # Exercise the runner without building a benchmark database. psql() { if [[ "${STUB_RESULT:-1}" == 1 ]]; then echo "FSB_RESULT: shape=single sel=0.1 limit=10 scans=1" \ "off_ms=2 off_passes=3 on_ms=1 on_passes=1 speedup=2" fi return "${STUB_STATUS:-0}" } export -f psql status=0 STUB_STATUS=3 bash "$TEST_DIR/run.sh" >"$TEST_DIR/failure.log" 2>&1 || status=$? [[ "$status" == 3 ]] || fail "psql failure was not propagated" if compgen -G "$TEST_DIR/results/*.json" >/dev/null; then fail "failed run published metrics" fi if STUB_RESULT=0 bash "$TEST_DIR/run.sh" >"$TEST_DIR/empty.log" 2>&1; then fail "empty result was accepted" fi if compgen -G "$TEST_DIR/results/*.json" >/dev/null; then fail "empty run published metrics" fi bash "$TEST_DIR/run.sh" >"$TEST_DIR/success.log" 2>&1 compgen -G "$TEST_DIR/results/*.json" >/dev/null || fail "successful run did not publish metrics" grep -q '"bm25_scans": 1' "$TEST_DIR"/results/*.json || fail "successful run lost scan metrics" echo "filtered-seed benchmark tests passed" timescale-pg_textsearch-b63fe58/test/scripts/gen_nul_lexeme_bm25vector.py000077500000000000000000000050071525762047700270440ustar00rootroot00000000000000#!/usr/bin/env python3 # # gen_nul_lexeme_bm25vector.py — emit a COPY BINARY frame containing a # row with a hand-encoded v2 bm25vector whose lexeme carries an # embedded NUL byte. Used by test/sql/vector_v1_rejected.sql to verify # that such values are rejected at the boundary rather than producing a # silently truncated text rendering. # # A NUL can never appear in a lexeme built from tokenized text or the # text input parser (both operate on NUL-terminated cstrings), so this # value is only reachable via a crafted binary input. # # Output (to stdout): COPY BINARY framing + 1 row [int4 id, v2 # bm25vector]. Hard-coded payload: id=42, index='compat_idx', # entries=[(freq=1, lex='bad\0lex')]. import struct import sys def maxalign(n: int, a: int = 8) -> int: return (n + a - 1) & ~(a - 1) def varint(n: int) -> bytes: out = bytearray() while True: b = n & 0x7F n >>= 7 if n: out.append(b | 0x80) else: out.append(b) return bytes(out) idx_name = b"compat_idx" inl = len(idx_name) # The lexeme deliberately contains an embedded NUL in the middle so the # entry passes the buffer-bounds checks but must be rejected by the # explicit NUL check. lexeme = b"bad\x00lex" freq = 1 # v2 layout (see src/types/vector.h): # int32 vl_len_ # char magic[4] = "BM25" # uint8 version = 2, uint8 reserved[3] = 0 # int32 index_name_len # int32 entry_count # char index_name[] + '\0' + padding to MAXALIGN(index_name_len + 1) # per entry (variable-length, no padding): # varint frequency # varint lexeme_len # char lexeme[lexeme_len] hdr_size = 20 idx_block = maxalign(inl + 1) entry = varint(freq) + varint(len(lexeme)) + lexeme total = hdr_size + idx_block + len(entry) # tpvector_send writes the first 4 bytes via pq_sendint32 (big-endian) # = total size, then pq_sendbytes the rest of the value as raw bytes # (host byte order on the wire). Mirror that here. v2 = struct.pack(">I", total) v2 += b"BM25" v2 += struct.pack("=B", 2) + b"\x00" * 3 v2 += struct.pack("=ii", inl, 1) v2 += idx_name + b"\x00" + b"\x00" * (idx_block - inl - 1) v2 += entry # COPY BINARY framing (network byte order for header / length prefixes). sig = b"PGCOPY\n\xff\r\n\x00" flags = struct.pack(">I", 0) hdrext = struct.pack(">I", 0) ncols = struct.pack(">H", 2) id_col = struct.pack(">I", 4) + struct.pack(">i", 42) vec_col = struct.pack(">I", len(v2)) + v2 trailer = struct.pack(">h", -1) sys.stdout.buffer.write(sig + flags + hdrext + ncols + id_col + vec_col + trailer) timescale-pg_textsearch-b63fe58/test/scripts/gen_v1_bm25vector.py000077500000000000000000000035631525762047700252420ustar00rootroot00000000000000#!/usr/bin/env python3 # # gen_v1_bm25vector.py — emit a COPY BINARY frame containing a row # with a hand-encoded legacy v1 bm25vector. Used by # test/sql/vector_v1_rejected.sql to verify that v1 inputs are # detected and rejected with a clear error (rather than silently # misinterpreted as v2). # # Output (to stdout): COPY BINARY framing + 1 row [int4 id, v1 # bm25vector]. Hard-coded payload: id=42, index='compat_idx', # entries=[(freq=1, lex='alpha'), (freq=2, lex='bravo')]. import struct import sys def maxalign(n: int, a: int = 8) -> int: return (n + a - 1) & ~(a - 1) idx_name = b"compat_idx" inl = len(idx_name) entries = [(1, b"alpha"), (2, b"bravo")] # v1 layout: # int32 vl_len_ # int32 index_name_len # int32 entry_count # char index_name[] + '\0' + padding to MAXALIGN # per entry: # int32 frequency # int32 lexeme_len # char lexeme[] # padding to MAXALIGN(8 + lexeme_len) hdr_size = 12 idx_block = maxalign(inl + 1) total = hdr_size + idx_block + sum(maxalign(8 + len(lex)) for _, lex in entries) # tpvector_send writes the first 4 bytes via pq_sendint32 (big- # endian), then pq_sendbytes the rest of the value as raw bytes # (which on the wire is host-byte-order). v1 = struct.pack(">I", total) v1 += struct.pack("=ii", inl, len(entries)) v1 += idx_name + b"\x00" + b"\x00" * (idx_block - inl - 1) for freq, lex in entries: block = struct.pack("=ii", freq, len(lex)) + lex v1 += block + b"\x00" * (maxalign(len(block)) - len(block)) # COPY BINARY framing (network byte order for header / length prefixes). sig = b"PGCOPY\n\xff\r\n\x00" flags = struct.pack(">I", 0) hdrext = struct.pack(">I", 0) ncols = struct.pack(">H", 2) id_col = struct.pack(">I", 4) + struct.pack(">i", 42) vec_col = struct.pack(">I", len(v1)) + v1 trailer = struct.pack(">h", -1) sys.stdout.buffer.write(sig + flags + hdrext + ncols + id_col + vec_col + trailer) timescale-pg_textsearch-b63fe58/test/scripts/hypertable.sh000077500000000000000000000135221525762047700241300ustar00rootroot00000000000000#!/bin/bash # Test BM25 index behavior with TimescaleDB hypertables # # This test is OPTIONAL - it only runs if TimescaleDB is installed. # It tests that the planner hook correctly handles CustomScan nodes # (like TimescaleDB's ConstraintAwareAppend) when computing BM25 scores. # # The test verifies that scores in SELECT expressions are correctly # replaced with stub functions that retrieve cached scores from the # index scan, rather than using standalone scoring (which would use # the parent hypertable's empty index statistics). set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PGPORT="${PGPORT:-5432}" PGHOST="${PGHOST:-localhost}" PSQL="psql -p $PGPORT -h $PGHOST -v ON_ERROR_STOP=1" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color echo_info() { echo -e "${YELLOW}[INFO]${NC} $1"; } echo_ok() { echo -e "${GREEN}[OK]${NC} $1"; } echo_fail() { echo -e "${RED}[FAIL]${NC} $1"; } # Check if TimescaleDB is available check_timescaledb() { if $PSQL -d postgres -tAc "SELECT 1 FROM pg_available_extensions WHERE name = 'timescaledb'" 2>/dev/null | grep -q 1; then return 0 else return 1 fi } # Main test main() { echo_info "Checking for TimescaleDB..." if ! check_timescaledb; then echo_info "TimescaleDB not available - skipping hypertable tests" echo_ok "Test skipped (TimescaleDB not installed)" exit 0 fi echo_info "TimescaleDB found - running hypertable tests" # Create test database TEST_DB="hypertable_test_$$" $PSQL -d postgres -c "DROP DATABASE IF EXISTS $TEST_DB" 2>/dev/null || true $PSQL -d postgres -c "CREATE DATABASE $TEST_DB" cleanup() { $PSQL -d postgres -c "DROP DATABASE IF EXISTS $TEST_DB" 2>/dev/null || true } trap cleanup EXIT # Setup extensions and test table $PSQL -d $TEST_DB <<'EOF' -- Setup extensions CREATE EXTENSION IF NOT EXISTS timescaledb; CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create hypertable CREATE TABLE hyper_docs ( created_at TIMESTAMPTZ NOT NULL, content TEXT NOT NULL ); SELECT create_hypertable('hyper_docs', 'created_at', chunk_time_interval => INTERVAL '1 month'); -- Insert test data across multiple chunks INSERT INTO hyper_docs (created_at, content) SELECT '2024-01-15'::timestamptz + (i * INTERVAL '15 days'), CASE (i % 5) WHEN 0 THEN 'database query optimization techniques' WHEN 1 THEN 'full text search engine' WHEN 2 THEN 'postgresql database administration' WHEN 3 THEN 'cloud computing services' WHEN 4 THEN 'machine learning algorithms' END FROM generate_series(0, 19) i; -- Create BM25 index CREATE INDEX hyper_docs_bm25_idx ON hyper_docs USING bm25(content) WITH (text_config='english'); EOF echo_info "Test 1: Verify BM25 index scans are used" # Check that the plan uses BM25 index scans (may be via CustomScan or MergeAppend) PLAN=$($PSQL -d $TEST_DB -tA <<'EOF' EXPLAIN (COSTS OFF) SELECT content, -(content <@> to_bm25query('database', 'hyper_docs_bm25_idx')) as score FROM hyper_docs ORDER BY content <@> to_bm25query('database', 'hyper_docs_bm25_idx') LIMIT 5; EOF ) if echo "$PLAN" | grep -q "Index Scan using.*hyper_docs_bm25_idx"; then echo_ok "Query plan uses BM25 index scans" else echo_fail "Query plan does not use BM25 index scans" echo "$PLAN" exit 1 fi echo_info "Test 2: Verify scores are non-zero" # Query and verify scores RESULT=$($PSQL -d $TEST_DB -tA <<'EOF' SELECT COUNT(*) as cnt FROM ( SELECT content, -(content <@> to_bm25query('database', 'hyper_docs_bm25_idx')) as score FROM hyper_docs ORDER BY content <@> to_bm25query('database', 'hyper_docs_bm25_idx') LIMIT 5 ) t WHERE score > 0; EOF ) NONZERO_COUNT=$(echo "$RESULT" | tr -d '[:space:]') if [ "$NONZERO_COUNT" -gt 0 ]; then echo_ok "Scores are non-zero ($NONZERO_COUNT results with positive scores)" else echo_fail "All scores are zero - planner hook may not be handling CustomScan correctly" # Show actual scores for debugging $PSQL -d $TEST_DB <<'EOF' SELECT content, -(content <@> to_bm25query('database', 'hyper_docs_bm25_idx')) as score FROM hyper_docs ORDER BY content <@> to_bm25query('database', 'hyper_docs_bm25_idx') LIMIT 5; EOF exit 1 fi echo_info "Test 3: Verify scores with time-based filter (common hypertable pattern)" # This pattern often triggers TimescaleDB's CustomScan (ConstraintAwareAppend) PLAN=$($PSQL -d $TEST_DB -tA <<'EOF' EXPLAIN (COSTS OFF) SELECT content, -(content <@> to_bm25query('database', 'hyper_docs_bm25_idx')) as score FROM hyper_docs WHERE created_at >= '2024-03-01' AND created_at < '2024-06-01' ORDER BY content <@> to_bm25query('database', 'hyper_docs_bm25_idx') LIMIT 5; EOF ) # Check if CustomScan is in the plan (may or may not be depending on TimescaleDB version) if echo "$PLAN" | grep -q "Custom Scan"; then echo_info "Query uses CustomScan (ConstraintAwareAppend) - testing CustomScan path" else echo_info "Query uses standard plan nodes - CustomScan not triggered" fi RESULT=$($PSQL -d $TEST_DB -tA <<'EOF' SELECT COUNT(*) as cnt FROM ( SELECT content, -(content <@> to_bm25query('database', 'hyper_docs_bm25_idx')) as score FROM hyper_docs WHERE created_at >= '2024-03-01' AND created_at < '2024-06-01' ORDER BY content <@> to_bm25query('database', 'hyper_docs_bm25_idx') LIMIT 5 ) t WHERE score > 0; EOF ) NONZERO_COUNT=$(echo "$RESULT" | tr -d '[:space:]') if [ "$NONZERO_COUNT" -gt 0 ]; then echo_ok "Scores with time filter are non-zero ($NONZERO_COUNT results)" else echo_fail "Scores with time filter are zero" exit 1 fi echo "" echo_ok "All hypertable tests passed!" } main "$@" timescale-pg_textsearch-b63fe58/test/scripts/inline_compaction_locking.sh000077500000000000000000000463061525762047700271770ustar00rootroot00000000000000#!/bin/bash # # Regression tests for spill-time inline compaction admission. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55461 TEST_DB=inline_compaction_locking_test DATA_DIR="${SCRIPT_DIR}/../tmp_inline_compaction_locking" SOCKET_DIR="${TMPDIR:-/tmp}/pgts_inline_${TEST_PORT}" LOGFILE="${DATA_DIR}/postgres.log" CLIENT_DIR="${DATA_DIR}/clients" TEST_CASE="${1:-all}" HAS_INJECTION_POINTS=0 RED='\033[0;31m' GREEN='\033[0;32m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } fail() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } cleanup() { local exit_code=$? jobs -p | xargs -r kill 2>/dev/null || true exec 3>&- 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" "${SOCKET_DIR}" exit "${exit_code}" } trap cleanup EXIT INT TERM setup_test_db() { rm -rf "${DATA_DIR}" "${SOCKET_DIR}" mkdir -p "${DATA_DIR}" "${SOCKET_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 mkdir -p "${CLIENT_DIR}" cat >>"${DATA_DIR}/postgresql.conf" </dev/null createdb -h "${SOCKET_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null if psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -qAt \ -c "SELECT count(*) = 2 FROM pg_available_extensions WHERE name IN ('injection_points', 'pg_textsearch_test');" | grep -qx t; then psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test;" >/dev/null HAS_INJECTION_POINTS=1 fi } PSQL="psql -h ${SOCKET_DIR} -p ${TEST_PORT} -d ${TEST_DB} -qAt \ -v ON_ERROR_STOP=1" wait_for_file_text() { local file="$1" local text="$2" local description="$3" local deadline=$((SECONDS + 15)) while ((SECONDS < deadline)); do if grep -Fq "${text}" "${file}" 2>/dev/null; then return fi sleep 0.1 done tail -n 30 "${file}" 2>/dev/null || true fail "timed out waiting for ${description}" } wait_for_reindex_wait() { local application_name="$1" local index_name="$2" local deadline=$((SECONDS + 15)) local waiting=0 while ((SECONDS < deadline)); do waiting=$($PSQL -c " SELECT count(*) FROM pg_locks l JOIN pg_stat_activity a USING (pid) WHERE a.application_name = '${application_name}' AND l.relation = '${index_name}'::regclass AND l.mode = 'ShareUpdateExclusiveLock' AND l.granted AND a.wait_event_type = 'Lock';") if [ "${waiting:-0}" -eq 1 ]; then return fi sleep 0.1 done fail "REINDEX CONCURRENTLY did not reach its writer wait" } skip_without_injection_points() { local description="$1" if [ "${HAS_INJECTION_POINTS}" -eq 1 ]; then return 1 fi log "Skipping ${description}: injection points are unavailable" return 0 } test_reindex_deadlock() { local fifo="${CLIENT_DIR}/writer.fifo" local writer_log="${CLIENT_DIR}/writer.log" local reindex_log="${CLIENT_DIR}/reindex.log" local writer_pid local reindex_pid log "Testing DML spill against REINDEX CONCURRENTLY" $PSQL <<'SQL' >/dev/null CREATE TABLE inline_deadlock (id integer PRIMARY KEY, body text NOT NULL); INSERT INTO inline_deadlock VALUES (1, 'initial alpha document'); CREATE INDEX inline_deadlock_idx ON inline_deadlock USING bm25(body) WITH (text_config='english', compaction='inline'); SQL mkfifo "${fifo}" PGAPPNAME=pgts-inline-writer \ $PSQL <"${fifo}" >"${writer_log}" 2>&1 & writer_pid=$! exec 3>"${fifo}" printf '%s\n' \ "BEGIN;" \ "SET LOCAL pg_textsearch.memtable_pages_threshold = 0;" \ "UPDATE inline_deadlock SET body = body || ' first' WHERE id = 1;" \ "SELECT 'writer-ready';" >&3 wait_for_file_text "${writer_log}" "writer-ready" "writer setup" PGAPPNAME=pgts-inline-reindex \ $PSQL -c "REINDEX INDEX CONCURRENTLY inline_deadlock_idx;" \ >"${reindex_log}" 2>&1 & reindex_pid=$! wait_for_reindex_wait pgts-inline-reindex inline_deadlock_idx printf '%s\n' \ "SET LOCAL pg_textsearch.memtable_pages_threshold = 1;" \ "UPDATE inline_deadlock SET body = body || ' second' WHERE id = 1;" \ "SELECT 'writer-finished';" \ "COMMIT;" >&3 exec 3>&- wait "${writer_pid}" || { cat "${writer_log}" cat "${reindex_log}" fail "writer aborted while REINDEX CONCURRENTLY held maintenance admission" } wait "${reindex_pid}" || { cat "${writer_log}" cat "${reindex_log}" fail "REINDEX CONCURRENTLY aborted during spill-time compaction" } grep -Fq "writer-finished" "${writer_log}" || fail "writer did not complete its spill-triggering update" if grep -qi "deadlock detected" "${writer_log}" "${reindex_log}"; then fail "spill-time inline compaction deadlocked with REINDEX CONCURRENTLY" fi $PSQL -c "DROP TABLE inline_deadlock CASCADE;" >/dev/null } test_full_l0_compacts_then_spills() { local chain_records local level_counts local ranked_count if skip_without_injection_points \ "automatic spill at a full but compactable L0"; then return fi log "Testing automatic spill at a full but compactable L0" $PSQL <<'SQL' >/dev/null CREATE TABLE inline_capacity (id integer PRIMARY KEY, body text NOT NULL); CREATE INDEX inline_capacity_idx ON inline_capacity USING bm25(body) WITH (text_config='english', compaction='manual'); SELECT pg_textsearch_test_attach_segment_limit(2); SET pg_textsearch.segments_per_level = 2; INSERT INTO inline_capacity VALUES (1, 'capacity alpha one'); SELECT bm25_spill_index('inline_capacity_idx'); INSERT INTO inline_capacity VALUES (2, 'capacity alpha two'); SELECT bm25_spill_index('inline_capacity_idx'); ALTER INDEX inline_capacity_idx SET (compaction='inline'); SET pg_textsearch.memtable_pages_threshold = 1; INSERT INTO inline_capacity VALUES (3, 'capacity alpha three'); SQL chain_records=$($PSQL -c " SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('inline_capacity_idx');") if [ "${chain_records:-0}" -ne 0 ]; then fail "full-L0 spill did not drain the chain after making room" fi level_counts=$($PSQL -c " SELECT bm25_level_counts('inline_capacity_idx'::regclass);") if [ "${level_counts}" != "{0,1,0,0,0,0,0,0}" ]; then fail "full-L0 spill did not consolidate the resulting segments" fi ranked_count=$($PSQL -c " SELECT count(*) FROM ( SELECT 1 FROM inline_capacity ORDER BY body <@> to_bm25query( 'alpha', 'inline_capacity_idx') ) ranked;") if [ "${ranked_count}" -ne 3 ]; then fail "full-L0 spill lost indexed documents" fi $PSQL <<'SQL' >/dev/null RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.segments_per_level; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); DROP TABLE inline_capacity CASCADE; SQL } # # VACUUM identifies dead documents from published segments alone, so # its phase-1 spill must leave the chain empty even when level 0 is at # its segment-count limit. A spill deferred back to the caller strands # the dead records in the chain, where they survive the vacuum and can # be scored against a recycled ctid. # test_vacuum_drains_chain_at_full_l0() { local chain_records local level_counts local ranked_count if skip_without_injection_points \ "VACUUM draining the memtable chain at a full L0"; then return fi log "Testing VACUUM drains the memtable chain at a full L0" $PSQL <<'SQL' >/dev/null CREATE TABLE vacuum_capacity (id integer PRIMARY KEY, body text NOT NULL); CREATE INDEX vacuum_capacity_idx ON vacuum_capacity USING bm25(body) WITH (text_config='english', compaction='manual'); SELECT pg_textsearch_test_attach_segment_limit(2); SET pg_textsearch.segments_per_level = 2; INSERT INTO vacuum_capacity VALUES (1, 'capacity alpha one'); SELECT bm25_spill_index('vacuum_capacity_idx'); INSERT INTO vacuum_capacity VALUES (2, 'capacity alpha two'); SELECT bm25_spill_index('vacuum_capacity_idx'); ALTER INDEX vacuum_capacity_idx SET (compaction='inline'); INSERT INTO vacuum_capacity VALUES (3, 'capacity victimtoken three'); INSERT INTO vacuum_capacity VALUES (4, 'capacity alpha four'); DELETE FROM vacuum_capacity WHERE id = 3; VACUUM vacuum_capacity; SQL chain_records=$($PSQL -c " SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('vacuum_capacity_idx');") if [ "${chain_records:-0}" -ne 0 ]; then fail "VACUUM left dead records in the memtable chain at a full L0" fi level_counts=$($PSQL -c " SELECT bm25_level_counts('vacuum_capacity_idx'::regclass);") if [ "${level_counts}" != "{0,1,0,0,0,0,0,0}" ]; then fail "VACUUM did not consolidate the resulting segments" fi ranked_count=$($PSQL -c " SELECT count(*) FROM ( SELECT 1 FROM vacuum_capacity ORDER BY body <@> to_bm25query( 'alpha', 'vacuum_capacity_idx') ) ranked;") if [ "${ranked_count}" -ne 3 ]; then fail "VACUUM spill lost indexed documents" fi $PSQL <<'SQL' >/dev/null RESET pg_textsearch.segments_per_level; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); DROP TABLE vacuum_capacity CASCADE; SQL } test_irreducible_full_l0_errors() { local mode="$1" local error_log="${CLIENT_DIR}/irreducible_${mode}.log" if skip_without_injection_points \ "irreducible full L0 fail-closed test (${mode})"; then return fi log "Testing an irreducible full L0 remains fail-closed (${mode})" if $PSQL -v mode="${mode}" >"${error_log}" 2>&1 <<'SQL' CREATE TABLE inline_irreducible ( id integer PRIMARY KEY, body text NOT NULL); CREATE INDEX inline_irreducible_idx ON inline_irreducible USING bm25(body) WITH (text_config='english', compaction='manual'); SET pg_textsearch.max_segment_size = '1MB'; SELECT pg_textsearch_test_attach_segment_limit(2); SET pg_textsearch.segments_per_level = 2; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..1 LOOP INSERT INTO inline_irreducible SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 32) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('inline_irreducible_idx'); END LOOP; END $$; -- Background mode needs pg_durable to pass ALTER INDEX admission; the -- spill path only reads the relopt, so set it directly. UPDATE pg_catalog.pg_class SET reloptions = array_replace(reloptions, 'compaction=manual', 'compaction=' || :'mode') WHERE oid = 'inline_irreducible_idx'::regclass; SET pg_textsearch.memtable_pages_threshold = 1; INSERT INTO inline_irreducible VALUES (5000, 'irreducible capacity trigger'); SQL then fail "irreducible full L0 (${mode}) silently deferred the spill" fi if ! grep -Fq "bm25 segment count limit reached at level 0" \ "${error_log}"; then cat "${error_log}" fail "irreducible full L0 (${mode}) failed for an unexpected reason" fi $PSQL <<'SQL' >/dev/null SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); DROP TABLE inline_irreducible CASCADE; SQL } test_managed_reindex_admission() { local fifo="${CLIENT_DIR}/managed_holder.fifo" local holder_log="${CLIENT_DIR}/managed_holder.log" local managed_log="${CLIENT_DIR}/managed_step.log" local reindex_log="${CLIENT_DIR}/managed_reindex.log" local holder_pid local managed_pid local reindex_pid local deadline local target local target_index_oid local target_db_oid local target_spc_oid local target_relfilenumber local target_owner_oid log "Testing managed compaction declines REINDEX maintenance admission" $PSQL <<'SQL' >/dev/null CREATE TABLE managed_reindex ( id integer PRIMARY KEY, body text NOT NULL); INSERT INTO managed_reindex VALUES (1, 'managed alpha document'); CREATE INDEX managed_reindex_idx ON managed_reindex USING bm25(body) WITH (text_config='english', compaction='manual'); UPDATE pg_catalog.pg_class SET reloptions = reloptions || ARRAY['compaction=background'] WHERE oid = 'managed_reindex_idx'::regclass; SQL target=$($PSQL -F '|' -c " SELECT c.oid, d.oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace), pg_relation_filenode(c.oid), c.relowner FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'managed_reindex_idx'::regclass;") IFS='|' read -r target_index_oid target_db_oid target_spc_oid \ target_relfilenumber target_owner_oid <<<"${target}" mkfifo "${fifo}" PGAPPNAME=pgts-managed-holder \ $PSQL <"${fifo}" >"${holder_log}" 2>&1 & holder_pid=$! exec 3>"${fifo}" printf '%s\n' \ "BEGIN;" \ "UPDATE managed_reindex SET body = body || ' held' WHERE id = 1;" \ "SELECT 'holder-ready';" >&3 wait_for_file_text "${holder_log}" "holder-ready" "managed holder" PGAPPNAME=pgts-managed-reindex \ $PSQL -c "REINDEX INDEX CONCURRENTLY managed_reindex_idx;" \ >"${reindex_log}" 2>&1 & reindex_pid=$! wait_for_reindex_wait pgts-managed-reindex managed_reindex_idx PGAPPNAME=pgts-managed-step $PSQL -c " SELECT bm25_compact_step_if_current( ${target_index_oid}::oid, ${target_db_oid}::oid, ${target_spc_oid}::oid, ${target_relfilenumber}::oid, ${target_owner_oid}::oid);" >"${managed_log}" 2>&1 & managed_pid=$! deadline=$((SECONDS + 3)) while kill -0 "${managed_pid}" 2>/dev/null && ((SECONDS < deadline)); do sleep 0.05 done if kill -0 "${managed_pid}" 2>/dev/null; then fail "managed compaction blocked behind REINDEX maintenance admission" fi wait "${managed_pid}" || { cat "${managed_log}" fail "managed compaction errored while maintenance was unavailable" } if ! grep -Fxq "f" "${managed_log}"; then cat "${managed_log}" fail "managed compaction did not report deferred work" fi printf '%s\n' "COMMIT;" >&3 exec 3>&- wait "${holder_pid}" || { cat "${holder_log}" fail "managed holder exited with an error" } wait "${reindex_pid}" || { cat "${reindex_log}" fail "managed REINDEX CONCURRENTLY exited with an error" } $PSQL -c "DROP TABLE managed_reindex CASCADE;" >/dev/null } # # REINDEX CONCURRENTLY never takes the maintenance lock, so the public # compaction entry points run to completion inside the transaction it # is waiting on. # test_public_compaction_reindex_admission() { local fifo="${CLIENT_DIR}/public.fifo" local caller_log="${CLIENT_DIR}/public_caller.log" local reindex_log="${CLIENT_DIR}/public_reindex.log" local caller_pid local reindex_pid log "Testing public compaction proceeds alongside REINDEX CONCURRENTLY" $PSQL <<'SQL' >/dev/null CREATE TABLE public_compact (id integer PRIMARY KEY, body text NOT NULL); INSERT INTO public_compact VALUES (1, 'public alpha document'); CREATE INDEX public_compact_idx ON public_compact USING bm25(body) WITH (text_config='english', compaction='manual'); SQL mkfifo "${fifo}" PGAPPNAME=pgts-public-caller \ $PSQL <"${fifo}" >"${caller_log}" 2>&1 & caller_pid=$! exec 3>"${fifo}" printf '%s\n' \ "BEGIN;" \ "UPDATE public_compact SET body = body || ' held' WHERE id = 1;" \ "SELECT 'caller-ready';" >&3 wait_for_file_text "${caller_log}" "caller-ready" "public caller" PGAPPNAME=pgts-public-reindex \ $PSQL -c "REINDEX INDEX CONCURRENTLY public_compact_idx;" \ >"${reindex_log}" 2>&1 & reindex_pid=$! wait_for_reindex_wait pgts-public-reindex public_compact_idx printf '%s\n' \ "DO \$\$ BEGIN PERFORM bm25_compact('public_compact_idx'::regclass); RAISE NOTICE 'compact-ran'; EXCEPTION WHEN lock_not_available THEN RAISE NOTICE 'compact-declined'; END \$\$;" \ "DO \$\$ BEGIN PERFORM bm25_compact_step('public_compact_idx'::regclass); RAISE NOTICE 'step-ran'; EXCEPTION WHEN lock_not_available THEN RAISE NOTICE 'step-declined'; END \$\$;" \ "DO \$\$ BEGIN PERFORM bm25_force_merge('public_compact_idx'); RAISE NOTICE 'merge-ran'; EXCEPTION WHEN lock_not_available THEN RAISE NOTICE 'merge-declined'; END \$\$;" \ "SELECT 'caller-finished';" \ "COMMIT;" >&3 exec 3>&- wait "${caller_pid}" || { cat "${caller_log}" cat "${reindex_log}" fail "public compaction blocked behind REINDEX maintenance admission" } wait "${reindex_pid}" || { cat "${caller_log}" cat "${reindex_log}" fail "REINDEX CONCURRENTLY aborted during public compaction" } if grep -qi "deadlock detected" "${caller_log}" "${reindex_log}"; then fail "public compaction deadlocked with REINDEX CONCURRENTLY" fi # # The entry points neither block nor decline, so each reports that # it ran. # for marker in compact-ran step-ran merge-ran; do grep -Fq "${marker}" "${caller_log}" || { cat "${caller_log}" fail "public compaction did not report ${marker}" } done $PSQL -c "DROP TABLE public_compact CASCADE;" >/dev/null } setup_test_db case "${TEST_CASE}" in reindex) test_reindex_deadlock ;; capacity) test_full_l0_compacts_then_spills ;; vacuum) test_vacuum_drains_chain_at_full_l0 ;; irreducible) test_irreducible_full_l0_errors inline test_irreducible_full_l0_errors background ;; injection) test_full_l0_compacts_then_spills test_vacuum_drains_chain_at_full_l0 test_irreducible_full_l0_errors inline test_irreducible_full_l0_errors background ;; managed) test_managed_reindex_admission ;; public) test_public_compaction_reindex_admission ;; all) test_reindex_deadlock test_full_l0_compacts_then_spills test_vacuum_drains_chain_at_full_l0 test_irreducible_full_l0_errors inline test_irreducible_full_l0_errors background test_managed_reindex_admission test_public_compaction_reindex_admission ;; *) fail "unknown test case: ${TEST_CASE}" ;; esac log "Inline compaction locking tests passed" timescale-pg_textsearch-b63fe58/test/scripts/logical_replication.sh000077500000000000000000000272221525762047700257760ustar00rootroot00000000000000#!/bin/bash # # Logical replication test for pg_textsearch # # Tests: # 1. Initial table sync copies data; index built on subscriber works # 2. Ongoing INSERTs replicate and become searchable # 3. DELETEs replicate and are removed from search results # 4. UPDATEs replicate and search reflects new content # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PUB_PORT=55438 SUB_PORT=55439 TEST_DB=logical_repl_test PUB_DIR="${SCRIPT_DIR}/../tmp_logical_pub" SUB_DIR="${SCRIPT_DIR}/../tmp_logical_sub" PUB_LOG="${PUB_DIR}/postgres.log" SUB_LOG="${SUB_DIR}/postgres.log" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}" } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}" } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}" exit 1 } cleanup() { local exit_code=$? log "Cleaning up test environment..." if [ $exit_code -ne 0 ]; then warn "Test failed - dumping logs:" if [ -f "${PUB_DIR}/log/postgres.log" ]; then echo "=== PUBLISHER LOG (last 80 lines) ===" tail -80 "${PUB_DIR}/log/postgres.log" 2>/dev/null || true fi if [ -f "${SUB_DIR}/log/postgres.log" ]; then echo "=== SUBSCRIBER LOG (last 80 lines) ===" tail -80 "${SUB_DIR}/log/postgres.log" 2>/dev/null || true fi fi for dir in "${SUB_DIR}" "${PUB_DIR}"; do if [ -f "${dir}/postmaster.pid" ]; then pg_ctl stop -D "${dir}" -m fast 2>/dev/null || pg_ctl stop -D "${dir}" -m immediate 2>/dev/null \ || true fi done rm -rf "${PUB_DIR}" "${SUB_DIR}" exit $exit_code } trap cleanup EXIT INT TERM pub_sql() { psql -p "${PUB_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } pub_sql_quiet() { psql -p "${PUB_PORT}" -d "${TEST_DB}" -tA -c "$1" 2>/dev/null } sub_sql() { psql -p "${SUB_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } sub_sql_quiet() { psql -p "${SUB_PORT}" -d "${TEST_DB}" -tA -c "$1" 2>/dev/null } search_count() { local port=$1 local query=$2 local index=$3 psql -p "${port}" -d "${TEST_DB}" -tA -c " SELECT count(*) FROM ( SELECT id FROM docs WHERE content <@> to_bm25query('${query}', '${index}') < 0 ORDER BY content <@> to_bm25query('${query}', '${index}') LIMIT 100 ) t;" 2>/dev/null } # Wait for subscriber to sync up to publisher's current state wait_for_sync() { local max_wait=${1:-15} local i=0 log "Waiting for logical replication sync (max ${max_wait}s)..." while [ $i -lt $max_wait ]; do # Check subscription state local state state=$(sub_sql_quiet \ "SELECT srsubstate FROM pg_subscription_rel LIMIT 1;") local pub_count pub_count=$(pub_sql_quiet "SELECT count(*) FROM docs;") local sub_count sub_count=$(sub_sql_quiet "SELECT count(*) FROM docs;") if [ "$pub_count" = "$sub_count" ] && [ -n "$pub_count" ]; then log "Subscriber synced: ${sub_count} rows" return 0 fi sleep 1 i=$((i + 1)) done warn "Subscriber may not have synced after ${max_wait}s" warn " Publisher rows: $(pub_sql_quiet 'SELECT count(*) FROM docs;')" warn " Subscriber rows: $(sub_sql_quiet 'SELECT count(*) FROM docs;')" return 1 } setup_publisher() { log "Setting up publisher on port ${PUB_PORT}..." rm -rf "${PUB_DIR}" mkdir -p "${PUB_DIR}" initdb -D "${PUB_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >> "${PUB_DIR}/postgresql.conf" << EOF port = ${PUB_PORT} shared_buffers = 128MB max_connections = 20 log_min_messages = notice logging_collector = on log_filename = 'postgres.log' shared_preload_libraries = 'pg_textsearch' wal_level = logical max_replication_slots = 5 max_wal_senders = 5 EOF echo "local replication all trust" >> "${PUB_DIR}/pg_hba.conf" echo "host replication all 127.0.0.1/32 trust" >> \ "${PUB_DIR}/pg_hba.conf" echo "host all all 127.0.0.1/32 trust" >> \ "${PUB_DIR}/pg_hba.conf" pg_ctl start -D "${PUB_DIR}" -l "${PUB_LOG}" -w createdb -p "${PUB_PORT}" "${TEST_DB}" psql -p "${PUB_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null } setup_subscriber() { log "Setting up subscriber on port ${SUB_PORT}..." rm -rf "${SUB_DIR}" mkdir -p "${SUB_DIR}" initdb -D "${SUB_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >> "${SUB_DIR}/postgresql.conf" << EOF port = ${SUB_PORT} shared_buffers = 128MB max_connections = 20 log_min_messages = notice logging_collector = on log_filename = 'postgres.log' shared_preload_libraries = 'pg_textsearch' EOF pg_ctl start -D "${SUB_DIR}" -l "${SUB_LOG}" -w createdb -p "${SUB_PORT}" "${TEST_DB}" psql -p "${SUB_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null } # --------------------------------------------------------------- # Test 1: Initial sync + index on subscriber # --------------------------------------------------------------- test_initial_sync() { log "=== Test 1: Initial sync and subscriber index ===" # Create table and data on publisher log "Creating table and data on publisher..." pub_sql " CREATE TABLE docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); " >/dev/null pub_sql " INSERT INTO docs (content) VALUES ('database systems and query optimization'), ('full text search algorithms'), ('postgresql extension development guide'), ('bm25 ranking algorithm implementation'), ('distributed systems architecture'), ('machine learning algorithms overview'), ('natural language processing methods'), ('search engine optimization techniques'), ('cloud computing infrastructure'), ('microservices design patterns'); " >/dev/null # Create publication pub_sql "CREATE PUBLICATION docs_pub FOR TABLE docs;" >/dev/null # Create matching table on subscriber (no data yet) sub_sql " CREATE TABLE docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); " >/dev/null # Create subscription (this triggers initial sync) sub_sql " CREATE SUBSCRIPTION docs_sub CONNECTION 'host=127.0.0.1 port=${PUB_PORT} dbname=${TEST_DB}' PUBLICATION docs_pub; " wait_for_sync # Now create bm25 index on subscriber log "Creating bm25 index on subscriber..." sub_sql " CREATE INDEX docs_bm25_idx ON docs USING bm25(content) WITH (text_config='english'); " # Also create bm25 index on publisher for comparison pub_sql " CREATE INDEX docs_bm25_idx ON docs USING bm25(content) WITH (text_config='english'); " >/dev/null # Compare search results local pub_count pub_count=$(search_count "${PUB_PORT}" "algorithms" "docs_bm25_idx") local sub_count sub_count=$(search_count "${SUB_PORT}" "algorithms" "docs_bm25_idx") log "Publisher search 'algorithms': ${pub_count} results" log "Subscriber search 'algorithms': ${sub_count} results" if [ -z "$sub_count" ] || [ "$sub_count" -lt 1 ]; then error "Subscriber search returned no results!" fi if [ "$sub_count" != "$pub_count" ]; then error "Result count mismatch: pub=${pub_count} sub=${sub_count}" fi log "Test 1 PASSED: Initial sync and subscriber index work" } # --------------------------------------------------------------- # Test 2: Ongoing INSERTs replicate and become searchable # --------------------------------------------------------------- test_insert_replication() { log "=== Test 2: INSERT replication ===" log "Inserting new documents on publisher..." pub_sql " INSERT INTO docs (content) VALUES ('advanced algorithms for sorting and searching'), ('graph algorithms and network analysis'), ('parallel algorithms on modern hardware'); " >/dev/null wait_for_sync local pub_count pub_count=$(search_count "${PUB_PORT}" "algorithms" "docs_bm25_idx") local sub_count sub_count=$(search_count "${SUB_PORT}" "algorithms" "docs_bm25_idx") log "Publisher 'algorithms' after inserts: ${pub_count}" log "Subscriber 'algorithms' after inserts: ${sub_count}" if [ "$sub_count" != "$pub_count" ]; then error "Result mismatch after inserts: pub=${pub_count} \ sub=${sub_count}" fi log "Test 2 PASSED: INSERT replication works" } # --------------------------------------------------------------- # Test 3: DELETEs replicate and are removed from search # --------------------------------------------------------------- test_delete_replication() { log "=== Test 3: DELETE replication ===" local pre_count pre_count=$(search_count "${SUB_PORT}" "algorithms" "docs_bm25_idx") log "Subscriber 'algorithms' before delete: ${pre_count}" # Delete one of the algorithm documents pub_sql "DELETE FROM docs WHERE id = 2;" >/dev/null wait_for_sync local pub_count pub_count=$(search_count "${PUB_PORT}" "algorithms" "docs_bm25_idx") local sub_count sub_count=$(search_count "${SUB_PORT}" "algorithms" "docs_bm25_idx") log "Publisher 'algorithms' after delete: ${pub_count}" log "Subscriber 'algorithms' after delete: ${sub_count}" if [ "$sub_count" != "$pub_count" ]; then error "Result mismatch after delete: pub=${pub_count} \ sub=${sub_count}" fi # Verify the count decreased if [ "$sub_count" -ge "$pre_count" ]; then error "DELETE not reflected in search results \ (before: ${pre_count}, after: ${sub_count})" fi log "Test 3 PASSED: DELETE replication works" } # --------------------------------------------------------------- # Test 4: UPDATEs replicate and search reflects new content # --------------------------------------------------------------- test_update_replication() { log "=== Test 4: UPDATE replication ===" # Update a document to contain 'algorithms' log "Updating document on publisher..." pub_sql " UPDATE docs SET content = 'updated document about algorithms' WHERE id = 1; " >/dev/null wait_for_sync local pub_count pub_count=$(search_count "${PUB_PORT}" "algorithms" "docs_bm25_idx") local sub_count sub_count=$(search_count "${SUB_PORT}" "algorithms" "docs_bm25_idx") log "Publisher 'algorithms' after update: ${pub_count}" log "Subscriber 'algorithms' after update: ${sub_count}" if [ "$sub_count" != "$pub_count" ]; then error "Result mismatch after update: pub=${pub_count} \ sub=${sub_count}" fi # Verify the updated doc appears in search local updated_content updated_content=$(sub_sql_quiet " SELECT content FROM docs WHERE id = 1;") if [ "$updated_content" != "updated document about algorithms" ]; then error "UPDATE not replicated: content='${updated_content}'" fi log "Test 4 PASSED: UPDATE replication works" } main() { log "Starting pg_textsearch logical replication test..." command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" command -v initdb >/dev/null 2>&1 || error "initdb not found" setup_publisher setup_subscriber test_initial_sync test_insert_replication test_delete_replication test_update_replication log "All logical replication tests passed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/memtable_stale_tail.sh000077500000000000000000000115151525762047700257600ustar00rootroot00000000000000#!/bin/bash # # A corrupt metapage tail must fail promptly, not trap writers in a retry loop. # set -euo pipefail PG_CONFIG="${PG_CONFIG:-pg_config}" export PATH="$("${PG_CONFIG}" --bindir):${PATH}" TEST_DIR=$(mktemp -d "${TMPDIR:-/tmp}/pg_textsearch_stale_tail.XXXXXX") DATA_DIR="${TEST_DIR}/data" LOGFILE="${TEST_DIR}/postgres.log" export PGHOST="${TEST_DIR}" export PGPORT=55469 export PGDATABASE=postgres clients=() cleanup() { local status=$? trap - EXIT INT TERM if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate -w >/dev/null fi for pid in "${clients[@]}"; do wait "${pid}" 2>/dev/null || true done if [ "${status}" -ne 0 ]; then for logfile in "${TEST_DIR}"/*.log; do [ -f "${logfile}" ] || continue echo "==> ${logfile} <==" >&2 tail -n 60 "${logfile}" >&2 done fi rm -rf "${TEST_DIR}" exit "${status}" } trap cleanup EXIT INT TERM fail() { echo "ERROR: $*" >&2 exit 1 } sql() { psql -XqAt -v ON_ERROR_STOP=1 "$@" } initdb -D "${DATA_DIR}" -A trust --no-locale >/dev/null cat >>"${DATA_DIR}/postgresql.conf" </dev/null sql <<'SQL' CREATE EXTENSION pg_textsearch; CREATE TABLE docs (id int, body text); CREATE INDEX docs_bm25 ON docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO docs SELECT i, (SELECT string_agg(md5(i::text || ':' || j::text), ' ') FROM generate_series(1, 60) j) FROM generate_series(1, 400) i; SQL relation=$(sql -c "SELECT pg_relation_filepath('docs_bm25')") block_size=$(sql -c "SHOW block_size") checksums=$(sql -c "SHOW data_checksums") pg_ctl stop -D "${DATA_DIR}" -m fast -w >/dev/null if [ "${checksums}" = on ]; then pg_checksums --disable -D "${DATA_DIR}" >/dev/null fi # The frozen v6 prefix is 108 bytes; v7 appends head and tail block numbers. # PostgreSQL's page header is 24 bytes. Only modify our stopped test cluster. python3 - "${DATA_DIR}/${relation}" "${block_size}" <<'PY' import os import struct import sys with open(sys.argv[1], "r+b") as index: index.seek(24 + 108) head, tail = struct.unpack("=II", index.read(8)) assert head != tail and head != 0xFFFFFFFF and tail != 0xFFFFFFFF index.seek(head * int(sys.argv[2]) + 24 + 12) assert struct.unpack("=I", index.read(4))[0] != 0xFFFFFFFF index.seek(24 + 112) index.write(struct.pack("=I", head)) index.flush() os.fsync(index.fileno()) PY pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w >/dev/null queries=( "INSERT INTO docs VALUES (401, 'normal short document')" "INSERT INTO docs SELECT 402, string_agg(md5(j::text), ' ') FROM generate_series(1, 1000) j" "UPDATE docs SET body = 'changed indexed document' WHERE id = 1" ) labels=(regular oversized update) for i in "${!queries[@]}"; do sql -v VERBOSITY=verbose -c \ "SET statement_timeout = '5s'; ${queries[i]}" \ >"${TEST_DIR}/${labels[i]}.log" 2>&1 & clients+=("$!") done # statement_timeout cannot break the old loop while its LWLock is held. deadline=$((SECONDS + 10)) for i in "${!clients[@]}"; do while kill -0 "${clients[i]}" 2>/dev/null; do ((SECONDS < deadline)) || fail "${labels[i]} writer did not reject the stale tail" sleep 0.05 done if wait "${clients[i]}"; then fail "${labels[i]} writer accepted the corrupt tail" fi grep -Fq 'ERROR: XX002:' "${TEST_DIR}/${labels[i]}.log" || fail "${labels[i]} writer did not report index corruption" grep -Fq 'HINT: REINDEX the index.' "${TEST_DIR}/${labels[i]}.log" || fail "${labels[i]} writer did not provide a repair hint" done clients=() [ "$(sql -c 'SELECT count(*) FROM docs')" = 400 ] || fail "failed writes changed the table" # Error cleanup must release locks, and the advertised repair must work. sql -c "SET statement_timeout = '10s'; REINDEX INDEX docs_bm25" for query in "${queries[@]}"; do sql -c "SET statement_timeout = '5s'; ${query}" done [ "$(sql -c 'SELECT count(*) FROM docs')" = 402 ] || fail "writes after REINDEX did not succeed" # Healthy concurrent extensions must still retry, for both record sizes. for terms in 60 1000; do cat >"${TEST_DIR}/insert.sql" <"${TEST_DIR}/concurrent-${terms}.log" 2>&1 done [ "$(sql -c 'SELECT count(*) FROM docs')" = 802 ] || fail "concurrent extensions lost writes" echo "Stale-tail rejection, REINDEX repair, and concurrent extensions passed" timescale-pg_textsearch-b63fe58/test/scripts/mixed_update_query_benchmark_test.sh000077500000000000000000000235021525762047700307360ustar00rootroot00000000000000#!/usr/bin/env bash set -euo pipefail ROOT_DIR="$(cd "$(dirname "$0")/../.." && pwd)" RUNNER="$ROOT_DIR/benchmarks/datasets/msmarco/mixed-update-query/run.sh" fail() { echo "mixed update/query benchmark test failed: $*" >&2 exit 1 } [[ -x "$RUNNER" ]] || fail "runner is missing or not executable" bash -n "$RUNNER" if grep -nE "<<'EOSQL'.*\\|\\|[[:space:]]*$" "$RUNNER"; then fail "heredoc command continuation can leak shell text into SQL" fi grep -Fq -- "-F \$'\\t'" "$RUNNER" || fail "wait-event samples are not emitted as tab-separated data" grep -q "wait_summary.tsv" "$RUNNER" || fail "runner does not persist aggregated wait-event counts" grep -Fq ': >"$OUTPUT_DIR/waits.tsv"' "$RUNNER" || fail "runner does not create the wait sample file before monitoring" if grep -q "env | grep '\\^PG'" "$RUNNER"; then fail "runner can persist PostgreSQL credentials" fi if grep -q 'pg_database=\$PGDATABASE' "$RUNNER"; then fail "runner can persist credentials embedded in PGDATABASE" fi grep -q 'HELPER_SCHEMA="pgts_benchmark_$RUN_SUFFIX"' "$RUNNER" || fail "helper tables are not isolated in a benchmark schema" if grep -q 'HELPER_SCHEMA="pg_' "$RUNNER"; then fail "helper schema uses PostgreSQL's reserved pg_ prefix" fi grep -q "RUN_SUFFIX" "$RUNNER" || fail "helper table names are not unique per run" grep -q "output directory already exists" "$RUNNER" || fail "runner does not atomically claim the output directory" grep -q "READER_APP=.*RUN_SUFFIX" "$RUNNER" || fail "reader application names are not unique per run" grep -q "WRITER_APP=.*RUN_SUFFIX" "$RUNNER" || fail "writer application names are not unique per run" grep -q "wait sampler failed" "$RUNNER" || fail "wait sampler failures are not surfaced" grep -q 'kill -TERM "$RUNNER_PID"' "$RUNNER" || fail "wait sampler failure does not stop the workload" grep -q "wait -n -p" "$RUNNER" || fail "runner does not detect the first workload process failure" if grep -q "if ! wait" "$RUNNER"; then fail "negated wait commands lose the child exit status" fi grep -q "ACTIVE_PID" "$RUNNER" || fail "foreground benchmark phases are not signal-safe" grep -q "finished_role" "$RUNNER" || fail "runner does not preserve the primary workload failure" grep -q "cleanup_workload" "$RUNNER" || fail "runner does not clean up active clients and helper tables" help_output="$("$RUNNER" --help)" grep -q "READ_CLIENTS" <<<"$help_output" || fail "help does not document READ_CLIENTS" grep -q "UPDATE_RATE" <<<"$help_output" || fail "help does not document UPDATE_RATE" grep -q -- "--dry-run" <<<"$help_output" || fail "help does not document --dry-run" dry_run_output="$( TABLE=docs \ ID_COLUMN=id \ TEXT_COLUMN=body \ INDEX=docs_bm25 \ QUERY_TABLE=queries \ QUERY_COLUMN=query_text \ DURATION=42 \ READ_CLIENTS=7 \ UPDATE_RATE=123 \ UPDATE_TARGETS=321 \ OUTPUT_DIR=/tmp/mixed-results \ "$RUNNER" --dry-run )" for expected in \ "table=docs" \ "id_column=id" \ "text_column=body" \ "index=docs_bm25" \ "query_table=queries" \ "query_column=query_text" \ "duration_seconds=42" \ "read_clients=7" \ "update_rate=123" \ "update_targets=321" \ "output_dir=/tmp/mixed-results" do grep -q "^${expected}$" <<<"$dry_run_output" || fail "dry-run output is missing ${expected}" done qualified_output="$( TABLE=search.docs \ ID_COLUMN=id \ TEXT_COLUMN=body \ INDEX=search.docs_bm25 \ QUERY_TABLE=search.queries \ QUERY_COLUMN=query_text \ "$RUNNER" --dry-run )" grep -q '^table=search.docs$' <<<"$qualified_output" || fail "schema-qualified table names are rejected" grep -q '^index=search.docs_bm25$' <<<"$qualified_output" || fail "schema-qualified index names are rejected" invalid_output="$( TABLE='docs; DROP TABLE docs' "$RUNNER" --dry-run 2>&1 )" && fail "unsafe SQL identifier was accepted" grep -q "TABLE must be an optionally schema-qualified SQL name" \ <<<"$invalid_output" || fail "unsafe identifier error was not actionable" arithmetic_marker="/tmp/pgts-benchmark-arithmetic-$$" rm -f "$arithmetic_marker" if READ_CLIENTS="CPU_COUNT[\$(touch $arithmetic_marker)0]" \ "$RUNNER" --dry-run >/dev/null 2>&1 then fail "unsafe numeric input was accepted" fi if [[ -e "$arithmetic_marker" ]]; then rm -f "$arithmetic_marker" fail "READ_CLIENTS was evaluated before validation" fi STUB_TMP="$(mktemp -d)" trap 'rm -rf "$STUB_TMP"' EXIT mkdir "$STUB_TMP/bin" cat >"$STUB_TMP/bin/psql" <<'EOF' #!/usr/bin/env bash set -euo pipefail command_text="" file="" while (($#)); do case "$1" in -f) shift file="${1:-}" ;; -*c*) shift command_text="${1:-}" ;; esac shift || true done if [[ -n "$file" ]]; then echo "SETUP $file" >>"$STUB_LOG" [[ "${STUB_FAIL_SETUP:-0}" != 1 ]] || exit 8 exit 0 fi if [[ -n "$command_text" ]]; then case "$command_text" in *"DROP SCHEMA"*) echo "DROP $command_text" >>"$STUB_LOG" ;; *"SHOW server_version"*) echo "18.6" ;; *"extversion"*) echo "1.5.0" ;; *"count(*) FROM pgts_benchmark_"*) echo "10" ;; *"count(*) FROM "*) echo "1000" ;; esac exit 0 fi input="$(cat)" if [[ "$input" == *"FROM pg_stat_activity"* ]]; then [[ "${STUB_FAIL_SAMPLER:-0}" != 1 ]] || exit 9 printf '2026-01-01 00:00:00+00\tstub\tLWLock\ttapir_index_lock\t\n' elif [[ "$input" == *"current_database()"* ]]; then printf 'stubdb\tstubuser\tlocal\t5432\n' fi EOF cat >"$STUB_TMP/bin/pgbench" <<'EOF' #!/usr/bin/env bash set -euo pipefail trap 'exit 143' TERM log_prefix="" script="" while (($#)); do case "$1" in --log-prefix=*) log_prefix="${1#*=}" ;; -f) shift script="${1:-}" ;; esac shift || true done if [[ -n "$log_prefix" ]]; then printf '0 1 1000 0 0 0\n' >"$log_prefix.$$" fi if [[ "${STUB_FAIL_WRITER:-0}" == 1 && "${PGAPPNAME:-}" == pgts-writer-* ]] then exit 7 fi if [[ "${STUB_FAIL_WRITER:-0}" == 1 && "$log_prefix" == *mixed_reader_pgbench ]] then sleep 5 fi if [[ "${STUB_FAIL_SAMPLER:-0}" == 1 && "$log_prefix" == *mixed_*_pgbench ]] then sleep 5 fi if [[ "${STUB_LONG_WARMUP:-0}" == 1 && -z "$log_prefix" ]]; then sleep 5 fi printf '%s\n' \ "number of transactions actually processed: 10" \ "latency average = 1.000 ms" \ "tps = 10.000 (without initial connection time)" EOF chmod +x "$STUB_TMP/bin/psql" "$STUB_TMP/bin/pgbench" run_stubbed() { env \ PATH="$STUB_TMP/bin:$PATH" \ STUB_LOG="$STUB_TMP/psql.log" \ USER=stubuser \ PGDATABASE='postgresql://stubuser:secret@stubhost/stubdb' \ TABLE=search.docs \ ID_COLUMN=id \ TEXT_COLUMN=body \ INDEX=search.docs_bm25 \ QUERY_TABLE=search.queries \ QUERY_COLUMN=query_text \ DURATION=1 \ WARMUP=1 \ READ_CLIENTS=2 \ READ_THREADS=2 \ UPDATE_RATE=10 \ UPDATE_TARGETS=10 \ "$@" \ "$RUNNER" } stub_output="$STUB_TMP/results" run_stubbed OUTPUT_DIR="$stub_output" >"$STUB_TMP/normal.out" grep -q '^database=stubdb$' "$stub_output/metadata.env" || fail "metadata does not record the resolved database" if grep -q 'secret\|postgresql://' "$stub_output/metadata.env"; then fail "metadata persists credentials from PGDATABASE" fi for setting in table index duration_seconds read_clients update_rate; do grep -q "^${setting}=" "$stub_output/metadata.env" || fail "metadata omits $setting" done find "$stub_output" -name 'mixed_writer_pgbench.*' -print -quit | grep -q . || fail "writer transaction logs are not preserved" grep -q $'^samples\tapplication_name\twait_event_type\twait_event$' \ "$stub_output/wait_summary.tsv" || fail "wait summary is not headered TSV" grep -q '^DROP DROP SCHEMA' "$STUB_TMP/psql.log" || fail "successful run does not remove its helper schema" if run_stubbed OUTPUT_DIR="$stub_output" >"$STUB_TMP/collision.out" 2>&1 then fail "existing output directory was reused" fi grep -q 'output directory already exists' "$STUB_TMP/collision.out" || fail "output collision error is not actionable" : >"$STUB_TMP/psql.log" if run_stubbed STUB_FAIL_SETUP=1 OUTPUT_DIR="$STUB_TMP/setup-failure" \ >"$STUB_TMP/setup-failure.out" 2>&1 then fail "setup failure was ignored" fi if grep -q '^DROP ' "$STUB_TMP/psql.log"; then fail "runner drops a helper schema it did not successfully create" fi : >"$STUB_TMP/psql.log" if run_stubbed STUB_FAIL_WRITER=1 OUTPUT_DIR="$STUB_TMP/writer-failure" \ >"$STUB_TMP/writer-failure.out" 2>&1 then fail "writer failure was ignored" fi grep -q 'writer pgbench failed' "$STUB_TMP/writer-failure.out" || fail "terminated reader obscures the primary writer failure" grep -q '^DROP ' "$STUB_TMP/psql.log" || fail "writer failure does not clean up the helper schema" : >"$STUB_TMP/psql.log" if run_stubbed STUB_FAIL_SAMPLER=1 OUTPUT_DIR="$STUB_TMP/sampler-failure" \ >"$STUB_TMP/sampler-failure.out" 2>&1 then fail "wait sampler failure was ignored" fi grep -q 'wait sampler failed' "$STUB_TMP/sampler-failure.out" || fail "wait sampler failure is not reported" grep -q '^DROP ' "$STUB_TMP/psql.log" || fail "wait sampler failure does not clean up the helper schema" : >"$STUB_TMP/psql.log" env \ PATH="$STUB_TMP/bin:$PATH" \ STUB_LOG="$STUB_TMP/psql.log" \ STUB_LONG_WARMUP=1 \ USER=stubuser \ PGDATABASE=stubdb \ TABLE=search.docs \ ID_COLUMN=id \ TEXT_COLUMN=body \ INDEX=search.docs_bm25 \ QUERY_TABLE=search.queries \ QUERY_COLUMN=query_text \ DURATION=1 \ WARMUP=1 \ READ_CLIENTS=2 \ READ_THREADS=2 \ UPDATE_RATE=10 \ UPDATE_TARGETS=10 \ OUTPUT_DIR="$STUB_TMP/interrupted" \ "$RUNNER" \ >"$STUB_TMP/interrupted.out" 2>&1 & runner_pid=$! for _ in {1..40}; do [[ -f "$STUB_TMP/interrupted/warmup.log" ]] && break sleep 0.05 done [[ -f "$STUB_TMP/interrupted/warmup.log" ]] || fail "interruption test did not reach the warmup phase" kill -TERM "$runner_pid" if wait "$runner_pid"; then fail "interrupted runner exited successfully" fi grep -q '^DROP ' "$STUB_TMP/psql.log" || fail "signal interruption does not clean up the helper schema" echo "mixed update/query benchmark tests passed" timescale-pg_textsearch-b63fe58/test/scripts/multi_backend_reindex.sh000077500000000000000000002352311525762047700263130ustar00rootroot00000000000000#!/bin/bash # # multi_backend_reindex.sh — regression test for issue #390: # "bug: BM25 index returns stale heap TIDs after heap rewrite" # # Reproduces the multi-backend scenario in which one backend has already # touched the BM25 index (priming its private TpLocalIndexState cache) # when a second backend runs a statement that gives the index a new # relfilenode and rebuilds it from scratch. Several distinct DDL paths # trigger this: # # * ALTER TABLE ... ADD COLUMN ... GENERATED ALWAYS AS (...) STORED # * VACUUM FULL # * CLUSTER
USING # * REINDEX INDEX (index relfilenode only; heap intact) # * TRUNCATE followed by inserts with a different record layout (#515) # # On v1.2.0 (in-memory memtable in DSA) the first backend's # local_state_cache (`src/index/state.c`, `tp_get_local_index_state`) # was never invalidated because pg_textsearch registers no # `CacheRegisterRelcacheCallback`. Subsequent writes from that backend # went to the OLD memtable still living in DSA, and a later spill # wrote a segment of OLD (now stale) heap CTIDs into the NEW index # file. Queries returning those stale CTIDs then failed with: # # ERROR: could not read blocks N..N in file "base//": # read only 0 of 8192 bytes # # As of pg_textsearch 1.3.0 (#374/#375/#392) the memtable lives in # the index relation's own on-disk pages, so the new relfilenode is # automatically empty. The shared-state allocation and its LWLock # must remain stable across rewrites because backend-local wrappers # are intentionally cached for the backend lifetime. This test pins # both properties for every heap-rewrite / index-rebuild path. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" WORKTREE_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" # Allow override but default to a port not used by any other test script. # (cross-checked against test/scripts/*.sh: 55434–55450, 55454–55457, # 55460–55462 are taken; 55458 is free as of this writing.) TEST_PORT="${TEST_PORT:-55458}" TEST_DB=pg_textsearch_reindex_test # Unique per-PID data dir so concurrent invocations (e.g. accidental # parallel make) cannot rm -rf each other's clusters. DATA_DIR="${SCRIPT_DIR}/../tmp_reindex_test_${TEST_PORT}_$$" SOCKET_DIR="${WORKTREE_ROOT}/.s$$" LOGFILE="${DATA_DIR}/postgres.log" # Preserved server-log destination if the test fails — kept outside # DATA_DIR so it survives the cleanup `rm -rf`. CI can `cat` this on # failure for post-mortem. PRESERVED_LOGFILE="${SCRIPT_DIR}/../tmp_reindex_test_${TEST_PORT}_$$_postgres.log" PREPARE_CLIENT_PID= RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } cleanup() { local exit_code=$? # Disarm the trap so a signal during cleanup doesn't re-enter us. trap - EXIT INT TERM log "Cleaning up reindex test environment (exit code: $exit_code)..." stop_prepare_client || true # On failure, preserve postgres.log outside DATA_DIR so CI can # surface it for debugging — DATA_DIR is about to be wiped. if [ "${exit_code}" -ne 0 ] && [ -f "${LOGFILE}" ]; then cp "${LOGFILE}" "${PRESERVED_LOGFILE}" 2>/dev/null || true if [ -f "${PRESERVED_LOGFILE}" ]; then warn "Preserved server log: ${PRESERVED_LOGFILE}" fi fi if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast -w &>/dev/null || pg_ctl stop -D "${DATA_DIR}" -m immediate -w &>/dev/null || true fi rm -rf "${DATA_DIR}" rm -rf "${SOCKET_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up reindex test PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" mkdir -p "${SOCKET_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 # Pin spill thresholds so the only memtable spill in this test is # the one we force via bm25_spill_index() — guards against future # default changes that might cause an implicit spill before the # rewrite, hiding the bug being exercised. # Both GUCs accept 0 as "disable": tp_bulk_load_threshold short- # circuits at src/index/state.c when <= 0, and the page-threshold # auto-spill returns early when == 0 at src/access/build.c. cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 20 max_worker_processes = 8 max_parallel_workers = 4 max_parallel_maintenance_workers = 2 max_prepared_transactions = 10 shared_buffers = 128MB unix_socket_directories = '${SOCKET_DIR}' listen_addresses = '' log_min_messages = warning shared_preload_libraries = 'pg_textsearch' pg_textsearch.bulk_load_threshold = 0 pg_textsearch.memtable_pages_threshold = 0 EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w \ || error "Failed to start PostgreSQL" createdb -h "${SOCKET_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null } run_sql() { psql -X -v ON_ERROR_STOP=1 -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" 2>&1 } run_sql_quiet() { psql -X -v ON_ERROR_STOP=1 -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } # run_sql_value: scalar fetch. Captures stderr (2>&1) so that any # server-side ERROR surfaces in the returned string and downstream # regex checks can detect storage corruption. Numeric callers must # validate the format (e.g. `[[ $val =~ ^[0-9]+$ ]]`) because the # returned value may be an error message rather than a number. run_sql_value() { psql -X -v ON_ERROR_STOP=1 -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -tAc "$1" 2>&1 } run_sql_quiet_bounded() { timeout --signal=TERM --kill-after=2s 20s \ psql -X -v ON_ERROR_STOP=1 -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -q -c "$1" >/dev/null 2>&1 } run_sql_value_bounded() { timeout --signal=TERM --kill-after=2s 20s \ psql -X -v ON_ERROR_STOP=1 -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -tAc "$1" 2>&1 } prepare_client_is_running() { local state [ -n "${PREPARE_CLIENT_PID}" ] || return 1 state=$(ps -o stat= -p "${PREPARE_CLIENT_PID}" 2>/dev/null) || return 1 [[ "${state}" != Z* ]] } wait_for_prepare_client_exit() { local deadline=$((SECONDS + 10)) local client_pid="${PREPARE_CLIENT_PID}" [ -n "${client_pid}" ] || return 0 while prepare_client_is_running; do if [ "${SECONDS}" -ge "${deadline}" ]; then return 1 fi sleep 0.1 done wait "${client_pid}" 2>/dev/null || true PREPARE_CLIENT_PID= } stop_prepare_client() { local client_pid="${PREPARE_CLIENT_PID}" [ -n "${client_pid}" ] || return 0 kill "${client_pid}" 2>/dev/null || true if wait_for_prepare_client_exit; then return 0 fi kill -KILL "${client_pid}" 2>/dev/null || true wait "${client_pid}" 2>/dev/null || true PREPARE_CLIENT_PID= return 1 } wait_for_prepare_marker() { local marker_file="$1" local deadline=$((SECONDS + 10)) while [ ! -f "${marker_file}" ]; do if ! prepare_client_is_running; then return 1 fi if [ "${SECONDS}" -ge "${deadline}" ]; then return 1 fi sleep 0.05 done } # Build a committed index whose old relfilenode has no memtable chain. Each # rollback test then REINDEXes it, appends enough large records to put the new # cache cursor beyond the old file's EOF, and opens the cache through a normal # BM25 query. PostgreSQL restores the old relfilenode at the selected rollback # boundary, so the next normal query must reject the discarded-file cursor. prepare_cache_locator_rollback_fixture() { run_sql_quiet " DROP TABLE IF EXISTS cache_locator_docs CASCADE; CREATE TABLE cache_locator_docs ( id bigint PRIMARY KEY, content text NOT NULL ); INSERT INTO cache_locator_docs SELECT g, 'locatoranchor committed row ' || g FROM generate_series(1, 20) g; CREATE INDEX cache_locator_idx ON cache_locator_docs USING bm25 (content) WITH (text_config='english'); " } cache_locator_rollback_insert_sql() { cat << 'EOF' INSERT INTO cache_locator_docs SELECT 1000 + g, 'locatoranchor rollbackonly payload ' || ( SELECT string_agg('lex' || g || 'x' || s, ' ') FROM generate_series(1, 96) s ) FROM generate_series(1, 256) g; EOF } assert_cache_locator_rollback_output() { local test_name=$1 local output=$2 local baseline=$3 local session_status=$4 local old_filenode new_filenode restored_filenode local old_blocks new_blocks in_tx_hits local in_tx_bytes bytes_before_read bytes_after_read local committed_hits rollback_rows info "${test_name} output: ${output}" if [ "${session_status}" -ne 0 ]; then error "${test_name} failed while reading the restored relfilenode \ (status ${session_status}). Full output:\n${output}" fi old_filenode=$(echo "${output}" | sed -n \ 's/^.*locator_old_filenode=\([0-9]*\).*$/\1/p' | head -1) new_filenode=$(echo "${output}" | sed -n \ 's/^.*locator_new_filenode=\([0-9]*\).*$/\1/p' | head -1) restored_filenode=$(echo "${output}" | sed -n \ 's/^.*locator_restored_filenode=\([0-9]*\).*$/\1/p' | head -1) old_blocks=$(echo "${output}" | sed -n \ 's/^.*locator_old_blocks=\([0-9]*\).*$/\1/p' | head -1) new_blocks=$(echo "${output}" | sed -n \ 's/^.*locator_new_blocks=\([0-9]*\).*$/\1/p' | head -1) in_tx_hits=$(echo "${output}" | sed -n \ 's/^.*locator_in_tx_hits=\([0-9]*\).*$/\1/p' | head -1) in_tx_bytes=$(echo "${output}" | sed -n \ 's/^.*locator_in_tx_bytes=\([0-9]*\).*$/\1/p' | head -1) bytes_before_read=$(echo "${output}" | sed -n \ 's/^.*locator_bytes_before_read=\([0-9]*\).*$/\1/p' | head -1) committed_hits=$(echo "${output}" | sed -n \ 's/^.*locator_committed_hits=\([0-9]*\).*$/\1/p' | head -1) rollback_rows=$(echo "${output}" | sed -n \ 's/^.*locator_rollback_rows=\([0-9]*\).*$/\1/p' | head -1) bytes_after_read=$(echo "${output}" | sed -n \ 's/^.*locator_bytes_after_read=\([0-9]*\).*$/\1/p' | head -1) if ! [[ "${old_filenode}" =~ ^[0-9]+$ ]] \ || ! [[ "${new_filenode}" =~ ^[0-9]+$ ]] \ || ! [[ "${restored_filenode}" =~ ^[0-9]+$ ]] \ || [ "${new_filenode}" = "${old_filenode}" ] \ || [ "${restored_filenode}" != "${old_filenode}" ]; then error "${test_name} did not replace and restore the index file \ ('${old_filenode:-missing}' -> '${new_filenode:-missing}' -> \ '${restored_filenode:-missing}'). Full output:\n${output}" fi if ! [[ "${old_blocks}" =~ ^[0-9]+$ ]] \ || ! [[ "${new_blocks}" =~ ^[0-9]+$ ]] \ || [ "${new_blocks}" -le "${old_blocks}" ]; then error "${test_name} did not extend the discarded relfilenode past \ the restored file (${old_blocks:-missing} -> ${new_blocks:-missing} blocks). \ Full output:\n${output}" fi if [ "${in_tx_hits}" != "276" ]; then error "${test_name} did not build the replacement-file cache through \ the normal BM25 path (hits=${in_tx_hits:-missing}). Full output:\n${output}" fi if ! [[ "${in_tx_bytes}" =~ ^[0-9]+$ ]] \ || ! [[ "${bytes_before_read}" =~ ^[0-9]+$ ]] \ || [ "${in_tx_bytes}" -le "${baseline}" ] \ || [ "${bytes_before_read}" -ne "${in_tx_bytes}" ]; then error "${test_name} did not retain the charged replacement-file \ cache through rollback (${baseline} -> ${in_tx_bytes:-missing} -> \ ${bytes_before_read:-missing}). Full output:\n${output}" fi if [ "${committed_hits}" != "20" ] || [ "${rollback_rows}" != "0" ]; then error "${test_name} returned the wrong restored rows \ (committed=${committed_hits:-missing}, rolled_back=${rollback_rows:-missing}). \ Full output:\n${output}" fi if [ "${bytes_after_read}" != "${baseline}" ]; then error "${test_name} did not drain discarded-file cache accounting \ (${bytes_before_read:-missing} -> ${bytes_after_read:-missing}, \ baseline ${baseline}). Full output:\n${output}" fi if ! echo "${output}" | grep -qF "locator_complete"; then error "${test_name} did not complete the restored-file read. \ Full output:\n${output}" fi } # A top-level abort restores the old index relfilenode after a normal query has # populated the shared cache from the replacement file. run_cache_locator_top_level_rollback_test() { local output session_status baseline log "Test: cache locator survives top-level REINDEX rollback" prepare_cache_locator_rollback_fixture baseline=$(run_sql_value "SELECT bm25_cache_global_estimated_bytes();") set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_cache_enabled = on; SELECT 'locator_old_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_old_blocks=' || pg_relation_size('cache_locator_idx') / current_setting('block_size')::bigint; BEGIN; REINDEX INDEX cache_locator_idx; $(cache_locator_rollback_insert_sql) SELECT 'locator_new_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_new_blocks=' || pg_relation_size('cache_locator_idx') / current_setting('block_size')::bigint; SELECT 'locator_in_tx_hits=' || count(*) FROM ( SELECT id FROM cache_locator_docs ORDER BY content <@> to_bm25query('locatoranchor', 'cache_locator_idx') LIMIT 1000 ) q; SELECT 'locator_in_tx_bytes=' || bm25_cache_global_estimated_bytes(); ROLLBACK; SELECT 'locator_restored_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_bytes_before_read=' || bm25_cache_global_estimated_bytes(); SELECT 'locator_committed_hits=' || count(*) FROM ( SELECT id FROM cache_locator_docs ORDER BY content <@> to_bm25query('locatoranchor', 'cache_locator_idx') LIMIT 1000 ) q; SELECT 'locator_rollback_rows=' || count(*) FROM cache_locator_docs WHERE content LIKE '%rollbackonly%'; SELECT 'locator_bytes_after_read=' || bm25_cache_global_estimated_bytes(); SELECT 'locator_complete'; EOF ) session_status=$? set -e assert_cache_locator_rollback_output \ "Top-level REINDEX rollback" "${output}" "${baseline}" \ "${session_status}" run_sql_quiet "DROP TABLE cache_locator_docs CASCADE;" log "✅ top-level rollback rebuilt the cache from the restored file" } # Rolling back to a savepoint switches the relation back to the old file while # the outer transaction and backend-local wrapper remain active. run_cache_locator_savepoint_rollback_test() { local output session_status baseline log "Test: cache locator survives savepoint REINDEX rollback" prepare_cache_locator_rollback_fixture baseline=$(run_sql_value "SELECT bm25_cache_global_estimated_bytes();") set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_cache_enabled = on; SELECT 'locator_old_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_old_blocks=' || pg_relation_size('cache_locator_idx') / current_setting('block_size')::bigint; BEGIN; SAVEPOINT before_reindex; REINDEX INDEX cache_locator_idx; $(cache_locator_rollback_insert_sql) SELECT 'locator_new_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_new_blocks=' || pg_relation_size('cache_locator_idx') / current_setting('block_size')::bigint; SELECT 'locator_in_tx_hits=' || count(*) FROM ( SELECT id FROM cache_locator_docs ORDER BY content <@> to_bm25query('locatoranchor', 'cache_locator_idx') LIMIT 1000 ) q; SELECT 'locator_in_tx_bytes=' || bm25_cache_global_estimated_bytes(); ROLLBACK TO SAVEPOINT before_reindex; SELECT 'locator_restored_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_bytes_before_read=' || bm25_cache_global_estimated_bytes(); SELECT 'locator_committed_hits=' || count(*) FROM ( SELECT id FROM cache_locator_docs ORDER BY content <@> to_bm25query('locatoranchor', 'cache_locator_idx') LIMIT 1000 ) q; SELECT 'locator_rollback_rows=' || count(*) FROM cache_locator_docs WHERE content LIKE '%rollbackonly%'; SELECT 'locator_bytes_after_read=' || bm25_cache_global_estimated_bytes(); SELECT 'locator_complete'; COMMIT; EOF ) session_status=$? set -e assert_cache_locator_rollback_output \ "Savepoint REINDEX rollback" "${output}" "${baseline}" \ "${session_status}" run_sql_quiet "DROP TABLE cache_locator_docs CASCADE;" log "✅ savepoint rollback rebuilt the cache from the restored file" } # Prepared rollback selects the old relfilenode in a different backend after # the preparing backend has populated shared cache state from the new file. run_cache_locator_prepared_rollback_test() { local prepare_output read_output output session_status baseline local gid prepared_count log "Test: cache locator survives ROLLBACK PREPARED after REINDEX" prepare_cache_locator_rollback_fixture baseline=$(run_sql_value "SELECT bm25_cache_global_estimated_bytes();") gid="cache_locator_reindex_${TEST_PORT}_$$" prepare_output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_cache_enabled = on; SELECT 'locator_old_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_old_blocks=' || pg_relation_size('cache_locator_idx') / current_setting('block_size')::bigint; BEGIN; REINDEX INDEX cache_locator_idx; $(cache_locator_rollback_insert_sql) SELECT 'locator_new_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_new_blocks=' || pg_relation_size('cache_locator_idx') / current_setting('block_size')::bigint; SELECT 'locator_in_tx_hits=' || count(*) FROM ( SELECT id FROM cache_locator_docs ORDER BY content <@> to_bm25query('locatoranchor', 'cache_locator_idx') LIMIT 1000 ) q; SELECT 'locator_in_tx_bytes=' || bm25_cache_global_estimated_bytes(); PREPARE TRANSACTION '${gid}'; EOF ) prepared_count=$(run_sql_value " SELECT count(*) FROM pg_prepared_xacts WHERE gid = '${gid}'; ") if [ "${prepared_count}" != "1" ]; then error "REINDEX cache-locator transaction was not prepared. \ Full output:\n${prepare_output}" fi run_sql_quiet "ROLLBACK PREPARED '${gid}';" set +e read_output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" --set ON_ERROR_STOP=1 2>&1 << 'EOF' \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_cache_enabled = on; SELECT 'locator_restored_filenode=' || pg_relation_filenode('cache_locator_idx'); SELECT 'locator_bytes_before_read=' || bm25_cache_global_estimated_bytes(); SELECT 'locator_committed_hits=' || count(*) FROM ( SELECT id FROM cache_locator_docs ORDER BY content <@> to_bm25query('locatoranchor', 'cache_locator_idx') LIMIT 1000 ) q; SELECT 'locator_rollback_rows=' || count(*) FROM cache_locator_docs WHERE content LIKE '%rollbackonly%'; SELECT 'locator_bytes_after_read=' || bm25_cache_global_estimated_bytes(); SELECT 'locator_complete'; EOF ) session_status=$? set -e output="${prepare_output}"$'\n'"${read_output}" assert_cache_locator_rollback_output \ "ROLLBACK PREPARED after REINDEX" "${output}" "${baseline}" \ "${session_status}" run_sql_quiet "DROP TABLE cache_locator_docs CASCADE;" log "✅ prepared rollback rebuilt the cache from the restored file" } # DROP removes the runtime registry entry immediately. If an initial CREATE # is dropped in a savepoint and that DROP is rolled back, first access must # reconstruct the CREATE ownership from PostgreSQL's restored relation state. # A top-level abort must then remove the reconstructed registry/cache state and # restore the exact cluster-global accounting baseline. run_initial_create_drop_rollback_abort_test() { local output session_status local baseline cache_observation bytes_after index_count log "Test: rolled-back DROP preserves initial-CREATE abort ownership" run_sql_quiet " DROP TABLE IF EXISTS create_drop_abort_docs CASCADE; CREATE TABLE create_drop_abort_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); INSERT INTO create_drop_abort_docs (content) SELECT 'create drop abort seed ' || g FROM generate_series(1, 25) g; " baseline=$(run_sql_value "SELECT bm25_cache_global_estimated_bytes();") set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on BEGIN; CREATE INDEX create_drop_abort_idx ON create_drop_abort_docs USING bm25 (content) WITH (text_config='english'); SAVEPOINT drop_index; DROP INDEX create_drop_abort_idx; ROLLBACK TO drop_index; INSERT INTO create_drop_abort_docs (content) VALUES ('create drop abort charged cache tail'); SELECT 'create_drop_abort_cache=' || result || '|' || estimated_bytes FROM bm25_cache_cold_build('create_drop_abort_idx'); ROLLBACK; SELECT 'create_drop_abort_bytes=' || bm25_cache_global_estimated_bytes(); SELECT 'create_drop_abort_index_count=' || count(*) FROM pg_class WHERE relname = 'create_drop_abort_idx'; EOF ) session_status=$? set -e info "Rolled-back DROP top-level abort output: ${output}" if [ "${session_status}" -ne 0 ]; then error "Rolled-back DROP top-level abort session failed \ (status ${session_status}). Full output:\n${output}" fi cache_observation=$(echo "${output}" | sed -n \ 's/^.*create_drop_abort_cache=\([^[:space:]]*\).*$/\1/p' | head -1) bytes_after=$(echo "${output}" | sed -n \ 's/^.*create_drop_abort_bytes=\([0-9]*\).*$/\1/p' | head -1) index_count=$(echo "${output}" | sed -n \ 's/^.*create_drop_abort_index_count=\([0-9]*\).*$/\1/p' | head -1) if ! [[ "${cache_observation}" =~ ^OK\|[1-9][0-9]*$ ]]; then error "Rolled-back DROP setup did not charge the reconstructed \ initial-CREATE cache (got '${cache_observation:-missing}'). \ Full output:\n${output}" fi if [ "${bytes_after}" != "${baseline}" ]; then error "Top-level abort after rolled-back DROP leaked cache \ accounting (${baseline} -> ${bytes_after:-missing}). \ Full output:\n${output}" fi if [ "${index_count}" != "0" ]; then error "Top-level abort after rolled-back DROP retained the \ initially-created index (count=${index_count:-missing})." fi run_sql_quiet "DROP TABLE create_drop_abort_docs;" log "✅ rolled-back DROP retained initial-CREATE abort ownership" } # A successful REINDEX replaces the relfilenode with an empty memtable chain. # The stable shared state's old page count must not make the first post-REINDEX # page look over threshold and spill prematurely. run_warm_reindex_chain_count_test() { local chain_pages_before chain_pages_after hits log "Test: warm successful REINDEX reseeds chain-page accounting" run_sql_quiet " DROP TABLE IF EXISTS warm_chain_docs CASCADE; CREATE TABLE warm_chain_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); CREATE INDEX warm_chain_docs_bm25 ON warm_chain_docs USING bm25 (content) WITH (text_config='english'); SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO warm_chain_docs (content) SELECT 'warm chain before reindex ' || g || ' ' || repeat('filler ', 4) FROM generate_series(1, 200) g; " chain_pages_before=$(run_sql_value " SELECT count(*) FROM bm25_memtable_chain('warm_chain_docs_bm25'); ") if ! [[ "${chain_pages_before}" =~ ^[0-9]+$ ]] \ || [ "${chain_pages_before}" -le 1 ]; then error "Warm REINDEX setup did not create a multi-page chain \ (pages=${chain_pages_before:-missing})." fi run_sql_quiet "REINDEX INDEX warm_chain_docs_bm25;" run_sql_quiet " SET pg_textsearch.memtable_pages_threshold = 2; INSERT INTO warm_chain_docs (content) VALUES ('warm chain first page after reindex'); " chain_pages_after=$(run_sql_value " SELECT count(*) FROM bm25_memtable_chain('warm_chain_docs_bm25'); ") hits=$(run_sql_value " SELECT count(*) FROM ( SELECT 1 FROM warm_chain_docs ORDER BY content <@> to_bm25query('warm', 'warm_chain_docs_bm25') ) q; ") if [ "${chain_pages_after}" != "1" ]; then error "Successful warm REINDEX reused the old chain-page count; \ the first new page was spilled (before=${chain_pages_before}, \ after=${chain_pages_after:-missing})." fi if [ "${hits}" != "201" ]; then error "Warm REINDEX chain-count test returned ${hits:-missing} \ rows instead of 201." fi run_sql_quiet "DROP TABLE warm_chain_docs CASCADE;" log "✅ warm successful REINDEX used the current relfilenode count" } # If a threshold consumer runs after REINDEX but before transaction abort, it # may reseed against the replacement relfilenode. Rollback must make that # cached count stale again so the restored multi-page chain is recounted. run_warm_reindex_reseed_before_abort_test() { local output session_status local chain_pages_before chain_pages_during chain_pages_after hits log "Test: REINDEX abort invalidates an in-transaction reseed" run_sql_quiet " DROP TABLE IF EXISTS warm_abort_chain_docs CASCADE; CREATE TABLE warm_abort_chain_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); CREATE INDEX warm_abort_chain_docs_bm25 ON warm_abort_chain_docs USING bm25 (content) WITH (text_config='english'); SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO warm_abort_chain_docs (content) SELECT 'warm abort chain before reindex ' || g || ' ' || repeat('filler ', 4) FROM generate_series(1, 200) g; " chain_pages_before=$(run_sql_value " SELECT count(*) FROM bm25_memtable_chain('warm_abort_chain_docs_bm25'); ") if ! [[ "${chain_pages_before}" =~ ^[0-9]+$ ]] \ || [ "${chain_pages_before}" -le 1 ]; then error "Warm REINDEX abort setup did not create a multi-page chain \ (pages=${chain_pages_before:-missing})." fi set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_pages_threshold = ${chain_pages_before}; BEGIN; REINDEX INDEX warm_abort_chain_docs_bm25; INSERT INTO warm_abort_chain_docs (content) VALUES ('warm abort replacement relfilenode reseed'); SELECT 'warm_abort_chain_during=' || count(*) FROM bm25_memtable_chain('warm_abort_chain_docs_bm25'); ROLLBACK; INSERT INTO warm_abort_chain_docs (content) VALUES ('warm abort restored relfilenode threshold'); SELECT 'warm_abort_chain_after=' || count(*) FROM bm25_memtable_chain('warm_abort_chain_docs_bm25'); SELECT 'warm_abort_hits=' || count(*) FROM ( SELECT 1 FROM warm_abort_chain_docs ORDER BY content <@> to_bm25query('abort', 'warm_abort_chain_docs_bm25') ) q; EOF ) session_status=$? set -e info "Warm REINDEX reseed-before-abort output: ${output}" if [ "${session_status}" -ne 0 ]; then error "Warm REINDEX reseed-before-abort session failed \ (status ${session_status}). Full output:\n${output}" fi chain_pages_during=$(echo "${output}" | sed -n \ 's/^.*warm_abort_chain_during=\([0-9]*\).*$/\1/p' | head -1) chain_pages_after=$(echo "${output}" | sed -n \ 's/^.*warm_abort_chain_after=\([0-9]*\).*$/\1/p' | head -1) hits=$(echo "${output}" | sed -n \ 's/^.*warm_abort_hits=\([0-9]*\).*$/\1/p' | head -1) if [ "${chain_pages_during}" != "1" ]; then error "REINDEX replacement chain was not reseeded before abort \ (pages=${chain_pages_during:-missing}). Full output:\n${output}" fi if [ "${chain_pages_after}" != "0" ]; then error "REINDEX abort left the replacement relfilenode count \ attached to the restored ${chain_pages_before}-page chain \ (pages=${chain_pages_after:-missing}). Full output:\n${output}" fi if [ "${hits}" != "201" ]; then error "Warm REINDEX reseed-before-abort test returned \ ${hits:-missing} rows instead of 201." fi run_sql_quiet "DROP TABLE warm_abort_chain_docs CASCADE;" log "✅ REINDEX abort invalidated the replacement relfilenode count" } # A restart clears the shared registry while leaving the index relation on # disk. The first post-restart REINDEX must therefore attach cold state # without claiming initial-CREATE ownership, and transaction abort must leave # that state usable in the same backend. Its page-count heuristic starts at # zero, so rollback must force a lazy recount of the restored nonempty chain # before the next threshold decision. run_cold_registry_reindex_rollback_test() { local output session_status hits chain_pages_before chain_pages_after log "Test: cold REINDEX abort reseeds restored chain-page accounting" run_sql_quiet " DROP TABLE IF EXISTS cold_docs CASCADE; CREATE TABLE cold_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); INSERT INTO cold_docs (content) SELECT 'cold registry rollback ' || g FROM generate_series(1, 225) g; CREATE INDEX cold_docs_bm25 ON cold_docs USING bm25 (content) WITH (text_config='english'); SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO cold_docs (content) SELECT 'cold registry rollback tail ' || g || ' ' || repeat('filler ', 4) FROM generate_series(1, 200) g; " chain_pages_before=$(run_sql_value " SELECT count(*) FROM bm25_memtable_chain('cold_docs_bm25'); ") if ! [[ "${chain_pages_before}" =~ ^[0-9]+$ ]] \ || [ "${chain_pages_before}" -le 1 ]; then error "Cold REINDEX abort setup did not create a multi-page chain \ (pages=${chain_pages_before:-missing})." fi pg_ctl restart -D "${DATA_DIR}" -m fast -l "${LOGFILE}" -w \ >/dev/null 2>&1 || error "Failed to restart PostgreSQL" set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_pages_threshold = 2; BEGIN; REINDEX INDEX cold_docs_bm25; SELECT 'cold_reindex_built_before_abort'; \set ON_ERROR_STOP 0 SELECT 1 / 0; \set ON_ERROR_STOP 1 ROLLBACK; INSERT INTO cold_docs (content) VALUES ('cold registry rollback tail after abort'); SELECT 'cold_reindex_chain_pages=' || count(*) FROM bm25_memtable_chain('cold_docs_bm25'); SELECT 'cold_reindex_hits=' || count(*) FROM ( SELECT 1 FROM cold_docs ORDER BY content <@> to_bm25query('rollback', 'cold_docs_bm25') ) q; EOF ) session_status=$? set -e info "Cold-registry REINDEX session output: ${output}" if [ "${session_status}" -ne 0 ]; then error "Cold-registry REINDEX session failed after rollback \ (status ${session_status}). Full output:\n${output}" fi if ! echo "${output}" | grep -qF "cold_reindex_built_before_abort" \ || ! echo "${output}" | grep -qF "division by zero"; then error "Cold-registry REINDEX rollback was not fully exercised. \ Full output:\n${output}" fi hits=$(echo "${output}" | sed -n \ 's/^.*cold_reindex_hits=\([0-9]*\).*$/\1/p' | head -1) chain_pages_after=$(echo "${output}" | sed -n \ 's/^.*cold_reindex_chain_pages=\([0-9]*\).*$/\1/p' | head -1) if [ "${chain_pages_after}" != "0" ]; then error "Cold REINDEX abort left a zero heuristic attached to the \ restored ${chain_pages_before}-page chain; the next threshold insert \ did not spill (pages=${chain_pages_after:-missing}). \ Full output:\n${output}" fi if [ "${hits}" != "426" ]; then error "Cold-registry REINDEX rollback left the index unusable \ (hits=${hits:-missing}). Full output:\n${output}" fi run_sql_quiet "DROP TABLE cold_docs CASCADE;" log "✅ cold REINDEX abort recounted and spilled the restored chain" } # REINDEX replaces only the current backend's wrapper. Transaction-local # bulk-load accounting accumulated before a savepoint must survive that # replacement and the savepoint rollback so PRE_COMMIT still spills. run_reindex_bulk_counter_preservation_test() { local output session_status local chain_pages_before_commit chain_pages_after_commit hits log "Test: REINDEX wrapper replacement preserves bulk-load counters" run_sql_quiet " DROP TABLE IF EXISTS bulk_counter_docs CASCADE; CREATE TABLE bulk_counter_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); CREATE INDEX bulk_counter_docs_bm25 ON bulk_counter_docs USING bm25 (content) WITH (text_config='english'); " set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 1; BEGIN; INSERT INTO bulk_counter_docs (content) VALUES ('bulk counter survives savepoint reindex rollback'); SAVEPOINT reindex_savepoint; REINDEX INDEX bulk_counter_docs_bm25; SELECT 'bulk_counter_reindex_complete'; ROLLBACK TO reindex_savepoint; SELECT 'bulk_counter_chain_before_commit=' || count(*) FROM bm25_memtable_chain('bulk_counter_docs_bm25'); COMMIT; SELECT 'bulk_counter_chain_after_commit=' || count(*) FROM bm25_memtable_chain('bulk_counter_docs_bm25'); SELECT 'bulk_counter_hits=' || count(*) FROM ( SELECT 1 FROM bulk_counter_docs ORDER BY content <@> to_bm25query('counter', 'bulk_counter_docs_bm25') ) q; EOF ) session_status=$? set -e info "REINDEX bulk-counter session output: ${output}" if [ "${session_status}" -ne 0 ] \ || ! echo "${output}" | grep -qF "bulk_counter_reindex_complete"; then error "REINDEX bulk-counter session did not complete the savepoint \ rollback scenario. Full output:\n${output}" fi chain_pages_before_commit=$(echo "${output}" | sed -n \ 's/^.*bulk_counter_chain_before_commit=\([0-9]*\).*$/\1/p' \ | head -1) chain_pages_after_commit=$(echo "${output}" | sed -n \ 's/^.*bulk_counter_chain_after_commit=\([0-9]*\).*$/\1/p' \ | head -1) hits=$(echo "${output}" | sed -n \ 's/^.*bulk_counter_hits=\([0-9]*\).*$/\1/p' | head -1) if ! [[ "${chain_pages_before_commit}" =~ ^[1-9][0-9]*$ ]]; then error "REINDEX bulk-counter setup did not restore a nonempty chain \ before PRE_COMMIT (pages=${chain_pages_before_commit:-missing})." fi if [ "${chain_pages_after_commit}" != "0" ]; then error "REINDEX wrapper replacement lost terms_added_this_xact; \ bulk_load_threshold=1 did not spill at PRE_COMMIT \ (${chain_pages_before_commit} -> ${chain_pages_after_commit:-missing})." fi if [ "${hits}" != "1" ]; then error "REINDEX bulk-counter test returned ${hits:-missing} rows \ instead of 1." fi run_sql_quiet "DROP TABLE bulk_counter_docs CASCADE;" log "✅ REINDEX wrapper preserved PRE_COMMIT bulk-load accounting" } run_prepare_bulk_state_case() { local case_name="$1" local outcome="$2" local table_name="$3" local index_name="$4" local prepared_term="$5" local controller_term="$6" local gid="pgts_prepare_bulk_${case_name}_${TEST_PORT}_$$" local ready_file="${DATA_DIR}/prepare_bulk_${case_name}.ready" local release_file="${DATA_DIR}/prepare_bulk_${case_name}.release" local output_file="${DATA_DIR}/prepare_bulk_${case_name}.out" local failure= local chain_after_prepare chain_before_unrelated chain_after_unrelated local backend_before backend_after prepared_count local prepared_heap_count prepared_index_count local controller_heap_count controller_index_count local output rm -f "${ready_file}" "${release_file}" "${output_file}" PGAPPNAME="pgts_prepare_bulk_${case_name}" \ timeout --signal=TERM --kill-after=2s 45s \ psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 >"${output_file}" 2>&1 << EOF & \set QUIET on SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 1; SELECT pg_backend_pid() AS prepare_backend \gset BEGIN; INSERT INTO ${table_name} (id, content) VALUES (1, '${prepared_term} prepared payload'); PREPARE TRANSACTION '${gid}'; \echo prepare_backend_before=:prepare_backend \! : > '${ready_file}'; while [ ! -f '${release_file}' ]; do sleep 0.05; done BEGIN; INSERT INTO prepare_bulk_plain (case_name) VALUES ('${case_name}'); SELECT pg_backend_pid() AS unrelated_backend \gset COMMIT; \echo prepare_backend_after=:unrelated_backend \echo prepare_unrelated_commit_complete EOF PREPARE_CLIENT_PID=$! if ! wait_for_prepare_marker "${ready_file}"; then output=$(cat "${output_file}" 2>/dev/null || true) stop_prepare_client || true prepared_count=$(run_sql_value_bounded " SELECT count(*) FROM pg_prepared_xacts WHERE gid = '${gid}'; " || true) if [ "${prepared_count}" = "1" ]; then run_sql_quiet_bounded "ROLLBACK PREPARED '${gid}';" || true fi error "PREPARE bulk-state ${case_name} client did not reach the \ post-PREPARE barrier. Full output:\n${output}" fi chain_after_prepare=$(run_sql_value_bounded " SELECT count(*) FROM bm25_memtable_chain('${index_name}'); ") if [ "${chain_after_prepare}" != "0" ]; then failure="${failure} PREPARE left ${chain_after_prepare} chain pages;" fi run_sql_quiet_bounded " SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO ${table_name} (id, content) VALUES (2, '${controller_term} controller payload'); " chain_before_unrelated=$(run_sql_value_bounded " SELECT count(*) FROM bm25_memtable_chain('${index_name}'); ") if ! [[ "${chain_before_unrelated}" =~ ^[1-9][0-9]*$ ]]; then failure="${failure} controller insert did not create a chain;" fi : >"${release_file}" if ! wait_for_prepare_client_exit; then failure="${failure} persistent client did not exit within 10 seconds;" stop_prepare_client || true fi output=$(cat "${output_file}" 2>/dev/null || true) backend_before=$(echo "${output}" | sed -n \ 's/^prepare_backend_before=\([0-9]*\)$/\1/p' | head -1) backend_after=$(echo "${output}" | sed -n \ 's/^prepare_backend_after=\([0-9]*\)$/\1/p' | head -1) if ! [[ "${backend_before}" =~ ^[0-9]+$ ]] || [ "${backend_before}" != "${backend_after}" ]; then failure="${failure} PREPARE and unrelated transaction used different backends;" fi if ! echo "${output}" | grep -qF "prepare_unrelated_commit_complete"; then failure="${failure} unrelated transaction did not commit;" fi chain_after_unrelated=$(run_sql_value_bounded " SELECT count(*) FROM bm25_memtable_chain('${index_name}'); ") if [ "${chain_after_unrelated}" != "${chain_before_unrelated}" ]; then failure="${failure} unrelated commit changed chain pages \ ${chain_before_unrelated}->${chain_after_unrelated};" fi prepared_count=$(run_sql_value_bounded " SELECT count(*) FROM pg_prepared_xacts WHERE gid = '${gid}'; ") if [ "${prepared_count}" = "1" ]; then if [ "${outcome}" = "commit" ]; then run_sql_quiet_bounded "COMMIT PREPARED '${gid}';" else run_sql_quiet_bounded "ROLLBACK PREPARED '${gid}';" fi else failure="${failure} prepared transaction count was ${prepared_count};" fi prepared_heap_count=$(run_sql_value_bounded " SELECT count(*) FROM ${table_name} WHERE content LIKE '%${prepared_term}%'; ") prepared_index_count=$(run_sql_value_bounded " SELECT count(*) FROM ( SELECT id FROM ${table_name} ORDER BY content <@> to_bm25query('${prepared_term}', '${index_name}') LIMIT 10 ) hits JOIN ${table_name} docs USING (id) WHERE docs.content LIKE '%${prepared_term}%'; ") controller_heap_count=$(run_sql_value_bounded " SELECT count(*) FROM ${table_name} WHERE content LIKE '%${controller_term}%'; ") controller_index_count=$(run_sql_value_bounded " SELECT count(*) FROM ( SELECT id FROM ${table_name} ORDER BY content <@> to_bm25query('${controller_term}', '${index_name}') LIMIT 10 ) hits JOIN ${table_name} docs USING (id) WHERE docs.content LIKE '%${controller_term}%'; ") if [ "${outcome}" = "commit" ]; then if [ "${prepared_heap_count}" != "1" ] || [ "${prepared_index_count}" != "1" ]; then failure="${failure} COMMIT PREPARED lost the prepared row \ (heap=${prepared_heap_count}, index=${prepared_index_count});" fi elif [ "${prepared_heap_count}" != "0" ] || [ "${prepared_index_count}" != "0" ]; then failure="${failure} ROLLBACK PREPARED retained the prepared row \ (heap=${prepared_heap_count}, index=${prepared_index_count});" fi if [ "${controller_heap_count}" != "1" ] || [ "${controller_index_count}" != "1" ]; then failure="${failure} controller row was not preserved \ (heap=${controller_heap_count}, index=${controller_index_count});" fi info "PREPARE bulk-state ${case_name} output: ${output}" if [ -n "${failure}" ]; then warn "PREPARE bulk-state ${case_name} failures:${failure}" return 1 fi log "✅ PREPARE ${outcome} isolated terminal bulk state" } # PREPARE must spill the preparing transaction while errors can still abort # preparation, then clear backend-local counters before that backend starts a # new transaction. A controller insert after the PREPARE barrier leaves a # fresh chain; the unrelated transaction must not spill that other backend's # work. Both two-phase outcomes must retain normal heap/index visibility. run_prepare_bulk_state_reset_test() { local failed=0 log "Test: PREPARE spills and resets backend-local bulk state" run_sql_quiet " DROP TABLE IF EXISTS prepare_bulk_commit_docs CASCADE; DROP TABLE IF EXISTS prepare_bulk_rollback_docs CASCADE; DROP TABLE IF EXISTS prepare_bulk_plain; CREATE TABLE prepare_bulk_commit_docs ( id bigint PRIMARY KEY, content text NOT NULL ); CREATE INDEX prepare_bulk_commit_docs_bm25 ON prepare_bulk_commit_docs USING bm25 (content) WITH (text_config='english'); CREATE TABLE prepare_bulk_rollback_docs ( id bigint PRIMARY KEY, content text NOT NULL ); CREATE INDEX prepare_bulk_rollback_docs_bm25 ON prepare_bulk_rollback_docs USING bm25 (content) WITH (text_config='english'); CREATE TABLE prepare_bulk_plain (case_name text); " if ! run_prepare_bulk_state_case \ commit \ commit \ prepare_bulk_commit_docs \ prepare_bulk_commit_docs_bm25 \ preparedcommitonly \ controllercommitonly; then failed=1 fi if ! run_prepare_bulk_state_case \ rollback \ rollback \ prepare_bulk_rollback_docs \ prepare_bulk_rollback_docs_bm25 \ preparedrollbackonly \ controllerrollbackonly; then failed=1 fi run_sql_quiet " DROP TABLE prepare_bulk_commit_docs CASCADE; DROP TABLE prepare_bulk_rollback_docs CASCADE; DROP TABLE prepare_bulk_plain; " if [ "${failed}" -ne 0 ]; then error "PREPARE bulk-state isolation regression failed" fi } # Initial CREATE ownership lives only in the creating backend, so it cannot be # serialized into a prepared transaction. A normally committed CREATE must # clear that ownership so a later transaction in the same backend can prepare. run_prepare_initial_create_test() { local reject_gid="pg_textsearch_create_reject_${TEST_PORT}_$$" local later_gid="pg_textsearch_after_create_${TEST_PORT}_$$" local output session_status prepared_count local baseline cache_observation bytes_after log "Test: PREPARE rejects reconstructed initial-CREATE ownership" run_sql_quiet " DROP TABLE IF EXISTS prepare_docs CASCADE; DROP TABLE IF EXISTS prepare_plain; CREATE TABLE prepare_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); INSERT INTO prepare_docs (content) SELECT 'prepare create ownership ' || g FROM generate_series(1, 25) g; CREATE TABLE prepare_plain (id int); " baseline=$(run_sql_value "SELECT bm25_cache_global_estimated_bytes();") set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF BEGIN; CREATE INDEX prepare_docs_bm25 ON prepare_docs USING bm25 (content) WITH (text_config='english'); SAVEPOINT drop_index; DROP INDEX prepare_docs_bm25; ROLLBACK TO drop_index; INSERT INTO prepare_docs (content) VALUES ('prepare reconstructed ownership charged cache tail'); SELECT 'prepare_drop_rollback_cache=' || result || '|' || estimated_bytes FROM bm25_cache_cold_build('prepare_docs_bm25'); PREPARE TRANSACTION '${reject_gid}'; EOF ) session_status=$? set -e info "Initial-CREATE PREPARE output: ${output}" cache_observation=$(echo "${output}" | sed -n \ 's/^.*prepare_drop_rollback_cache=\([^[:space:]]*\).*$/\1/p' \ | head -1) if ! [[ "${cache_observation}" =~ ^OK\|[1-9][0-9]*$ ]]; then error "PREPARE setup did not charge the reconstructed \ initial-CREATE cache (got '${cache_observation:-missing}'). \ Full output:\n${output}" fi if [ "${session_status}" -eq 0 ]; then run_sql_quiet "ROLLBACK PREPARED '${reject_gid}';" error "PREPARE TRANSACTION accepted initial-CREATE ownership \ after DROP INDEX rollback; the reproduction was cleaned with \ ROLLBACK PREPARED. Full output:\n${output}" fi if ! echo "${output}" | grep -qF \ "cannot prepare a transaction that created a pg_textsearch index"; then error "PREPARE TRANSACTION failed for an unexpected reason. \ Full output:\n${output}" fi prepared_count=$(run_sql_value " SELECT count(*) FROM pg_prepared_xacts WHERE gid = '${reject_gid}'; ") if [ "${prepared_count}" != "0" ]; then run_sql_quiet "ROLLBACK PREPARED '${reject_gid}';" error "Rejected initial CREATE still left a prepared transaction." fi bytes_after=$(run_sql_value "SELECT bm25_cache_global_estimated_bytes();") if [ "${bytes_after}" != "${baseline}" ]; then error "Rejected PREPARE after rolled-back DROP leaked cache \ accounting (${baseline} -> ${bytes_after})." fi if [ "$(run_sql_value " SELECT count(*) FROM pg_class WHERE relname = 'prepare_docs_bm25'; ")" != "0" ]; then error "Rejected initial CREATE did not roll back its index." fi set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF CREATE INDEX prepare_docs_bm25 ON prepare_docs USING bm25 (content) WITH (text_config='english'); SELECT 'committed_create_complete'; BEGIN; INSERT INTO prepare_plain VALUES (1); PREPARE TRANSACTION '${later_gid}'; SELECT 'later_prepare_succeeded'; EOF ) session_status=$? set -e info "Post-CREATE PREPARE output: ${output}" if [ "${session_status}" -ne 0 ] \ || ! echo "${output}" | grep -qF "committed_create_complete" \ || ! echo "${output}" | grep -qF "later_prepare_succeeded"; then error "Normal CREATE commit left ownership that affected a later \ transaction. Full output:\n${output}" fi prepared_count=$(run_sql_value " SELECT count(*) FROM pg_prepared_xacts WHERE gid = '${later_gid}'; ") if [ "${prepared_count}" != "1" ]; then error "Later transaction was not prepared successfully." fi run_sql_quiet "ROLLBACK PREPARED '${later_gid}';" run_sql_quiet " DROP TABLE prepare_docs CASCADE; DROP TABLE prepare_plain; " log "✅ PREPARE ownership guard and commit clearing passed" } # Verify both REINDEX abort boundaries. A build-time failure must leave the # old cache untouched. If REINDEX finishes and the surrounding transaction # later aborts, the cache may remain discarded, but the stable shared state # must still serve the restored on-disk index without leaking accounting. run_failed_reindex_test() { local output session_status local generation_before generation_after_failure local generation_after_rollback local cache_bytes_before cache_bytes_after_failure local cache_bytes_after_rollback cache_bytes_after_rebuild local hits_after_failure hits_after_rollback log "Test: failed and rolled-back REINDEX preserve stable state" run_sql_quiet " DROP TABLE IF EXISTS docs CASCADE; DROP FUNCTION IF EXISTS maybe_fail_reindex(text); CREATE FUNCTION maybe_fail_reindex(value text) RETURNS text LANGUAGE plpgsql IMMUTABLE PARALLEL UNSAFE AS \$\$ BEGIN IF current_setting( 'tapir_test.reindex_fail', true ) = 'on' THEN RAISE EXCEPTION 'forced reindex failure'; END IF; RETURN value; END \$\$; CREATE TABLE docs ( id bigserial PRIMARY KEY, content text NOT NULL ); INSERT INTO docs (content) SELECT 'row ' || g || ' failure rollback' FROM generate_series(1, 200) g; CREATE INDEX docs_bm25 ON docs USING bm25 (maybe_fail_reindex(content)) WITH (text_config='english'); " set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on INSERT INTO docs (content) SELECT 'tail ' || g || ' failure rollback cached before failed reindex' FROM generate_series(201, 225) g; SELECT 'failed_reindex_cache_bytes_before=' || estimated_bytes FROM bm25_cache_cold_build('docs_bm25'); SELECT 'failed_reindex_generation_before=' || bm25_cache_bump_spill_generation('docs_bm25'); SELECT 'failed_reindex_summary_before=' || replace(bm25_summarize_index('docs_bm25'), E'\n', ' '); \! if psql -X -h "${SOCKET_DIR}" -p ${TEST_PORT} -d "${TEST_DB}" -v ON_ERROR_STOP=1 -q -c "SET tapir_test.reindex_fail = 'on'; REINDEX INDEX docs_bm25;" 2>&1; then echo B-reindex-unexpected-success; else echo B-reindex-failed-as-expected; fi SELECT 'failed_reindex_summary_after=' || replace(bm25_summarize_index('docs_bm25'), E'\n', ' '); SELECT 'failed_reindex_generation_after_failure=' || bm25_cache_bump_spill_generation('docs_bm25'); SELECT 'failed_reindex_cache_bytes_after_failure=' || bm25_cache_global_estimated_bytes(); SELECT 'failed_reindex_hits_after_failure=' || count(*) FROM ( SELECT 1 FROM docs ORDER BY maybe_fail_reindex(content) <@> to_bm25query('rollback', 'docs_bm25') ) q; \! if psql -X -h "${SOCKET_DIR}" -p ${TEST_PORT} -d "${TEST_DB}" -v ON_ERROR_STOP=1 -q -c "BEGIN; REINDEX INDEX docs_bm25; SELECT 'B-reindex-built-before-abort'; SELECT 1 / 0;" 2>&1; then echo B-postbuild-abort-unexpected-success; else echo B-postbuild-abort-observed; fi SELECT 'postbuild_abort_summary=' || replace(bm25_summarize_index('docs_bm25'), E'\n', ' '); SELECT 'failed_reindex_generation_after_rollback=' || bm25_cache_bump_spill_generation('docs_bm25'); SELECT 'failed_reindex_cache_bytes_after_rollback=' || bm25_cache_global_estimated_bytes(); SELECT 'failed_reindex_hits_after_rollback=' || count(*) FROM ( SELECT 1 FROM docs ORDER BY maybe_fail_reindex(content) <@> to_bm25query('rollback', 'docs_bm25') ) q; SELECT 'failed_reindex_cache_bytes_after_rebuild=' || bm25_cache_global_estimated_bytes(); EOF ) session_status=$? set -e info "Failed-REINDEX session output: ${output}" if [ "${session_status}" -ne 0 ]; then error "Backend A failed after the expected REINDEX error \ (status ${session_status}). Full output:\n${output}" fi if ! echo "${output}" | grep -qF "forced reindex failure"; then error "Nested REINDEX did not report the injected failure. \ Full output:\n${output}" fi if ! echo "${output}" | grep -qF "B-reindex-failed-as-expected"; then error "Nested REINDEX failure status was not observed. \ Full output:\n${output}" fi if echo "${output}" | grep -qF "B-reindex-unexpected-success"; then error "Nested REINDEX unexpectedly succeeded. Full output:\n${output}" fi if ! echo "${output}" | grep -qF "B-reindex-built-before-abort" \ || ! echo "${output}" | grep -qF "division by zero" \ || ! echo "${output}" | grep -qF "B-postbuild-abort-observed"; then error "Post-build transaction abort was not exercised fully. \ Full output:\n${output}" fi if echo "${output}" | grep -qF "B-postbuild-abort-unexpected-success"; then error "Post-build transaction unexpectedly committed. \ Full output:\n${output}" fi local marker for marker in \ "failed_reindex_summary_before=" \ "failed_reindex_summary_after=" \ "postbuild_abort_summary="; do if ! echo "${output}" | grep -qF "${marker}"; then error "Failed REINDEX test missed required marker '${marker}'. \ Full output:\n${output}" fi done generation_before=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_generation_before=\([0-9]*\).*$/\1/p' \ | head -1) generation_after_failure=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_generation_after_failure=\([0-9]*\).*$/\1/p' \ | head -1) generation_after_rollback=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_generation_after_rollback=\([0-9]*\).*$/\1/p' \ | head -1) if ! [[ "${generation_before}" =~ ^[0-9]+$ ]] \ || ! [[ "${generation_after_failure}" =~ ^[0-9]+$ ]] \ || ! [[ "${generation_after_rollback}" =~ ^[0-9]+$ ]]; then error "Could not parse failed-REINDEX generations \ ('${generation_before}' -> '${generation_after_failure}' -> \ '${generation_after_rollback}')." fi if [ "${generation_after_failure}" -ne $((generation_before + 1)) ]; then error "Failed REINDEX changed or replaced the pre-existing \ shared state generation (${generation_before} -> \ ${generation_after_failure})." fi if [ "${generation_after_rollback}" -ne \ $((generation_after_failure + 2)) ]; then error "Aborting after REINDEX finalization replaced or discarded \ the pre-existing shared state (${generation_after_failure} -> \ ${generation_after_rollback})." fi cache_bytes_before=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_cache_bytes_before=\([0-9]*\).*$/\1/p' \ | head -1) cache_bytes_after_failure=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_cache_bytes_after_failure=\([0-9]*\).*$/\1/p' \ | head -1) cache_bytes_after_rollback=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_cache_bytes_after_rollback=\([0-9]*\).*$/\1/p' \ | head -1) cache_bytes_after_rebuild=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_cache_bytes_after_rebuild=\([0-9]*\).*$/\1/p' \ | head -1) if ! [[ "${cache_bytes_before}" =~ ^[0-9]+$ ]] \ || ! [[ "${cache_bytes_after_failure}" =~ ^[0-9]+$ ]] \ || ! [[ "${cache_bytes_after_rollback}" =~ ^[0-9]+$ ]] \ || ! [[ "${cache_bytes_after_rebuild}" =~ ^[0-9]+$ ]]; then error "Could not parse failed-REINDEX cache accounting \ ('${cache_bytes_before}', '${cache_bytes_after_failure}', \ '${cache_bytes_after_rollback}', '${cache_bytes_after_rebuild}')." fi if [ "${cache_bytes_before}" -le 0 ]; then error "Failed REINDEX setup did not populate the runtime cache." fi if [ "${cache_bytes_after_failure}" -ne "${cache_bytes_before}" ]; then error "Build-time REINDEX failure changed the existing cache \ accounting (${cache_bytes_before} -> ${cache_bytes_after_failure})." fi if [ "${cache_bytes_after_rollback}" -ne 0 ] \ || [ "${cache_bytes_after_rebuild}" -ne "${cache_bytes_before}" ]; then error "Post-finalization REINDEX abort leaked or failed to rebuild \ cache accounting (${cache_bytes_before} -> \ ${cache_bytes_after_rollback} -> ${cache_bytes_after_rebuild})." fi hits_after_failure=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_hits_after_failure=\([0-9]*\).*$/\1/p' \ | head -1) hits_after_rollback=$(echo "${output}" | sed -n \ 's/^.*failed_reindex_hits_after_rollback=\([0-9]*\).*$/\1/p' \ | head -1) if [ "${hits_after_failure}" != "225" ] \ || [ "${hits_after_rollback}" != "225" ]; then error "Failed or rolled-back REINDEX left the old index unusable \ (after failure=${hits_after_failure:-missing}, \ after rollback=${hits_after_rollback:-missing})." fi run_sql_quiet " DROP TABLE docs CASCADE; DROP FUNCTION maybe_fail_reindex(text); " log "✅ failed and rolled-back REINDEX preserved stable state" } # prepare_docs_table — fresh, intentionally-bloated docs table. # # 30k rows + fillfactor=10 → ~4000-6000 heap pages. DELETE of the # tail leaves ~1000 live rows but the heap retains its page count # until rewrite (autovacuum is disabled at table level). This makes # the OLD heap span many blocks while the NEW heap (post-rewrite) is # small — so any stale CTID retained by a backend with a primed # cache will point well beyond the new heap's EOF, surfacing the # "could not read blocks" bug deterministically on v1.2.0. prepare_docs_table() { run_sql_quiet " DROP TABLE IF EXISTS docs CASCADE; CREATE TABLE docs ( id bigserial PRIMARY KEY, content text NOT NULL ) WITH (autovacuum_enabled=false, fillfactor=10); INSERT INTO docs (content) SELECT 'row ' || g || ' lorem ipsum dolor sit amet ' || md5(g::text) FROM generate_series(1, 30000) g; CREATE INDEX docs_bm25 ON docs USING bm25 (content) WITH (text_config='english'); " run_sql_quiet "DELETE FROM docs WHERE id > 1000;" run_sql_quiet "ANALYZE docs;" } prepare_parallel_docs_table() { run_sql_quiet " DROP TABLE IF EXISTS docs CASCADE; CREATE TABLE docs ( id bigserial PRIMARY KEY, content text NOT NULL ) WITH (autovacuum_enabled=false); INSERT INTO docs (content) SELECT 'parallel row ' || g || ' lorem ipsum dolor sit amet ' || repeat(md5(g::text), 2) FROM generate_series(1, 100000) g; ALTER TABLE docs SET (parallel_workers = 2); ANALYZE docs; SET max_parallel_maintenance_workers = 2; SET max_parallel_workers = 4; SET maintenance_work_mem = '256MB'; SET min_parallel_table_scan_size = 0; CREATE INDEX docs_bm25 ON docs USING bm25 (content) WITH (text_config='english'); " } # run_rewrite_test test_name rewrite_sql expect_heap_shrink rewrite_count # require_parallel # # Drive both backends from one psql session: Backend A primes its # TpLocalIndexState cache via INSERT, samples pre-rewrite heap size, # then spawns Backend B (psql via `\!`) to perform the supplied # rewrite, then continues using its (potentially stale) cached state # to INSERT 50 'moonlight' rows and force a spill. # # After the heredoc completes, all SETUP INVARIANTS are checked # BEFORE any bug-class assertion (so a degraded test environment # can't masquerade as a bug-fix regression): # - no storage-corruption errors in session output # - all session markers were emitted # - bm25_spill_index returned a real segment root # - pre_heap_pages >= 1000 (bloat sufficient) # - heap actually shrank (skipped when expect_heap_shrink=0) # # Then the BUG-CLASS ASSERTIONS run: # - docs_persisted == live heap row count # - top-100 'lorem' index scan returns exactly 100 (no errors) # - exactly 50 live 'moonlight' rows are returned through the index # # Args: # $1: human-readable test name # $2: SQL Backend B should run (the heap/index rewrite) # $3: 1 if the rewrite is expected to compact the heap, 0 otherwise # (REINDEX INDEX only touches the index relfilenode) # $4: number of rewrites to run while Backend A retains its cached state # $5: 1 if every rewrite must launch a parallel worker, 0 otherwise run_rewrite_test() { local test_name="$1" local rewrite_sql="$2" local expect_heap_shrink="$3" local rewrite_count="$4" local require_parallel="$5" local rewrite_steps="" local rewrite_no log "Test: ${test_name} (issue #390)" if [ "${require_parallel}" = "1" ]; then prepare_parallel_docs_table else prepare_docs_table fi for ((rewrite_no = 1; rewrite_no <= rewrite_count; rewrite_no++)); do rewrite_steps+="\\! if psql -X -h \"${SOCKET_DIR}\"" rewrite_steps+=" -p ${TEST_PORT} -d \"${TEST_DB}\"" rewrite_steps+=" -v ON_ERROR_STOP=1 -q" rewrite_steps+=" -c \"${rewrite_sql}\"" rewrite_steps+="; then echo B-rewrite-ok-${rewrite_no};" rewrite_steps+=" else echo B-rewrite-failed-${rewrite_no}; fi" rewrite_steps+=$'\n' rewrite_steps+="SELECT 'post_rewrite_summary_${rewrite_no}=' ||" rewrite_steps+=$'\n' rewrite_steps+=" replace(bm25_summarize_index(" rewrite_steps+="'docs_bm25'), E'\\\\n', ' ');" rewrite_steps+=$'\n' rewrite_steps+="SELECT 'post_rewrite_generation_${rewrite_no}=' ||" rewrite_steps+=$'\n' rewrite_steps+=" bm25_cache_bump_spill_generation(" rewrite_steps+="'docs_bm25');" rewrite_steps+=$'\n' done # Bash heredoc with unquoted EOF so ${SOCKET_DIR}, ${TEST_PORT}, # ${TEST_DB}, and ${rewrite_steps} all expand at script-render time. # Backend A stops on its first SQL error. Each nested Backend B # command emits a success marker only if its rewrite exits zero; # psql's \! does not propagate nested command status reliably. # # NOTE on shell quoting: ${rewrite_sql} is interpolated INSIDE # double quotes after `\! psql ... -c`. Any SQL string passed # here must therefore contain no unescaped double quotes. All # four current callers use only single-quoted SQL literals — if # a future case needs double quotes, switch the inner -c to a # here-string fed to psql instead. local output session_status set +e output=$(psql -X -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ --set ON_ERROR_STOP=1 2>&1 << EOF \set QUIET on \pset footer off \pset tuples_only on -- Backend A: prime local_state_cache by inserting into the index. -- The 'unicorn' term is unique to these rows so we can isolate them -- later. 500 rows extend the heap further, so any CTID captured here -- will reference a high block number. INSERT INTO docs (content) SELECT 'pre ' || g || ' unicorn rainbow sparkle ' || md5(('h'||g)::text) FROM generate_series(1, 500) g; -- Sample pre-rewrite heap pages HERE, immediately before the -- rewrite, so the post-rewrite "heap shrank" invariant has zero -- slack (matches the exact moment Backend B is about to act on). SELECT 'pre_heap_pages=' || (pg_relation_size('docs')::bigint / 8192); SELECT 'pre_rewrite_generation=' || bm25_cache_bump_spill_generation('docs_bm25'); SELECT 'A-primed' AS marker; -- Backend B: one or more heap/index rewrites. Each runs in a separate -- psql process via \! while Backend A's session stays open and retains -- the local-state wrapper cached before the first rewrite. ${rewrite_steps} -- Backend A is still the same session. Its cached TpLocalIndexState -- must remain valid because REINDEX resets the shared state in place. SELECT 'A-post-rewrite' AS marker; -- Write through the same stable wrapper. INSERT INTO docs (content) SELECT 'late ' || g || ' lorem moonlight ' || md5(('l'||g)::text) FROM generate_series(1, 50) g; -- Force a spill. On the v1.2.0 bug this is what materialises the -- corruption: the OLD in-memory memtable (with OLD CTIDs) gets -- written into the NEW index file as a stale L0 segment. -- bm25_spill_index returns the new L0 segment's root block number -- (always > 0 because block 0 is the metapage) or NULL if nothing -- was spilled. SELECT 'spill_segment_root=' || bm25_spill_index('docs_bm25'); SELECT 'A-spilled' AS marker; EOF ) session_status=$? set -e info "Backend A session output: ${output}" # ----- SETUP INVARIANT CHECKS (must pass before bug assertions) ----- # 1. Backend A must itself exit successfully. This catches SQL # errors before marker/count parsing can accidentally mask them. if [ "${session_status}" -ne 0 ]; then error "Backend A session exited with status ${session_status}. \ Full output:\n${output}" fi # 2. No corruption or generic SQL errors may appear anywhere, # including output from nested Backend B psql processes. if echo "${output}" | grep -qiE "could not read blocks|invalid \ (page|segment)|XX001|(^|[[:space:]])ERROR:"; then error "Backend session produced an SQL/storage error. \ Output:\n${output}" fi if echo "${output}" | grep -qF "B-rewrite-failed-"; then error "A nested rewrite command failed. Output:\n${output}" fi # 3. Session and every nested rewrite must emit their required # markers. In particular, a failed nested psql cannot be hidden # by outer psql continuing after \!. local marker for marker in "A-primed" "A-post-rewrite" "A-spilled"; do if ! echo "${output}" | grep -qF "${marker}"; then error "Backend A session never reached marker \ '${marker}'. Full output:\n${output}" fi done for ((rewrite_no = 1; rewrite_no <= rewrite_count; rewrite_no++)); do for marker in \ "B-rewrite-ok-${rewrite_no}" \ "post_rewrite_summary_${rewrite_no}=" \ "post_rewrite_generation_${rewrite_no}="; do if ! echo "${output}" | grep -qF "${marker}"; then error "Rewrite ${rewrite_no} never reached required marker \ '${marker}'. Full output:\n${output}" fi done done # 4. A designated parallel case must prove every nested REINDEX # actually launched at least one worker. if [ "${require_parallel}" = "1" ]; then local parallel_launches parallel_launches=$(echo "${output}" | grep -Ec "parallel index build: launched [1-9][0-9]* " || true) if [ "${parallel_launches}" -ne "${rewrite_count}" ]; then error "Expected ${rewrite_count} parallel rewrites, observed \ ${parallel_launches}. Full output:\n${output}" fi fi # 5. The spill generation belongs to the stable shared state. # It must advance monotonically across every rewrite; resetting # to 1 proves the shared state was destructively replaced. local previous_generation current_generation previous_generation=$(echo "${output}" | sed -n \ 's/^.*pre_rewrite_generation=\([0-9]*\).*$/\1/p' | head -1) if ! [[ "${previous_generation}" =~ ^[0-9]+$ ]]; then error "Failed to parse pre-rewrite spill generation \ (got '${previous_generation}')." fi for ((rewrite_no = 1; rewrite_no <= rewrite_count; rewrite_no++)); do current_generation=$(echo "${output}" | sed -n \ "s/^.*post_rewrite_generation_${rewrite_no}=\\([0-9]*\\).*$/\\1/p" \ | head -1) if ! [[ "${current_generation}" =~ ^[0-9]+$ ]]; then error "Failed to parse spill generation after rewrite \ ${rewrite_no} (got '${current_generation}')." fi if [ "${current_generation}" -le "${previous_generation}" ]; then error "Shared-state spill generation did not advance across \ rewrite ${rewrite_no} (${previous_generation} -> ${current_generation}); \ the registry state was replaced instead of reset in place." fi previous_generation="${current_generation}" done # 6. Spill must have produced a real segment. bm25_spill_index # returns a BlockNumber (always > 0 for a non-empty spill) or # NULL when no segment was written. local spill_segment_root spill_segment_root=$(echo "${output}" \ | sed -n 's/^.*spill_segment_root=\([0-9]*\).*$/\1/p' | head -1) if [ -z "${spill_segment_root}" ] \ || [ "${spill_segment_root}" -le 0 ]; then error "bm25_spill_index returned no segment root \ (got '${spill_segment_root:-empty}'); the late inserts should have \ populated the memtable before the explicit spill." fi info "Spill segment root block: ${spill_segment_root}" # 7. Pre-rewrite heap page count (parsed from session output). local pre_heap_pages pre_heap_pages=$(echo "${output}" \ | sed -n 's/^.*pre_heap_pages=\([0-9]*\).*$/\1/p' | head -1) if ! [[ "${pre_heap_pages}" =~ ^[0-9]+$ ]]; then error "Failed to obtain pre-rewrite heap page count from \ session output (got '${pre_heap_pages}'). Test setup broken." fi info "Pre-rewrite heap pages: ${pre_heap_pages}" # 8. Bloat threshold. Promoted from warn to error: the entire # test thesis depends on the OLD heap dwarfing the NEW one. # Without enough bloat, stale CTIDs may still land on valid # blocks and the bug becomes silently invisible. if [ "${pre_heap_pages}" -lt 1000 ]; then error "Pre-rewrite heap is too small (${pre_heap_pages} \ pages, need >= 1000). Test setup is degenerate — bloat is insufficient \ for stale-CTID detection." fi # 9. Heap-rewrite invariant: confirms the rewrite actually # happened. Skipped for REINDEX INDEX (touches only the index # relfilenode, not the heap). if [ "${expect_heap_shrink}" = "1" ]; then local heap_size_now heap_size_now=$(run_sql_value \ "SELECT pg_relation_size('docs')/8192;") if ! [[ "${heap_size_now}" =~ ^[0-9]+$ ]]; then error "Failed to obtain post-rewrite heap size \ (got '${heap_size_now}')." fi info "Post-rewrite heap pages: ${heap_size_now}" if [ "${heap_size_now}" -ge "${pre_heap_pages}" ]; then error "Heap was not compacted by '${test_name}' \ (pre=${pre_heap_pages}, post=${heap_size_now}). Test setup invalid — \ either bloat was insufficient or the rewrite did not happen." fi fi # ----- BUG-CLASS ASSERTIONS (only after setup is verified) ----- # A1. docs_persisted must equal live heap row count. The bug # inflates the segment doc count above the actual row count # because the stale memtable was spilled as a second L0 # segment whose entries reference long-dead CTIDs. local heap_rows seg_docs heap_rows=$(run_sql_value "SELECT count(*) FROM docs;") seg_docs=$(run_sql_value " SELECT (regexp_match( bm25_summarize_index('docs_bm25'), 'docs_persisted: ([0-9]+)' ))[1]::bigint; ") info "Heap live rows: ${heap_rows}, docs_persisted: ${seg_docs}" if [ -z "${seg_docs}" ]; then error "Failed to parse docs_persisted from \ bm25_summarize_index output:\n$(run_sql \ "SELECT bm25_summarize_index('docs_bm25');")" fi if [ "${seg_docs}" != "${heap_rows}" ]; then error "BUG (issue #390): docs_persisted (${seg_docs}) does \ not match live heap row count (${heap_rows}). Excess docs indicate \ a stale segment was written from a pre-rewrite memtable.\n\nFull \ summary:\n$(run_sql "SELECT bm25_summarize_index('docs_bm25');")" fi # A2. Top-100 'lorem' scan. The bug manifests as "could not read # blocks" when the stale segment's top-k results point past # the new heap's EOF. Wrap in `set +e/-e` so psql's non-zero # exit on the corruption error does NOT abort before we # inspect the captured stderr. local scan_q set +e scan_q=$(run_sql_value " SELECT count(*)::text FROM ( SELECT id FROM docs ORDER BY content <@> to_bm25query('lorem', 'docs_bm25') LIMIT 100 ) s; ") set -e if echo "${scan_q}" | grep -qiE "could not read blocks|invalid \ (page|segment)|error"; then error "BUG (issue #390): top-100 'lorem' index scan failed: \ ${scan_q}" fi if [ "${scan_q}" != "100" ]; then error "BUG (issue #390): top-100 index scan for 'lorem' \ returned '${scan_q}' rows, expected 100." fi # A3. Post-rewrite late-insert visibility: 'moonlight' appears # only in the 50 late rows Backend A inserted via the # (potentially stale) cache. md5-hex output uses only [0-9a-f] # so 'moonlight' (m,n,i,g,h,t) cannot accidentally appear in # any pre-rewrite row's md5 suffix — the inner-join LIKE # filter is therefore provably tight. Single query handles # both the error-detection and count-verification roles # (no double-scan). local moonlight_count set +e moonlight_count=$(run_sql_value " SELECT count(*)::text FROM ( SELECT id FROM docs ORDER BY content <@> to_bm25query('moonlight', 'docs_bm25') LIMIT 100 ) s JOIN docs d ON d.id = s.id WHERE d.content LIKE '%moonlight%'; ") set -e if echo "${moonlight_count}" | grep -qiE "could not read blocks\ |invalid (page|segment)|error"; then error "BUG (issue #390): post-rewrite 'moonlight' query \ failed: ${moonlight_count}" fi if [ "${moonlight_count}" != "50" ]; then error "BUG (issue #390): expected 50 live 'moonlight' rows \ via BM25 index, got '${moonlight_count}'. The post-rewrite late \ inserts were not correctly indexed in the new file." fi run_sql_quiet "DROP TABLE docs CASCADE;" log "✅ ${test_name}: no stale CTIDs after rewrite" } run_warm_cache_rewrite_test() { local action="$1" local rewrite_file="${DATA_DIR}/warm_cache_rewrite.sql" local output local marker local query_term=search log "Test: warm memtable cache across ${action} (#515)" run_sql_quiet " CREATE TABLE cursor_docs (id integer, content text); CREATE INDEX cursor_idx ON cursor_docs USING bm25(content) WITH (text_config='simple'); INSERT INTO cursor_docs SELECT g, 'search token' || g FROM generate_series(1, 1000) g; " if [ "${action}" = reindex ]; then cat >"${rewrite_file}" <<'SQL' SET pg_textsearch.compress_segments = off; REINDEX INDEX cursor_idx; SQL else query_term=replacement42 cat >"${rewrite_file}" <<'SQL' TRUNCATE cursor_docs; INSERT INTO cursor_docs SELECT g, 'search token' || g || ' ' || (SELECT string_agg('replacement' || n, ' ' ORDER BY n) FROM generate_series(0, 59) n) FROM generate_series(1, 1000) g; SQL fi # Keep the reader connection alive across the other backend's rewrite. if ! output=$(PGHOST="${SOCKET_DIR}" PGPORT="${TEST_PORT}" \ PGDATABASE="${TEST_DB}" psql -X -qAt -v ON_ERROR_STOP=1 2>&1 </dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" setup_test_db run_warm_cache_rewrite_test reindex run_warm_cache_rewrite_test truncate run_cache_locator_top_level_rollback_test run_cache_locator_savepoint_rollback_test run_cache_locator_prepared_rollback_test run_initial_create_drop_rollback_abort_test run_warm_reindex_chain_count_test run_warm_reindex_reseed_before_abort_test run_cold_registry_reindex_rollback_test run_reindex_bulk_counter_preservation_test run_prepare_bulk_state_reset_test run_prepare_initial_create_test run_failed_reindex_test # Cover the heap/index rewrite paths from issue #390 as a class, # then exercise repeated serial and actual parallel REINDEX while # Backend A retains the same TpLocalIndexState wrapper. run_rewrite_test \ "ALTER TABLE ADD COLUMN ... GENERATED STORED" \ "ALTER TABLE docs ADD COLUMN content_tsv tsvector GENERATED \ ALWAYS AS (to_tsvector('english', content)) STORED;" \ 1 \ 1 \ 0 run_rewrite_test \ "VACUUM FULL docs" \ "VACUUM FULL docs;" \ 1 \ 1 \ 0 run_rewrite_test \ "CLUSTER docs USING docs_pkey" \ "CLUSTER docs USING docs_pkey;" \ 1 \ 1 \ 0 run_rewrite_test \ "repeated serial REINDEX INDEX docs_bm25" \ "SET max_parallel_maintenance_workers = 0; \ REINDEX INDEX docs_bm25;" \ 0 \ 3 \ 0 run_rewrite_test \ "parallel REINDEX INDEX docs_bm25" \ "SET max_parallel_maintenance_workers = 2; \ SET max_parallel_workers = 4; \ SET maintenance_work_mem = '256MB'; \ SET min_parallel_table_scan_size = 0; \ REINDEX INDEX docs_bm25;" \ 0 \ 1 \ 1 log "🎉 multi_backend_reindex.sh: all tests passed" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/multi_index.sh000077500000000000000000001024041525762047700243100ustar00rootroot00000000000000#!/bin/bash # # Multi-index, multi-user, multi-schema concurrency tests. # Verifies that concurrent memtable operations work correctly # across privilege boundaries, schemas, and memory pressure. # # Test scenarios: # 1. Multiple bm25 indexes on the same table # 2. Cross-schema indexes with concurrent operations # 3. Non-owner INSERT triggering auto-spill # 4. Cross-user memory pressure spill # 5. Concurrent spills from multiple backends # 6. Query during cross-index spill # 7. Mixed workload across many indexes # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55440 TEST_DB=pg_textsearch_multi_index_test DATA_DIR="${SCRIPT_DIR}/../tmp_multi_index_test" LOGFILE="${DATA_DIR}/postgres.log" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$($PG_CONFIG --bindir)" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } PASS_COUNT=0 FAIL_COUNT=0 pass() { PASS_COUNT=$((PASS_COUNT + 1)) log "PASS: $1" } fail() { FAIL_COUNT=$((FAIL_COUNT + 1)) echo -e "${RED}[$(date '+%H:%M:%S')] FAIL: $1${NC}" } cleanup() { local exit_code=$? log "Cleaning up multi-index test (exit code: $exit_code)..." jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m fast \ &>/dev/null || "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m immediate \ &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up test PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" "${PGBINDIR}/initdb" -D "${DATA_DIR}" \ --auth-local=trust --auth-host=trust >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 40 shared_buffers = 128MB unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' log_min_messages = warning shared_preload_libraries = 'pg_textsearch' pg_textsearch.bulk_load_threshold = 0 pg_textsearch.memtable_pages_threshold = 0 EOF "${PGBINDIR}/pg_ctl" start -D "${DATA_DIR}" -l "${LOGFILE}" \ -w || error "Failed to start PostgreSQL" "${PGBINDIR}/createdb" -h "${DATA_DIR}" -p "${TEST_PORT}" \ "${TEST_DB}" run_sql_quiet "CREATE EXTENSION pg_textsearch VERSION '1.5.1';" } run_sql() { "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" 2>&1 } run_sql_quiet() { "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } run_sql_value() { "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -tAc "$1" 2>/dev/null } # Run SQL as a specific role. run_sql_as() { local role="$1" local sql="$2" "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -U "${role}" -d "${TEST_DB}" -c "$sql" 2>&1 } run_sql_as_quiet() { local role="$1" local sql="$2" "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -U "${role}" -d "${TEST_DB}" -c "$sql" >/dev/null 2>&1 } run_sql_as_value() { local role="$1" local sql="$2" "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -U "${role}" -d "${TEST_DB}" -tAc "$sql" 2>/dev/null } # Reload server configuration after ALTER SYSTEM. reload_conf() { run_sql_quiet "SELECT pg_reload_conf();" # Short pause for backends to pick up new GUC values. sleep 0.2 } # Return the number of on-disk segments for an index. # Always runs as superuser (bm25_summarize_index is revoked # from PUBLIC). Returns 0 when no segments exist. get_segment_count() { local index_name="$1" local summary summary=$(run_sql_value \ "SELECT bm25_summarize_index('$index_name');") local count count=$(echo "$summary" \ | grep -oE 'Total: [0-9]+ segments' \ | grep -oE '[0-9]+' || echo 0) echo "${count:-0}" } # Assert that an index has at least one on-disk segment, # proving that an auto-spill occurred. assert_spilled() { local label="$1" local index_name="$2" local seg_ct seg_ct=$(get_segment_count "$index_name") if [ "$seg_ct" -gt 0 ]; then pass "${label}: auto-spill confirmed (${seg_ct} segments)" else fail "${label}: expected segments > 0 but got ${seg_ct}" fi } # ------------------------------------------------------- # Test 1: Multiple bm25 indexes on the same table # # Two indexes on different columns of the same table. # Insert, spill one, query both, spill the other. # ------------------------------------------------------- test_multi_index_same_table() { log "Test 1: Multiple bm25 indexes on the same table" run_sql_quiet "CREATE TABLE multi_col ( id serial PRIMARY KEY, title text, body text );" run_sql_quiet "CREATE INDEX idx_title ON multi_col USING bm25(title) WITH (text_config='english');" run_sql_quiet "CREATE INDEX idx_body ON multi_col USING bm25(body) WITH (text_config='english');" # Insert documents with distinct terms per column. run_sql_quiet "INSERT INTO multi_col (title, body) SELECT 'alpha_' || i || ' bravo_' || (i % 10), 'charlie_' || i || ' delta_' || (i % 10) FROM generate_series(1, 500) i;" # Both indexes should be queryable before any spill. local title_ct title_ct=$(run_sql_value \ "SELECT count(*) FROM multi_col WHERE title <@> 'bravo_3'::bm25query < 0;") if [ "$title_ct" -gt 0 ]; then pass "Test 1a: title index returns results before spill" else fail "Test 1a: title index returned 0 rows" fi local body_ct body_ct=$(run_sql_value \ "SELECT count(*) FROM multi_col WHERE body <@> 'delta_3'::bm25query < 0;") if [ "$body_ct" -gt 0 ]; then pass "Test 1b: body index returns results before spill" else fail "Test 1b: body index returned 0 rows" fi # Spill one index, verify both still work. run_sql_quiet "SELECT bm25_spill_index('idx_title');" title_ct=$(run_sql_value \ "SELECT count(*) FROM multi_col WHERE title <@> 'bravo_3'::bm25query < 0;") body_ct=$(run_sql_value \ "SELECT count(*) FROM multi_col WHERE body <@> 'delta_3'::bm25query < 0;") if [ "$title_ct" -gt 0 ] && [ "$body_ct" -gt 0 ]; then pass "Test 1c: both indexes work after spilling title" else fail "Test 1c: title=$title_ct body=$body_ct after title spill" fi # Spill the other index, then insert more, verify consistency. run_sql_quiet "SELECT bm25_spill_index('idx_body');" run_sql_quiet "INSERT INTO multi_col (title, body) SELECT 'echo_' || i, 'foxtrot_' || i FROM generate_series(1, 200) i;" local echo_ct echo_ct=$(run_sql_value \ "SELECT count(*) FROM multi_col WHERE title <@> 'echo'::bm25query < 0;") local fox_ct fox_ct=$(run_sql_value \ "SELECT count(*) FROM multi_col WHERE body <@> 'foxtrot'::bm25query < 0;") if [ "$echo_ct" -eq 200 ] && [ "$fox_ct" -eq 200 ]; then pass "Test 1d: post-spill inserts visible in both indexes" else fail "Test 1d: echo=$echo_ct foxtrot=$fox_ct (expected 200)" fi run_sql_quiet "DROP TABLE multi_col CASCADE;" log "Test 1 complete" } # ------------------------------------------------------- # Test 2: Cross-schema indexes with concurrent operations # # Indexes in two different schemas; concurrent inserts # from parallel backends, then verify query results. # ------------------------------------------------------- test_cross_schema() { log "Test 2: Cross-schema concurrent operations" run_sql_quiet "CREATE SCHEMA schema_a;" run_sql_quiet "CREATE SCHEMA schema_b;" run_sql_quiet "CREATE TABLE schema_a.docs ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE TABLE schema_b.docs ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_a ON schema_a.docs USING bm25(content) WITH (text_config='english');" run_sql_quiet "CREATE INDEX idx_b ON schema_b.docs USING bm25(content) WITH (text_config='english');" # Concurrent inserts into both schemas from parallel backends. "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " INSERT INTO schema_a.docs (content) SELECT 'schema_a_term_' || i || ' common_' || (i % 20) FROM generate_series(1, 1000) i; " >/dev/null 2>&1 & local pid_a=$! "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " INSERT INTO schema_b.docs (content) SELECT 'schema_b_term_' || i || ' common_' || (i % 20) FROM generate_series(1, 1000) i; " >/dev/null 2>&1 & local pid_b=$! wait $pid_a || error "Schema A insert failed" wait $pid_b || error "Schema B insert failed" # Verify both indexes return correct results. local ct_a ct_a=$(run_sql_value \ "SELECT count(*) FROM schema_a.docs WHERE content <@> to_bm25query('common_5', 'schema_a.idx_a') < 0;") local ct_b ct_b=$(run_sql_value \ "SELECT count(*) FROM schema_b.docs WHERE content <@> to_bm25query('common_5', 'schema_b.idx_b') < 0;") if [ "$ct_a" -gt 0 ] && [ "$ct_b" -gt 0 ]; then pass "Test 2a: both schemas return results ($ct_a, $ct_b)" else fail "Test 2a: schema_a=$ct_a schema_b=$ct_b" fi # Spill one schema's index, verify the other is unaffected. run_sql_quiet "SELECT bm25_spill_index('schema_a.idx_a');" ct_b=$(run_sql_value \ "SELECT count(*) FROM schema_b.docs WHERE content <@> to_bm25query('common_5', 'schema_b.idx_b') < 0;") if [ "$ct_b" -gt 0 ]; then pass "Test 2b: schema_b unaffected by schema_a spill" else fail "Test 2b: schema_b returned 0 after schema_a spill" fi # Insert more into schema_b while schema_a is on disk. run_sql_quiet "INSERT INTO schema_b.docs (content) SELECT 'xylophone_' || i FROM generate_series(1, 200) i;" local extra_ct extra_ct=$(run_sql_value \ "SELECT count(*) FROM schema_b.docs WHERE content <@> to_bm25query('xylophone', 'schema_b.idx_b') < 0;") if [ "$extra_ct" -eq 200 ]; then pass "Test 2c: cross-schema post-spill inserts work" else fail "Test 2c: xylophone count=$extra_ct (expected 200)" fi run_sql_quiet "DROP SCHEMA schema_a CASCADE;" run_sql_quiet "DROP SCHEMA schema_b CASCADE;" log "Test 2 complete" } # ------------------------------------------------------- # Test 3: Non-owner INSERT triggers auto-spill # # Owner creates table + index. A non-owner role with # INSERT privilege inserts enough data to trigger the # per-index auto-spill. Verify data integrity for both # owner and non-owner queries. # ------------------------------------------------------- test_non_owner_spill() { log "Test 3: Non-owner INSERT triggers auto-spill" # Tighten per-index limit so spill triggers quickly. run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" reload_conf # Create roles. run_sql_quiet "DO \$\$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'idx_owner') THEN REASSIGN OWNED BY idx_owner TO CURRENT_USER; DROP OWNED BY idx_owner CASCADE; DROP ROLE idx_owner; END IF; IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'inserter') THEN REASSIGN OWNED BY inserter TO CURRENT_USER; DROP OWNED BY inserter CASCADE; DROP ROLE inserter; END IF; END \$\$;" run_sql_quiet "CREATE ROLE idx_owner LOGIN;" run_sql_quiet "CREATE ROLE inserter LOGIN;" # Owner creates table and index. run_sql_quiet "GRANT CREATE ON SCHEMA public TO idx_owner;" run_sql_as_quiet idx_owner \ "CREATE TABLE owned_docs (id serial PRIMARY KEY, content text);" run_sql_as_quiet idx_owner \ "CREATE INDEX idx_owned ON owned_docs USING bm25(content) WITH (text_config='english');" # Grant INSERT and SELECT to the inserter role. run_sql_as_quiet idx_owner \ "GRANT INSERT, SELECT ON owned_docs TO inserter;" run_sql_as_quiet idx_owner \ "GRANT USAGE, SELECT ON SEQUENCE owned_docs_id_seq TO inserter;" # Inserter loads enough data to trigger per-index soft limit. # With 4MB limit, per-index soft = 512kB. run_sql_as_quiet inserter \ "INSERT INTO owned_docs (content) SELECT 'nonowner_' || i || ' term_' || (i % 50) || ' ' || repeat('pad_', 10) FROM generate_series(1, 3000) i;" assert_spilled "Test 3" idx_owned # Both roles should see correct results. local owner_ct owner_ct=$(run_sql_as_value idx_owner \ "SELECT count(*) FROM owned_docs WHERE content <@> 'term_7'::bm25query < 0;") local inserter_ct inserter_ct=$(run_sql_as_value inserter \ "SELECT count(*) FROM owned_docs WHERE content <@> 'term_7'::bm25query < 0;") if [ "$owner_ct" -gt 0 ] && [ "$owner_ct" = "$inserter_ct" ]; then pass "Test 3a: owner and inserter see same results ($owner_ct)" else fail "Test 3a: owner=$owner_ct inserter=$inserter_ct" fi # Verify inserter cannot call bm25_spill_index (owner check). local spill_err spill_err=$(run_sql_as inserter \ "SELECT bm25_spill_index('idx_owned');" 2>&1 || true) if echo "$spill_err" | grep -qi "must be owner"; then pass "Test 3b: non-owner blocked from explicit spill" else fail "Test 3b: expected 'must be owner', got: $spill_err" fi # Insert more as inserter, verify auto-spill keeps working. run_sql_as_quiet inserter \ "INSERT INTO owned_docs (content) SELECT 'wave2_' || i FROM generate_series(1, 500) i;" local wave2_ct wave2_ct=$(run_sql_as_value inserter \ "SELECT count(*) FROM owned_docs WHERE content <@> 'wave2'::bm25query < 0;") if [ "$wave2_ct" -eq 500 ]; then pass "Test 3c: second insert wave visible after auto-spill" else fail "Test 3c: wave2 count=$wave2_ct (expected 500)" fi # Cleanup. run_sql_quiet "DROP TABLE owned_docs CASCADE;" run_sql_quiet "DROP OWNED BY idx_owner CASCADE;" run_sql_quiet "DROP OWNED BY inserter CASCADE;" run_sql_quiet "DROP ROLE idx_owner;" run_sql_quiet "DROP ROLE inserter;" run_sql_quiet "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" reload_conf log "Test 3 complete" } # ------------------------------------------------------- # Test 4: Cross-user memory pressure spill # # Two users each own an index. User A inserts heavily. # Global memory pressure triggers spill of User B's index. # Verify User B's index is intact afterward. # ------------------------------------------------------- test_cross_user_pressure_spill() { log "Test 4: Cross-user memory pressure spill" # Tight global limit: 4MB total, per-index soft = 512kB. run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" reload_conf # Create two owner roles. run_sql_quiet "DO \$\$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'user_a') THEN REASSIGN OWNED BY user_a TO CURRENT_USER; DROP OWNED BY user_a CASCADE; DROP ROLE user_a; END IF; IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'user_b') THEN REASSIGN OWNED BY user_b TO CURRENT_USER; DROP OWNED BY user_b CASCADE; DROP ROLE user_b; END IF; END \$\$;" run_sql_quiet "CREATE ROLE user_a LOGIN;" run_sql_quiet "CREATE ROLE user_b LOGIN;" run_sql_quiet "GRANT CREATE ON SCHEMA public TO user_a;" run_sql_quiet "GRANT CREATE ON SCHEMA public TO user_b;" # Each user creates their own table and index. run_sql_as_quiet user_b \ "CREATE TABLE docs_b (id serial PRIMARY KEY, content text);" run_sql_as_quiet user_b \ "CREATE INDEX idx_b ON docs_b USING bm25(content) WITH (text_config='english');" run_sql_as_quiet user_a \ "CREATE TABLE docs_a (id serial PRIMARY KEY, content text);" run_sql_as_quiet user_a \ "CREATE INDEX idx_a ON docs_a USING bm25(content) WITH (text_config='english');" # User B seeds their index (small load, stays in memtable). run_sql_as_quiet user_b \ "INSERT INTO docs_b (content) SELECT 'userb_doc_' || i || ' topic_' || (i % 30) FROM generate_series(1, 500) i;" # Verify User B's data is queryable. local b_before b_before=$(run_sql_as_value user_b \ "SELECT count(*) FROM docs_b WHERE content <@> 'topic_5'::bm25query < 0;") if [ "$b_before" -le 0 ]; then fail "Test 4 setup: user_b has no results" # Continue anyway to see if spill path works. fi # User A floods their index to blow through global soft limit. # This should trigger eviction of the largest memtable # (possibly user_b's), or spill user_a's own. run_sql_as_quiet user_a \ "INSERT INTO docs_a (content) SELECT 'usera_flood_' || i || ' ' || repeat('pad_', 15) FROM generate_series(1, 5000) i;" # At least one index must have been auto-spilled. local seg_a seg_b seg_a=$(get_segment_count idx_a) seg_b=$(get_segment_count idx_b) if [ "$((seg_a + seg_b))" -gt 0 ]; then pass "Test 4: auto-spill confirmed (idx_a=${seg_a}, idx_b=${seg_b} segments)" else fail "Test 4: no auto-spill occurred (idx_a=${seg_a}, idx_b=${seg_b})" fi # Verify User B's index is still intact and queryable. local b_after b_after=$(run_sql_as_value user_b \ "SELECT count(*) FROM docs_b WHERE content <@> 'topic_5'::bm25query < 0;") if [ "$b_after" = "$b_before" ]; then pass "Test 4a: user_b results intact after user_a pressure ($b_after)" else fail "Test 4a: user_b before=$b_before after=$b_after" fi # User A's data should also be fully queryable. local a_ct a_ct=$(run_sql_as_value user_a \ "SELECT count(*) FROM docs_a WHERE content <@> 'usera_flood'::bm25query < 0;") if [ "$a_ct" -eq 5000 ]; then pass "Test 4b: user_a data intact ($a_ct rows)" else fail "Test 4b: user_a count=$a_ct (expected 5000)" fi # User B inserts more after the pressure event. run_sql_as_quiet user_b \ "INSERT INTO docs_b (content) SELECT 'post_pressure_' || i FROM generate_series(1, 200) i;" local post_ct post_ct=$(run_sql_as_value user_b \ "SELECT count(*) FROM docs_b WHERE content <@> 'post_pressure'::bm25query < 0;") if [ "$post_ct" -eq 200 ]; then pass "Test 4c: user_b inserts work after pressure spill" else fail "Test 4c: post_pressure count=$post_ct (expected 200)" fi # Cleanup. run_sql_quiet "DROP TABLE docs_a CASCADE;" run_sql_quiet "DROP TABLE docs_b CASCADE;" run_sql_quiet "DROP OWNED BY user_a CASCADE;" run_sql_quiet "DROP OWNED BY user_b CASCADE;" run_sql_quiet "DROP ROLE user_a;" run_sql_quiet "DROP ROLE user_b;" run_sql_quiet "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" reload_conf log "Test 4 complete" } # ------------------------------------------------------- # Test 5: Concurrent spills from multiple backends # # Multiple backends insert into different indexes and # hit the spill threshold around the same time. Verify # no deadlocks and data integrity. # ------------------------------------------------------- test_concurrent_spills() { log "Test 5: Concurrent spills from multiple backends" # Tight per-index limit so each backend spills. run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" reload_conf local N=4 for i in $(seq 1 $N); do run_sql_quiet "CREATE TABLE spill_t${i} ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_spill_${i} ON spill_t${i} USING bm25(content) WITH (text_config='english');" done # Launch parallel inserts — each backend floods its own index. local pids=() for i in $(seq 1 $N); do "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " INSERT INTO spill_t${i} (content) SELECT 'idx${i}_term_' || j || ' ' || repeat('fill_', 8) FROM generate_series(1, 2000) j; " >/dev/null 2>&1 & pids+=($!) done local any_fail=0 for pid in "${pids[@]}"; do if ! wait "$pid"; then any_fail=1 fi done if [ "$any_fail" -eq 1 ]; then fail "Test 5a: one or more concurrent inserts failed" else pass "Test 5a: all concurrent inserts completed" fi # Verify each index has all its data AND was auto-spilled. local all_ok=1 local total_segments=0 for i in $(seq 1 $N); do local ct ct=$(run_sql_value \ "SELECT count(*) FROM spill_t${i} WHERE content <@> 'idx${i}_term'::bm25query < 0;") if [ "$ct" -ne 2000 ]; then fail "Test 5b: spill_t${i} count=$ct (expected 2000)" all_ok=0 fi local seg seg=$(get_segment_count "idx_spill_${i}") total_segments=$((total_segments + seg)) done if [ "$all_ok" -eq 1 ]; then pass "Test 5b: all $N indexes have correct counts" fi if [ "$total_segments" -ge "$N" ]; then pass "Test 5c: auto-spills confirmed ($total_segments total segments across $N indexes)" else fail "Test 5c: expected >= $N segments, got $total_segments" fi # Cleanup. for i in $(seq 1 $N); do run_sql_quiet "DROP TABLE spill_t${i} CASCADE;" done run_sql_quiet "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" reload_conf log "Test 5 complete" } # ------------------------------------------------------- # Test 6: Query during cross-index spill # # Backend A continuously queries index_1 while Backend B # inserts into index_2 under tight memory, potentially # triggering cross-index spill of index_1. # ------------------------------------------------------- test_query_during_spill() { log "Test 6: Query during cross-index spill" run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" reload_conf # Set up two tables with indexes. run_sql_quiet "CREATE TABLE query_target ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_qt ON query_target USING bm25(content) WITH (text_config='english');" run_sql_quiet "INSERT INTO query_target (content) SELECT 'target_' || i || ' keyword_' || (i % 20) FROM generate_series(1, 1000) i;" run_sql_quiet "CREATE TABLE spill_driver ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_sd ON spill_driver USING bm25(content) WITH (text_config='english');" # Expected result count for validation. local expected_ct expected_ct=$(run_sql_value \ "SELECT count(*) FROM query_target WHERE content <@> 'keyword_5'::bm25query < 0;") # Backend A: query loop. local query_ok_file="${DATA_DIR}/query_ok" echo "0" > "$query_ok_file" ( local failures=0 for round in $(seq 1 20); do local ct ct=$("${PGBINDIR}/psql" -h "${DATA_DIR}" \ -p "${TEST_PORT}" -d "${TEST_DB}" -tAc \ "SELECT count(*) FROM query_target WHERE content <@> 'keyword_5'::bm25query < 0;" 2>/dev/null) if [ "$ct" != "$expected_ct" ]; then failures=$((failures + 1)) fi sleep 0.05 done echo "$failures" > "$query_ok_file" ) & local query_pid=$! # Backend B: flood spill_driver to create memory pressure. "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " INSERT INTO spill_driver (content) SELECT 'driver_' || i || ' ' || repeat('bulk_', 15) FROM generate_series(1, 5000) i; " >/dev/null 2>&1 & local insert_pid=$! wait $insert_pid || warn "spill_driver insert had an error" wait $query_pid local query_failures query_failures=$(cat "$query_ok_file") if [ "$query_failures" -eq 0 ]; then pass "Test 6a: queries returned correct results during spill" else fail "Test 6a: $query_failures query rounds returned wrong count" fi # Verify that memory pressure actually triggered spills. local seg_qt seg_sd seg_qt=$(get_segment_count idx_qt) seg_sd=$(get_segment_count idx_sd) if [ "$((seg_qt + seg_sd))" -gt 0 ]; then pass "Test 6b: auto-spill confirmed (qt=${seg_qt}, sd=${seg_sd} segments)" else fail "Test 6b: no auto-spill occurred (qt=${seg_qt}, sd=${seg_sd})" fi run_sql_quiet "DROP TABLE query_target CASCADE;" run_sql_quiet "DROP TABLE spill_driver CASCADE;" run_sql_quiet "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" reload_conf log "Test 6 complete" } # ------------------------------------------------------- # Test 7: Mixed workload across many indexes # # Create several indexes across different schemas and # roles, then run a mixed workload of inserts, queries, # and explicit spills from multiple backends. # ------------------------------------------------------- test_mixed_workload() { log "Test 7: Mixed workload across many indexes" run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" reload_conf # Create schemas and roles. run_sql_quiet "DO \$\$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mix_owner') THEN REASSIGN OWNED BY mix_owner TO CURRENT_USER; DROP OWNED BY mix_owner CASCADE; DROP ROLE mix_owner; END IF; IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'mix_writer') THEN REASSIGN OWNED BY mix_writer TO CURRENT_USER; DROP OWNED BY mix_writer CASCADE; DROP ROLE mix_writer; END IF; END \$\$;" run_sql_quiet "CREATE SCHEMA mix_s1;" run_sql_quiet "CREATE SCHEMA mix_s2;" run_sql_quiet "CREATE ROLE mix_owner LOGIN;" run_sql_quiet "CREATE ROLE mix_writer LOGIN;" run_sql_quiet "GRANT ALL ON SCHEMA mix_s1 TO mix_owner;" run_sql_quiet "GRANT USAGE ON SCHEMA mix_s1 TO mix_writer;" run_sql_quiet "GRANT USAGE ON SCHEMA mix_s2 TO mix_writer;" # Owner creates tables and indexes in different schemas. run_sql_as_quiet mix_owner \ "CREATE TABLE mix_s1.t1 (id serial PRIMARY KEY, content text);" run_sql_as_quiet mix_owner \ "CREATE INDEX idx_m1 ON mix_s1.t1 USING bm25(content) WITH (text_config='english');" # Superuser creates a table in mix_s2. run_sql_quiet "CREATE TABLE mix_s2.t2 (id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_m2 ON mix_s2.t2 USING bm25(content) WITH (text_config='english');" # Public schema table. run_sql_quiet "CREATE TABLE mix_pub (id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_mp ON mix_pub USING bm25(content) WITH (text_config='english');" # Grant insert privileges. run_sql_as_quiet mix_owner \ "GRANT INSERT, SELECT ON mix_s1.t1 TO mix_writer;" run_sql_as_quiet mix_owner \ "GRANT USAGE, SELECT ON SEQUENCE mix_s1.t1_id_seq TO mix_writer;" run_sql_quiet "GRANT INSERT, SELECT ON mix_s2.t2 TO mix_writer;" run_sql_quiet "GRANT USAGE, SELECT ON SEQUENCE mix_s2.t2_id_seq TO mix_writer;" run_sql_quiet "GRANT INSERT, SELECT ON mix_pub TO mix_writer;" run_sql_quiet "GRANT USAGE, SELECT ON SEQUENCE mix_pub_id_seq TO mix_writer;" # Launch concurrent workload: 3 backends inserting into # different tables, 1 backend querying. # Each backend inserts 3000 rows with padding to exceed # the 512kB per-index soft limit (4MB / 8). "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -U mix_writer -d "${TEST_DB}" -c " INSERT INTO mix_s1.t1 (content) SELECT 'schema1_' || i || ' ' || repeat('w_', 10) FROM generate_series(1, 3000) i; " >/dev/null 2>&1 & local pid1=$! "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -U mix_writer -d "${TEST_DB}" -c " INSERT INTO mix_s2.t2 (content) SELECT 'schema2_' || i || ' ' || repeat('x_', 10) FROM generate_series(1, 3000) i; " >/dev/null 2>&1 & local pid2=$! "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " INSERT INTO mix_pub (content) SELECT 'public_' || i || ' ' || repeat('y_', 10) FROM generate_series(1, 3000) i; " >/dev/null 2>&1 & local pid3=$! # Concurrent query loop while inserts are running. local query_results="${DATA_DIR}/mix_query_results" ( for round in $(seq 1 15); do "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -tAc " SELECT count(*) FROM mix_pub WHERE content <@> 'public'::bm25query < 0; " >> "$query_results" 2>/dev/null sleep 0.1 done ) & local qpid=$! wait $pid1 || warn "mix_s1 insert had issues" wait $pid2 || warn "mix_s2 insert had issues" wait $pid3 || warn "mix_pub insert had issues" wait $qpid # Verify final counts across all indexes. local ct1 ct1=$(run_sql_value \ "SELECT count(*) FROM mix_s1.t1 WHERE content <@> to_bm25query('schema1', 'mix_s1.idx_m1') < 0;") local ct2 ct2=$(run_sql_value \ "SELECT count(*) FROM mix_s2.t2 WHERE content <@> to_bm25query('schema2', 'mix_s2.idx_m2') < 0;") local ct3 ct3=$(run_sql_value \ "SELECT count(*) FROM mix_pub WHERE content <@> 'public'::bm25query < 0;") if [ "$ct1" -eq 3000 ] && [ "$ct2" -eq 3000 ] \ && [ "$ct3" -eq 3000 ]; then pass "Test 7a: all three indexes have 3000 rows" else fail "Test 7a: s1=$ct1 s2=$ct2 pub=$ct3 (expected 3000)" fi # Verify auto-spills happened under the 8MB global limit. local seg_m1 seg_m2 seg_mp seg_m1=$(get_segment_count 'mix_s1.idx_m1') seg_m2=$(get_segment_count 'mix_s2.idx_m2') seg_mp=$(get_segment_count idx_mp) local total_seg=$((seg_m1 + seg_m2 + seg_mp)) if [ "$total_seg" -gt 0 ]; then pass "Test 7b: auto-spill confirmed (m1=${seg_m1}, m2=${seg_m2}, mp=${seg_mp})" else fail "Test 7b: no auto-spill occurred across 3 indexes" fi # Verify query results were monotonically non-decreasing # (the count should never go backward). if [ -f "$query_results" ]; then local prev=0 local monotonic=1 while IFS= read -r line; do if [ -n "$line" ] && [ "$line" -lt "$prev" ]; then monotonic=0 break fi prev="$line" done < "$query_results" if [ "$monotonic" -eq 1 ]; then pass "Test 7c: query counts monotonically non-decreasing" else fail "Test 7c: query count went backward" fi else warn "Test 7b: no query results file" fi # Cleanup. run_sql_quiet "DROP TABLE mix_pub CASCADE;" run_sql_quiet "DROP SCHEMA mix_s1 CASCADE;" run_sql_quiet "DROP SCHEMA mix_s2 CASCADE;" run_sql_quiet "DROP OWNED BY mix_owner CASCADE;" run_sql_quiet "DROP OWNED BY mix_writer CASCADE;" run_sql_quiet "DROP ROLE mix_owner;" run_sql_quiet "DROP ROLE mix_writer;" run_sql_quiet "ALTER SYSTEM RESET pg_textsearch.memtable_pages_threshold;" reload_conf log "Test 7 complete" } # ------------------------------------------------------- # Main # ------------------------------------------------------- log "==========================================" log "Multi-index / multi-user / multi-schema" log "concurrency tests for pg_textsearch" log "==========================================" setup_test_db test_multi_index_same_table test_cross_schema test_non_owner_spill test_cross_user_pressure_spill test_concurrent_spills test_query_during_spill test_mixed_workload log "==========================================" if [ "$FAIL_COUNT" -gt 0 ]; then error "$FAIL_COUNT test(s) FAILED, $PASS_COUNT passed" else log "All tests passed ($PASS_COUNT assertions)" fi timescale-pg_textsearch-b63fe58/test/scripts/nonblocking_compaction.sh000077500000000000000000001140721525762047700265120ustar00rootroot00000000000000#!/bin/bash # # Deterministic coverage for the unlocked compaction build phase. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG}" --bindir)" export PATH="${PGBINDIR}:${PATH}" TEST_PORT=55466 TEST_DB=nonblocking_compaction_test DATA_DIR="${SCRIPT_DIR}/../tmp_nonblocking_compaction" SOCKET_DIR="${SCRIPT_DIR}/.nbc_sock" LOGFILE="${DATA_DIR}/postgres.log" CLIENT_DIR="${DATA_DIR}/client_logs" KEEP_DIR="${SCRIPT_DIR}/../tmp_nonblocking_compaction_logs" POINT_AFTER_SELECT='pg-textsearch-compaction-after-select' POINT_SOURCE_ESTIMATE='pg-textsearch-compaction-source-estimate' POINT_BEFORE_PUBLISH='pg-textsearch-compaction-before-publish' POINT_AFTER_RESTAMP='pg-textsearch-compaction-after-restamp' RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # Diagnostics go to stderr: helpers are called from inside command # substitutions, whose stdout is captured rather than shown. log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}" >&2; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}" >&2; } sql() { psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 "$@" } diagnose() { exec 3>&1 1>&2 warn "pg_stat_activity:" sql -F '|' -c " SELECT pid, application_name, state, coalesce(wait_event_type, ''), coalesce(wait_event, ''), left(query, 100) FROM pg_stat_activity WHERE datname = current_database() ORDER BY pid;" 2>&1 || true warn "server log tail:" tail -n 80 "${LOGFILE}" 2>/dev/null || true warn "client log tails:" for file in "${CLIENT_DIR}"/*.log; do [ -e "${file}" ] || continue echo "==> ${file} <==" tail -n 20 "${file}" || true done exec 1>&3 3>&- } fail() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}" >&2 diagnose exit 1 } cleanup() { local exit_code=$? local pid trap - EXIT INT TERM for pid in $(jobs -p); do kill "${pid}" 2>/dev/null || true done if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate >/dev/null 2>&1 || true fi if [ "${exit_code}" -ne 0 ] && [ -d "${DATA_DIR}" ]; then rm -rf "${KEEP_DIR}" mkdir -p "${KEEP_DIR}" cp "${LOGFILE}" "${KEEP_DIR}/" 2>/dev/null || true cp -r "${CLIENT_DIR}" "${KEEP_DIR}/" 2>/dev/null || true warn "Preserved failure logs in ${KEEP_DIR}" fi rm -rf "${DATA_DIR}" "${SOCKET_DIR}" exit "${exit_code}" } trap cleanup EXIT INT TERM setup_cluster() { log "Setting up dedicated PostgreSQL cluster..." rm -rf "${DATA_DIR}" "${SOCKET_DIR}" "${KEEP_DIR}" mkdir -p "${DATA_DIR}" "${SOCKET_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 || fail "initdb failed" mkdir -p "${CLIENT_DIR}" cat >>"${DATA_DIR}/postgresql.conf" </dev/null } seed_index() { local table_name=$1 local index_name=$2 local token=$3 local batch sql -c " CREATE TABLE ${table_name} ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX ${index_name} ON ${table_name} USING bm25(body) WITH (text_config = 'english', compaction = 'off');" >/dev/null for batch in 1 2 3 4; do sql -c " INSERT INTO ${table_name}(body) SELECT 'common ${token} batch${batch} document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('${index_name}');" >/dev/null done assert_graph "${index_name}" "{4,0,0,0,0,0,0,0}" assert_index_scan_plan "${table_name}" "${index_name}" "${token}" } seed_all_indexes() { log "Seeding independent indexes with deterministic compaction debt..." seed_index scan_docs scan_idx scancase seed_index planning_docs planning_idx planningcase seed_index restamp_docs restamp_idx restampcase seed_index insert_docs insert_idx insertcase seed_index spill_docs spill_idx spillcase seed_index serial_docs serial_idx serialcase seed_index parallel_a_docs parallel_a_idx parallelacase seed_index parallel_b_docs parallel_b_idx parallelbcase seed_index publication_docs publication_idx publicationcase seed_index cancel_docs cancel_idx cancelcase for phase in output_data page_index tombstone; do seed_index "${phase}_docs" "${phase}_idx" "${phase}case" seed_index "${phase}_control_docs" "${phase}_control_idx" \ "${phase}case" done } test_inline_policy_runs_one_pass() { local batch log "Case: visible inline compaction runs one bounded pass..." sql -c " CREATE TABLE inline_once_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX inline_once_idx ON inline_once_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off');" >/dev/null for batch in 1 2 3 4 5 6 7; do sql -c " INSERT INTO inline_once_docs(body) SELECT 'inlineonce batch${batch} document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('inline_once_idx');" >/dev/null done sql -c " ALTER INDEX inline_once_idx SET (compaction = 'inline'); INSERT INTO inline_once_docs(body) SELECT 'inlineonce batch8 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('inline_once_idx');" >/dev/null assert_graph inline_once_idx "{4,1,0,0,0,0,0,0}" } index_oid() { sql -c "SELECT '${1}'::regclass::oid;" } graph() { sql -c "SELECT bm25_level_counts('${1}'::regclass)::text;" } pending_free() { sql -c "SELECT bm25_pending_free_pages('${1}');" } relation_blocks() { sql -c " SELECT pg_relation_size('${1}'::regclass, 'main') / current_setting('block_size')::bigint;" } segment_topology() { local index_name=$1 local roots roots=$(sql -c "SELECT bm25_summarize_index('${index_name}');" | sed -nE \ 's/^ L([0-7]) Segment ([0-9]+): block=([0-9]+),.*/L\1:\2=\3/p' | paste -sd, -) printf '%s|%s\n' "$(graph "${index_name}")" "${roots}" } ranked_signature() { local table_name=$1 local index_name=$2 local token=$3 sql -c " SET enable_seqscan = off; SELECT md5(string_agg( id::text || ':' || encode(float8send(score::double precision), 'hex'), ',' ORDER BY score, id)) FROM ( SELECT id, body <@> to_bm25query( '${token}', '${index_name}') AS score FROM ${table_name} ORDER BY body <@> to_bm25query( '${token}', '${index_name}'), id LIMIT 1000 ) ranked;" } assert_graph() { local index_name=$1 local expected=$2 local actual actual=$(graph "${index_name}") [ "${actual}" = "${expected}" ] || fail "${index_name} graph is ${actual}, expected ${expected}" } assert_index_scan_plan() { local table_name=$1 local index_name=$2 local token=$3 local plan local expected="Index Scan using ${index_name} on ${table_name}" plan=$(sql -c " SET enable_seqscan = off; EXPLAIN (COSTS off) SELECT id FROM ${table_name} ORDER BY body <@> to_bm25query('${token}', '${index_name}') LIMIT 1000;") if ! grep -Fq "${expected}" <<<"${plan}"; then warn "Unexpected ranked-query plan for ${index_name}:" echo "${plan}" fail "ranked assertions would not use ${index_name}" fi } ranked_count() { local table_name=$1 local index_name=$2 local token=$3 sql -c " SET enable_seqscan = off; SELECT count(*) FROM ( SELECT id FROM ${table_name} ORDER BY body <@> to_bm25query( '${token}', '${index_name}') LIMIT 1000 ) ranked;" } assert_all_documents() { local table_name=$1 local index_name=$2 local token=$3 local heap_count local index_count local mismatch_count heap_count=$(sql -c "SELECT count(*) FROM ${table_name};") index_count=$(ranked_count "${table_name}" "${index_name}" "${token}") [ "${index_count}" = "${heap_count}" ] || fail "${index_name} returned ${index_count}/${heap_count} documents" mismatch_count=$(sql -c " SET enable_seqscan = off; WITH ranked AS MATERIALIZED ( SELECT id FROM ${table_name} ORDER BY body <@> to_bm25query('${token}', '${index_name}') LIMIT 1000 ), missing AS ( SELECT id FROM ${table_name} EXCEPT SELECT id FROM ranked ), extra AS ( SELECT id FROM ranked EXCEPT SELECT id FROM ${table_name} ) SELECT (SELECT count(*) FROM missing) + (SELECT count(*) FROM extra);") [ "${mismatch_count}" = "0" ] || fail "${index_name} ranked results differ from heap document IDs" } STARTED_PID= # Run bm25_compact_step() in the background with zero or more injection # points armed to wait. The points are attached by the compacting # session itself, so injection_points_set_local() scopes them to that # backend and detaches them when it exits. start_compaction() { local app_name=$1 local index_name=$2 local points=$3 local output_file=$4 local attach="" local point for point in ${points}; do attach+="SELECT injection_points_attach('${point}', 'wait');" done PGAPPNAME="${app_name}" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SET statement_timeout = '90s'; SELECT injection_points_set_local(); ${attach} SELECT bm25_compact_step('${index_name}'::regclass);" \ >"${output_file}" 2>&1 & STARTED_PID=$! } start_sql() { local app_name=$1 local statement=$2 local output_file=$3 PGAPPNAME="${app_name}" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 -c "${statement}" \ >"${output_file}" 2>&1 & STARTED_PID=$! } backend_pid() { local app_name=$1 local deadline=$((SECONDS + 10)) local pid while ((SECONDS < deadline)); do pid=$(sql -c " SELECT pid FROM pg_stat_activity WHERE application_name = '${app_name}' ORDER BY backend_start DESC LIMIT 1;" 2>/dev/null || true) if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return fi sleep 0.05 done fail "backend ${app_name} did not appear within 10 seconds" } # Resolve a backend PID for a client that is allowed to finish before it # is observed. Prints nothing once the client has exited unseen. backend_pid_if_running() { local app_name=$1 local client_pid=$2 local deadline=$((SECONDS + 10)) local alive local pid while ((SECONDS < deadline)); do alive=0 kill -0 "${client_pid}" 2>/dev/null && alive=1 pid=$(sql -c " SELECT pid FROM pg_stat_activity WHERE application_name = '${app_name}' ORDER BY backend_start DESC LIMIT 1;" 2>/dev/null || true) if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return 0 fi ((alive)) || return 0 sleep 0.05 done fail "backend ${app_name} did not appear within 10 seconds" } # A backend parked in an injection point reports the point name as a # custom wait event, so its state is observable directly instead of # being inferred from log output. injection_waiting() { local point=$1 local backend=$2 sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = ${backend} AND wait_event_type = 'InjectionPoint' AND wait_event = '${point}' );" 2>/dev/null || true } injection_waiter_count() { local point=$1 sql -c " SELECT count(*) FROM pg_stat_activity WHERE wait_event_type = 'InjectionPoint' AND wait_event = '${point}';" 2>/dev/null || echo "" } wait_for_injection() { local point=$1 local backend=$2 local deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do if [ "$(injection_waiting "${point}" "${backend}")" = "t" ]; then log "Backend ${backend} is waiting at ${point}" return fi sleep 0.05 done fail "backend ${backend} did not reach injection point ${point}" } assert_still_paused() { local point=$1 local backend=$2 local label=$3 [ "$(injection_waiting "${point}" "${backend}")" = "t" ] || fail "${label} did not overlap the ${point} wait" } # Wake exactly one waiter. injection_points_wakeup() releases a single # registered waiter per call, so confirm the population drops before any # later call, which would otherwise re-signal the same slot. release_injection() { local point=$1 local before local deadline before=$(injection_waiter_count "${point}") [[ "${before}" =~ ^[0-9]+$ ]] && ((before >= 1)) || fail "no backend is waiting at ${point}" sql -c "SELECT injection_points_wakeup('${point}');" >/dev/null || fail "could not wake injection point ${point}" deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do [ "$(injection_waiter_count "${point}")" = "$((before - 1))" ] && return sleep 0.05 done fail "waiter count at ${point} did not fall to $((before - 1))" } wait_for_exit() { local pid=$1 local timeout_seconds=$2 local label=$3 local deadline=$((SECONDS + timeout_seconds)) while kill -0 "${pid}" 2>/dev/null; do ((SECONDS < deadline)) || fail "${label} did not finish within ${timeout_seconds} seconds" sleep 0.05 done } wait_success() { local pid=$1 local timeout_seconds=$2 local label=$3 local output_file=$4 wait_for_exit "${pid}" "${timeout_seconds}" "${label}" if ! wait "${pid}"; then tail -n 40 "${output_file}" 2>/dev/null || true fail "${label} failed" fi } require_completion_during_pause() { local operation_pid=$1 local compactor_pid=$2 local label=$3 local output_file=$4 local point=$5 local backend=$6 wait_success "${operation_pid}" 3 "${label}" "${output_file}" kill -0 "${compactor_pid}" 2>/dev/null || fail "${label} completed only after compaction left its pause" assert_still_paused "${point}" "${backend}" "${label}" log "${label} completed while compaction was still paused" } wait_for_spill_queue_or_completion() { local operation_pid=$1 local backend=$2 local blocker=$3 local deadline=$((SECONDS + 3)) local blocked while ((SECONDS < deadline)); do blocked=$(sql -c " SELECT wait_event_type = 'LWLock' AND wait_event = 'tapir_index_lock' FROM pg_stat_activity WHERE pid = ${backend};" \ 2>/dev/null || true) if [ "${blocked}" = "t" ]; then log "Observed spill backend ${backend} queued behind ${blocker}" return fi if ! kill -0 "${operation_pid}" 2>/dev/null; then log "Spill completed before it needed to queue" return fi sleep 0.05 done fail "spill neither queued nor completed within 3 seconds" } wait_for_publication_barrier_waiter() { local operation_pid=$1 local backend=$2 local oid=$3 local deadline=$((SECONDS + 3)) local blocked while ((SECONDS < deadline)); do blocked=$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_locks WHERE pid = ${backend} AND locktype = 'object' AND classid = 'pg_am'::regclass AND objid = ${oid} AND objsubid = 4 AND mode = 'ShareLock' AND NOT granted );" 2>/dev/null || true) if [ "${blocked}" = "t" ]; then log "Observed spill backend ${backend} queued on publication barrier" return fi kill -0 "${operation_pid}" 2>/dev/null || fail "spill completed without waiting on publication barrier" sleep 0.05 done fail "spill did not queue on publication barrier within 3 seconds" } test_scan_progress() { local compactor_output="${CLIENT_DIR}/scan_compactor.log" local scan_output="${CLIENT_DIR}/scan_reader.log" local compactor_pid local reader_pid local backend local oid log "Case: ranked scan progresses during unlocked build..." oid=$(index_oid scan_idx) start_compaction pgts-scan-compactor scan_idx \ "${POINT_AFTER_SELECT}" "${compactor_output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-scan-compactor) wait_for_injection "${POINT_AFTER_SELECT}" "${backend}" start_sql pgts-scan-reader " SET enable_seqscan = off; SELECT count(*) FROM ( SELECT id FROM scan_docs ORDER BY body <@> to_bm25query('scancase', 'scan_idx') LIMIT 1000 ) ranked;" "${scan_output}" reader_pid=${STARTED_PID} require_completion_during_pause \ "${reader_pid}" "${compactor_pid}" "ranked scan" "${scan_output}" \ "${POINT_AFTER_SELECT}" "${backend}" [ "$(tail -n 1 "${scan_output}")" = "32" ] || fail "ranked scan did not return all 32 documents" release_injection "${POINT_AFTER_SELECT}" wait_success "${compactor_pid}" 10 "scan compactor" "${compactor_output}" assert_all_documents scan_docs scan_idx scancase } test_source_estimation_progress() { local compactor_output="${CLIENT_DIR}/planning_compactor.log" local spill_output="${CLIENT_DIR}/planning_spill.log" local scan_output="${CLIENT_DIR}/planning_reader.log" local compactor_pid local spiller_pid local spiller_backend local reader_pid local backend local oid local reader_count log "Case: ranked scan passes a spill queued during source estimation..." oid=$(index_oid planning_idx) start_compaction \ pgts-planning-compactor planning_idx \ "${POINT_SOURCE_ESTIMATE} ${POINT_AFTER_SELECT}" \ "${compactor_output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-planning-compactor) wait_for_injection "${POINT_SOURCE_ESTIMATE}" "${backend}" start_sql pgts-planning-spill " INSERT INTO planning_docs(body) SELECT 'common planningcase concurrent prefix ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('planning_idx');" "${spill_output}" spiller_pid=${STARTED_PID} spiller_backend=$(backend_pid_if_running \ pgts-planning-spill "${spiller_pid}") if [ -n "${spiller_backend}" ]; then wait_for_spill_queue_or_completion \ "${spiller_pid}" "${spiller_backend}" "${backend}" else log "Spill completed before it needed to queue" fi start_sql pgts-planning-reader " SET enable_seqscan = off; SELECT count(*) FROM ( SELECT id FROM planning_docs ORDER BY body <@> to_bm25query( 'planningcase', 'planning_idx') LIMIT 1000 ) ranked;" "${scan_output}" reader_pid=${STARTED_PID} require_completion_during_pause \ "${reader_pid}" "${compactor_pid}" "source-estimation ranked scan" \ "${scan_output}" "${POINT_SOURCE_ESTIMATE}" "${backend}" reader_count=$(tail -n 1 "${scan_output}") [[ "${reader_count}" =~ ^[0-9]+$ ]] || fail "source-estimation ranked scan returned '${reader_count}'" require_completion_during_pause \ "${spiller_pid}" "${compactor_pid}" "source-estimation spill" \ "${spill_output}" "${POINT_SOURCE_ESTIMATE}" "${backend}" release_injection "${POINT_SOURCE_ESTIMATE}" wait_for_injection "${POINT_AFTER_SELECT}" "${backend}" release_injection "${POINT_AFTER_SELECT}" wait_success "${compactor_pid}" 15 \ "planning compactor" "${compactor_output}" assert_graph planning_idx "{1,1,0,0,0,0,0,0}" assert_all_documents planning_docs planning_idx planningcase } test_scan_progress_after_restamp() { local compactor_output="${CLIENT_DIR}/restamp_compactor.log" local scan_output="${CLIENT_DIR}/restamp_reader.log" local compactor_pid local reader_pid local backend local oid log "Case: ranked scan progresses after detached reclaim restamping..." oid=$(index_oid restamp_idx) start_compaction pgts-restamp-compactor restamp_idx \ "${POINT_AFTER_RESTAMP}" "${compactor_output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-restamp-compactor) wait_for_injection "${POINT_AFTER_RESTAMP}" "${backend}" start_sql pgts-restamp-reader " SET enable_seqscan = off; SELECT count(*) FROM ( SELECT id FROM restamp_docs ORDER BY body <@> to_bm25query('restampcase', 'restamp_idx') LIMIT 1000 ) ranked;" "${scan_output}" reader_pid=${STARTED_PID} require_completion_during_pause \ "${reader_pid}" "${compactor_pid}" "post-restamp ranked scan" \ "${scan_output}" "${POINT_AFTER_RESTAMP}" "${backend}" [ "$(tail -n 1 "${scan_output}")" = "32" ] || fail "post-restamp ranked scan did not return all 32 documents" release_injection "${POINT_AFTER_RESTAMP}" wait_success "${compactor_pid}" 10 \ "restamp compactor" "${compactor_output}" assert_all_documents restamp_docs restamp_idx restampcase } test_insert_progress() { local compactor_output="${CLIENT_DIR}/insert_compactor.log" local insert_output="${CLIENT_DIR}/insert_writer.log" local compactor_pid local writer_pid local backend local oid log "Case: insert progresses during unlocked build..." oid=$(index_oid insert_idx) start_compaction pgts-insert-compactor insert_idx \ "${POINT_AFTER_SELECT}" "${compactor_output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-insert-compactor) wait_for_injection "${POINT_AFTER_SELECT}" "${backend}" start_sql pgts-insert-writer " INSERT INTO insert_docs(body) VALUES ('common insertcase live insert') RETURNING id;" "${insert_output}" writer_pid=${STARTED_PID} require_completion_during_pause \ "${writer_pid}" "${compactor_pid}" "insert" "${insert_output}" \ "${POINT_AFTER_SELECT}" "${backend}" release_injection "${POINT_AFTER_SELECT}" wait_success "${compactor_pid}" 10 \ "insert compactor" "${compactor_output}" assert_all_documents insert_docs insert_idx insertcase } test_spill_prefix_progress() { local compactor_output="${CLIENT_DIR}/spill_compactor.log" local spill_output="${CLIENT_DIR}/spill_writer.log" local compactor_pid local spiller_pid local backend local oid log "Case: concurrent spill prefix survives cross-level consolidation..." sql -c " SELECT bm25_compact('spill_idx'); SELECT pg_textsearch_test_move_level('spill_idx', 1, 7); INSERT INTO spill_docs(body) SELECT 'common spillcase below threshold ' || gs FROM generate_series(1, 32) gs; SELECT bm25_spill_index('spill_idx');" >/dev/null assert_graph spill_idx "{1,0,0,0,0,0,0,1}" oid=$(index_oid spill_idx) start_compaction pgts-spill-compactor spill_idx \ "${POINT_AFTER_SELECT}" "${compactor_output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-spill-compactor) wait_for_injection "${POINT_AFTER_SELECT}" "${backend}" start_sql pgts-spill-writer " INSERT INTO spill_docs(body) SELECT 'common spillcase concurrent prefix ' || gs FROM generate_series(1, 5) gs; SELECT bm25_spill_index('spill_idx');" "${spill_output}" spiller_pid=${STARTED_PID} require_completion_during_pause \ "${spiller_pid}" "${compactor_pid}" "spill" "${spill_output}" \ "${POINT_AFTER_SELECT}" "${backend}" release_injection "${POINT_AFTER_SELECT}" wait_success "${compactor_pid}" 10 \ "spill compactor" "${compactor_output}" assert_graph spill_idx "{1,1,0,0,0,0,0,0}" assert_all_documents spill_docs spill_idx spillcase } test_same_index_serialization() { local first_output="${CLIENT_DIR}/serial_first.log" local second_output="${CLIENT_DIR}/serial_second.log" local first_pid local second_pid local first_backend local second_backend local oid local lock_proof local deadline log "Case: same-index maintenance serializes..." oid=$(index_oid serial_idx) start_compaction pgts-serial-first serial_idx \ "${POINT_AFTER_SELECT}" "${first_output}" first_pid=${STARTED_PID} first_backend=$(backend_pid pgts-serial-first) wait_for_injection "${POINT_AFTER_SELECT}" "${first_backend}" start_compaction pgts-serial-second serial_idx \ "" "${second_output}" second_pid=${STARTED_PID} second_backend=$(backend_pid pgts-serial-second) deadline=$((SECONDS + 3)) lock_proof=f while ((SECONDS < deadline)); do lock_proof=$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity activity JOIN pg_locks pending ON pending.pid = activity.pid WHERE activity.pid = ${second_backend} AND activity.state = 'active' AND ${first_backend} = ANY (pg_blocking_pids(activity.pid)) AND pending.locktype = 'object' AND pending.classid = 'pg_am'::regclass AND pending.objid = ${oid} AND pending.objsubid = 3 AND pending.mode = 'ExclusiveLock' AND NOT pending.granted );" 2>/dev/null || true) if [ "${lock_proof}" = "t" ]; then break fi sleep 0.05 done [ "${lock_proof}" = "t" ] || fail "second compactor was not blocked by backend ${first_backend} on the serial_idx maintenance lock" kill -0 "${first_pid}" 2>/dev/null || fail "first compactor left its pause before serialization proof" assert_still_paused \ "${POINT_AFTER_SELECT}" "${first_backend}" "same-index compactor" kill -0 "${second_pid}" 2>/dev/null || fail "second same-index compactor completed during the first pause" release_injection "${POINT_AFTER_SELECT}" wait_success "${first_pid}" 10 "first serial compactor" "${first_output}" wait_success "${second_pid}" 10 "second serial compactor" "${second_output}" assert_all_documents serial_docs serial_idx serialcase } test_different_index_overlap() { local a_output="${CLIENT_DIR}/parallel_a.log" local b_output="${CLIENT_DIR}/parallel_b.log" local a_pid local b_pid local a_backend local b_backend log "Case: different indexes compact concurrently..." # A point name is a cluster-wide registry entry, so the two # compactors park at different phases of the same unlocked build # rather than sharing one name. start_compaction pgts-parallel-a parallel_a_idx \ "${POINT_AFTER_SELECT}" "${a_output}" a_pid=${STARTED_PID} a_backend=$(backend_pid pgts-parallel-a) wait_for_injection "${POINT_AFTER_SELECT}" "${a_backend}" start_compaction pgts-parallel-b parallel_b_idx \ "${POINT_SOURCE_ESTIMATE}" "${b_output}" b_pid=${STARTED_PID} b_backend=$(backend_pid pgts-parallel-b) wait_for_injection "${POINT_SOURCE_ESTIMATE}" "${b_backend}" kill -0 "${a_pid}" 2>/dev/null || fail "index A finished before index B started its own build" assert_still_paused \ "${POINT_AFTER_SELECT}" "${a_backend}" "different-index compactor" release_injection "${POINT_AFTER_SELECT}" release_injection "${POINT_SOURCE_ESTIMATE}" wait_success "${a_pid}" 10 "parallel compactor A" "${a_output}" wait_success "${b_pid}" 10 "parallel compactor B" "${b_output}" assert_all_documents parallel_a_docs parallel_a_idx parallelacase assert_all_documents parallel_b_docs parallel_b_idx parallelbcase } test_publication_barrier_progress() { local compactor_output="${CLIENT_DIR}/publication_compactor.log" local first_spill_output="${CLIENT_DIR}/publication_first_spill.log" local second_spill_output="${CLIENT_DIR}/publication_second_spill.log" local compactor_pid local spiller_pid local spiller_backend local backend local oid local retry_marker local retry_count log "Case: publication barrier prevents spill identity races..." oid=$(index_oid publication_idx) start_compaction \ pgts-publication-compactor publication_idx \ "${POINT_AFTER_SELECT} ${POINT_BEFORE_PUBLISH}" \ "${compactor_output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-publication-compactor) wait_for_injection "${POINT_AFTER_SELECT}" "${backend}" start_sql pgts-publication-first-spill " INSERT INTO publication_docs(body) SELECT 'common publicationcase first prefix ' || gs FROM generate_series(1, 5) gs; SELECT bm25_spill_index('publication_idx');" "${first_spill_output}" spiller_pid=${STARTED_PID} require_completion_during_pause \ "${spiller_pid}" "${compactor_pid}" "first publication spill" \ "${first_spill_output}" "${POINT_AFTER_SELECT}" "${backend}" release_injection "${POINT_AFTER_SELECT}" wait_for_injection "${POINT_BEFORE_PUBLISH}" "${backend}" start_sql pgts-publication-second-spill " INSERT INTO publication_docs(body) SELECT 'common publicationcase second prefix ' || gs FROM generate_series(1, 7) gs; SELECT bm25_spill_index('publication_idx');" "${second_spill_output}" spiller_pid=${STARTED_PID} spiller_backend=$(backend_pid pgts-publication-second-spill) wait_for_publication_barrier_waiter \ "${spiller_pid}" "${spiller_backend}" "${oid}" assert_still_paused \ "${POINT_BEFORE_PUBLISH}" "${backend}" "publication barrier spill" release_injection "${POINT_BEFORE_PUBLISH}" wait_success "${compactor_pid}" 15 \ "publication compactor" "${compactor_output}" wait_success "${spiller_pid}" 10 \ "second publication spill" "${second_spill_output}" retry_marker="pg_textsearch compaction publication retry for index ${oid} backend ${backend}" retry_count=$(grep -Fc "${retry_marker}" "${LOGFILE}" || true) [ "${retry_count}" = "0" ] || fail "publication retried ${retry_count} times despite its barrier" assert_graph publication_idx "{2,1,0,0,0,0,0,0}" assert_all_documents \ publication_docs publication_idx publicationcase } test_cancel_before_publish() { local output="${CLIENT_DIR}/cancel_compactor.log" local compactor_pid local backend local oid local before_graph local before_topology local before_signature local cancel_result log "Case: cancellation before publication preserves the old graph..." oid=$(index_oid cancel_idx) before_graph=$(graph cancel_idx) before_topology=$(segment_topology cancel_idx) before_signature=$(ranked_signature cancel_docs cancel_idx cancelcase) start_compaction pgts-cancel-compactor cancel_idx \ "${POINT_BEFORE_PUBLISH}" "${output}" compactor_pid=${STARTED_PID} backend=$(backend_pid pgts-cancel-compactor) wait_for_injection "${POINT_BEFORE_PUBLISH}" "${backend}" cancel_result=$(sql -c "SELECT pg_cancel_backend(${backend});") [ "${cancel_result}" = "t" ] || fail "pg_cancel_backend(${backend}) returned ${cancel_result}" wait_for_exit "${compactor_pid}" 10 "cancelled compactor" if wait "${compactor_pid}"; then fail "cancelled compactor unexpectedly succeeded" fi grep -Fq "canceling statement due to user request" "${output}" || fail "compactor did not report query cancellation" [ "$(graph cancel_idx)" = "${before_graph}" ] || fail "cancellation changed the published segment graph" [ "$(segment_topology cancel_idx)" = "${before_topology}" ] || fail "cancellation changed the ordered segment-root topology" [ "$(ranked_signature cancel_docs cancel_idx cancelcase)" = \ "${before_signature}" ] || fail "cancellation changed ranked IDs or scores" assert_all_documents cancel_docs cancel_idx cancelcase } test_cancel_after_allocation() { local phase=$1 local table_name=$2 local index_name=$3 local control_table=$4 local control_index=$5 local token=$6 local app_name="pgts-${phase}-cancel" local output="${CLIENT_DIR}/${phase}_cancel.log" local compactor_pid local backend local oid local before_graph local before_pending local before_topology local before_signature local before_blocks local cancelled_blocks local final_blocks local control_before_blocks local control_final_blocks local cancel_result log "Case: cancellation after ${phase} allocation discards owned pages..." oid=$(index_oid "${index_name}") before_graph=$(graph "${index_name}") before_pending=$(pending_free "${index_name}") before_topology=$(segment_topology "${index_name}") before_signature=$( ranked_signature "${table_name}" "${index_name}" "${token}") before_blocks=$(relation_blocks "${index_name}") control_before_blocks=$(relation_blocks "${control_index}") start_compaction \ "${app_name}" "${index_name}" \ "pg-textsearch-compaction-alloc-${phase}" \ "${output}" compactor_pid=${STARTED_PID} backend=$(backend_pid "${app_name}") wait_for_injection "pg-textsearch-compaction-alloc-${phase}" "${backend}" cancel_result=$(sql -c "SELECT pg_cancel_backend(${backend});") [ "${cancel_result}" = "t" ] || fail "pg_cancel_backend(${backend}) returned ${cancel_result}" wait_for_exit "${compactor_pid}" 10 "${phase} cancelled compactor" if wait "${compactor_pid}"; then fail "${phase} cancelled compactor unexpectedly succeeded" fi grep -Fq "canceling statement due to user request" "${output}" || fail "${phase} compactor did not report query cancellation" [ "$(graph "${index_name}")" = "${before_graph}" ] || fail "${phase} cancellation changed the published segment graph" [ "$(pending_free "${index_name}")" = "${before_pending}" ] || fail "${phase} cancellation changed the pending-free count" [ "$(segment_topology "${index_name}")" = "${before_topology}" ] || fail "${phase} cancellation changed the ordered segment-root topology" [ "$(ranked_signature \ "${table_name}" "${index_name}" "${token}")" = \ "${before_signature}" ] || fail "${phase} cancellation changed ranked IDs or scores" assert_all_documents "${table_name}" "${index_name}" "${token}" cancelled_blocks=$(relation_blocks "${index_name}") [ "${cancelled_blocks}" -ge "${before_blocks}" ] || fail "${phase} cancellation unexpectedly shrank the index" sql -c "SELECT bm25_compact_step('${index_name}'::regclass);" >/dev/null sql -c "SELECT bm25_compact_step('${control_index}'::regclass);" >/dev/null assert_graph "${index_name}" "{0,1,0,0,0,0,0,0}" assert_graph "${control_index}" "{0,1,0,0,0,0,0,0}" assert_all_documents "${table_name}" "${index_name}" "${token}" assert_all_documents "${control_table}" "${control_index}" "${token}" final_blocks=$(relation_blocks "${index_name}") control_final_blocks=$(relation_blocks "${control_index}") [ $((final_blocks - before_blocks)) -eq \ $((control_final_blocks - control_before_blocks)) ] || fail "${phase} discarded pages were not reused by later compaction" sql -c " INSERT INTO ${table_name}(body) VALUES ('common ${token} post-cancel spill'); SELECT bm25_spill_index('${index_name}');" >/dev/null assert_graph "${index_name}" "{1,1,0,0,0,0,0,0}" assert_all_documents "${table_name}" "${index_name}" "${token}" } test_partial_allocation_cancellation() { test_cancel_after_allocation \ output-data output_data_docs output_data_idx \ output_data_control_docs output_data_control_idx output_datacase test_cancel_after_allocation \ page-index page_index_docs page_index_idx \ page_index_control_docs page_index_control_idx page_indexcase test_cancel_after_allocation \ tombstone tombstone_docs tombstone_idx \ tombstone_control_docs tombstone_control_idx tombstonecase } main() { if [ ! -f "$(pg_config --pkglibdir)/injection_points.so" ]; then log "Skipping: PostgreSQL was built without injection points" exit 0 fi setup_cluster seed_all_indexes test_inline_policy_runs_one_pass test_source_estimation_progress test_scan_progress test_scan_progress_after_restamp test_insert_progress test_spill_prefix_progress test_same_index_serialization test_different_index_overlap test_publication_barrier_progress test_cancel_before_publish test_partial_allocation_cancellation log "All deterministic non-blocking compaction cases passed" } main "$@" timescale-pg_textsearch-b63fe58/test/scripts/nonblocking_spill.sh000077500000000000000000000543521525762047700255050ustar00rootroot00000000000000#!/bin/bash # # Verify that runtime spill construction blocks memtable writers, not readers. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG}" --bindir)" export PATH="${PGBINDIR}:${PATH}" TEST_PORT=55468 TEST_DB=nonblocking_spill_test DATA_DIR="${SCRIPT_DIR}/../tmp_nonblocking_spill" SOCKET_DIR="${SCRIPT_DIR}/.nbs_sock" LOGFILE="${DATA_DIR}/postgres.log" CLIENT_DIR="${DATA_DIR}/clients" POINT_SPILL_BEFORE_FINALIZE='pg-textsearch-spill-before-finalize' POINT_TOMBSTONE_AFTER_UNLINK='pg-textsearch-tombstone-after-unlink' POINT_BEFORE_TRUNCATE='pg-textsearch-force-merge-before-truncate' POINT_AFTER_INSPECTION='pg-textsearch-truncate-after-inspection' fail() { echo "ERROR: $*" >&2 tail -n 80 "${LOGFILE}" 2>/dev/null || true for file in "${CLIENT_DIR}"/*.log; do [ -e "${file}" ] || continue echo "==> ${file} <==" >&2 tail -n 30 "${file}" >&2 || true done exit 1 } cleanup() { local status=$? local pid trap - EXIT INT TERM for pid in $(jobs -p); do kill "${pid}" 2>/dev/null || true done if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" "${SOCKET_DIR}" exit "${status}" } trap cleanup EXIT INT TERM sql() { psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 "$@" } backend_pid() { local app_name=$1 local deadline=$((SECONDS + 10)) local pid while ((SECONDS < deadline)); do pid=$(sql -c " SELECT pid FROM pg_stat_activity WHERE application_name = '${app_name}' ORDER BY backend_start DESC LIMIT 1;" 2>/dev/null || true) if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return fi sleep 0.05 done fail "backend ${app_name} did not appear" } wait_for_injection() { local backend=$1 local deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do if [ "$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = ${backend} AND wait_event_type = 'InjectionPoint' AND wait_event = '${POINT_SPILL_BEFORE_FINALIZE}' );")" = "t" ]; then return fi sleep 0.05 done fail "spill did not reach ${POINT_SPILL_BEFORE_FINALIZE}" } wait_for_wait_event() { local backend=$1 local wait_event=$2 local deadline=$((SECONDS + 10)) local actual while ((SECONDS < deadline)); do if [ "$(sql -c " SELECT wait_event FROM pg_stat_activity WHERE pid = ${backend};")" = "${wait_event}" ]; then return fi sleep 0.05 done actual=$(sql -c " SELECT wait_event_type || ':' || wait_event FROM pg_stat_activity WHERE pid = ${backend};") fail "backend ${backend} did not wait on ${wait_event} (got ${actual})" } wait_for_exit() { local pid=$1 local seconds=$2 local label=$3 local deadline=$((SECONDS + seconds)) while kill -0 "${pid}" 2>/dev/null; do ((SECONDS < deadline)) || fail "${label} did not finish" sleep 0.05 done wait "${pid}" || fail "${label} failed" } rm -rf "${DATA_DIR}" "${SOCKET_DIR}" mkdir -p "${SOCKET_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null mkdir -p "${CLIENT_DIR}" cat >>"${DATA_DIR}/postgresql.conf" </dev/null || fail "PostgreSQL startup failed" createdb -h "${SOCKET_DIR}" -p "${TEST_PORT}" "${TEST_DB}" sql -c " CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE TABLE spill_docs (id integer PRIMARY KEY, body text NOT NULL); CREATE INDEX spill_idx ON spill_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO spill_docs SELECT gs, 'alpha document ' || gs FROM generate_series(1, 500) gs;" >/dev/null PGAPPNAME=spill-builder sql -c " SET statement_timeout = '60s'; SELECT injection_points_set_local(); SELECT injection_points_attach( '${POINT_SPILL_BEFORE_FINALIZE}', 'wait'); SELECT bm25_spill_index('spill_idx');" \ >"${CLIENT_DIR}/spill.log" 2>&1 & spill_client=$! spill_backend=$(backend_pid spill-builder) wait_for_injection "${spill_backend}" PGAPPNAME=spill-reader sql -c " SELECT count(*) FROM ( SELECT 1 FROM spill_docs ORDER BY body <@> to_bm25query('alpha', 'spill_idx') LIMIT 500 ) ranked;" >"${CLIENT_DIR}/reader.log" 2>&1 & reader_client=$! wait_for_exit "${reader_client}" 3 "ranked reader" grep -qx '500' "${CLIENT_DIR}/reader.log" || fail "ranked reader did not see the stable chain" PGAPPNAME=spill-writer sql -c " INSERT INTO spill_docs VALUES (501, 'alpha document 501');" \ >"${CLIENT_DIR}/writer.log" 2>&1 & writer_client=$! writer_backend=$(backend_pid spill-writer) wait_for_wait_event "${writer_backend}" tapir_memtable_write_lock sql -c "SELECT injection_points_wakeup( '${POINT_SPILL_BEFORE_FINALIZE}');" >/dev/null wait_for_exit "${spill_client}" 10 "spill" wait_for_exit "${writer_client}" 10 "blocked writer" final_count=$(sql -c " SELECT count(*) FROM ( SELECT 1 FROM spill_docs ORDER BY body <@> to_bm25query('alpha', 'spill_idx') LIMIT 501 ) ranked;") [ "${final_count}" = "501" ] || fail "expected 501 ranked rows after spill, got ${final_count}" sql -c " CREATE TABLE spill_cancel_docs ( id integer PRIMARY KEY, body text NOT NULL ); CREATE INDEX spill_cancel_idx ON spill_cancel_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO spill_cancel_docs SELECT gs, 'cancelterm document ' || gs FROM generate_series(1, 500) gs;" >/dev/null if PGAPPNAME=spill-cancel sql -c " SELECT injection_points_set_local(); SELECT injection_points_attach( '${POINT_SPILL_BEFORE_FINALIZE}', 'error'); SELECT bm25_spill_index('spill_cancel_idx');" \ >"${CLIENT_DIR}/cancel.log" 2>&1; then fail "injected spill error completed successfully" fi cancel_chain_count=$(sql -c " SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('spill_cancel_idx');") cancel_graph=$(sql -c " SELECT bm25_level_counts('spill_cancel_idx'::regclass)::text;") cancel_ranked_count=$(sql -c " SELECT count(*) FROM ( SELECT 1 FROM spill_cancel_docs ORDER BY body <@> to_bm25query('cancelterm', 'spill_cancel_idx') LIMIT 500 ) ranked;") [ "${cancel_chain_count}" = "500" ] || fail "failed spill state: chain=${cancel_chain_count} graph=${cancel_graph} ranked=${cancel_ranked_count}" [ "${cancel_graph}" = "{0,0,0,0,0,0,0,0}" ] || fail "failed spill published a segment" [ "${cancel_ranked_count}" = "500" ] || fail "failed spill made chain documents unqueryable" cancel_blocks_after_error=$(sql -c " SELECT pg_relation_size('spill_cancel_idx'::regclass, 'main') / current_setting('block_size')::integer;") sql -c "SELECT bm25_spill_index('spill_cancel_idx');" >/dev/null cancel_graph=$(sql -c " SELECT bm25_level_counts('spill_cancel_idx'::regclass)::text;") cancel_blocks_after_retry=$(sql -c " SELECT pg_relation_size('spill_cancel_idx'::regclass, 'main') / current_setting('block_size')::integer;") [ "${cancel_graph}" = "{1,0,0,0,0,0,0,0}" ] || fail "spill after cancellation did not publish one segment" [ "${cancel_blocks_after_retry}" = "${cancel_blocks_after_error}" ] || fail "spill retry extended from ${cancel_blocks_after_error} to ${cancel_blocks_after_retry} blocks instead of reusing discarded output" sql -c " CREATE TABLE published_docs (id integer PRIMARY KEY, body text NOT NULL); CREATE INDEX published_idx ON published_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO published_docs SELECT gs, 'publishedterm document ' || gs FROM generate_series(1, 500) gs;" >/dev/null if sql -c " SELECT injection_points_set_local(); SELECT injection_points_attach( 'pg-textsearch-after-spill-finalize', 'error'); SELECT bm25_spill_index('published_idx');" \ >"${CLIENT_DIR}/published_error.log" 2>&1; then fail "post-publication error did not fire" fi grep -q 'error triggered for injection point' \ "${CLIENT_DIR}/published_error.log" || fail "post-publication spill failed for an unexpected reason" published_graph=$(sql -c " SELECT bm25_level_counts('published_idx'::regclass)::text;") [ "${published_graph}" = "{1,0,0,0,0,0,0,0}" ] || fail "post-publication error lost the segment" [ "$(sql -c " SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('published_idx');")" = "0" ] || fail "post-publication error left the old chain published" published_ids=$(sql -c " SET enable_seqscan = off; SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM published_docs ORDER BY body <@> to_bm25query('publishedterm', 'published_idx') LIMIT 500 ) ranked;") [ "${published_ids}" = "$(seq -s, 1 500)" ] || fail "post-publication error damaged the published segment" sql -c " CREATE TABLE reclaim_docs (id integer PRIMARY KEY, body text NOT NULL); CREATE INDEX reclaim_idx ON reclaim_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual');" >/dev/null for batch in 1 2; do sql -c " INSERT INTO reclaim_docs SELECT (${batch} - 1) * 200 + gs, 'reclaimterm batch ${batch} document ' || gs FROM generate_series(1, 200) gs; SELECT bm25_spill_index('reclaim_idx');" >/dev/null done sql -c "SELECT bm25_compact_step('reclaim_idx'::regclass);" >/dev/null [ "$(sql -c "SELECT bm25_pending_free_pages('reclaim_idx');")" -gt 0 ] || fail "first compaction did not create deferred reclaim work" for batch in 3 4; do sql -c " INSERT INTO reclaim_docs SELECT (${batch} - 1) * 200 + gs, 'reclaimterm batch ${batch} document ' || gs FROM generate_series(1, 200) gs; SELECT bm25_spill_index('reclaim_idx');" >/dev/null done PGAPPNAME=reclaim-builder sql -c " SET statement_timeout = '60s'; SELECT injection_points_set_local(); SELECT injection_points_attach( '${POINT_TOMBSTONE_AFTER_UNLINK}', 'wait'); SELECT bm25_compact_step('reclaim_idx'::regclass);" \ >"${CLIENT_DIR}/reclaim.log" 2>&1 & reclaim_client=$! reclaim_backend=$(backend_pid reclaim-builder) deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do if [ "$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = ${reclaim_backend} AND wait_event_type = 'InjectionPoint' AND wait_event = '${POINT_TOMBSTONE_AFTER_UNLINK}' );")" = "t" ]; then break fi sleep 0.05 done reclaim_wait_event=$(sql -c " SELECT wait_event FROM pg_stat_activity WHERE pid = ${reclaim_backend};") [ "${reclaim_wait_event}" = "${POINT_TOMBSTONE_AFTER_UNLINK}" ] || fail "reclaim did not pause after tombstone unlink" PGAPPNAME=reclaim-reader sql -c " SELECT count(*) FROM ( SELECT 1 FROM reclaim_docs ORDER BY body <@> to_bm25query('reclaimterm', 'reclaim_idx') LIMIT 800 ) ranked;" >"${CLIENT_DIR}/reclaim_reader.log" 2>&1 & reclaim_reader=$! wait_for_exit "${reclaim_reader}" 3 "reader during tombstone free" grep -qx '800' "${CLIENT_DIR}/reclaim_reader.log" || fail "reader during tombstone free returned wrong results" sql -c "SELECT injection_points_wakeup( '${POINT_TOMBSTONE_AFTER_UNLINK}');" >/dev/null wait_for_exit "${reclaim_client}" 20 "reclaim compaction" sql -c " CREATE EXTENSION pg_textsearch_test; CREATE TABLE capacity_docs (id integer, body text); CREATE INDEX capacity_idx ON capacity_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO capacity_docs VALUES (0, 'capacityterm'); SELECT bm25_spill_index('capacity_idx'); INSERT INTO capacity_docs SELECT gs, 'capacityterm document ' || gs FROM generate_series(1, 500) gs; SELECT pg_textsearch_test_attach_segment_limit(1); DO \$\$ DECLARE before_bytes bigint := pg_relation_size('capacity_idx'); BEGIN BEGIN PERFORM bm25_spill_index('capacity_idx'); RAISE EXCEPTION 'full L0 spill unexpectedly succeeded'; EXCEPTION WHEN program_limit_exceeded THEN NULL; END; IF pg_relation_size('capacity_idx') <> before_bytes THEN RAISE EXCEPTION 'full L0 spill allocated unreachable output'; END IF; END \$\$; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit');" >/dev/null # A shutdown spill must not enter the blocking acquisition at either phase. for phase in freeze publish; do sql -c " CREATE TABLE shutdown_${phase}_docs (id integer, body text); CREATE INDEX shutdown_${phase}_idx ON shutdown_${phase}_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO shutdown_${phase}_docs SELECT gs, 'shutdownterm document ' || gs FROM generate_series(1, 500) gs;" >/dev/null point='pg-textsearch-index-lock-exclusive-waiter' if [ "${phase}" = publish ]; then point="${POINT_SPILL_BEFORE_FINALIZE}" fi PGAPPNAME=shutdown-target sql -c " SELECT count(*) FROM ( SELECT 1 FROM shutdown_${phase}_docs ORDER BY body <@> to_bm25query( 'shutdownterm', 'shutdown_${phase}_idx') LIMIT 1 ) ranked; SELECT pg_sleep(60);" >"${CLIENT_DIR}/shutdown_${phase}.log" 2>&1 & shutdown_client=$! shutdown_backend=$(backend_pid shutdown-target) wait_for_wait_event "${shutdown_backend}" PgSleep sql -c "SELECT injection_points_attach('${point}', 'wait');" >/dev/null sql -c "SELECT pg_terminate_backend(${shutdown_backend}, 5000);" \ >"${CLIENT_DIR}/terminate_${phase}.log" 2>&1 & terminate_client=$! if [ "${phase}" = publish ]; then wait_for_injection "${shutdown_backend}" sql -c " SELECT injection_points_attach( 'pg-textsearch-index-lock-exclusive-waiter', 'wait'); SELECT injection_points_wakeup( '${POINT_SPILL_BEFORE_FINALIZE}');" >/dev/null fi wait_for_exit "${terminate_client}" 7 "shutdown ${phase} termination" grep -qx 't' "${CLIENT_DIR}/terminate_${phase}.log" || fail "shutdown ${phase} used a blocking index acquisition" if wait "${shutdown_client}"; then fail "terminated shutdown ${phase} client unexpectedly succeeded" fi sql -c "SELECT injection_points_detach('${point}');" >/dev/null if [ "${phase}" = publish ]; then sql -c "SELECT injection_points_detach( 'pg-textsearch-index-lock-exclusive-waiter');" >/dev/null fi [ "$(sql -c " SELECT bm25_level_counts('shutdown_${phase}_idx'::regclass)::text; ")" = "{1,0,0,0,0,0,0,0}" ] || fail "shutdown ${phase} did not publish its uncontended spill" done # Removing the truncator's writer gate must fail both lock-order checks. # VACUUM leaves a large recyclable suffix behind a small surviving segment. for ordering in spill_first truncate_first; do sql -c " CREATE TABLE ${ordering}_docs (id integer PRIMARY KEY, body text); INSERT INTO ${ordering}_docs SELECT gs, 'truncateterm original ' || gs FROM generate_series(1, 500) gs; CREATE INDEX ${ordering}_idx ON ${ordering}_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO ${ordering}_docs SELECT gs, 'discardterm ' || md5(gs::text) FROM generate_series(1001, 11000) gs; SELECT bm25_spill_index('${ordering}_idx'); DELETE FROM ${ordering}_docs WHERE id > 1000;" >/dev/null sql -c "VACUUM ${ordering}_docs;" >/dev/null sql -c "SELECT txid_current();" >/dev/null sql -c "SELECT txid_current();" >/dev/null sql -c "VACUUM ${ordering}_docs;" >/dev/null [ "$(sql -c " SELECT bm25_pending_free_pages('${ordering}_idx');")" = "0" ] || fail "${ordering}: fixture still has parked pages" PGAPPNAME=truncate-merge sql -c " SET statement_timeout = '60s'; SELECT injection_points_set_local(); SELECT injection_points_attach('${POINT_BEFORE_TRUNCATE}', 'wait'); SELECT injection_points_attach('${POINT_AFTER_INSPECTION}', 'wait'); SELECT bm25_force_merge('${ordering}_idx');" \ >"${CLIENT_DIR}/${ordering}_merge.log" 2>&1 & merge_client=$! merge_backend=$(backend_pid truncate-merge) wait_for_wait_event "${merge_backend}" "${POINT_BEFORE_TRUNCATE}" # The force-merge's initial spill is already over. Add a fresh chain # without letting a session-exit spill consume it. sql -c " INSERT INTO ${ordering}_docs SELECT gs, 'truncateterm pending ' || gs FROM generate_series(501, 1000) gs; CREATE TABLE ${ordering}_expected AS SELECT id, body <@> to_bm25query( 'truncateterm', '${ordering}_idx') AS score FROM ${ordering}_docs ORDER BY body <@> to_bm25query('truncateterm', '${ordering}_idx') LIMIT 1000;" >/dev/null bytes_before=$(sql -c " SELECT pg_relation_size('${ordering}_idx');") if [ "${ordering}" = truncate_first ]; then sql -c "SELECT injection_points_wakeup( '${POINT_BEFORE_TRUNCATE}');" >/dev/null wait_for_wait_event "${merge_backend}" "${POINT_AFTER_INSPECTION}" fi PGAPPNAME=truncate-spill sql -c " SET statement_timeout = '60s'; SELECT injection_points_set_local(); SELECT injection_points_attach( '${POINT_SPILL_BEFORE_FINALIZE}', 'wait'); SELECT bm25_spill_index('${ordering}_idx');" \ >"${CLIENT_DIR}/${ordering}_spill.log" 2>&1 & spill_client=$! spill_backend=$(backend_pid truncate-spill) if [ "${ordering}" = spill_first ]; then wait_for_injection "${spill_backend}" # Construction has released the index lock, but not the writer # gate. Truncation must wait for that in-flight allocation owner. sql -c "SELECT injection_points_wakeup( '${POINT_BEFORE_TRUNCATE}');" >/dev/null wait_for_wait_event "${merge_backend}" tapir_memtable_write_lock sql -c "SELECT injection_points_wakeup( '${POINT_SPILL_BEFORE_FINALIZE}');" >/dev/null wait_for_exit "${spill_client}" 10 "spill before truncation" wait_for_wait_event "${merge_backend}" "${POINT_AFTER_INSPECTION}" sql -c "SELECT injection_points_wakeup( '${POINT_AFTER_INSPECTION}');" >/dev/null wait_for_exit "${merge_client}" 10 "merge after spill" else # An inspected suffix is still in the FSM. No spill may start # allocating it before RelationTruncate completes. wait_for_wait_event "${spill_backend}" tapir_memtable_write_lock sql -c "SELECT injection_points_wakeup( '${POINT_AFTER_INSPECTION}');" >/dev/null wait_for_exit "${merge_client}" 10 "merge before spill" wait_for_injection "${spill_backend}" sql -c "SELECT injection_points_wakeup( '${POINT_SPILL_BEFORE_FINALIZE}');" >/dev/null wait_for_exit "${spill_client}" 10 "spill after truncation" fi bytes_after=$(sql -c " SELECT pg_relation_size('${ordering}_idx');") [ "${bytes_after}" -lt "${bytes_before}" ] || fail "${ordering}: fixture did not exercise suffix truncation" done # Read every segment page map (including root/page-index references), then # compare every ranked ID and score with the pre-spill chain result. check_truncated_indexes() { local ordering for ordering in spill_first truncate_first; do sql -c " SET enable_seqscan = off; DO \$\$ DECLARE summary text := bm25_summarize_index('${ordering}_idx'); BEGIN IF summary !~ 'docs_persisted: 1000' OR (SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('${ordering}_idx')) <> 0 THEN RAISE EXCEPTION 'invalid post-truncate graph: %', summary; END IF; IF EXISTS ( WITH actual AS ( SELECT id, body <@> to_bm25query( 'truncateterm', '${ordering}_idx') AS score FROM ${ordering}_docs ORDER BY body <@> to_bm25query( 'truncateterm', '${ordering}_idx') LIMIT 1000 ) (SELECT * FROM actual EXCEPT ALL SELECT * FROM ${ordering}_expected) UNION ALL (SELECT * FROM ${ordering}_expected EXCEPT ALL SELECT * FROM actual) ) THEN RAISE EXCEPTION 'truncation changed ranked results'; END IF; IF (SELECT count(*) FROM ${ordering}_expected) <> 1000 THEN RAISE EXCEPTION 'incomplete pre-spill query result'; END IF; END \$\$;" >/dev/null || fail "${ordering}: graph/result check failed" done } check_truncated_indexes # Spill is a physical change in a read-only SQL transaction. Force a later # commit record to flush its WAL before testing recovery of the new graph. sql -c " UPDATE truncate_first_expected SET score = score WHERE id = 1;" \ >/dev/null pg_ctl stop -D "${DATA_DIR}" -m immediate -w >/dev/null pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w >/dev/null || fail "PostgreSQL recovery startup failed" check_truncated_indexes echo "nonblocking spill test passed" timescale-pg_textsearch-b63fe58/test/scripts/parallel_build_bulkread.sh000077500000000000000000000061511525762047700266150ustar00rootroot00000000000000#!/bin/bash # Parallel builds should reuse a bulk-read ring, not fill shared_buffers. set -euo pipefail PG_CONFIG="${PG_CONFIG:-pg_config}" export PATH="$("${PG_CONFIG}" --bindir):${PATH}" TEST_DIR=$(mktemp -d "${TMPDIR:-/tmp}/pgts_bulkread.XXXXXX") DATA_DIR="${TEST_DIR}/data" export PGHOST="${TEST_DIR}" export PGPORT="${TEST_PORT:-55479}" export PGDATABASE=postgres cleanup() { local status=$? trap - EXIT if [ -f "${DATA_DIR}/postmaster.pid" ]; then if ! pg_ctl stop -D "${DATA_DIR}" -m fast -w >/dev/null; then status=1 fi fi if [ "${status}" -eq 0 ]; then rm -rf "${TEST_DIR}" else echo "FAIL: evidence preserved in ${TEST_DIR}" >&2 fi exit "${status}" } trap cleanup EXIT trap 'exit 130' INT trap 'exit 143' TERM sql() { psql -XqAt -v ON_ERROR_STOP=1 "$@" } initdb -D "${DATA_DIR}" -A trust --no-locale >/dev/null pg_ctl start -D "${DATA_DIR}" -l "${TEST_DIR}/server.log" -w \ -o "-k ${PGHOST} -p ${PGPORT} -c listen_addresses='' \ -c shared_preload_libraries=pg_textsearch \ -c shared_buffers=16MB -c max_worker_processes=4 \ -c max_parallel_workers=2 \ -c max_parallel_maintenance_workers=2" >/dev/null sql >"${TEST_DIR}/fixture.log" 2>&1 <<'SQL' CREATE EXTENSION pg_textsearch; CREATE TABLE docs (id integer, body text) WITH (parallel_workers=2); INSERT INTO docs SELECT i, repeat('database search benchmark ', 12) FROM generate_series(1, 150000) i; ANALYZE docs; DO $$ BEGIN IF pg_relation_size('docs') <= pg_size_bytes(current_setting('shared_buffers')) / 4 THEN RAISE EXCEPTION 'fixture too small for bulk-read strategy'; END IF; END $$; SELECT pg_stat_reset_shared('io'); SQL sql >"${TEST_DIR}/build.log" 2>&1 <<'SQL' SET maintenance_work_mem = '64MB'; CREATE INDEX docs_bm25 ON docs USING bm25(body) WITH (text_config='english'); SET enable_seqscan = off; DO $$ DECLARE hits bigint; distinct_ids bigint; BEGIN SELECT count(*), count(DISTINCT id) INTO hits, distinct_ids FROM ( SELECT id FROM docs ORDER BY body <@> to_bm25query('database', 'docs_bm25') LIMIT 150000 ) ranked; IF hits <> 150000 OR distinct_ids <> 150000 THEN RAISE EXCEPTION 'incorrect build: % hits, % distinct IDs', hits, distinct_ids; END IF; END $$; SQL grep -Fq 'launched 2 of 2 requested workers' "${TEST_DIR}/build.log" || { echo "ERROR: expected two parallel build workers" >&2 exit 1 } # pg_stat_io groups parallel workers under "background worker". # Their statistics can arrive just after CREATE INDEX returns. for attempt in $(seq 1 100); do reuses=$(sql -c "SELECT coalesce(sum(reuses), 0) FROM pg_stat_io WHERE backend_type = 'background worker' AND object = 'relation' AND context = 'bulkread'") if [ "${reuses}" -gt 0 ]; then echo "PASS: parallel workers reused bulk-read buffers (${reuses})" exit 0 fi sleep 0.1 done sql -c "SELECT * FROM pg_stat_io WHERE backend_type='background worker'" \ >"${TEST_DIR}/io.log" echo "ERROR: parallel build did not reuse bulk-read buffers" >&2 exit 1 timescale-pg_textsearch-b63fe58/test/scripts/parallel_vacuum.sh000077500000000000000000000255671525762047700251610ustar00rootroot00000000000000#!/bin/bash # # Parallel VACUUM enters parallel mode before calling index bulk-delete, # including for indexes processed by the leader. pg_textsearch maintenance # must therefore avoid assigning a transaction ID while removing dead docs. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG}" --bindir)" export PATH="${PGBINDIR}:${PATH}" TEST_PORT=55461 TEST_DB=parallel_vacuum_test DATA_DIR="${SCRIPT_DIR}/../tmp_parallel_vacuum" SOCKET_DIR= LOGFILE="${DATA_DIR}/postgres.log" MODE="${1:-normal}" PRELOAD_LIBRARIES=pg_textsearch cleanup() { local exit_code=$? trap - EXIT INT TERM if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" if [ -n "${SOCKET_DIR}" ]; then rm -rf "${SOCKET_DIR}" fi exit "${exit_code}" } trap cleanup EXIT INT TERM if [ "${MODE}" = "injection" ]; then [ -f "$("${PG_CONFIG}" --pkglibdir)/injection_points.so" ] || { echo "injection_points.so is required for injection mode" >&2 exit 1 } PRELOAD_LIBRARIES='pg_textsearch,injection_points' fi SOCKET_DIR="$(mktemp -d /tmp/pgts-parallel-vacuum.XXXXXX)" rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >>"${DATA_DIR}/postgresql.conf" </dev/null; then cat "${LOGFILE}" >&2 exit 1 fi createdb -h "${SOCKET_DIR}" -p "${TEST_PORT}" "${TEST_DB}" PSQL=( psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 ) "${PSQL[@]}" <<'SQL' >/dev/null CREATE EXTENSION pg_textsearch; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE spill_docs ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL ); CREATE INDEX spill_docs_bm25 ON spill_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); DO $$ BEGIN FOR batch IN 1..8 LOOP INSERT INTO spill_docs (body) SELECT 'parallel spill batch ' || batch || ' document ' || gs FROM generate_series(1, 500) gs; PERFORM bm25_spill_index('spill_docs_bm25'); END LOOP; END $$; ALTER INDEX spill_docs_bm25 SET (compaction = 'inline'); INSERT INTO spill_docs (body) SELECT 'parallel spill pending document ' || gs || ' ' || repeat(md5(gs::text), 4) FROM generate_series(1, 20000) gs; DELETE FROM spill_docs WHERE id <= 5000; CREATE TABLE docs ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL ); CREATE INDEX docs_bm25 ON docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO docs (body) SELECT 'parallel vacuum document ' || gs || ' ' || repeat(md5(gs::text), 4) FROM generate_series(1, 20000) gs; SELECT bm25_spill_index('docs_bm25'); DELETE FROM docs; SQL if [ "${MODE}" = "injection" ]; then "${PSQL[@]}" <<'SQL' >/dev/null CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE legacy_docs ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL ); CREATE INDEX legacy_docs_bm25 ON legacy_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO legacy_docs (body) SELECT 'parallel legacy document ' || gs || ' ' || repeat(md5(gs::text), 4) FROM generate_series(1, 20000) gs; SELECT pg_textsearch_test_attach_legacy_segment(1000000); SELECT bm25_spill_index('legacy_docs_bm25'); SELECT injection_points_detach('pg-textsearch-legacy-segment'); SQL legacy_root=$( "${PSQL[@]}" -c \ "SELECT (regexp_match( bm25_summarize_index('legacy_docs_bm25'), 'L0 Segment 1: block=([0-9]+)'))[1];" ) "${PSQL[@]}" -c \ "DELETE FROM legacy_docs WHERE id <= 5000;" >/dev/null legacy_pending_before=$( "${PSQL[@]}" -c \ "SELECT bm25_pending_free_pages('legacy_docs_bm25');" ) if ! legacy_vacuum_output=$( "${PSQL[@]}" <<'SQL' 2>&1 SET min_parallel_index_scan_size = 0; VACUUM (PARALLEL 1, VERBOSE) legacy_docs; SQL ); then echo "${legacy_vacuum_output}" >&2 exit 1 fi if ! grep -q "launched 1 parallel vacuum worker" \ <<<"${legacy_vacuum_output}"; then echo "parallel legacy VACUUM did not launch a worker" >&2 echo "${legacy_vacuum_output}" >&2 exit 1 fi legacy_remaining=$( "${PSQL[@]}" -c " SELECT count(*) FROM ( SELECT id FROM legacy_docs ORDER BY body <@> to_bm25query( 'parallel legacy', 'legacy_docs_bm25') ) ranked;" ) if [ "${legacy_remaining}" != "15000" ]; then echo "parallel legacy VACUUM retained ${legacy_remaining}/15000 documents" \ >&2 exit 1 fi legacy_summary=$( "${PSQL[@]}" -c \ "SELECT bm25_summarize_index('legacy_docs_bm25');" ) legacy_new_root=$( sed -n 's/.*L0 Segment 1: block=\([0-9][0-9]*\).*/\1/p' \ <<<"${legacy_summary}" ) if [ -z "${legacy_new_root}" ]; then echo "parallel legacy VACUUM summary has no L0 segment root" >&2 echo "${legacy_summary}" >&2 exit 1 fi if [ "${legacy_new_root}" = "${legacy_root}" ]; then echo "parallel legacy VACUUM did not replace root ${legacy_root}" >&2 exit 1 fi legacy_pending_after=$( "${PSQL[@]}" -c \ "SELECT bm25_pending_free_pages('legacy_docs_bm25');" ) if [ "${legacy_pending_after}" -le "${legacy_pending_before}" ]; then echo "parallel legacy VACUUM pending-free pages did not increase: " \ "${legacy_pending_before} -> ${legacy_pending_after}" >&2 exit 1 fi fi spill_memtable_before=$( "${PSQL[@]}" -c \ "SELECT count(*) FROM bm25_memtable_chain('spill_docs_bm25');" ) if [ "${spill_memtable_before}" -le 0 ]; then echo "parallel spill VACUUM setup has no pending memtable pages" >&2 exit 1 fi if ! spill_vacuum_output=$( "${PSQL[@]}" <<'SQL' 2>&1 SET min_parallel_index_scan_size = 0; VACUUM (PARALLEL 1, VERBOSE) spill_docs; SQL ); then echo "${spill_vacuum_output}" >&2 exit 1 fi if ! grep -q "launched 1 parallel vacuum worker" \ <<<"${spill_vacuum_output}"; then echo "parallel spill VACUUM did not launch a worker" >&2 echo "${spill_vacuum_output}" >&2 exit 1 fi spill_memtable_after=$( "${PSQL[@]}" -c \ "SELECT count(*) FROM bm25_memtable_chain('spill_docs_bm25');" ) if [ "${spill_memtable_after}" -ne 0 ]; then echo "parallel VACUUM did not spill ${spill_memtable_after} memtable pages" \ >&2 exit 1 fi spill_remaining=$( "${PSQL[@]}" -c " SELECT count(*) FROM ( SELECT id FROM spill_docs ORDER BY body <@> to_bm25query( 'parallel spill', 'spill_docs_bm25') ) ranked;" ) if [ "${spill_remaining}" != "19000" ]; then echo "parallel VACUUM spill retained ${spill_remaining}/19000 documents" \ >&2 exit 1 fi if ! vacuum_output=$( "${PSQL[@]}" <<'SQL' 2>&1 SET min_parallel_index_scan_size = 0; VACUUM (PARALLEL 1, VERBOSE) docs; SQL ); then echo "${vacuum_output}" >&2 exit 1 fi if ! grep -q "launched 1 parallel vacuum worker" <<<"${vacuum_output}"; then echo "parallel VACUUM did not launch a worker" >&2 echo "${vacuum_output}" >&2 exit 1 fi remaining=$( "${PSQL[@]}" -c " SELECT count(*) FROM ( SELECT id FROM docs ORDER BY body <@> to_bm25query( 'parallel vacuum', 'docs_bm25') ) ranked;" ) if [ "${remaining}" != "0" ]; then echo "parallel VACUUM left ${remaining} deleted documents searchable" >&2 exit 1 fi parallel_levels=$( "${PSQL[@]}" -c "SELECT bm25_level_counts('docs_bm25');" ) if [ "${parallel_levels}" != "{1,0,0,0,0,0,0,0}" ]; then echo "parallel VACUUM left unexpected levels ${parallel_levels}" >&2 exit 1 fi parallel_summary=$( "${PSQL[@]}" -c "SELECT bm25_summarize_index('docs_bm25');" ) if ! grep -q "alive=0, dead=20000" <<<"${parallel_summary}"; then echo "parallel VACUUM did not persist the all-dead alive bitmap" >&2 echo "${parallel_summary}" >&2 exit 1 fi "${PSQL[@]}" -c "VACUUM (PARALLEL 0) docs;" >/dev/null serial_levels=$( "${PSQL[@]}" -c "SELECT bm25_level_counts('docs_bm25');" ) if [ "${serial_levels}" != "{0,0,0,0,0,0,0,0}" ]; then echo "serial VACUUM left empty segment levels ${serial_levels}" >&2 exit 1 fi "${PSQL[@]}" <<'SQL' >/dev/null ALTER TABLE docs ADD COLUMN label text; CREATE TABLE control_docs ( label text NOT NULL, body text NOT NULL ); CREATE INDEX control_docs_bm25 ON control_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO docs (label, body) VALUES ('short', 'needle alpha beta gamma delta'), ('long', 'needle needle needle needle needle' || repeat(' filler', 45)); INSERT INTO control_docs SELECT label, body FROM docs; SELECT bm25_spill_index('docs_bm25'); SELECT bm25_spill_index('control_docs_bm25'); SQL cleanup_order=$( "${PSQL[@]}" -c " SELECT string_agg(label, ',' ORDER BY score, label) FROM ( SELECT label, body <@> to_bm25query( 'needle', 'docs_bm25') AS score FROM docs ORDER BY body <@> to_bm25query( 'needle', 'docs_bm25') ) ranked;" ) control_order=$( "${PSQL[@]}" -c " SELECT string_agg(label, ',' ORDER BY score, label) FROM ( SELECT label, body <@> to_bm25query( 'needle', 'control_docs_bm25') AS score FROM control_docs ORDER BY body <@> to_bm25query( 'needle', 'control_docs_bm25') ) ranked;" ) if [ "${control_order}" != "long,short" ]; then echo "control corpus did not produce sensitive order: ${control_order}" >&2 exit 1 fi if [ "${cleanup_order}" != "${control_order}" ]; then echo "post-cleanup order ${cleanup_order} differs from control ${control_order}" \ >&2 exit 1 fi echo "Parallel VACUUM test passed" timescale-pg_textsearch-b63fe58/test/scripts/partial_concurrent_read.sh000077500000000000000000000150041525762047700266570ustar00rootroot00000000000000#!/bin/bash # # Regression test for issue #404: the standalone <@> scoring operator # (used on a seqscan) opened segment pages without the per-index LWLock, # so a concurrent spill/merge could free and recycle those blocks under # the reader, yielding "invalid segment header". # # The reader forces the standalone path (enable_indexscan=off) on a # partial index while writers and spill/merge run concurrently. Before the # fix this fails within seconds; after it, it completes cleanly. # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55441 TEST_DB=partial_concurrent_read_test DATA_DIR="${SCRIPT_DIR}/../tmp_partial_concurrent_read" LOGFILE="${DATA_DIR}/postgres.log" ERR_DIR="${DATA_DIR}/client_logs" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } cleanup() { local exit_code=$? log "Cleaning up (exit code: $exit_code)..." jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 mkdir -p "${ERR_DIR}" cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' shared_preload_libraries = 'pg_textsearch' logging_collector = on log_directory = '.' log_filename = 'postgres.log' autovacuum = on EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w -o "-p ${TEST_PORT}" || \ error "Failed to start PostgreSQL" createdb -h "${DATA_DIR}" -p ${TEST_PORT} ${TEST_DB} psql -h "${DATA_DIR}" -p ${TEST_PORT} -d ${TEST_DB} \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null log "Test database ready" } PSQL="psql -h ${DATA_DIR} -p ${TEST_PORT} -d ${TEST_DB} -qAt -v ON_ERROR_STOP=1" seed_data() { log "Creating selective partial index and seeding data..." $PSQL <<'SQL' >/dev/null SET client_min_messages=warning; CREATE TABLE docs (id bigserial PRIMARY KEY, content text NOT NULL, lang text NOT NULL); CREATE INDEX docs_de_bm25 ON docs USING bm25(content) WITH (text_config='german') WHERE lang = 'de'; INSERT INTO docs (content, lang) SELECT repeat('postgres bm25 search ', 50) || gs, CASE WHEN gs % 2 = 0 THEN 'de' ELSE 'en' END FROM generate_series(1, 4000) gs; SELECT bm25_spill_index('docs_de_bm25'); SELECT bm25_force_merge('docs_de_bm25'); SQL # The test only covers #404 if the reader runs as a Seq Scan # (standalone <@> scoring); fail loudly if the planner picks otherwise. local plan plan=$($PSQL -c "SET enable_indexscan=off; SET enable_bitmapscan=off; EXPLAIN (COSTS off) SELECT id FROM docs WHERE lang='de' ORDER BY content <@> to_bm25query('postgres bm25', 'docs_de_bm25') LIMIT 20") if ! echo "$plan" | grep -qi "Seq Scan"; then warn "Reader plan was not a Seq Scan:" echo "$plan" error "TEST SETUP FAILED: reader no longer exercises the standalone path" fi } writer() { for i in $(seq 1 120); do $PSQL -c "SET statement_timeout='60s'; SET lock_timeout='30s'; INSERT INTO docs (content, lang) SELECT 'writer postgres bm25 ' || gs, 'de' FROM generate_series(1, 30) gs" \ >>"${ERR_DIR}/writer.log" 2>&1 || return 10 done } merger() { local tag=$1 for i in $(seq 1 200); do $PSQL -c "DO \$merge\$ DECLARE spilled bigint; BEGIN spilled := bm25_spill_index('docs_de_bm25'); IF spilled > 0 THEN RAISE NOTICE 'concurrent spill completed'; END IF; BEGIN PERFORM bm25_force_merge('docs_de_bm25'); RAISE NOTICE 'concurrent force merge completed'; EXCEPTION WHEN lock_not_available THEN NULL; END; END \$merge\$" \ >>"${ERR_DIR}/merger_${tag}.log" 2>&1 || return 30 done } # "Partial" is incidental: the trigger is the standalone <@> seqscan path # (forced by enable_indexscan=off), not any partial-index-specific code. reader() { local tag=$1 for i in $(seq 1 300); do $PSQL -c "SET enable_indexscan=off; SET enable_bitmapscan=off; SET statement_timeout='60s'; SET lock_timeout='30s'; SELECT count(*) FROM ( SELECT id FROM docs WHERE lang='de' ORDER BY content <@> to_bm25query('postgres bm25', 'docs_de_bm25') LIMIT 20 ) s" >>"${ERR_DIR}/reader_${tag}.log" 2>&1 || return 40 done } run_test() { log "Running concurrent writer + two mergers + readers (standalone path)..." writer & w_pid=$! merger a & ma_pid=$! merger b & mb_pid=$! reader a & ra_pid=$! reader b & rb_pid=$! local failed=0 wait $w_pid || { warn "writer failed"; failed=1; } wait $ma_pid || { warn "merger a failed"; failed=1; } wait $mb_pid || { warn "merger b failed"; failed=1; } wait $ra_pid || { warn "reader a failed"; failed=1; } wait $rb_pid || { warn "reader b failed"; failed=1; } # Surface the actual error for diagnosis. if grep -RIl "invalid segment header" "${ERR_DIR}" >/dev/null 2>&1; then warn "Found 'invalid segment header' in client logs:" grep -RIn "invalid segment header" "${ERR_DIR}" | head -5 error "TEST FAILED: issue #404 reproduced (invalid segment header)" fi if [ "$failed" -ne 0 ]; then warn "Client logs:" tail -n 20 "${ERR_DIR}"/*.log 2>/dev/null || true error "TEST FAILED: a concurrent client exited with an error" fi if ! grep -Rq "concurrent spill completed" "${ERR_DIR}"; then error "TEST FAILED: no concurrent spill completed" fi if ! grep -Rq "concurrent force merge completed" "${ERR_DIR}"; then error "TEST FAILED: no concurrent force merge completed" fi log "TEST PASSED: partial-index standalone reads survived concurrent spill/merge" } # Main setup_test_db seed_data run_test log "All tests passed!" exit 0 timescale-pg_textsearch-b63fe58/test/scripts/recovery.sh000077500000000000000000000324761525762047700236400ustar00rootroot00000000000000#!/bin/bash # # Crash recovery test script for pg_textsearch # This script simulates actual crashes and verifies recovery functionality # set -eu SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55435 TEST_DB=crash_recovery_test TEST_HOST=127.0.0.1 DATA_DIR="${SCRIPT_DIR}/../tmp_crash_test" LOGFILE="${DATA_DIR}/postgres.log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # No Color log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}" } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}" } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}" exit 1 } cleanup() { local status=$? log "Cleaning up test environment..." if [ "${status}" -ne 0 ]; then for logfile in "${LOGFILE}" "${DATA_DIR}/log/postgres.log"; do if [ -f "${logfile}" ]; then echo "=== ${logfile} (last 80 lines) ===" tail -80 "${logfile}" 2>/dev/null || true fi done fi if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast &>/dev/null || pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" exit "$status" } # Set up cleanup trap trap cleanup EXIT INT TERM setup_test_db() { log "Setting up temporary PostgreSQL instance..." # Clean up any existing test data rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" # Initialize test cluster initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 local lib_name="pg_textsearch" # Configure test instance cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} listen_addresses = '${TEST_HOST}' unix_socket_directories = '' log_statement = 'all' shared_buffers = 128MB max_connections = 20 log_min_messages = notice logging_collector = on log_filename = 'postgres.log' shared_preload_libraries = '${lib_name}' restart_after_crash = off EOF # Start PostgreSQL log "Starting test PostgreSQL instance on port ${TEST_PORT}..." pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w # Create test database createdb -h "${TEST_HOST}" -p "${TEST_PORT}" "${TEST_DB}" # Install extension psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null } run_sql() { if [ "$#" -eq 0 ]; then psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" \ 2>/dev/null else psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "$1" 2>/dev/null fi } run_sql_file() { psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -f "$1" 2>/dev/null } simulate_crash() { log "Simulating PostgreSQL crash (SIGKILL)..." # Get PostgreSQL process PID POSTGRES_PID=$(head -1 "${DATA_DIR}/postmaster.pid") if [ -n "$POSTGRES_PID" ] && kill -0 "$POSTGRES_PID" 2>/dev/null; then # Kill PostgreSQL immediately (simulates crash) kill -9 "$POSTGRES_PID" # Wait for process to die sleep 2 # Clean up PID file if it still exists rm -f "${DATA_DIR}/postmaster.pid" log "PostgreSQL crashed (PID: $POSTGRES_PID)" else error "Could not find PostgreSQL process to kill" fi } restart_postgres() { log "Restarting PostgreSQL after crash..." # Start PostgreSQL again pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w -t 30 # Verify connection works psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "SELECT 1;" >/dev/null log "PostgreSQL restarted successfully" } test_crash_recovery() { log "Running crash recovery test..." # Phase 1: Set up initial data log "Phase 1: Creating initial test data..." cat << 'EOF' | run_sql DROP TABLE IF EXISTS crash_test_docs CASCADE; CREATE TABLE crash_test_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL, category VARCHAR(50) ); INSERT INTO crash_test_docs (content, category) VALUES ('database systems and query optimization performance', 'tech'), ('full text search algorithms implementation', 'tech'), ('postgresql extension development guide', 'tech'), ('crash recovery mechanisms testing', 'tech'), ('bm25 ranking algorithm implementation details', 'tech'), ('advanced database indexing structures', 'tech'), ('search engine optimization techniques', 'tech'), ('distributed systems architecture patterns', 'tech'), ('machine learning algorithms overview', 'ai'), ('natural language processing methods', 'ai'), ('computer vision applications', 'ai'), ('deep learning neural networks', 'ai'), ('data mining and analytics', 'data'), ('big data processing frameworks', 'data'), ('real time streaming systems', 'data'), ('cloud computing infrastructure', 'cloud'), ('microservices architecture design', 'cloud'), ('container orchestration platforms', 'cloud'), ('security protocols and encryption', 'security'), ('network security best practices', 'security'); CREATE INDEX crash_idx ON crash_test_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); EOF # Verify initial search works and capture baseline results log "Verifying initial search functionality..." # Test 1: Search for 'database' - should return documents with database-related content INITIAL_RESULTS=$(run_sql "SELECT id, content FROM crash_test_docs ORDER BY content <@> to_bm25query('database', 'crash_idx') LIMIT 5;" | grep -E "^\s*[0-9]+\s*\|" || echo "") # Count results and validate INITIAL_COUNT=$(echo "$INITIAL_RESULTS" | wc -l | tr -d ' ') if [ "$INITIAL_COUNT" -eq 0 ]; then error "Initial search returned no results - this indicates a problem" fi log "Initial search test passed (found $INITIAL_COUNT documents)" log "Sample results: $(echo "$INITIAL_RESULTS" | head -2 | tr '\n' '; ')..." # Store expected results for comparison after recovery echo "$INITIAL_RESULTS" > "${DATA_DIR}/expected_results.txt" # Phase 2: Simulate crash during operation log "Phase 2: Simulating crash..." # Add more data that will be in WAL but not necessarily flushed run_sql "INSERT INTO crash_test_docs (content, category) VALUES ('crash recovery test document with database terms', 'test');" & INSERT_PID=$! # Give the insert a moment to start sleep 0.1 # Crash PostgreSQL simulate_crash # Wait for insert process to complete (it will fail due to crash) wait $INSERT_PID 2>/dev/null || true # Phase 3: Recovery log "Phase 3: Testing recovery..." restart_postgres # Verify data consistency after crash log "Verifying data consistency after recovery..." # Test same query as before crash RECOVERY_RESULTS=$(run_sql "SELECT id, content FROM crash_test_docs ORDER BY content <@> to_bm25query('database', 'crash_idx') LIMIT 5;" | grep -E "^\s*[0-9]+\s*\|" || echo "") RECOVERY_COUNT=$(echo "$RECOVERY_RESULTS" | wc -l | tr -d ' ') if [ "$RECOVERY_COUNT" -eq 0 ]; then error "Recovery search returned no results - recovery failed!" fi # Compare results with pre-crash baseline if [ -f "${DATA_DIR}/expected_results.txt" ]; then EXPECTED_RESULTS=$(cat "${DATA_DIR}/expected_results.txt") if [ "$RECOVERY_RESULTS" != "$EXPECTED_RESULTS" ]; then warn "Search results differ after recovery:" warn "Expected: $(echo "$EXPECTED_RESULTS" | head -2 | tr '\n' '; ')..." warn "Got: $(echo "$RECOVERY_RESULTS" | head -2 | tr '\n' '; ')..." # Don't fail here as insert during crash might have succeeded else log "Search results match pre-crash baseline perfectly" fi fi log "Recovery search test passed (found $RECOVERY_COUNT documents)" # Test that we can still insert and search log "Testing post-recovery functionality..." run_sql "INSERT INTO crash_test_docs (content, category) VALUES ('post recovery test document with algorithms', 'test');" # Test search for 'algorithms' which should find multiple documents POST_RECOVERY_RESULTS=$(run_sql "SELECT id, content FROM crash_test_docs ORDER BY content <@> to_bm25query('algorithms', 'crash_idx') LIMIT 5;" | grep -E "^\s*[0-9]+\s*\|" || echo "") POST_RECOVERY_COUNT=$(echo "$POST_RECOVERY_RESULTS" | wc -l | tr -d ' ') if [ "$POST_RECOVERY_COUNT" -eq 0 ]; then error "Post-recovery search returned no results - functionality broken!" fi log "Post-recovery insert and search test passed (found $POST_RECOVERY_COUNT documents)" log "Sample results: $(echo "$POST_RECOVERY_RESULTS" | head -2 | tr '\n' '; ')..." # Test debug dump functionality after recovery - this exposes the bug log "Phase 4: Testing debug dump functionality after recovery..." # This is the critical test that should expose the recovery bug DEBUG_RESULT=$(run_sql "SELECT bm25_dump_index('crash_idx');" 2>&1 | head -5 || echo "FAILED: Debug dump crashed") if echo "$DEBUG_RESULT" | grep -q "FAILED\|ERROR\|connection.*lost"; then error "Debug dump failed after recovery - this exposes the recovery bug!" fi log "Debug dump test passed - recovery working correctly" # Test index rebuild scenario log "Phase 5: Testing index rebuild after crash..." # Drop and recreate index to test rebuild from heap run_sql "DROP INDEX crash_idx;" run_sql "CREATE INDEX crash_idx_rebuilt ON crash_test_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" # Test rebuild with 'systems' query to verify full functionality REBUILD_RESULTS=$(run_sql "SELECT id, content FROM crash_test_docs ORDER BY content <@> to_bm25query('systems', 'crash_idx_rebuilt') LIMIT 5;" | grep -E "^\s*[0-9]+\s*\|" || echo "") REBUILD_COUNT=$(echo "$REBUILD_RESULTS" | wc -l | tr -d ' ') if [ "$REBUILD_COUNT" -eq 0 ]; then error "Index rebuild search returned no results - rebuild failed!" fi log "Index rebuild test passed (found $REBUILD_COUNT documents)" log "Sample results: $(echo "$REBUILD_RESULTS" | head -2 | tr '\n' '; ')..." } # Cover the bootstrap path's empty-index branch: an index with no # docs (no segments, no docid pages, metap.total_docs=0) opened on # the first backend access after a server restart fires # tp_rebuild_index_from_disk's empty-index branch # (state.c). This is otherwise hard to reach from SQL because # CREATE INDEX populates the registry — only a server restart # (or fresh standby) leaves the registry empty and the index # non-empty-on-disk-but-also-totally-empty. test_empty_index_rebuild() { log "Running empty-index rebuild test..." # Phase 1: create an empty index (table has no rows). log "Phase 1: create empty bm25 index" run_sql "DROP TABLE IF EXISTS empty_rebuild_test CASCADE;" >/dev/null run_sql "CREATE TABLE empty_rebuild_test (id SERIAL PRIMARY KEY, content TEXT);" >/dev/null run_sql "CREATE INDEX empty_rebuild_idx ON empty_rebuild_test USING bm25(content) WITH (text_config='english');" >/dev/null # Phase 2: clean restart. Stop drops the per-backend registry; # the next backend's first access has to bootstrap. log "Phase 2: restart server" pg_ctl stop -D "${DATA_DIR}" -m fast >/dev/null 2>&1 pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w >/dev/null 2>&1 psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -c "SELECT 1;" >/dev/null # Phase 3: first query against the empty index — drives # bootstrap. Empty-index branch should leave shared state # registered with total_docs=0. log "Phase 3: first query post-restart fires bootstrap" local count count=$(psql -tA -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " SELECT count(*) FROM ( SELECT id FROM empty_rebuild_test ORDER BY content <@> to_bm25query('alpha', 'empty_rebuild_idx') LIMIT 100 ) t;") if [ "${count}" != "0" ]; then error "Empty-index query returned '${count}', expected 0" fi # Phase 4: verify the index is functional post-bootstrap. An # insert should be queryable via the same backend's now-warm # local state. log "Phase 4: post-bootstrap insert + query" psql -h "${TEST_HOST}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "INSERT INTO empty_rebuild_test (content) VALUES ('alpha bravo charlie');" >/dev/null count=$(psql -tA -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " SELECT count(*) FROM ( SELECT id FROM empty_rebuild_test ORDER BY content <@> to_bm25query('alpha', 'empty_rebuild_idx') LIMIT 100 ) t;") if [ "${count}" != "1" ]; then error "Post-bootstrap query returned '${count}', expected 1" fi log "Empty-index rebuild test passed" } main() { log "Starting pg_textsearch crash recovery test..." # Check if we have necessary tools command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found in PATH" command -v psql >/dev/null 2>&1 || error "psql not found in PATH" command -v createdb >/dev/null 2>&1 || error "createdb not found in PATH" command -v initdb >/dev/null 2>&1 || error "initdb not found in PATH" # Run the test setup_test_db test_empty_index_rebuild test_crash_recovery log "✅ All crash recovery tests passed!" exit 0 } # Only run main if script is executed directly if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication.sh000077500000000000000000000401041525762047700242760ustar00rootroot00000000000000#!/bin/bash # # replication.sh — physical (streaming) replication smoke test for # pg_textsearch. Sets up a primary + hot-standby pair via # pg_basebackup, exercises the index across the basic write/replay # cycle, and asserts the standby sees the same results as the # primary. # # Tests: # 1. Standby starts up with replicated bm25 index (basebackup # includes the index file). # 2. BM25 search queries work on the hot standby (read-side # smoke test). # 3. Primary inserts replicate via WAL and a primary-side spill # reaches the standby (exercises the #342 fix in this PR via # the memtable-spill path). # 4. Standby can be promoted; index still works on the new # primary. # 5. Long-lived standby backend sees primary inserts that arrive # via WAL replay during the backend's lifetime. Currently # FAILS — see #345 (per-backend cache not invalidated by WAL # replay). # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55436 STANDBY_PORT=55437 TEST_DB=replication_test PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # --------------------------------------------------------------- # Test 1: Build index on primary, then create standby, query it # --------------------------------------------------------------- test_basic_standby_queries() { log "=== Test 1: Basic standby queries ===" log "Creating table and index on primary..." primary_sql " CREATE TABLE docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); " >/dev/null primary_sql " INSERT INTO docs (content) VALUES ('database systems and query optimization'), ('full text search algorithms'), ('postgresql extension development guide'), ('bm25 ranking algorithm implementation'), ('distributed systems architecture'), ('machine learning algorithms overview'), ('natural language processing methods'), ('search engine optimization techniques'), ('cloud computing infrastructure'), ('microservices design patterns'); " >/dev/null primary_sql " CREATE INDEX docs_bm25_idx ON docs USING bm25(content) WITH (text_config='english'); " # Verify search works on primary local primary_count primary_count=$(search_count "${PRIMARY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Primary search for 'algorithms': ${primary_count} results" if [ -z "$primary_count" ] || [ "$primary_count" -lt 1 ]; then error "Primary search returned no results!" fi # Now create the standby (after data is loaded, avoids slow # checkpoint during basebackup of empty cluster) setup_standby wait_for_standby_catchup log "Running search on standby..." local standby_result standby_result=$(standby_sql " SELECT id, content <@> to_bm25query('algorithms', 'docs_bm25_idx') AS score FROM docs WHERE content <@> to_bm25query('algorithms', 'docs_bm25_idx') < 0 ORDER BY content <@> to_bm25query('algorithms', 'docs_bm25_idx') LIMIT 10; ") echo "$standby_result" local standby_count standby_count=$(search_count "${STANDBY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Standby search for 'algorithms': ${standby_count} results" if [ -z "$standby_count" ] || [ "$standby_count" -lt 1 ]; then error "Standby search returned no results!" fi if [ "$standby_count" != "$primary_count" ]; then error "Result count mismatch: primary=${primary_count} \ standby=${standby_count}" fi log "Test 1 PASSED: Standby queries match primary" } # --------------------------------------------------------------- # Test 2: Insert on primary, verify replication to standby # --------------------------------------------------------------- test_ongoing_replication() { log "=== Test 2: Ongoing replication ===" log "Inserting new documents on primary..." primary_sql " INSERT INTO docs (content) VALUES ('advanced algorithms for sorting and searching'), ('graph algorithms and network analysis'), ('parallel algorithms on modern hardware'); " >/dev/null wait_for_standby_catchup log "Searching on standby after new inserts..." local standby_count standby_count=$(search_count "${STANDBY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Standby search for 'algorithms' after inserts: \ ${standby_count} results" # Should find more than before (original 2 + 3 new = 5) if [ "$standby_count" -lt 4 ]; then error "Expected at least 4 results, got ${standby_count}" fi # Verify counts match primary local primary_count primary_count=$(search_count "${PRIMARY_PORT}" docs content \ "algorithms" "docs_bm25_idx") if [ "$standby_count" != "$primary_count" ]; then error "Result count mismatch after inserts: \ primary=${primary_count} standby=${standby_count}" fi log "Test 2 PASSED: Ongoing replication works" } # --------------------------------------------------------------- # Test 3: Spill to segments on primary, verify on standby # --------------------------------------------------------------- test_segment_replication() { log "=== Test 3: Segment replication ===" log "Forcing memtable spill on primary..." primary_sql "SELECT bm25_spill_index('docs_bm25_idx');" wait_for_standby_catchup log "Searching on standby after segment spill..." local standby_count standby_count=$(search_count "${STANDBY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Standby search after spill: ${standby_count} results" local primary_count primary_count=$(search_count "${PRIMARY_PORT}" docs content \ "algorithms" "docs_bm25_idx") if [ "$standby_count" != "$primary_count" ]; then error "Result mismatch after spill: \ primary=${primary_count} standby=${standby_count}" fi log "Test 3 PASSED: Segment replication works" } # --------------------------------------------------------------- # Test 4: Promote standby and verify read/write functionality # --------------------------------------------------------------- test_standby_promotion() { log "=== Test 4: Standby promotion ===" # Capture pre-promotion search state local pre_count pre_count=$(search_count "${STANDBY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Pre-promotion search count: ${pre_count}" log "Promoting standby to primary..." pg_ctl promote -D "${STANDBY_DIR}" -w # Wait for promotion to complete local i=0 while [ $i -lt 10 ]; do local in_recovery in_recovery=$(psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -tA \ -c "SELECT pg_is_in_recovery();" 2>/dev/null) if [ "$in_recovery" = "f" ]; then break fi sleep 1 i=$((i + 1)) done local in_recovery in_recovery=$(psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -tA \ -c "SELECT pg_is_in_recovery();" 2>/dev/null) if [ "$in_recovery" != "f" ]; then error "Standby did not complete promotion!" fi log "Standby promoted successfully" # Test read queries on promoted standby log "Testing search on promoted standby..." local promoted_count promoted_count=$(search_count "${STANDBY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Search results on promoted standby: ${promoted_count}" if [ -z "$promoted_count" ] || [ "$promoted_count" -lt 1 ]; then error "Search failed on promoted standby!" fi # Test write on promoted standby log "Testing insert on promoted standby..." psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -c " INSERT INTO docs (content) VALUES ('post-promotion test document about algorithms'); " >/dev/null 2>&1 local post_insert_count post_insert_count=$(search_count "${STANDBY_PORT}" docs content \ "algorithms" "docs_bm25_idx") log "Search after insert on promoted standby: \ ${post_insert_count}" if [ "$post_insert_count" -le "$promoted_count" ]; then error "Insert on promoted standby not reflected in search \ (before: ${promoted_count}, after: ${post_insert_count})" fi log "Test 4 PASSED: Standby promotion works" } # --------------------------------------------------------------- # Test 5: Long-lived standby backend sees new primary inserts # # This test exposes the memtable-staleness bug: a backend that # stays alive across primary inserts rebuilds its memtable once # (on first scan) and then never updates it because WAL replay # does not call into our aminsert path. # --------------------------------------------------------------- test_long_lived_backend_staleness() { log "=== Test 5: Long-lived backend staleness ===" primary_sql " DROP TABLE IF EXISTS staleness_docs CASCADE; CREATE TABLE staleness_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO staleness_docs (content) VALUES ('initial document about quantum mechanics'), ('initial document about general relativity'); CREATE INDEX staleness_idx ON staleness_docs USING bm25(content) WITH (text_config='english'); " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM ( SELECT id FROM staleness_docs ORDER BY content <@> to_bm25query('quantum', 'staleness_idx') LIMIT 100 ) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO staleness_docs (content) VALUES ('quantum entanglement and bell inequalities'); \" >/dev/null wait_for_standby_catchup" log "Initial standby count: ${LL_BEFORE}" log "Standby count after primary insert: ${LL_AFTER}" if [ "${LL_BEFORE}" != "2" ]; then error "Expected initial count=2, got '${LL_BEFORE}'" fi # The bug manifests in two ways: # 1. Silent staleness: LL_AFTER == LL_BEFORE (memtable not updated). # 2. Segment-block invalidation: LL_AFTER contains 'ERROR' # (long-lived backend has a stale view of segment storage). # Once fixed, LL_AFTER should be LL_BEFORE + 1 (the new doc is visible). if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "BUG (expected): long-lived standby backend did not see \ new primary insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "Test 5 PASSED: Long-lived backend sees new primary inserts" } # --------------------------------------------------------------- # Test 6: Segment graph snapshots vs concurrent WAL replay # --------------------------------------------------------------- # Standalone <@> scoring captures the metapage, the memtable chain # endpoint and every published segment root as one snapshot, then walks # only the captured block numbers. On a primary the reader's per-index # LWLock already excludes spill publication, so the recovery-mode # capture path is unreachable there. A standby has no such protection: # there is no custom rmgr, so WAL replay republishes the graph without # taking the per-index lock. # # This exercises that path -- roughly two thousand recovery-mode # captures -- against a primary that is spilling and merging, and # asserts no reader error and no torn or corrupt segment read. The # debug pause GUCs widen the capture windows so replay interleaves. # # It does not prove the root walk itself is atomic: the pause hooks # bracket the capture, not the individual level chains, so replay # cannot be forced to land between two levels from a shell test. test_standby_snapshot_race() { log "=== Test 6: Standby snapshot vs concurrent replay ===" primary_sql " DROP TABLE IF EXISTS race_docs CASCADE; CREATE TABLE race_docs ( id bigserial PRIMARY KEY, content text NOT NULL ); CREATE INDEX race_idx ON race_docs USING bm25(content) WITH (text_config='english'); INSERT INTO race_docs (content) SELECT 'postgres bm25 snapshot race ' || gs FROM generate_series(1, 30) gs; SELECT bm25_spill_index('race_idx'); INSERT INTO race_docs (content) SELECT 'postgres bm25 second batch ' || gs FROM generate_series(1, 30) gs; SELECT bm25_spill_index('race_idx'); " >/dev/null wait_for_standby_catchup local plan plan=$(standby_sql "EXPLAIN (COSTS off) SELECT content <@> to_bm25query('postgres bm25', 'race_idx') FROM race_docs") if ! echo "${plan}" | grep -qi 'Seq Scan'; then echo "${plan}" error "Test 6 SETUP FAILED: reader does not use standalone scoring" fi local readerlog="${STANDBY_DIR}/snapshot_race_reader.log" local standbylog="${STANDBY_DIR}/log/postgres.log" : > "${readerlog}" ( for _ in $(seq 1 10); do psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -qAt \ -v ON_ERROR_STOP=1 -c " SET enable_indexscan=off; SET enable_bitmapscan=off; SET statement_timeout='120s'; SET pg_textsearch.\ debug_segment_graph_snapshot_pause_before_lock_ms=1; SET pg_textsearch.\ debug_segment_graph_snapshot_pause_before_unlock_ms=8; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms=8; SELECT count(score) FROM ( SELECT content <@> to_bm25query('postgres bm25', 'race_idx') AS score FROM race_docs ) s" >>"${readerlog}" 2>&1 || exit 40 done ) & local reader_pid=$! ( for _ in $(seq 1 15); do psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" -qAt -c " INSERT INTO race_docs (content) SELECT 'racer postgres bm25 ' || gs FROM generate_series(1, 10) gs; SELECT bm25_spill_index('race_idx'); SELECT bm25_force_merge('race_idx'); " >/dev/null 2>&1 || true done ) & local writer_pid=$! local failed=0 wait ${reader_pid} || failed=1 wait ${writer_pid} || true local torn='shorter than its recorded count' torn="${torn}|longer than its recorded count" torn="${torn}|could not read BM25 segment root" torn="${torn}|segment-root count overflow" torn="${torn}|invalid segment header" if grep -IEq "${torn}" "${readerlog}" || \ grep -IEq "${torn}" "${standbylog}"; then grep -IEn "${torn}" "${readerlog}" "${standbylog}" | head -5 error "Test 6 FAILED: standby snapshot torn by concurrent replay" fi if [ "${failed}" -ne 0 ]; then tail -n 10 "${readerlog}" error "Test 6 FAILED: standby reader exited with an error" fi # Prove the run actually opened the window it claims to test. if ! grep -q "segment graph snapshot pause at before-unlock" \ "${standbylog}"; then error "Test 6 FAILED: the capture window never opened on the standby" fi log "Test 6 PASSED: standby snapshots survived concurrent replay" } main() { log "Starting pg_textsearch physical replication test..." command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" command -v pg_basebackup >/dev/null 2>&1 || \ error "pg_basebackup not found" command -v initdb >/dev/null 2>&1 || error "initdb not found" setup_primary test_basic_standby_queries test_ongoing_replication test_segment_replication test_long_lived_backend_staleness test_standby_snapshot_race test_standby_promotion log "All physical replication tests passed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_cascading.sh000077500000000000000000000135561525762047700263050ustar00rootroot00000000000000#!/bin/bash # # replication_cascading.sh — three-node cascading replication. # primary -> standby1 -> standby2 (standby2 is built via # pg_basebackup from standby1, not from the primary). Validates that # pg_textsearch behaves correctly when WAL flows through an # intermediate hop. # # Tests: # C1: standby2 sees primary's data via standby1 — basic two-hop # read. # C2: Long-lived standby2 backend should see primary inserts that # arrive over the two-hop WAL chain. Currently FAILS — the # #345 staleness bug propagates two hops just like one. # C3: Promotion chain — promote standby1 to primary; standby2 # (configured with recovery_target_timeline = latest) follows # the new timeline and remains queryable. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55448 STANDBY_PORT=55449 STANDBY2_PORT=55450 TEST_DB=replication_cascading PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_casc_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_casc_standby" STANDBY2_DIR="${SCRIPT_DIR}/../tmp_repl_casc_standby2" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # ------------------------------------------------------------------- # C1: Cascading basic query. # standby2 sees primary's data via standby1. # Expected: PASS today. # ------------------------------------------------------------------- test_cascading_basic_query() { log "=== C1: cascading basic query ===" primary_sql " DROP TABLE IF EXISTS c1_docs CASCADE; CREATE TABLE c1_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO c1_docs (content) SELECT 'cascade doc ' || i || ' alpha' FROM generate_series(1,30) i; CREATE INDEX c1_idx ON c1_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_node_catchup "${STANDBY_PORT}" wait_for_node_catchup "${STANDBY2_PORT}" local count count=$(search_count "${STANDBY2_PORT}" c1_docs content "alpha" \ "c1_idx") log " standby2 count: ${count} (expect 30)" if [ "${count}" != "30" ]; then error "C1: standby2 missing data (got ${count})" fi log "C1 PASSED" } # ------------------------------------------------------------------- # C2: Cascading long-lived staleness. # Long-lived backend on standby2 should see new primary inserts. # Expected: FAIL today (the bug propagates two hops). # ------------------------------------------------------------------- test_cascading_long_lived_staleness() { log "=== C2: cascading long-lived staleness (should FAIL) ===" primary_sql " DROP TABLE IF EXISTS c2_docs CASCADE; CREATE TABLE c2_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO c2_docs (content) VALUES ('initial cascade alpha'); CREATE INDEX c2_idx ON c2_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_node_catchup "${STANDBY_PORT}" wait_for_node_catchup "${STANDBY2_PORT}" local query=" SELECT count(*) FROM (SELECT id FROM c2_docs ORDER BY content <@> to_bm25query('alpha','c2_idx') LIMIT 100) t; " long_lived_before_after "${STANDBY2_PORT}" "${query}" \ "primary_sql \" INSERT INTO c2_docs (content) VALUES ('new cascade alpha bravo'); \" >/dev/null wait_for_node_catchup \"${STANDBY_PORT}\" wait_for_node_catchup \"${STANDBY2_PORT}\"" log " before=${LL_BEFORE}, after=${LL_AFTER}" if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "C2 BUG (expected): standby2 long-lived backend missed \ primary insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "C2 PASSED" } # ------------------------------------------------------------------- # C3: Cascading promotion chain. # Promote standby1 to primary; standby2 follows the new timeline. # Expected: PASS today (Postgres handles the timeline; bm25 just # rides on top). # ------------------------------------------------------------------- test_cascading_promotion_chain() { log "=== C3: cascading promotion chain ===" # standby2 needs recovery_target_timeline=latest so it picks up # the new timeline after standby1's promotion. cat >> "${STANDBY2_DIR}/postgresql.conf" </dev/null sleep 3 local count count=$(search_count "${STANDBY2_PORT}" c2_docs content "alpha" \ "c2_idx") log " standby2 count after chain promotion: ${count}" if [[ "${count}" == *ERROR* ]] || \ ! [[ "${count}" =~ ^[0-9]+$ ]] || \ [ "${count}" -lt 2 ]; then error "C3: standby2 not caught up after chain promotion \ (got ${count}, expected >= 2)" fi log "C3 PASSED" } main() { log "Starting pg_textsearch cascading replication tests..." check_required_tools setup_primary setup_standby setup_standby2 test_cascading_basic_query test_cascading_long_lived_staleness test_cascading_promotion_chain log "All cascading replication tests completed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_compat.sh000077500000000000000000000223471525762047700256520ustar00rootroot00000000000000#!/bin/bash # # replication_compat.sh — compatibility tests: # F1: physical + logical replication coexistence # F2: TimescaleDB hypertable replication # F3: pg_basebackup with bm25 index files set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55446 STANDBY_PORT=55447 LOGICAL_SUB_PORT=55448 TEST_DB=replication_compat PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_compat_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_compat_standby" LOGICAL_SUB_DIR="${SCRIPT_DIR}/../tmp_repl_compat_subscriber" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" # Custom cleanup that also handles the logical-subscriber instance. # Capture $? on entry: the wrapper runs commands before delegating # to repl_cleanup, which destroys the original exit code. Restore it # (via a subshell `exit`) so repl_cleanup's `local exit_code=$?` # captures the script's real exit status, not `rm -rf`'s. compat_cleanup() { local rc=$? if [ -n "${LOGICAL_SUB_DIR:-}" ] && \ [ -f "${LOGICAL_SUB_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${LOGICAL_SUB_DIR}" -m fast 2>/dev/null \ || true fi rm -rf "${LOGICAL_SUB_DIR:-}" (exit "$rc") repl_cleanup } trap compat_cleanup EXIT INT TERM # Bring up a third standalone instance to act as a logical # subscriber. setup_logical_subscriber() { log "Setting up logical subscriber on port ${LOGICAL_SUB_PORT}..." rm -rf "${LOGICAL_SUB_DIR}" mkdir -p "${LOGICAL_SUB_DIR}" initdb -D "${LOGICAL_SUB_DIR}" --auth-local=trust \ --auth-host=trust >/dev/null 2>&1 cat >> "${LOGICAL_SUB_DIR}/postgresql.conf" </dev/null } # ------------------------------------------------------------------- # F1: physical + logical replication coexistence. # Primary publishes a logical pub for one table, has a physical # standby, AND has a logical subscriber on a third instance. # All three views agree. # Expected: PASS today. # ------------------------------------------------------------------- test_logical_and_physical_coexist() { log "=== F1: logical + physical coexistence ===" # Need wal_level=logical on primary. psql -p "${PRIMARY_PORT}" -d postgres -c " ALTER SYSTEM SET wal_level = logical; " >/dev/null pg_ctl restart -D "${PRIMARY_DIR}" -m fast -w # Recreate standby with logical-level WAL. pg_ctl stop -D "${STANDBY_DIR}" -m fast 2>/dev/null || true rm -rf "${STANDBY_DIR}" primary_sql " DROP TABLE IF EXISTS f1_docs CASCADE; CREATE TABLE f1_docs (id INT PRIMARY KEY, content TEXT); INSERT INTO f1_docs VALUES (1, 'initial alpha bravo'), (2, 'second alpha charlie'); CREATE INDEX f1_idx ON f1_docs USING bm25(content) WITH (text_config='simple'); CREATE PUBLICATION f1_pub FOR TABLE f1_docs; " >/dev/null setup_standby setup_logical_subscriber psql -p "${LOGICAL_SUB_PORT}" -d "${TEST_DB}" -c " CREATE TABLE f1_docs (id INT PRIMARY KEY, content TEXT); " >/dev/null # CREATE SUBSCRIPTION with create_slot=true (the default) cannot # run inside a transaction block, so issue it on its own. psql -p "${LOGICAL_SUB_PORT}" -d "${TEST_DB}" -c \ "CREATE SUBSCRIPTION f1_sub CONNECTION 'host=127.0.0.1 port=${PRIMARY_PORT} dbname=${TEST_DB}' PUBLICATION f1_pub;" \ >/dev/null sleep 3 # initial sync # Insert on primary. primary_sql " INSERT INTO f1_docs VALUES (3, 'third alpha delta'); " >/dev/null wait_for_standby_catchup sleep 2 # let logical replication catch up # Subscriber: build the bm25 index now (logical only replicates # the heap). psql -p "${LOGICAL_SUB_PORT}" -d "${TEST_DB}" -c " CREATE INDEX IF NOT EXISTS f1_sub_idx ON f1_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null local primary_count standby_count subscriber_count primary_count=$(search_count "${PRIMARY_PORT}" f1_docs content \ "alpha" "f1_idx") standby_count=$(search_count "${STANDBY_PORT}" f1_docs content \ "alpha" "f1_idx") subscriber_count=$(search_count "${LOGICAL_SUB_PORT}" f1_docs \ content "alpha" "f1_sub_idx") log " primary=${primary_count} standby=${standby_count} \ subscriber=${subscriber_count}" if [ "${primary_count}" != "3" ] || \ [ "${standby_count}" != "3" ] || \ [ "${subscriber_count}" != "3" ]; then error "F1: counts disagree across nodes" fi log "F1 PASSED" } # ------------------------------------------------------------------- # F2: TimescaleDB hypertable replication. # Hypertable on primary has a bm25 index. Long-lived standby backend # should see new inserts to recent chunks. # Expected: FAIL today when actually run (memtable staleness on # hypertable inserts). # Skipped if TimescaleDB is not available OR not loaded via # shared_preload_libraries. Today the test is typically SKIPPED on # pg18-release because setup_primary in replication_lib.sh sets # SPL='pg_textsearch' only. # ------------------------------------------------------------------- test_timescaledb_hypertable_replication_long_lived() { log "=== F2: TimescaleDB hypertable (should FAIL today) ===" local has_ts has_ts=$(psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" -tA -c " SELECT count(*) FROM pg_available_extensions WHERE name='timescaledb';" 2>/dev/null) if [ "${has_ts}" != "1" ]; then warn "F2: TimescaleDB not available, skipping" return 0 fi # Try to actually load it. Fails if timescaledb is not in # shared_preload_libraries on the primary. if ! psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION IF NOT EXISTS timescaledb;" \ >/dev/null 2>&1; then warn "F2: TimescaleDB present but not in \ shared_preload_libraries — skipping. To actually exercise this \ test, add timescaledb to shared_preload_libraries in setup_primary \ (replication_lib.sh)." return 0 fi primary_sql " DROP TABLE IF EXISTS f2_events CASCADE; CREATE TABLE f2_events ( ts TIMESTAMPTZ NOT NULL, content TEXT ); SELECT create_hypertable('f2_events', 'ts', chunk_time_interval => INTERVAL '1 day'); INSERT INTO f2_events VALUES (now() - INTERVAL '12 hours', 'old event alpha'), (now() - INTERVAL '1 hour', 'recent event alpha'); CREATE INDEX f2_idx ON f2_events USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM (SELECT ts FROM f2_events ORDER BY content <@> to_bm25query('alpha','f2_idx') LIMIT 100) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO f2_events VALUES (now(), 'brand new event alpha bravo'); \" >/dev/null wait_for_standby_catchup" log " before=${LL_BEFORE}, after=${LL_AFTER}" if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "F2 BUG (expected): hypertable standby long-lived \ backend missed primary insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "F2 PASSED" } # ------------------------------------------------------------------- # F3: pg_basebackup with bm25 index files. # Standby was already created via pg_basebackup. Verify the bm25 # index file made it into the backup and is queryable on the # standby. (This is a sanity check that segment files are not # excluded from basebackup.) # Expected: PASS today. # ------------------------------------------------------------------- test_pg_basebackup_with_bm25_index() { log "=== F3: pg_basebackup includes bm25 index files ===" # Use the F1 table+index that's already on primary+standby. # Spill primary so segment files exist on disk. primary_sql "SELECT bm25_spill_index('f1_idx');" >/dev/null wait_for_standby_catchup # Wipe standby and rebuild from basebackup. pg_ctl stop -D "${STANDBY_DIR}" -m fast 2>/dev/null || true rm -rf "${STANDBY_DIR}" setup_standby wait_for_standby_catchup local count count=$(search_count "${STANDBY_PORT}" f1_docs content "alpha" \ "f1_idx") log " re-basebackuped standby count: ${count}" if [ "${count}" != "3" ]; then error "F3: re-basebackuped standby missing data (got ${count})" fi log "F3 PASSED" } main() { log "Starting pg_textsearch replication compat tests..." check_required_tools setup_primary setup_standby test_logical_and_physical_coexist test_timescaledb_hypertable_replication_long_lived test_pg_basebackup_with_bm25_index log "All replication compat tests completed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_concurrency.sh000077500000000000000000000232441525762047700267160ustar00rootroot00000000000000#!/bin/bash # # replication_concurrency.sh — concurrency tests for physical # replication. Each test holds N persistent psql sessions open # against the standby (long-lived backends), drives a write # workload on the primary, and asserts that all standby sessions # converge on the right answer. # # Tests: # B1: 8 concurrent long-lived standby readers each take a # before/after reading around a single primary insert. # Currently FAILS — exposes #345 in the multi-reader case. # B2: 4 long-lived standby readers query repeatedly during and # after a 5000-doc primary insert burst. All should converge # on 5000. Currently FAILS — same root cause as B1. # B3: Many standby readers under a primary spam-insert + spam- # spill workload. Validates no crashes, no deadlocks, no # errors in the standby log. PASSES today (vacuously — the # drain code that this would guard doesn't exist yet). set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55442 STANDBY_PORT=55443 TEST_DB=replication_concurrency PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_conc_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_conc_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # ------------------------------------------------------------------- # B1: N concurrent long-lived backends on standby. # Eight long-lived backends each query before+after a primary # insert. All should see the new row. # Expected: FAIL today (every long-lived backend goes stale). # ------------------------------------------------------------------- test_concurrent_standby_readers() { log "=== B1: 8 concurrent long-lived standby readers ===" primary_sql " DROP TABLE IF EXISTS b1_docs CASCADE; CREATE TABLE b1_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO b1_docs (content) VALUES ('initial doc alpha bravo'); CREATE INDEX b1_idx ON b1_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup # Spawn 8 reader subprocesses. Each opens its own long-lived # session, takes a 'before' reading, waits for a sync file, # takes an 'after' reading, and writes both to a result file. local sync_file sync_file=$(mktemp) rm "${sync_file}" # parent recreates after primary insert local results_dir results_dir=$(mktemp -d) local pids=() for i in $(seq 1 8); do ( # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" long_lived_open "${STANDBY_PORT}" local before before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM b1_docs ORDER BY content <@> to_bm25query('alpha', 'b1_idx') LIMIT 100) t;") # Wait until parent flips sync file. while [ ! -f "${sync_file}" ]; do sleep 0.1; done local after after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM b1_docs ORDER BY content <@> to_bm25query('alpha', 'b1_idx') LIMIT 100) t;") long_lived_close echo "${before} ${after}" > "${results_dir}/r_${i}" ) & pids+=($!) done # Give readers time to take 'before' readings. sleep 2 # Insert on primary, wait for replication, signal readers. primary_sql " INSERT INTO b1_docs (content) VALUES ('newly added alpha bravo charlie'); " >/dev/null wait_for_standby_catchup touch "${sync_file}" for p in "${pids[@]}"; do wait "$p"; done local fail_count=0 for i in $(seq 1 8); do local before after read -r before after < "${results_dir}/r_${i}" log " reader ${i}: before=${before}, after=${after}" if [[ "${after}" == *ERROR* ]] || \ ! [[ "${after}" =~ ^[0-9]+$ ]] || \ [ "${after}" -le "${before}" ]; then fail_count=$((fail_count + 1)) fi done rm -rf "${results_dir}" "${sync_file}" if [ "${fail_count}" -gt 0 ]; then error "B1 BUG (expected): ${fail_count}/8 standby readers \ missed the new primary insert" fi log "B1 PASSED" } # ------------------------------------------------------------------- # B2: Drain under insert burst. # Primary inserts 5000 docs; 4 standby long-lived readers # repeatedly query during and after. All should converge to seeing # all 5000. # Expected: FAIL today. # ------------------------------------------------------------------- test_drain_under_insert_burst() { log "=== B2: drain under insert burst ===" primary_sql " DROP TABLE IF EXISTS b2_docs CASCADE; CREATE TABLE b2_docs (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX b2_idx ON b2_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup local results_dir results_dir=$(mktemp -d) local stop_file stop_file=$(mktemp); rm "${stop_file}" local pids=() for i in 1 2 3 4; do ( # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" long_lived_open "${STANDBY_PORT}" local last=0 while [ ! -f "${stop_file}" ]; do last=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM b2_docs ORDER BY content <@> to_bm25query('burst','b2_idx') LIMIT 10000) t;") sleep 0.2 done # Final reading after stop signal. local final final=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM b2_docs ORDER BY content <@> to_bm25query('burst','b2_idx') LIMIT 10000) t;") long_lived_close echo "${last} ${final}" > "${results_dir}/r_${i}" ) & pids+=($!) done # Insert burst on primary in batches. for batch in $(seq 1 50); do primary_sql " INSERT INTO b2_docs (content) SELECT 'burst batch ${batch} doc ' || j FROM generate_series(1, 100) j; " >/dev/null done wait_for_standby_catchup sleep 1 touch "${stop_file}" for p in "${pids[@]}"; do wait "$p"; done local fail_count=0 for i in 1 2 3 4; do local last final read -r last final < "${results_dir}/r_${i}" log " reader ${i}: last_during=${last}, final_after=${final}" if [[ "${final}" == *ERROR* ]] || \ ! [[ "${final}" =~ ^[0-9]+$ ]] || \ [ "${final}" -lt "5000" ]; then fail_count=$((fail_count + 1)) fi done rm -rf "${results_dir}" "${stop_file}" if [ "${fail_count}" -gt 0 ]; then error "B2 BUG (expected): ${fail_count}/4 standby readers \ did not converge to 5000 after burst" fi log "B2 PASSED" } # ------------------------------------------------------------------- # B3: No deadlock under spam load. # Primary spam-inserts and spam-spills; many standby readers # repeatedly query. No process crashes, no deadlocks, no error # messages in standby log. # Expected: PASS today (passes vacuously — there's no drain code # to deadlock yet). After implementation, this guards the LWLock. # ------------------------------------------------------------------- test_no_deadlock_under_spam_load() { log "=== B3: no deadlock under spam load ===" primary_sql " DROP TABLE IF EXISTS b3_docs CASCADE; CREATE TABLE b3_docs (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX b3_idx ON b3_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup local stop_file stop_file=$(mktemp); rm "${stop_file}" # 4 standby readers local rd_pids=() for i in 1 2 3 4; do ( while [ ! -f "${stop_file}" ]; do psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -tA -c " SELECT count(*) FROM (SELECT id FROM b3_docs ORDER BY content <@> to_bm25query('spam','b3_idx') LIMIT 1000) t; " >/dev/null 2>&1 || true done ) & rd_pids+=($!) done # Primary writer: insert + occasional spill. for batch in $(seq 1 30); do primary_sql " INSERT INTO b3_docs (content) SELECT 'spam batch ${batch} doc ' || j FROM generate_series(1, 200) j; " >/dev/null if [ $((batch % 5)) -eq 0 ]; then primary_sql "SELECT bm25_spill_index('b3_idx');" >/dev/null fi done wait_for_standby_catchup sleep 2 touch "${stop_file}" for p in "${rd_pids[@]}"; do wait "$p" 2>/dev/null; done rm -f "${stop_file}" # Verify standby is still alive and logs clean of fatal errors. local alive alive=$(node_sql_quiet "${STANDBY_PORT}" "SELECT 1;") if [ "${alive}" != "1" ]; then error "B3: standby is dead after spam load" fi if grep -E 'PANIC|FATAL|deadlock' \ "${STANDBY_DIR}/log/postgres.log" >/dev/null 2>&1; then error "B3: standby log shows PANIC/FATAL/deadlock entries" fi log "B3 PASSED" } main() { log "Starting pg_textsearch replication concurrency tests..." check_required_tools setup_primary setup_standby test_concurrent_standby_readers test_drain_under_insert_burst test_no_deadlock_under_spam_load log "All replication concurrency tests completed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_correctness.sh000077500000000000000000000460501525762047700267160ustar00rootroot00000000000000#!/bin/bash # # replication_correctness.sh — physical replication correctness # tests focusing on long-lived standby backends across schema # variations and primary write events (insert/update/delete/spill/ # merge). # # Failure mode tracked here: long-lived standby backend staleness # (#345). A standby backend that stays alive across primary inserts # rebuilds its in-memory view once on first scan and is never # invalidated by WAL replay; subsequent primary inserts and segment # merges are invisible to it. Surfaces in this file as "0 results" # (or undercount) for documents that ARE visible on disk to the # standby. # # Before PR #343 there was a second surface here — long-lived # standby backends erroring on segment reads when the segments # arrived via WAL replay. That was #342 (page-index pages weren't # WAL-logged at all), fixed by PR #343. The remaining failures in # this file all map to #345. # # A1 PASSES today — it reads segments that existed in the initial # basebackup, so the long-lived backend's stale view is still # correct. All other tests FAIL today against #345. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55440 STANDBY_PORT=55441 TEST_DB=replication_correctness PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_corr_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_corr_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # ------------------------------------------------------------------- # Test A1: Long-lived read after primary spill. # After the primary spills its memtable, the long-lived standby # backend re-queries and sees the spilled (now-segment) data. # Expected: PASS today (segments replicate via GenericXLog). # ------------------------------------------------------------------- test_long_lived_read_after_primary_spill() { log "=== A1: read after primary spill ===" primary_sql " DROP TABLE IF EXISTS a1_docs CASCADE; CREATE TABLE a1_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO a1_docs (content) SELECT 'doc ' || i || ' alpha bravo charlie' FROM generate_series(1,200) i; CREATE INDEX a1_idx ON a1_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null setup_standby wait_for_standby_catchup long_lived_open "${STANDBY_PORT}" local before before=$(long_lived_query " SELECT count(*) FROM ( SELECT id FROM a1_docs ORDER BY content <@> to_bm25query('alpha', 'a1_idx') LIMIT 1000 ) t;") primary_sql "SELECT bm25_spill_index('a1_idx');" >/dev/null wait_for_standby_catchup local after after=$(long_lived_query " SELECT count(*) FROM ( SELECT id FROM a1_docs ORDER BY content <@> to_bm25query('alpha', 'a1_idx') LIMIT 1000 ) t;") long_lived_close log "before=${before} after=${after} (both should be 200)" if [ "${before}" != "200" ] || [ "${after}" != "200" ]; then error "A1: counts wrong (before=${before}, after=${after})" fi log "A1 PASSED" } # ------------------------------------------------------------------- # Test A3: Memtable+segment mix. # Primary has data both in a segment (post-spill) AND in the # memtable (post-spill inserts). Long-lived standby backend should # see ALL of it. # Expected: FAIL today — the segment portion replicates fine, but # the memtable portion is invisible to the long-lived backend. # ------------------------------------------------------------------- test_long_lived_memtable_segment_mix() { log "=== A3: memtable + segment mix (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS a3_docs CASCADE; CREATE TABLE a3_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO a3_docs (content) SELECT 'spilled doc ' || i || ' alpha' FROM generate_series(1,100) i; CREATE INDEX a3_idx ON a3_docs USING bm25(content) WITH (text_config='simple'); SELECT bm25_spill_index('a3_idx'); " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM ( SELECT id FROM a3_docs ORDER BY content <@> to_bm25query('alpha', 'a3_idx') LIMIT 1000 ) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO a3_docs (content) SELECT 'memtable doc ' || i || ' alpha' FROM generate_series(1,50) i; \" >/dev/null wait_for_standby_catchup" log "before=${LL_BEFORE} (expect 100), after=${LL_AFTER} (expect 150)" if [ "${LL_BEFORE}" != "100" ]; then error "A3: before count wrong (${LL_BEFORE})" fi if [ "${LL_AFTER}" != "150" ]; then error "A3 BUG (expected): standby long-lived backend missed \ memtable inserts (before=${LL_BEFORE}, after=${LL_AFTER}, expected=150)" fi log "A3 PASSED" } # ------------------------------------------------------------------- # Test A4: UPDATE replication (HOT and non-HOT). # UPDATE on primary changes the indexed text. Long-lived standby # backend should see the new content searchable. # Expected: FAIL today. # ------------------------------------------------------------------- test_long_lived_update_replication() { log "=== A4: UPDATE replication (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS a4_docs CASCADE; CREATE TABLE a4_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO a4_docs (content) SELECT 'original doc ' || i || ' xenon' FROM generate_series(1,20) i; CREATE INDEX a4_idx ON a4_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup long_lived_open "${STANDBY_PORT}" local xenon_before zircon_before xenon_before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM a4_docs ORDER BY content <@> to_bm25query('xenon', 'a4_idx') LIMIT 100) t;") zircon_before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM a4_docs ORDER BY content <@> to_bm25query('zircon', 'a4_idx') LIMIT 100) t;") primary_sql " UPDATE a4_docs SET content = 'replaced doc ' || id || ' zircon' WHERE id <= 10; " >/dev/null wait_for_standby_catchup local xenon_after zircon_after xenon_after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM a4_docs ORDER BY content <@> to_bm25query('xenon', 'a4_idx') LIMIT 100) t;") zircon_after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM a4_docs ORDER BY content <@> to_bm25query('zircon', 'a4_idx') LIMIT 100) t;") long_lived_close log "xenon: before=${xenon_before} (expect 20), after=${xenon_after} (expect 10)" log "zircon: before=${zircon_before} (expect 0), after=${zircon_after} (expect 10)" if [ "${xenon_after}" != "10" ] || [ "${zircon_after}" != "10" ]; then error "A4 BUG (expected): standby long-lived backend missed \ UPDATEs (xenon=${xenon_after}/expected 10, zircon=${zircon_after}/expected 10)" fi log "A4 PASSED" } # ------------------------------------------------------------------- # Test A5: DELETE + VACUUM replication. # Deleted+vacuumed rows should not appear in standby search results. # Expected: FAIL today — same WAL-replayed-segment issue as A2. # Long-lived standby backend cannot read segments created by primary # spill+merge that arrived via WAL replay. Will pass once the rmgr- # based replication implementation lands. # ------------------------------------------------------------------- test_long_lived_delete_vacuum() { log "=== A5: DELETE + VACUUM ===" primary_sql " DROP TABLE IF EXISTS a5_docs CASCADE; CREATE TABLE a5_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO a5_docs (content) SELECT 'doc ' || i || ' alpha' FROM generate_series(1,100) i; CREATE INDEX a5_idx ON a5_docs USING bm25(content) WITH (text_config='simple'); SELECT bm25_spill_index('a5_idx'); " >/dev/null wait_for_standby_catchup long_lived_open "${STANDBY_PORT}" local before before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM a5_docs ORDER BY content <@> to_bm25query('alpha', 'a5_idx') LIMIT 1000) t;") primary_sql "DELETE FROM a5_docs WHERE id <= 30;" >/dev/null primary_sql "VACUUM a5_docs;" >/dev/null wait_for_standby_catchup local after after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM a5_docs ORDER BY content <@> to_bm25query('alpha', 'a5_idx') LIMIT 1000) t;") long_lived_close log "before=${before} (expect 100), after=${after} (expect 70)" if [[ "${before}" == *ERROR* ]] || [[ "${after}" == *ERROR* ]] || \ [ "${before}" != "100" ] || [ "${after}" != "70" ]; then error "A5 BUG (expected): standby long-lived backend cannot read \ WAL-replayed segments (before=${before}, after=${after}, \ expected before=100 after=70)" fi log "A5 PASSED" } # ------------------------------------------------------------------- # Test E1: Partitioned table replication. # Long-lived standby backend should see new inserts to a # partitioned table. # Expected: FAIL today. # ------------------------------------------------------------------- test_long_lived_partitioned_table() { log "=== E1: partitioned table (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS e1_docs CASCADE; CREATE TABLE e1_docs ( id INT, tenant INT, content TEXT ) PARTITION BY RANGE (tenant); CREATE TABLE e1_docs_t1 PARTITION OF e1_docs FOR VALUES FROM (0) TO (10); CREATE TABLE e1_docs_t2 PARTITION OF e1_docs FOR VALUES FROM (10) TO (20); INSERT INTO e1_docs (id, tenant, content) SELECT i, i % 20, 'doc ' || i || ' alpha' FROM generate_series(1,40) i; CREATE INDEX e1_idx_t1 ON e1_docs_t1 USING bm25(content) WITH (text_config='simple'); CREATE INDEX e1_idx_t2 ON e1_docs_t2 USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup # Query against one partition's index directly. local query=" SELECT count(*) FROM (SELECT id FROM e1_docs_t1 ORDER BY content <@> to_bm25query('alpha', 'e1_idx_t1') LIMIT 100) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO e1_docs (id, tenant, content) VALUES (101, 5, 'newdoc 101 alpha'), (102, 7, 'newdoc 102 alpha'); \" >/dev/null wait_for_standby_catchup" log "before=${LL_BEFORE}, after=${LL_AFTER}" if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "E1 BUG (expected): standby long-lived backend missed \ partition insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "E1 PASSED" } # ------------------------------------------------------------------- # Test E2: Expression index. # Index on lower(content). Long-lived standby backend should see # new rows via the expression index. # Expected: FAIL today. # ------------------------------------------------------------------- test_long_lived_expression_index() { log "=== E2: expression index (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS e2_docs CASCADE; CREATE TABLE e2_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO e2_docs (content) VALUES ('Initial Document ALPHA'), ('Another Initial Doc BRAVO'); CREATE INDEX e2_idx ON e2_docs USING bm25(lower(content)) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM (SELECT id FROM e2_docs ORDER BY lower(content) <@> to_bm25query('alpha', 'e2_idx') LIMIT 100) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO e2_docs (content) VALUES ('Brand New Document ALPHA'); \" >/dev/null wait_for_standby_catchup" log "before=${LL_BEFORE} (expect 1), after=${LL_AFTER} (expect 2)" if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "E2 BUG (expected): expression-index standby long-lived \ backend missed insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "E2 PASSED" } # ------------------------------------------------------------------- # Test E3: Partial index. # Long-lived standby backend should see new rows that match the # WHERE predicate. # Expected: FAIL today. # ------------------------------------------------------------------- test_long_lived_partial_index() { log "=== E3: partial index (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS e3_docs CASCADE; CREATE TABLE e3_docs (id SERIAL PRIMARY KEY, content TEXT, published BOOL); INSERT INTO e3_docs (content, published) VALUES ('published initial alpha', true), ('draft initial alpha', false); CREATE INDEX e3_idx ON e3_docs USING bm25(content) WITH (text_config='simple') WHERE published; " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM (SELECT id FROM e3_docs WHERE published ORDER BY content <@> to_bm25query('alpha', 'e3_idx') LIMIT 100) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO e3_docs (content, published) VALUES ('newly published alpha', true); \" >/dev/null wait_for_standby_catchup" log "before=${LL_BEFORE} (expect 1), after=${LL_AFTER} (expect 2)" if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "E3 BUG (expected): partial-index standby long-lived \ backend missed insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "E3 PASSED" } # ------------------------------------------------------------------- # Test E4: text[] (array) index. # Long-lived standby backend should see new array-column inserts. # Expected: FAIL today. # ------------------------------------------------------------------- test_long_lived_text_array() { log "=== E4: text[] index (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS e4_docs CASCADE; CREATE TABLE e4_docs (id SERIAL PRIMARY KEY, tags TEXT[]); INSERT INTO e4_docs (tags) VALUES (ARRAY['initial','alpha','bravo']); CREATE INDEX e4_idx ON e4_docs USING bm25(tags) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM (SELECT id FROM e4_docs ORDER BY tags <@> to_bm25query('alpha', 'e4_idx') LIMIT 100) t; " long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" INSERT INTO e4_docs (tags) VALUES (ARRAY['new','alpha','charlie']); \" >/dev/null wait_for_standby_catchup" log "before=${LL_BEFORE} (expect 1), after=${LL_AFTER} (expect 2)" if [[ "${LL_AFTER}" == *ERROR* ]] || \ ! [[ "${LL_AFTER}" =~ ^[0-9]+$ ]] || \ [ "${LL_AFTER}" -le "${LL_BEFORE}" ]; then error "E4 BUG (expected): text[]-index standby long-lived \ backend missed insert (before=${LL_BEFORE}, after=${LL_AFTER})" fi log "E4 PASSED" } # ------------------------------------------------------------------- # Test E5: Multiple indexes on same table. # Two indexes on the same table (different columns or different # text_configs). Both should reflect new inserts to a long-lived # backend. # Expected: FAIL today (both indexes have stale memtables). # ------------------------------------------------------------------- test_long_lived_multiple_indexes_same_table() { log "=== E5: multiple indexes on same table (should FAIL today) ===" primary_sql " DROP TABLE IF EXISTS e5_docs CASCADE; CREATE TABLE e5_docs (id SERIAL PRIMARY KEY, title TEXT, body TEXT); INSERT INTO e5_docs (title, body) VALUES ('initial title alpha', 'initial body bravo'); CREATE INDEX e5_title_idx ON e5_docs USING bm25(title) WITH (text_config='simple'); CREATE INDEX e5_body_idx ON e5_docs USING bm25(body) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup long_lived_open "${STANDBY_PORT}" local title_before body_before title_before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM e5_docs ORDER BY title <@> to_bm25query('alpha','e5_title_idx') LIMIT 100) t;") body_before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM e5_docs ORDER BY body <@> to_bm25query('bravo','e5_body_idx') LIMIT 100) t;") primary_sql " INSERT INTO e5_docs (title, body) VALUES ('new title alpha', 'new body bravo'); " >/dev/null wait_for_standby_catchup local title_after body_after title_after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM e5_docs ORDER BY title <@> to_bm25query('alpha','e5_title_idx') LIMIT 100) t;") body_after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM e5_docs ORDER BY body <@> to_bm25query('bravo','e5_body_idx') LIMIT 100) t;") long_lived_close log "title: before=${title_before}, after=${title_after}" log "body: before=${body_before}, after=${body_after}" if [[ "${title_after}" == *ERROR* ]] || \ [[ "${body_after}" == *ERROR* ]] || \ ! [[ "${title_after}" =~ ^[0-9]+$ ]] || \ ! [[ "${body_after}" =~ ^[0-9]+$ ]] || \ [ "${title_after}" -le "${title_before}" ] || \ [ "${body_after}" -le "${body_before}" ]; then error "E5 BUG (expected): standby long-lived backend missed \ inserts on one or both indexes (title=${title_after}, body=${body_after})" fi log "E5 PASSED" } main() { log "Starting pg_textsearch replication correctness tests..." check_required_tools setup_primary test_long_lived_read_after_primary_spill test_long_lived_memtable_segment_mix test_long_lived_update_replication test_long_lived_delete_vacuum test_long_lived_partitioned_table test_long_lived_expression_index test_long_lived_partial_index test_long_lived_text_array test_long_lived_multiple_indexes_same_table log "All replication correctness tests completed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_failover.sh000077500000000000000000000176661525762047700262060ustar00rootroot00000000000000#!/bin/bash # # replication_failover.sh — standby-promotion tests. Brings up a # primary + streaming standby pair, leaves the standby in some # specific pre-promotion state, then promotes the standby and # checks the (new) primary still works. # # Tests: # D1: Promotion under active primary write load — primary is # still inserting when the standby is promoted. Post- # promotion the new primary should accept reads + writes. # D2: Promotion with an unspilled memtable on the primary — the # standby's view of the index includes WAL-replayed memtable # posting entries that have never been spilled to a segment. # Post-promotion this state must still be queryable. # D3: Post-promotion writes + searches — the new primary takes # fresh inserts, search returns them. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55444 STANDBY_PORT=55445 TEST_DB=replication_failover PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_fail_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_fail_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM wait_for_promotion() { local port=$1 local i=0 while [ $i -lt 15 ]; do local in_recovery in_recovery=$(node_sql_quiet "${port}" \ "SELECT pg_is_in_recovery();") if [ "$in_recovery" = "f" ]; then return 0 fi sleep 1 i=$((i + 1)) done error "Promotion did not complete on port ${port}" } # ------------------------------------------------------------------- # D1: Promotion under steady write load. # Primary actively inserting; promote standby; verify the promoted # standby (a) sees pre-promotion data, (b) accepts new writes. # Expected: FAIL today — segments created by primary spill while # the standby is already running (WAL-replayed) are not enumerable # on the standby even after promotion. bm25_summarize_index shows # corpus stats (total_docs=100) but no segments. Same bug class as # replication_correctness.sh A2/A5. Will pass once the rmgr-based # replication implementation lands. # ------------------------------------------------------------------- test_promotion_under_write_load() { log "=== D1: promotion under write load ===" primary_sql " DROP TABLE IF EXISTS d1_docs CASCADE; CREATE TABLE d1_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO d1_docs (content) SELECT 'doc ' || i || ' alpha' FROM generate_series(1,100) i; CREATE INDEX d1_idx ON d1_docs USING bm25(content) WITH (text_config='simple'); SELECT bm25_spill_index('d1_idx'); " >/dev/null wait_for_standby_catchup pg_ctl promote -D "${STANDBY_DIR}" -w wait_for_promotion "${STANDBY_PORT}" # Spilled docs should be visible (segment-on-disk). local count count=$(search_count "${STANDBY_PORT}" d1_docs content "alpha" \ "d1_idx") log " promoted-standby alpha count: ${count}" if [[ "${count}" == *ERROR* ]] || [ "${count}" != "100" ]; then error "D1 BUG (expected): WAL-replayed spilled segment not \ enumerable on promoted standby (got ${count}/100)" fi # Writes work. psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -c " INSERT INTO d1_docs (content) VALUES ('post-promotion alpha doc'); " >/dev/null local count2 count2=$(search_count "${STANDBY_PORT}" d1_docs content "alpha" \ "d1_idx") if [[ "${count2}" == *ERROR* ]] || \ ! [[ "${count2}" =~ ^[0-9]+$ ]] || \ [ "${count2}" -le "${count}" ]; then error "D1: writes on promoted standby not visible \ (${count} -> ${count2})" fi log "D1 PASSED" } # ------------------------------------------------------------------- # D2: Promotion with unspilled data. # Primary has un-spilled data in memtable; standby is replicated # but its memtable was rebuilt from docid pages (so it has the # data). Promote standby; verify all data is visible after promotion. # Expected: PASS today — the long-lived backend opened before any # inserts has its first query trigger memtable rebuild, and the # promotion + subsequent query sees post-rebuild state. The memtable # staleness bug does not trigger in this sequence (compare with # replication.sh test_long_lived_backend_staleness, which DOES open # the long-lived backend before the first insert and sees staleness). # ------------------------------------------------------------------- test_promotion_with_unspilled_memtable() { log "=== D2: promotion with unspilled memtable ===" # Reset: fresh primary+standby pair so this test is independent # of D1's promoted standby (which is no longer in recovery). pg_ctl stop -D "${STANDBY_DIR}" -m fast 2>/dev/null || true pg_ctl stop -D "${PRIMARY_DIR}" -m fast 2>/dev/null || true rm -rf "${PRIMARY_DIR}" "${STANDBY_DIR}" setup_primary primary_sql " DROP TABLE IF EXISTS d2_docs CASCADE; CREATE TABLE d2_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO d2_docs (content) SELECT 'doc ' || i || ' alpha' FROM generate_series(1,50) i; CREATE INDEX d2_idx ON d2_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null setup_standby wait_for_standby_catchup long_lived_open "${STANDBY_PORT}" local before before=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM d2_docs ORDER BY content <@> to_bm25query('alpha','d2_idx') LIMIT 200) t;") log " pre-promotion long-lived count: ${before} (expect 50)" # Insert more on primary — these stay in memtable, not spilled. primary_sql " INSERT INTO d2_docs (content) SELECT 'late doc ' || i || ' alpha' FROM generate_series(1,30) i; " >/dev/null wait_for_standby_catchup pg_ctl promote -D "${STANDBY_DIR}" -w wait_for_promotion "${STANDBY_PORT}" local after after=$(long_lived_query " SELECT count(*) FROM (SELECT id FROM d2_docs ORDER BY content <@> to_bm25query('alpha','d2_idx') LIMIT 200) t;") long_lived_close log " post-promotion long-lived count: ${after} (expect 80)" if [ "${after}" != "80" ]; then error "D2 BUG (likely): long-lived backend on promoted standby \ missed pre-promotion memtable inserts (${after}/80)" fi log "D2 PASSED" } # ------------------------------------------------------------------- # D3: Post-promotion writes + search via fresh backends. # Verify the standard "promoted standby is a normal primary" path # works for fresh psql connections. # Expected: PASS today. # ------------------------------------------------------------------- test_post_promotion_writes_and_search() { log "=== D3: post-promotion writes + search ===" psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -c " INSERT INTO d2_docs (content) VALUES ('promoted-mode doc alpha bravo'), ('another promoted doc alpha'); " >/dev/null local count count=$(search_count "${STANDBY_PORT}" d2_docs content "alpha" \ "d2_idx") log " search count after promotion-writes: ${count}" if [[ "${count}" == *ERROR* ]] || \ ! [[ "${count}" =~ ^[0-9]+$ ]] || \ [ "${count}" -lt 82 ]; then error "D3: post-promotion writes not visible (got ${count})" fi log "D3 PASSED" } main() { log "Starting pg_textsearch replication failover tests..." check_required_tools setup_primary primary_sql " DROP TABLE IF EXISTS d_warmup CASCADE; CREATE TABLE d_warmup (id INT, content TEXT); INSERT INTO d_warmup VALUES (1, 'warmup'); " >/dev/null setup_standby wait_for_standby_catchup test_promotion_under_write_load test_promotion_with_unspilled_memtable test_post_promotion_writes_and_search log "All replication failover tests completed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_issue_342.sh000077500000000000000000000117231525762047700261030ustar00rootroot00000000000000#!/bin/bash # # replication_issue_342.sh — targeted reproducers for issue #342: # "BM25 index queries fail on streaming replicas with # 'ERROR: invalid page index at block N'" # # Root cause per issue: page-index pages are written via # MarkBufferDirty + FlushRelationBuffers without GenericXLog, so they # never reach the standby. Three code paths are listed in the issue: # # 1. Serial CREATE INDEX — tp_write_segment_from_build_ctx # (src/access/build_context.c) # 2. Parallel CREATE INDEX — leader path # (src/access/build_parallel.c) # 3. Memtable spill / merge — write_page_index_internal # (src/segment/segment.c) # # At small data sizes (< ~150K docs) the serial-build path doesn't # trigger a visible failure because the segment's page-index fits in # a single block whose extension WAL covers the page implicitly. The # spill path triggers reliably because additional segment pages are # allocated post-build, exposing the missing GenericXLog coverage on # the page-index page. # # Broader coverage (UPDATE, partitioned, expression, partial, array, # multi-index) is in `replication_correctness.sh`. This file is the # minimal, focused reproducer pinned to the issue. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55454 STANDBY_PORT=55455 TEST_DB=replication_issue_342 PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_342_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_342_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # Count matching docs without the LIMIT 100 cap that search_count uses. search_count_full() { local port=$1 local table=$2 local column=$3 local query=$4 local index=$5 local limit=${6:-100000} psql -p "${port}" -d "${TEST_DB}" -tA -c " SELECT count(*) FROM ( SELECT id FROM ${table} ORDER BY ${column} <@> to_bm25query('${query}', '${index}') LIMIT ${limit} ) t;" 2>/dev/null } expect_count() { local label=$1 local got=$2 local expected=$3 if [[ "${got}" == *ERROR* ]] || \ [[ "${got}" == *"invalid page"* ]] || \ [[ "${got}" == *"invalid segment"* ]] || \ [[ "${got}" == *"could not read blocks"* ]] || \ ! [[ "${got}" =~ ^[0-9]+$ ]] || \ [ "${got}" != "${expected}" ]; then error "${label} BUG (expected via #342): standby returned \ '${got}', expected '${expected}'" fi } # ------------------------------------------------------------------- # Test: Memtable spill with standby streaming. # # Reproduces issue #342's "spill / level merge" path. Empty index is # built (replicates trivially via initial relation extension WAL), # then the primary inserts data and spills the memtable. The spill # exercises write_page_index_internal in segment.c, which today # writes the page-index page via MarkBufferDirty without GenericXLog. # The standby cannot read it. # # Symptom on un-fixed code: standby query errors with # ERROR: could not read blocks N..N in file "base/X/Y": read only 0 # (or the magic-byte variant if FSM recycled a stale page). # ------------------------------------------------------------------- test_memtable_spill_with_standby() { log "=== #342: memtable spill with standby streaming ===" primary_sql " DROP TABLE IF EXISTS chunks_contracts CASCADE; CREATE TABLE chunks_contracts ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); CREATE INDEX idx_chunks_contracts_content_bm25 ON chunks_contracts USING bm25(content) WITH (text_config='english'); INSERT INTO chunks_contracts (content) SELECT 'document ' || i || ' alpha bravo charlie ' || repeat(md5(i::text), 8) FROM generate_series(1, 3000) i; SELECT bm25_spill_index('idx_chunks_contracts_content_bm25'); " >/dev/null wait_for_standby_catchup local primary_count primary_count=$(search_count_full "${PRIMARY_PORT}" \ chunks_contracts content "alpha" \ "idx_chunks_contracts_content_bm25" 10000) log " primary count: ${primary_count}" if [ "${primary_count}" != "3000" ]; then error "#342: primary returned ${primary_count}/3000 — \ test setup is broken" fi local standby_count standby_count=$(search_count_full "${STANDBY_PORT}" \ chunks_contracts content "alpha" \ "idx_chunks_contracts_content_bm25" 10000) log " standby count: ${standby_count} (expect 3000)" expect_count "#342" "${standby_count}" "3000" log "#342 PASSED" } main() { log "Starting issue #342 reproducer..." check_required_tools setup_primary setup_standby wait_for_standby_catchup test_memtable_spill_with_standby log "Issue #342 reproducer PASSED — bug is fixed." exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_lib.sh000066400000000000000000000300711525762047700251230ustar00rootroot00000000000000#!/bin/bash # # replication_lib.sh — shared helpers for physical-replication tests. # # Sourced by test/scripts/replication*.sh files. Provides: # - two- and three-node setup primitives (initdb + pg_basebackup), # - one-shot SQL helpers against either node by port, # - LSN-based catchup waits, # - a FIFO-managed persistent psql session ("long-lived backend"), # used to reproduce #345 — the bug only manifests when the same # standby backend stays alive across primary inserts, since a # fresh connection rebuilds its in-memory cache from current # on-disk state on first scan. # # Required environment variables (set by the sourcing script BEFORE # this file is sourced): # # PRIMARY_PORT, STANDBY_PORT # PRIMARY_DIR, STANDBY_DIR # TEST_DB # # Optional: # STANDBY2_PORT, STANDBY2_DIR — set by replication_cascading.sh # # Colors for output. RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # Route all psql / pg_basebackup connections through a writable # socket directory by default. Callers may select TCP and disable # Unix sockets when their workspace path exceeds the platform limit. REPL_SOCKET_DIR=${REPL_SOCKET_DIR-/tmp} export PGHOST=${REPL_HOST:-${REPL_SOCKET_DIR}} # Injection points only exist on a server built with # --enable-injection-points. Sourcing scripts use this to skip the # cases that need a deterministic pause. HAS_INJECTION_POINTS=0 REPL_PRELOAD_LIBRARIES='pg_textsearch' if [ -f "$(pg_config --pkglibdir)/injection_points.so" ] && [ -f "$(pg_config --pkglibdir)/pg_textsearch_test.so" ]; then HAS_INJECTION_POINTS=1 REPL_PRELOAD_LIBRARIES='pg_textsearch,injection_points' fi # Diagnostics go to stderr: helpers are called from inside command # substitutions, whose stdout is captured rather than shown. log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}" >&2 } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}" >&2 } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}" >&2 exit 1 } # ------------------------------------------------------------------- # Cleanup. Each script's main() should `trap repl_cleanup EXIT INT TERM` # AFTER setting PRIMARY_DIR/STANDBY_DIR/STANDBY2_DIR. # ------------------------------------------------------------------- repl_cleanup() { local exit_code=$? log "Cleaning up replication test environment..." if [ $exit_code -ne 0 ]; then warn "Test failed - dumping logs:" for d in "${PRIMARY_DIR:-}" "${STANDBY_DIR:-}" \ "${STANDBY2_DIR:-}"; do [ -z "$d" ] && continue if [ -f "${d}/log/postgres.log" ]; then echo "=== $(basename "$d") LOG (last 80 lines) ===" tail -80 "${d}/log/postgres.log" 2>/dev/null || true fi done fi for dir in "${STANDBY2_DIR:-}" "${STANDBY_DIR:-}" \ "${PRIMARY_DIR:-}"; do [ -z "$dir" ] && continue if [ -f "${dir}/postmaster.pid" ]; then pg_ctl stop -D "${dir}" -m fast 2>/dev/null || pg_ctl stop -D "${dir}" -m immediate 2>/dev/null || \ true fi done for dir in "${PRIMARY_DIR:-}" "${STANDBY_DIR:-}" \ "${STANDBY2_DIR:-}"; do [ -z "$dir" ] || rm -rf "${dir}" done exit $exit_code } # ------------------------------------------------------------------- # SQL helpers. Each takes either a port (for arbitrary node) or # uses a default (PRIMARY_PORT / STANDBY_PORT). # ------------------------------------------------------------------- primary_sql() { psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } primary_sql_quiet() { psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" -tA -c "$1" 2>/dev/null } standby_sql() { psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } standby_sql_quiet() { psql -p "${STANDBY_PORT}" -d "${TEST_DB}" -tA -c "$1" 2>/dev/null } standby2_sql_quiet() { psql -p "${STANDBY2_PORT}" -d "${TEST_DB}" -tA -c "$1" 2>/dev/null } node_sql_quiet() { psql -p "$1" -d "${TEST_DB}" -tA -c "$2" 2>/dev/null } # Count search results using the index-scan pattern (CLAUDE.md says # WHERE-with-<@>-<-0 is an anti-pattern; ORDER BY ... LIMIT is right). # Optional 6th arg overrides the LIMIT (default 10000) for tests that # need to count more than 10K matching docs. search_count() { local port=$1 local table=$2 local column=$3 local query=$4 local index=$5 local limit=${6:-10000} psql -p "${port}" -d "${TEST_DB}" -tA -c " SELECT count(*) FROM ( SELECT id FROM ${table} ORDER BY ${column} <@> to_bm25query('${query}', '${index}') LIMIT ${limit} ) t;" 2>/dev/null } # ------------------------------------------------------------------- # Wait for a target node's pg_last_wal_replay_lsn to catch up to the # primary's pg_current_wal_lsn. # ------------------------------------------------------------------- wait_for_node_catchup() { local target_port=$1 local max_wait=${2:-15} local i=0 log "Waiting for node ${target_port} to catch up (max ${max_wait}s)..." while [ $i -lt $max_wait ]; do local primary_lsn primary_lsn=$(node_sql_quiet "${PRIMARY_PORT}" \ "SELECT pg_current_wal_lsn();") local target_lsn target_lsn=$(node_sql_quiet "${target_port}" \ "SELECT pg_last_wal_replay_lsn();") if [ -n "$primary_lsn" ] && [ -n "$target_lsn" ]; then local caught_up caught_up=$(node_sql_quiet "${target_port}" \ "SELECT '${target_lsn}'::pg_lsn >= '${primary_lsn}'::pg_lsn;") if [ "$caught_up" = "t" ]; then log "Node ${target_port} caught up at LSN ${target_lsn}" return 0 fi fi sleep 1 i=$((i + 1)) done warn "Node ${target_port} may not have caught up after ${max_wait}s" return 1 } # Compatibility shim for the existing replication.sh. wait_for_standby_catchup() { wait_for_node_catchup "${STANDBY_PORT}" "${1:-15}" } # ------------------------------------------------------------------- # Two-node setup helpers. setup_primary uses the script-supplied # PRIMARY_DIR/PRIMARY_PORT. # ------------------------------------------------------------------- setup_primary() { log "Setting up primary instance on port ${PRIMARY_PORT}..." rm -rf "${PRIMARY_DIR}" mkdir -p "${PRIMARY_DIR}" initdb -D "${PRIMARY_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >> "${PRIMARY_DIR}/postgresql.conf" <> "${PRIMARY_DIR}/pg_hba.conf" echo "host replication all 127.0.0.1/32 trust" >> \ "${PRIMARY_DIR}/pg_hba.conf" pg_ctl start -D "${PRIMARY_DIR}" \ -l "${PRIMARY_DIR}/postgres.log" -w createdb -p "${PRIMARY_PORT}" "${TEST_DB}" psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null if [ "${HAS_INJECTION_POINTS}" -eq 1 ]; then psql -p "${PRIMARY_PORT}" -d "${TEST_DB}" \ -c "CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test;" >/dev/null fi } # Standby is built from primary via pg_basebackup. Caller must # already have run setup_primary (and optionally loaded data). setup_standby() { log "Creating standby via pg_basebackup..." rm -rf "${STANDBY_DIR}" pg_basebackup -D "${STANDBY_DIR}" -p "${PRIMARY_PORT}" -R \ -X stream --checkpoint=fast >/dev/null 2>&1 cat >> "${STANDBY_DIR}/postgresql.conf" </dev/null 2>&1 cat >> "${STANDBY2_DIR}/postgresql.conf" < # result=$(long_lived_query "SELECT count(*) FROM ...;") # long_lived_close # # Only one long-lived session per shell at a time. # ------------------------------------------------------------------- long_lived_open() { local port=$1 LL_TMP_DIR=$(mktemp -d -t pg_ts_repl.XXXXXX) LL_IN_FIFO="${LL_TMP_DIR}/in" LL_OUT_FIFO="${LL_TMP_DIR}/out" mkfifo "${LL_IN_FIFO}" "${LL_OUT_FIFO}" # -tA: tuples-only, unaligned. -q: quiet (no banners between cmds). psql -p "${port}" -d "${TEST_DB}" -tAq \ < "${LL_IN_FIFO}" > "${LL_OUT_FIFO}" 2>&1 & LL_PID=$! # Hold input fifo open via fd 9 so psql doesn't see EOF. exec 9>"${LL_IN_FIFO}" # Open output FIFO on fd 8 so reads don't block on writer churn. exec 8<"${LL_OUT_FIFO}" } # Send SQL, read until sentinel. Returns query output (may be # multiple lines, may be empty). Caller checks for ERROR: lines. long_lived_query() { local sql=$1 local sentinel="__pg_ts_sentinel_$$_${RANDOM}__" echo "$sql" >&9 echo "SELECT '${sentinel}';" >&9 local result="" local line while IFS= read -r line <&8; do if [ "$line" = "$sentinel" ]; then break fi if [ -n "$result" ]; then result="${result}"$'\n'"${line}" else result="$line" fi done printf '%s' "$result" } long_lived_close() { exec 9>&- exec 8<&- wait "${LL_PID}" 2>/dev/null || true rm -rf "${LL_TMP_DIR}" LL_PID="" } # Helper that wraps the full pattern: open standby, query before, # do something on primary, query after on the SAME backend, return # both counts via globals LL_BEFORE and LL_AFTER, close. # # Args: # $1: port to open long-lived session on # $2: query to run before+after (must return a single value) # $3: bash command to execute between the two queries long_lived_before_after() { local port=$1 local query=$2 local between=$3 long_lived_open "${port}" LL_BEFORE=$(long_lived_query "${query}") eval "${between}" LL_AFTER=$(long_lived_query "${query}") long_lived_close } # ------------------------------------------------------------------- # Misc utilities. # ------------------------------------------------------------------- check_required_tools() { command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" command -v pg_basebackup >/dev/null 2>&1 || \ error "pg_basebackup not found" command -v initdb >/dev/null 2>&1 || error "initdb not found" } timescale-pg_textsearch-b63fe58/test/scripts/replication_memtable_dead_reclaim.sh000077500000000000000000000225131525762047700306210ustar00rootroot00000000000000#!/bin/bash # # replication_memtable_dead_reclaim.sh — physical replication coverage # for memtable spill DEAD stamping (issue #386). # # 1. GenericXLog DEAD + dead_fxid stamps replicate to the standby. # 2. Standby can query the index after spill (dead orphans are not # reachable from the metapage). # 3. Primary VACUUM reclaims DEAD pages locally (FSM not WAL-logged). # 4. Standby still lists the same dead pages until it runs local # reclaim; after promote + VACUUM, reclaim works there too. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55462 STANDBY_PORT=55463 TEST_DB=replication_memtable_dead PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_memtable_dead_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_memtable_dead_standby" IDX=dead_spill_idx SPILL_DEAD_COUNT=0 SPILL_MAIN_SIZE_PRIMARY=0 SPILL_MAIN_SIZE_STANDBY=0 # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM dead_page_count() { node_sql_quiet "$1" \ "SELECT count(*)::int FROM bm25_memtable_dead_pages('${IDX}');" } dead_flags_ok() { node_sql_quiet "$1" " SELECT bool_and((flags & 1) <> 0) AND bool_and(dead_fxid > 0) FROM bm25_memtable_dead_pages('${IDX}');" } chain_page_count() { node_sql_quiet "$1" \ "SELECT count(*)::int FROM bm25_memtable_chain('${IDX}');" } index_main_size() { node_sql_quiet "$1" " SELECT pg_relation_size('${IDX}'::regclass, 'main')::bigint;" } # DEAD flags stay on disk until a block is reallocated; prove reclaim # via main-fork growth on a second chain build (memtable_reclaim.sql). verify_fsm_reuse() { local port=$1 local dead=$2 local size_after_spill=$3 node_sql_quiet "${port}" \ "SET pg_textsearch.memtable_pages_threshold = 0;" >/dev/null node_sql_quiet "${port}" " INSERT INTO dead_spill_t (body) SELECT 'reuse ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i;" >/dev/null local size_after size_after=$(index_main_size "${port}") local growth=$((size_after - size_after_spill)) local bs budget bs=$(node_sql_quiet "${port}" \ "SELECT current_setting('block_size')::int;") budget=$((dead * bs)) log " main fork growth after re-insert: ${growth} bytes \ (budget ${budget} = ${dead} dead pages)" if [ "${growth}" -ge "${budget}" ]; then return 1 fi return 0 } test_dead_stamp_replicates_to_standby() { log "=== Test 1: DEAD stamps replicate via GenericXLog ===" primary_sql " DROP TABLE IF EXISTS dead_spill_t CASCADE; CREATE TABLE dead_spill_t (id serial PRIMARY KEY, body text); CREATE INDEX ${IDX} ON dead_spill_t USING bm25(body) WITH (text_config = 'english'); " >/dev/null wait_for_standby_catchup primary_sql " INSERT INTO dead_spill_t (body) SELECT 'dead repl ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; " >/dev/null wait_for_standby_catchup local live_pages live_pages=$(chain_page_count "${PRIMARY_PORT}") log " primary live chain pages before spill: ${live_pages}" if [ "${live_pages:-0}" -le 1 ]; then error "Test 1: expected multi-page memtable chain, got ${live_pages}" fi primary_sql "SELECT bm25_spill_index('${IDX}');" >/dev/null primary_sql "CHECKPOINT;" >/dev/null wait_for_standby_catchup local primary_dead standby_dead primary_dead=$(dead_page_count "${PRIMARY_PORT}") standby_dead=$(dead_page_count "${STANDBY_PORT}") log " primary dead pages: ${primary_dead} (expect ${live_pages})" log " standby dead pages: ${standby_dead} (expect ${live_pages})" if [ "${primary_dead}" != "${live_pages}" ]; then error "Test 1: primary dead count ${primary_dead}, expected ${live_pages}" fi if [ "${standby_dead}" != "${live_pages}" ]; then error "Test 1: standby dead count ${standby_dead}, expected ${live_pages}. \ DEAD GenericXLog did not replicate." fi local primary_flags standby_flags primary_flags=$(dead_flags_ok "${PRIMARY_PORT}") standby_flags=$(dead_flags_ok "${STANDBY_PORT}") if [ "${primary_flags}" != "t" ] || [ "${standby_flags}" != "t" ]; then error "Test 1: DEAD flag or dead_fxid missing (primary=${primary_flags}, \ standby=${standby_flags})" fi local live_after live_after=$(chain_page_count "${STANDBY_PORT}") if [ "${live_after}" != "0" ]; then error "Test 1: standby live chain after spill should be 0, got ${live_after}" fi SPILL_DEAD_COUNT="${primary_dead}" SPILL_MAIN_SIZE_PRIMARY=$(index_main_size "${PRIMARY_PORT}") SPILL_MAIN_SIZE_STANDBY=$(index_main_size "${STANDBY_PORT}") export SPILL_DEAD_COUNT SPILL_MAIN_SIZE_PRIMARY SPILL_MAIN_SIZE_STANDBY if [ "${SPILL_MAIN_SIZE_PRIMARY}" != "${SPILL_MAIN_SIZE_STANDBY}" ]; then error "Test 1: primary/standby main fork size mismatch after spill \ (${SPILL_MAIN_SIZE_PRIMARY} vs ${SPILL_MAIN_SIZE_STANDBY})" fi log "Test 1 PASSED: DEAD stamps match on primary and standby" } test_standby_query_after_spill() { log "=== Test 2: standby search after spill ===" local count count=$(search_count "${STANDBY_PORT}" dead_spill_t body \ "dead" "${IDX}") log " standby search hits: ${count} (expect > 0)" if [ -z "${count}" ] || [ "${count}" -lt 1 ]; then error "Test 2: standby query returned no rows after spill" fi log "Test 2 PASSED: standby queries work with replicated DEAD chain" } test_primary_reclaim_and_standby_still_dead() { log "=== Test 3: primary reclaim; standby keeps DEAD until local VACUUM ===" primary_sql "VACUUM ANALYZE dead_spill_t;" >/dev/null primary_sql "VACUUM ANALYZE dead_spill_t;" >/dev/null wait_for_standby_catchup local standby_dead standby_dead=$(dead_page_count "${STANDBY_PORT}") log " standby dead pages after primary VACUUM only: ${standby_dead} \ (expect ${SPILL_DEAD_COUNT}; FSM not WAL-logged)" if [ "${standby_dead}" != "${SPILL_DEAD_COUNT}" ]; then error "Test 3: standby dead count ${standby_dead}, \ expected ${SPILL_DEAD_COUNT}" fi local standby_size_after_primary_vacuum standby_size_after_primary_vacuum=$(index_main_size "${STANDBY_PORT}") log " standby main size after primary VACUUM only: \ ${standby_size_after_primary_vacuum} (expect ${SPILL_MAIN_SIZE_STANDBY})" if [ "${standby_size_after_primary_vacuum}" != \ "${SPILL_MAIN_SIZE_STANDBY}" ]; then error "Test 3: standby main fork grew/shrank after primary VACUUM \ only (${standby_size_after_primary_vacuum} vs ${SPILL_MAIN_SIZE_STANDBY}); \ FSM reclaim is not WAL-logged" fi if ! verify_fsm_reuse "${PRIMARY_PORT}" "${SPILL_DEAD_COUNT}" \ "${SPILL_MAIN_SIZE_PRIMARY}"; then error "Test 3: primary FSM reclaim did not reuse dead pages" fi primary_sql "CHECKPOINT;" >/dev/null wait_for_standby_catchup local primary_size standby_size_after_reuse primary_size=$(index_main_size "${PRIMARY_PORT}") standby_size_after_reuse=$(index_main_size "${STANDBY_PORT}") log " main size after primary re-insert: primary=${primary_size} \ standby=${standby_size_after_reuse}" if [ "${primary_size}" != "${standby_size_after_reuse}" ]; then error "Test 3: primary/standby main fork diverged after reuse WAL \ (${primary_size} vs ${standby_size_after_reuse})" fi log "Test 3 PASSED: primary FSM reclaim is local (not streamed)" } test_standby_reclaim_after_promote() { log "=== Test 4: standby reclaim after promote ===" pg_ctl promote -D "${STANDBY_DIR}" -w local in_recovery in_recovery=$(node_sql_quiet "${STANDBY_PORT}" \ "SELECT pg_is_in_recovery();") if [ "${in_recovery}" = "t" ]; then error "Test 4: standby still in recovery after promote" fi node_sql_quiet "${STANDBY_PORT}" \ "VACUUM ANALYZE dead_spill_t;" >/dev/null node_sql_quiet "${STANDBY_PORT}" \ "VACUUM ANALYZE dead_spill_t;" >/dev/null if ! verify_fsm_reuse "${STANDBY_PORT}" "${SPILL_DEAD_COUNT}" \ "${SPILL_MAIN_SIZE_STANDBY}"; then error "Test 4: promoted node FSM reclaim did not reuse dead pages" fi local count count=$(search_count "${STANDBY_PORT}" dead_spill_t body \ "dead" "${IDX}") if [ -z "${count}" ] || [ "${count}" -lt 1 ]; then error "Test 4: search failed on promoted node after reclaim" fi log "Test 4 PASSED: replica-local reclaim after promote" } main() { log "Starting memtable DEAD/reclaim replication tests..." check_required_tools setup_primary setup_standby wait_for_standby_catchup local failures="" test_dead_stamp_replicates_to_standby || \ failures="${failures} test_dead_stamp_replicates_to_standby" test_standby_query_after_spill || \ failures="${failures} test_standby_query_after_spill" test_primary_reclaim_and_standby_still_dead || \ failures="${failures} test_primary_reclaim_and_standby_still_dead" test_standby_reclaim_after_promote || \ failures="${failures} test_standby_reclaim_after_promote" if [ -n "${failures}" ]; then error "Memtable DEAD replication failures:${failures}" fi log "All memtable DEAD/reclaim replication tests passed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_parallel_build.sh000077500000000000000000000133621525762047700273370ustar00rootroot00000000000000#!/bin/bash # # replication_parallel_build.sh — parallel CREATE INDEX replication # test for issue #342. # # PR #343 fixed three WAL-silent paths. The # tp_write_segment_from_build_ctx (serial build) and # write_page_index_internal (spill / merge) paths are exercised by # replication_issue_342.sh and replication_correctness.sh at small # data scales. The parallel-build leader path # (src/access/build_parallel.c, write_merged_segment_to_sink + new # FULL_IMAGE pass) only runs for tables of >= TP_MIN_PARALLEL_TUPLES # (100K) rows when max_parallel_maintenance_workers > 0. # # This test inserts 150K rows on the primary, runs parallel CREATE # INDEX, waits for the standby to catch up, and asserts the standby # returns the same row count as the primary. # # On the un-fixed code path, the standby would either ERROR with # "could not read blocks" / "invalid page index" or return 0 results # from the un-WAL-logged merged segment. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55456 STANDBY_PORT=55457 TEST_DB=replication_parallel_build PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_pbuild_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_pbuild_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # Count matching docs without the LIMIT 100 cap that search_count # uses (we have 150K rows). search_count_full() { local port=$1 local table=$2 local column=$3 local query=$4 local index=$5 local limit=${6:-200000} psql -p "${port}" -d "${TEST_DB}" -tA -c " SELECT count(*) FROM ( SELECT id FROM ${table} ORDER BY ${column} <@> to_bm25query('${query}', '${index}') LIMIT ${limit} ) t;" 2>/dev/null } expect_count() { local label=$1 local got=$2 local expected=$3 if [[ "${got}" == *ERROR* ]] || \ [[ "${got}" == *"invalid page"* ]] || \ [[ "${got}" == *"invalid segment"* ]] || \ [[ "${got}" == *"could not read blocks"* ]] || \ ! [[ "${got}" =~ ^[0-9]+$ ]] || \ [ "${got}" != "${expected}" ]; then error "${label} BUG (expected via #343): standby returned \ '${got}', expected '${expected}'" fi } # ------------------------------------------------------------------- # Test: parallel CREATE INDEX with standby streaming. # # Reproduces the parallel-build leg of issue #342. >=100K rows # triggers parallel CREATE INDEX (TP_MIN_PARALLEL_TUPLES). The leader # merges N worker segments via write_merged_segment_to_sink; before # PR #343 the merged data pages reached disk on the primary but # never made it into WAL. # ------------------------------------------------------------------- test_parallel_create_index_with_standby() { log "=== parallel CREATE INDEX with standby streaming ===" primary_sql " DROP TABLE IF EXISTS pbuild_docs CASCADE; CREATE TABLE pbuild_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO pbuild_docs (content) SELECT 'document ' || i || ' alpha bravo charlie ' || repeat(md5(i::text), 4) FROM generate_series(1, 150000) i; ANALYZE pbuild_docs; " >/dev/null # Force parallel build. pg_textsearch's heuristic is gated on # reltuples >= TP_MIN_PARALLEL_TUPLES (100K) and on the planner # assigning workers via plan_create_index_workers. ANALYZE above # ensures reltuples reflects reality; min_parallel_table_scan_size # = 0 keeps the planner from declining parallelism for the small # heap. primary_sql " SET max_parallel_maintenance_workers = 4; SET maintenance_work_mem = '256MB'; SET min_parallel_table_scan_size = 0; CREATE INDEX pbuild_idx ON pbuild_docs USING bm25(content) WITH (text_config='english'); " >/dev/null # Sanity: the index build must have launched parallel workers, # otherwise we are testing the serial path (already covered by # other replication tests) and not the build_parallel.c fix. # pg_textsearch emits a NOTICE "parallel index build: launched # N of M requested workers" when tp_build_parallel runs; the # NOTICE goes to the server log at log_min_messages = notice. local launched launched=$(grep -oE "parallel index build: launched [0-9]+" \ "${PRIMARY_DIR}/log/postgres.log" 2>/dev/null | tail -1) if [ -z "${launched}" ] || \ [[ "${launched}" =~ launched\ 0$ ]]; then error "parallel build: primary did not launch parallel \ workers (log line: '${launched:-}'); test is not \ exercising build_parallel.c. Check max_parallel_maintenance_workers \ and reltuples on pbuild_docs." fi log " ${launched}" wait_for_standby_catchup 30 local primary_count primary_count=$(search_count_full "${PRIMARY_PORT}" \ pbuild_docs content "alpha" pbuild_idx 200000) log " primary count: ${primary_count}" if [ "${primary_count}" != "150000" ]; then error "parallel build: primary returned \ ${primary_count}/150000 — test setup is broken" fi local standby_count standby_count=$(search_count_full "${STANDBY_PORT}" \ pbuild_docs content "alpha" pbuild_idx 200000) log " standby count: ${standby_count} (expect 150000)" expect_count "parallel-build" "${standby_count}" "150000" log "parallel CREATE INDEX replication PASSED" } main() { log "Starting parallel-build replication test..." check_required_tools setup_primary setup_standby wait_for_standby_catchup test_parallel_create_index_with_standby log "Parallel-build replication test PASSED — \ build_parallel.c WAL fix verified." exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_segment_reclaim.sh000077500000000000000000000165761525762047700275340ustar00rootroot00000000000000#!/bin/bash # # replication_segment_reclaim.sh -- acceptance test for the deferred, # standby-safe reclaim of displaced segment pages (issue #380). # # Scenario (the two #380 acceptance criteria): # 1. A standby holding an open snapshot, with hot_standby_feedback=on, # holds back the primary's reclaim horizon. A merge on the primary # PARKS the displaced segment pages; a following VACUUM must NOT # recycle them while the standby snapshot is live -- so # bm25_pending_free_pages stays > 0 and the standby keeps answering # queries correctly (no "could not read block", no corruption). # 2. Once the standby snapshot is released and the horizon advances, # a VACUUM on the primary drains every parked page to the FSM # (bm25_pending_free_pages drops to 0). # # Uses the shared replication harness (replication_lib.sh). Asserts # explicitly; repl_cleanup preserves the failing exit status. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55438 STANDBY_PORT=55439 TEST_DB=segment_reclaim_repl_test PRIMARY_DIR="${SCRIPT_DIR}/../tmp_reclaim_repl_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_reclaim_repl_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # Wait until the primary sees the standby's feedback xmin (proves # hot_standby_feedback is propagating and will hold back the horizon). wait_for_backend_xmin() { local i=0 while [ $i -lt 20 ]; do local x x=$(primary_sql_quiet "SELECT backend_xmin FROM pg_stat_replication WHERE backend_xmin IS NOT NULL LIMIT 1;") if [ -n "$x" ]; then log "Primary sees standby backend_xmin=${x}" return 0 fi sleep 0.5 i=$((i + 1)) done return 1 } main() { check_required_tools setup_primary # Build a two-L0-segment index: CREATE INDEX writes one segment # directly; a second batch fills the memtable; a spill writes the # second segment. A later force-merge will displace both. log "Loading data and spilling a second segment on the primary..." primary_sql "CREATE TABLE rec (id int, body text) WITH (autovacuum_enabled = false);" >/dev/null primary_sql "INSERT INTO rec SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 4000) g;" >/dev/null primary_sql "CREATE INDEX rec_idx ON rec USING bm25 (body) WITH (text_config = 'english');" >/dev/null primary_sql "INSERT INTO rec SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(4001, 8000) g;" >/dev/null spilled=$(primary_sql_quiet "SELECT bm25_spill_index('rec_idx') > 0;") [ "$spilled" = "t" ] || error "spill did not write a segment (got '$spilled')" # Bring up the standby (basebackup snapshots the two L0 segments), # with hot_standby_feedback on and frequent status updates so the # standby's snapshot xmin reaches the primary quickly. setup_standby cat >> "${STANDBY_DIR}/postgresql.conf" </dev/null wait_for_standby_catchup # Sanity: the standby answers the index query before the merge. pre=$(search_count "${STANDBY_PORT}" rec body "alpha" "rec_idx") [ -n "$pre" ] && [ "$pre" -gt 0 ] || error "standby query returned no rows pre-merge (got '$pre')" log "Standby answered pre-merge query: ${pre} rows." # Open a long-lived REPEATABLE READ snapshot on the standby and hold # it open across the merge. This is what pins the primary's horizon. long_lived_open "${STANDBY_PORT}" long_lived_query "BEGIN ISOLATION LEVEL REPEATABLE READ;" >/dev/null held=$(long_lived_query "SELECT count(*) FROM (SELECT id FROM rec ORDER BY body <@> to_bm25query('alpha', 'rec_idx') LIMIT 50) s;") [ -n "$held" ] && [ "$held" -gt 0 ] || error "standby long-lived query returned no rows (got '$held')" log "Standby holds a REPEATABLE READ snapshot (query saw ${held} rows)." wait_for_backend_xmin || error "primary never saw standby backend_xmin (hot_standby_feedback not propagating)" # Merge on the primary parks the displaced pages; force the merge's # WAL to disk (force-merge assigns no xid) so it streams to the # standby; then VACUUM attempts to drain. The live standby snapshot # must hold the horizon back, so the parked pages stay parked. log "Forcing merge + VACUUM on the primary (standby snapshot live)..." primary_sql "SELECT bm25_force_merge('rec_idx');" >/dev/null primary_sql "CREATE TABLE flush_marker (x int);" >/dev/null primary_sql "VACUUM rec;" >/dev/null parked_held=$(primary_sql_quiet "SELECT bm25_pending_free_pages('rec_idx');") case "$parked_held" in ''|*[!0-9]*) error "pending_free_pages not an integer: '$parked_held'" ;; esac [ "$parked_held" -gt 0 ] || error "parked pages were recycled while standby snapshot was live (got $parked_held, expected > 0)" log "PASS: $parked_held page(s) stay parked while standby snapshot is live." # The standby must replay the merge and keep answering correctly -- # the displaced pages it may still reference were NOT recycled. wait_for_standby_catchup post=$(search_count "${STANDBY_PORT}" rec body "alpha" "rec_idx") [ -n "$post" ] && [ "$post" -gt 0 ] || error "standby query failed/empty after merge replay (got '$post')" log "PASS: standby answers correctly after merge replay (${post} rows)." # The held snapshot itself still reads cleanly (no torn pages). still=$(long_lived_query "SELECT count(*) FROM (SELECT id FROM rec ORDER BY body <@> to_bm25query('beta', 'rec_idx') LIMIT 50) s;") [[ "$still" == *ERROR* ]] && error "standby snapshot query errored after merge: ${still}" [ -n "$still" ] && [ "$still" -gt 0 ] || error "standby snapshot query returned no rows after merge (got '$still')" log "PASS: held standby snapshot still reads cleanly (${still} rows)." # Release the snapshot; the primary's horizon can now advance. long_lived_query "COMMIT;" >/dev/null long_lived_close # Advance the xid horizon past the merge stamp and VACUUM again; # every parked page must now be reclaimed. Retry to absorb feedback # propagation lag after the standby releases its xmin. log "Releasing snapshot; draining on the primary..." drained="" for _ in 1 2 3 4 5 6 7 8 9 10; do primary_sql_quiet "SELECT txid_current();" >/dev/null primary_sql_quiet "SELECT txid_current();" >/dev/null primary_sql "VACUUM rec;" >/dev/null drained=$(primary_sql_quiet "SELECT bm25_pending_free_pages('rec_idx');") [ "$drained" = "0" ] && break sleep 0.5 done [ "$drained" = "0" ] || error "VACUUM did not drain parked pages after snapshot release (got '$drained', expected 0)" log "PASS: all parked pages reclaimed after snapshot release (now 0)." # Both nodes still healthy. wait_for_standby_catchup pcnt=$(search_count "${PRIMARY_PORT}" rec body "gamma" "rec_idx") scnt=$(search_count "${STANDBY_PORT}" rec body "gamma" "rec_idx") [ "$pcnt" = "$scnt" ] || error "primary/standby result mismatch after reclaim: primary=$pcnt standby=$scnt" log "PASS: primary and standby agree after reclaim ($pcnt rows)." log "✅ All standby segment-reclaim acceptance checks passed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/replication_spill_paths.sh000077500000000000000000000507711525762047700267130ustar00rootroot00000000000000#!/bin/bash # # replication_spill_paths.sh — verify that the SPILL WAL records # emitted from the non-`tp_do_spill` spill paths actually replicate # end-to-end. Two tests: # # 1. tp_bulk_load_spill_check (PRE_COMMIT bulk-load threshold) # A fat single-transaction insert crosses bulk_load_threshold; # PRE_COMMIT triggers a spill on the primary. The standby's # memtable must be empty afterwards (cleared by SPILL redo); # the segment must hold all the docs. # # 2. tp_spill_memtable_if_needed (VACUUM bulkdelete) # INSERT then DELETE then VACUUM. VACUUM's bulkdelete phase # always spills any non-empty memtable. The standby's memtable # must be empty afterwards; the segment holds the docs (with # half marked dead by the alive-bitset update). # # Both tests assert directly against bm25_summarize_index on the # standby (BMW query results turn out to mask memtable+segment # overlap in some cases). The bug they catch is "primary spill # replicates segment data + metapage but not the memtable-clear # signal" — visible only as a memtable+segment doc-count # divergence on the standby. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55460 STANDBY_PORT=55461 TEST_DB=replication_spill_paths PRIMARY_DIR="${SCRIPT_DIR}/../tmp_repl_spill_paths_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_repl_spill_paths_standby" # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" trap repl_cleanup EXIT INT TERM # --------------------------------------------------------------- # Test 1: bulk-load PRE_COMMIT spill replication # # Tunes the GUCs so the only spill trigger that fires is the # bulk-load PRE_COMMIT path: # - bulk_load_threshold = 500 terms (low, easy to cross) # - memtable_pages_threshold = 0 (disabled) # Then runs a 2000-row INSERT in a single transaction. Each row # carries enough unique terms that terms_added_this_xact crosses # the threshold by COMMIT time, so tp_bulk_load_spill_check fires # at PRE_COMMIT. # --------------------------------------------------------------- test_bulk_load_spill_replication() { log "=== Test 1: bulk-load PRE_COMMIT spill replication ===" primary_sql " DROP TABLE IF EXISTS bulk_docs CASCADE; CREATE TABLE bulk_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); CREATE INDEX bulk_docs_idx ON bulk_docs USING bm25(content) WITH (text_config='english'); " >/dev/null wait_for_standby_catchup local query=" SELECT count(*) FROM ( SELECT id FROM bulk_docs ORDER BY content <@> to_bm25query('alpha', 'bulk_docs_idx') LIMIT 100000 ) t; " # bulk_load_threshold and memtable_pages_threshold are SUSET — # set per-session in the same psql -c that runs the INSERT so # PRE_COMMIT sees them. long_lived_before_after "${STANDBY_PORT}" "${query}" \ "primary_sql \" SET pg_textsearch.bulk_load_threshold = 500; SET pg_textsearch.memtable_pages_threshold = 0; BEGIN; INSERT INTO bulk_docs (content) SELECT 'alpha bravo charlie delta echo ' || 'foxtrot golf hotel india juliet ' || md5(g::text) || ' doc_' || g FROM generate_series(1, 2000) g; COMMIT; \" >/dev/null wait_for_standby_catchup" log " initial standby count: ${LL_BEFORE}" log " post-insert count: ${LL_AFTER}" if [ "${LL_BEFORE}" != "0" ]; then error "Test 1: expected initial 0, got '${LL_BEFORE}'" fi # Long-lived backend must see all 2000 docs (this fails before # PR 2 — INSERT_TERMS not replicated to the cached memtable). if [ "${LL_AFTER}" != "2000" ]; then error "Test 1: expected post-insert count 2000, \ got '${LL_AFTER}'" fi # Direct check on standby state via bm25_summarize_index. The # bulk-load PRE_COMMIT spill on the primary should have written # the SPILL WAL record, whose redo on the standby clears the # standby's memtable. Replay independently lands the segment on # the standby's disk via GenericXLog. Final state on the # standby must be: memtable empty (cleared by SPILL redo), # segment populated (received via segment-data WAL). # # Without the SPILL WAL emit (the bug this test catches), # nothing would clear the standby memtable; it would retain # all 2000 docs from INSERT_TERMS replay, AND the segment # would also have 2000 — visible as a memtable+segment overlap # in bm25_summarize_index even though the BMW query happens to # mask the duplication elsewhere. local standby_summary standby_summary=$(standby_sql_quiet " SELECT bm25_summarize_index('bulk_docs_idx');") local standby_memtable_docs local standby_segment_docs standby_memtable_docs=$(echo "${standby_summary}" \ | awk '/^Memtable:/{f=1;next} f && /documents:/{print $2; exit}') standby_segment_docs=$(echo "${standby_summary}" \ | awk '/^Segments:/{f=1;next} \ f && /Total:/{ \ for(i=1;i<=NF;i++) if($i ~ /^[0-9]+$/ && $(i+1) ~ /docs/){print $i; exit} \ }') log " standby memtable docs: ${standby_memtable_docs:-} (expect 0)" log " standby segment docs: ${standby_segment_docs:-} (expect 2000)" if [ "${standby_memtable_docs:-x}" != "0" ]; then log " --- standby summary (for diagnosis) ---" echo "${standby_summary}" | while IFS= read -r line; do log " S| ${line}" done error "Test 1: standby memtable should be empty after \ PRE_COMMIT spill replicates (got '${standby_memtable_docs}'). \ SPILL WAL is not being emitted from tp_bulk_load_spill_check, \ or its redo is not clearing the standby memtable." fi if [ "${standby_segment_docs:-0}" != "2000" ]; then error "Test 1: standby segment should hold all 2000 docs \ post-spill, got '${standby_segment_docs}'" fi log "Test 1 PASSED: bulk-load PRE_COMMIT spill replicates" } # --------------------------------------------------------------- # Test 2: VACUUM-triggered spill replication # # VACUUM's bulkdelete phase always calls tp_spill_memtable_if_needed # with min_postings=1 (vacuum.c:766) — i.e., spills any non-empty # memtable so the bulkdelete pass operates on segment storage only. # This is one of the spill paths PR 2 added tp_xlog_spill to. # # The test inserts a small number of docs (well under the bulk-load # threshold), DELETEs some, then VACUUMs. The VACUUM's bulkdelete # spill fires deterministically. With SPILL WAL emitted, redo # clears the standby's memtable; the standby's bm25_summarize_index # memtable count drops to 0. Without the WAL emit, the standby's # memtable retains all the inserted docs. # --------------------------------------------------------------- test_vacuum_spill_replication() { log "=== Test 2: VACUUM-triggered spill replication ===" primary_sql " DROP TABLE IF EXISTS vac_docs CASCADE; CREATE TABLE vac_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); CREATE INDEX vac_docs_idx ON vac_docs USING bm25(content) WITH (text_config='english'); " >/dev/null wait_for_standby_catchup # Open a long-lived backend on the standby BEFORE inserts so # the standby's registry gets populated and INSERT_TERMS redo # actually applies records to the memtable. Without this, the # standby's first access happens after VACUUM has already # cleared docid pages on the primary, so the rebuild walks an # empty docid chain and the memtable winds up empty # vacuously — independent of whether SPILL replicated. long_lived_open "${STANDBY_PORT}" # Initial query forces tp_rebuild_index_from_disk on this # backend, which registers the empty index. long_lived_query "SELECT bm25_summarize_index('vac_docs_idx');" \ >/dev/null primary_sql " INSERT INTO vac_docs (content) SELECT 'gamma delta epsilon zeta eta ' || md5(g::text) FROM generate_series(1, 200) g; " >/dev/null wait_for_standby_catchup # Sanity: standby memtable now populated via INSERT_TERMS replay. local pre_vacuum_summary pre_vacuum_memtable pre_vacuum_summary=$(long_lived_query \ "SELECT bm25_summarize_index('vac_docs_idx');") pre_vacuum_memtable=$(echo "${pre_vacuum_summary}" \ | awk '/^Memtable:/{f=1;next} f && /documents:/{print $2; exit}') log " standby memtable docs pre-VACUUM: ${pre_vacuum_memtable:-} (expect 200)" if [ "${pre_vacuum_memtable:-0}" != "200" ]; then long_lived_close error "Test 2: standby memtable did not populate via \ INSERT_TERMS replay (got '${pre_vacuum_memtable}'); test setup \ broken." fi primary_sql "DELETE FROM vac_docs WHERE id <= 100;" >/dev/null primary_sql "VACUUM vac_docs;" >/dev/null # Force a checkpoint after VACUUM so all VACUUM-generated WAL # is flushed to disk before we wait for standby catchup. primary_sql "CHECKPOINT;" >/dev/null wait_for_standby_catchup local post_vacuum_summary post_vacuum_memtable post_vacuum_segments post_vacuum_summary=$(long_lived_query \ "SELECT bm25_summarize_index('vac_docs_idx');") post_vacuum_memtable=$(echo "${post_vacuum_summary}" \ | awk '/^Memtable:/{f=1;next} f && /documents:/{print $2; exit}') post_vacuum_segments=$(echo "${post_vacuum_summary}" \ | awk '/^Segments:/{f=1;next} \ f && /Total:/{ \ for(i=1;i<=NF;i++) if($i ~ /^[0-9]+$/ && $(i+1) ~ /docs/){print $i; exit} \ }') log " standby memtable docs post-VACUUM: ${post_vacuum_memtable:-} (expect 0)" log " standby segment docs post-VACUUM: ${post_vacuum_segments:-} (expect 200)" long_lived_close if [ "${post_vacuum_memtable:-x}" != "0" ]; then log " --- post-VACUUM standby summary ---" echo "${post_vacuum_summary}" | while IFS= read -r line; do log " S| ${line}" done error "Test 2: standby memtable should be empty after \ VACUUM-triggered spill replicates (got '${post_vacuum_memtable}'). \ SPILL WAL is not being emitted from tp_spill_memtable_if_needed, \ or its redo is not clearing the standby memtable." fi if [ "${post_vacuum_segments:-0}" != "200" ]; then error "Test 2: standby segment should hold all 200 docs \ post-VACUUM, got '${post_vacuum_segments}'" fi log "Test 2 PASSED: VACUUM-triggered spill replicates" } # --------------------------------------------------------------- # Test 3: cold-start corpus stats after no-spill inserts (#350) # # Standby cold-start path: a fresh backend opens an index whose # registry slot is empty (no other backend has touched this index # yet), so tp_get_local_index_state -> tp_rebuild_index_from_disk # walks the docid pages to repopulate the in-memory memtable. # # Pre-#350 fix the rebuild path overwrote shared->total_docs with # metap->total_docs unconditionally. metap is only synced inside # the spill paths, so for a no-spill index it sits at 0 and the # overwrite zeroed the count we just walked. BM25 and BMW both # require total_docs > 0 to score, so every query returned no # rows — the externally visible PITR-into-mid-burst symptom from # #350. # # The reproducer is simpler than full PITR: just insert without # spilling and let a *new* standby backend rebuild from docid # pages. The bug is in the rebuild's overwrite, not in PITR per # se. # --------------------------------------------------------------- test_cold_start_corpus_stats() { log "=== Test 3: cold-start corpus stats (no-spill rebuild) ===" primary_sql " DROP TABLE IF EXISTS cs_docs CASCADE; CREATE TABLE cs_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); CREATE INDEX cs_docs_idx ON cs_docs USING bm25(content) WITH (text_config='english'); -- Insert without spilling. The default -- bulk_load_threshold (100k terms) and the chain-page -- threshold (64 pages) are nowhere near 200 docs of -- 6 words each, so this stays purely in the memtable. INSERT INTO cs_docs (content) SELECT 'iota kappa lambda mu nu ' || md5(g::text) FROM generate_series(1, 200) g; " >/dev/null wait_for_standby_catchup # Confirm primary has the docs (sanity). local primary_total primary_total=$(primary_sql_quiet \ "SELECT count(*) FROM cs_docs;") if [ "${primary_total}" != "200" ]; then error "Test 3: primary didn't get 200 docs (got \ ${primary_total})" fi # Open a *new* standby backend — the registry on the standby # is empty for this index (no prior touch), so the very first # query in this session goes through tp_rebuild_index_from_disk. # The rebuild emits an INFO ("Recovering pg_textsearch index # ... from disk") that PostgreSQL always ships to the client # regardless of client_min_messages, so the long_lived_query # output may include both the INFO line and the numeric count. # Extract the last numeric line. long_lived_open "${STANDBY_PORT}" local raw raw=$(long_lived_query " SELECT count(*) FROM ( SELECT id FROM cs_docs ORDER BY content <@> to_bm25query('iota', 'cs_docs_idx') LIMIT 10000 ) t;") local count count=$(printf '%s\n' "${raw}" | awk '/^[0-9]+$/ {n=$0} END {print n}') log " standby cold-start query count: ${count} (expect 200)" long_lived_close if [ "${count}" != "200" ]; then error "Test 3 (#350): standby cold-start query returned \ ${count}, expected 200. The rebuild path overwrote the docid-walk \ count with stale metap->total_docs (no spill -> metap=0)." fi log "Test 3 PASSED: cold-start corpus stats reconstructed correctly" } # --------------------------------------------------------------- # Test 4: long-lived standby read across primary segment merge (#349) # # Exercise the MERGE_LINKAGE end-to-end path: drive enough spills # on the primary (segments_per_level=8 → 9th spill triggers L0 # compaction), then a 10th, while a long-lived standby backend # queries before and after. Asserts both reads return the # expected counts. # # Caveat: the original #349 race (a backend mid-walk on the L0 # chain when redo unlinks the source segments) is timing-sensitive # and cannot be reliably triggered in CI by sequential queries — # the walk completes in microseconds and wait_for_standby_catchup # always lets redo finish before the next query begins. The fix # is structural: merge linkage now goes through a custom rmgr # record whose redo holds LW_EXCLUSIVE on the per-index lock, # blocking against concurrent scans (LW_SHARED) until they # release. This test exists to catch regressions in the # happy-path replication of a merge — if MERGE_LINKAGE is wrong # (forgets a metap field, miscounts shrinkage, etc.) the post- # merge query count diverges. # --------------------------------------------------------------- test_long_lived_read_after_segment_merge() { log "=== Test 4: long-lived read after segment merge (#349) ===" primary_sql " DROP TABLE IF EXISTS sm_docs CASCADE; CREATE TABLE sm_docs (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX sm_idx ON sm_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null # Force several level-0 spills. With segments_per_level=8 # (default) we need at least 9 to trigger compaction on the # 9th spill. local i for i in 1 2 3 4 5 6 7 8 9; do primary_sql " INSERT INTO sm_docs (content) SELECT 'merge doc ' || j || ' alpha bravo' FROM generate_series(1,100) j; SELECT bm25_spill_index('sm_idx'); " >/dev/null done wait_for_standby_catchup local primary_total primary_total=$(primary_sql_quiet "SELECT count(*) FROM sm_docs;") if [ "${primary_total}" != "900" ]; then error "Test 4: primary expected 900 docs, got ${primary_total}" fi # Open the long-lived standby backend now — registers the # index in the standby's registry so subsequent INSERT_TERMS # redo applies, and so the backend caches a metap snapshot # straddling the upcoming merge. long_lived_open "${STANDBY_PORT}" local raw before raw=$(long_lived_query " SELECT count(*) FROM ( SELECT id FROM sm_docs ORDER BY content <@> to_bm25query('alpha', 'sm_idx') LIMIT 10000 ) t;") before=$(printf '%s\n' "${raw}" | awk '/^[0-9]+$/ {n=$0} END {print n}') # Trigger a level-0 compaction by adding more segments past # the threshold. primary_sql " INSERT INTO sm_docs (content) SELECT 'merge doc extra ' || j || ' alpha' FROM generate_series(1,50) j; SELECT bm25_spill_index('sm_idx'); " >/dev/null wait_for_standby_catchup local raw_after after raw_after=$(long_lived_query " SELECT count(*) FROM ( SELECT id FROM sm_docs ORDER BY content <@> to_bm25query('alpha', 'sm_idx') LIMIT 10000 ) t;") after=$(printf '%s\n' "${raw_after}" | awk '/^[0-9]+$/ {n=$0} END {print n}') long_lived_close log " long-lived count before merge: ${before} (expect 900)" log " long-lived count after merge: ${after} (expect 950)" if [ "${before}" != "900" ]; then error "Test 4 (#349): pre-merge long-lived count ${before}, \ expected 900" fi if [ "${after}" != "950" ]; then error "Test 4 (#349): post-merge long-lived count ${after}, \ expected 950. The MERGE_LINKAGE redo on the standby is not \ serializing against in-flight scans (LW_EXCLUSIVE not held), or \ the linkage isn't being WAL-emitted as a custom record." fi log "Test 4 PASSED: long-lived read after merge stays consistent" } # # Test 5: write-path functions are rejected on a standby # # bm25_spill_index() and bm25_force_merge() mutate index pages, so both # must refuse to run during recovery rather than failing deep in the # write path with a low-level WAL error. # test_write_functions_rejected_on_standby() { log "Test 5: write-path functions rejected on standby" primary_sql " DROP TABLE IF EXISTS sp5_docs CASCADE; CREATE TABLE sp5_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO sp5_docs (content) SELECT 'doc ' || i || ' alpha bravo' FROM generate_series(1,50) i; CREATE INDEX sp5_idx ON sp5_docs USING bm25(content) WITH (text_config='simple'); " >/dev/null wait_for_standby_catchup local spill_out force_out spill_out=$(standby_sql "SELECT bm25_spill_index('sp5_idx');") force_out=$(standby_sql "SELECT bm25_force_merge('sp5_idx');") log " spill on standby: ${spill_out}" log " force merge on standby: ${force_out}" if [[ "${spill_out}" != *"cannot run during recovery"* ]]; then error "Test 5: bm25_spill_index() was not rejected on the \ standby: ${spill_out}" fi if [[ "${force_out}" != *"cannot run during recovery"* ]]; then error "Test 5: bm25_force_merge() was not rejected on the \ standby: ${force_out}" fi log "Test 5 PASSED: write-path functions refuse to run during recovery" } main() { log "Starting spill-paths replication test..." check_required_tools setup_primary setup_standby wait_for_standby_catchup # Run each subtest in a subshell so a failure in one doesn't # short-circuit the others. Useful for the verify-coverage # workflow (deliberate bug-reverts) — every test should still # report independently. Track failures and exit non-zero at # the end if any failed. local failures="" ( test_bulk_load_spill_replication ) || \ failures="${failures} test_bulk_load_spill_replication" ( test_vacuum_spill_replication ) || \ failures="${failures} test_vacuum_spill_replication" ( test_cold_start_corpus_stats ) || \ failures="${failures} test_cold_start_corpus_stats" ( test_long_lived_read_after_segment_merge ) || \ failures="${failures} test_long_lived_read_after_segment_merge" ( test_write_functions_rejected_on_standby ) || \ failures="${failures} test_write_functions_rejected_on_standby" if [ -n "${failures}" ]; then error "Spill-paths failures:${failures}" fi log "All spill-paths replication tests passed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/rls_ddl_locking.sh000077500000000000000000000612421525762047700251240ustar00rootroot00000000000000#!/bin/bash # # Verify that RLS/BM25 DDL serializes before PostgreSQL takes relation locks. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT="${TEST_PORT:-55463}" TEST_DB=pg_textsearch_rls_locking_test DATA_DIR="${SCRIPT_DIR}/../tmp_rls_locking_${TEST_PORT}_$$" LOGFILE="${DATA_DIR}/postgres.log" SESSION_A_INPUT="${DATA_DIR}/session_a.in" SESSION_A_OUTPUT="${DATA_DIR}/session_a.out" SESSION_B_OUTPUT="${DATA_DIR}/session_b.out" SESSION_C_OUTPUT="${DATA_DIR}/session_c.out" SESSION_A_PID= SESSION_B_PID= SESSION_C_PID= PARTITION_ORDER_FAILED=0 cleanup() { local exit_code=$? trap - EXIT INT TERM exec 3>&- 2>/dev/null || true if [ -n "${SESSION_A_PID}" ] && kill -0 "${SESSION_A_PID}" 2>/dev/null; then kill "${SESSION_A_PID}" 2>/dev/null || true wait "${SESSION_A_PID}" 2>/dev/null || true fi if [ -n "${SESSION_B_PID}" ] && kill -0 "${SESSION_B_PID}" 2>/dev/null; then kill "${SESSION_B_PID}" 2>/dev/null || true wait "${SESSION_B_PID}" 2>/dev/null || true fi if [ -n "${SESSION_C_PID}" ] && kill -0 "${SESSION_C_PID}" 2>/dev/null; then kill "${SESSION_C_PID}" 2>/dev/null || true wait "${SESSION_C_PID}" 2>/dev/null || true fi if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast -w >/dev/null 2>&1 || pg_ctl stop -D "${DATA_DIR}" -m immediate -w >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" exit "${exit_code}" } trap cleanup EXIT INT TERM run_value() { psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -tAc "$1" } wait_for_true() { local query=$1 local description=$2 local i for i in $(seq 1 200); do if [ "$(run_value "${query}")" = "t" ]; then return 0 fi sleep 0.05 done echo "Timed out waiting for ${description}" >&2 return 1 } start_rls_holder() { local table_name=$1 rm -f "${SESSION_A_INPUT}" "${SESSION_A_OUTPUT}" mkfifo "${SESSION_A_INPUT}" PGAPPNAME=rls-lock-session-a \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 < "${SESSION_A_INPUT}" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! exec 3> "${SESSION_A_INPUT}" printf '%s\n' \ "SET pg_textsearch.allow_rls = off;" \ "BEGIN;" \ "ALTER TABLE ${table_name} ENABLE ROW LEVEL SECURITY;" >&3 wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-a' AND a.state = 'idle in transaction' AND l.locktype = 'relation' AND l.relation = '${table_name}'::regclass AND l.mode = 'AccessExclusiveLock' AND l.granted ); " "session A to hold ${table_name}" } release_session_a() { printf '%s\n' "COMMIT;" "\\q" >&3 exec 3>&- wait "${SESSION_A_PID}" SESSION_A_PID= } mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 20 shared_buffers = 128MB unix_socket_directories = '${DATA_DIR}' listen_addresses = '' log_min_messages = warning shared_preload_libraries = 'pg_textsearch' EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w >/dev/null createdb -h "${DATA_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' CREATE EXTENSION pg_textsearch; CREATE TABLE lock_parent (id integer, content text); CREATE TABLE lock_child () INHERITS (lock_parent); CREATE TABLE noop_inherit_parent (id integer); CREATE TABLE noop_inherit_child () INHERITS (noop_inherit_parent); CREATE TABLE noop_reindex (id integer); CREATE INDEX noop_reindex_idx ON noop_reindex(id); CREATE TABLE extension_replace_parent (id integer, content text); CREATE TABLE extension_replace_child () INHERITS (extension_replace_parent); SQL rm -f "${SESSION_A_INPUT}" "${SESSION_A_OUTPUT}" mkfifo "${SESSION_A_INPUT}" PGAPPNAME=rls-extension-replace-session-a \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 < "${SESSION_A_INPUT}" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! exec 3> "${SESSION_A_INPUT}" printf '%s\n' \ "BEGIN;" \ "DROP EXTENSION pg_textsearch CASCADE;" \ "CREATE EXTENSION pg_textsearch;" >&3 wait_for_true " SELECT state = 'idle in transaction' FROM pg_stat_activity WHERE application_name = 'rls-extension-replace-session-a'; " "session A to replace the extension" PGAPPNAME=rls-extension-replace-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = off; BEGIN; ALTER TABLE extension_replace_parent ENABLE ROW LEVEL SECURITY; SELECT pg_sleep(5); COMMIT;" \ > "${SESSION_B_OUTPUT}" 2>&1 & SESSION_B_PID=$! wait_for_true " SELECT wait_event_type = 'Lock' FROM pg_stat_activity WHERE application_name = 'rls-extension-replace-session-b'; " "session B to wait for extension replacement" release_session_a wait_for_true " SELECT wait_event_type = 'Timeout' FROM pg_stat_activity WHERE application_name = 'rls-extension-replace-session-b'; " "session B to finish RLS enablement" PGAPPNAME=rls-extension-replace-session-c \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = off; CREATE INDEX extension_replace_child_idx ON extension_replace_child USING bm25(content) WITH (text_config='english');" \ > "${SESSION_C_OUTPUT}" 2>&1 & SESSION_C_PID=$! set +e wait "${SESSION_B_PID}" session_b_status=$? SESSION_B_PID= wait "${SESSION_C_PID}" session_c_status=$? SESSION_C_PID= set -e if [ "${session_b_status}" -ne 0 ]; then echo "RLS enablement failed after extension replacement" >&2 cat "${SESSION_B_OUTPUT}" >&2 exit 1 fi if [ "${session_c_status}" -eq 0 ]; then echo "Extension replacement changed the RLS DDL lock identity" >&2 exit 1 fi if ! grep -q "BM25 indexes are not allowed on row-level security" \ "${SESSION_C_OUTPUT}"; then echo "Missing expected post-replacement RLS rejection" >&2 cat "${SESSION_C_OUTPUT}" >&2 exit 1 fi extension_replace_state=$(run_value " SELECT relrowsecurity, to_regclass('extension_replace_child_idx') IS NULL FROM pg_class WHERE oid = 'extension_replace_parent'::regclass; ") if [ "${extension_replace_state}" != "t|t" ]; then echo "Extension replacement allowed opposite hierarchy mutations" >&2 exit 1 fi rm -f "${SESSION_A_INPUT}" "${SESSION_A_OUTPUT}" mkfifo "${SESSION_A_INPUT}" PGAPPNAME=rls-noop-session \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 < "${SESSION_A_INPUT}" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! exec 3> "${SESSION_A_INPUT}" printf '%s\n' \ "SET pg_textsearch.allow_rls = off;" \ "BEGIN;" \ "ALTER TABLE IF EXISTS missing_rls_table ENABLE ROW LEVEL SECURITY;" \ "CREATE TABLE IF NOT EXISTS noop_inherit_child () INHERITS (noop_inherit_parent);" \ "REINDEX INDEX noop_reindex_idx;" >&3 wait_for_true " SELECT state = 'idle in transaction' FROM pg_stat_activity WHERE application_name = 'rls-noop-session'; " "no-op RLS command to finish" noop_object_locks=$(run_value " SELECT count(*) FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-noop-session' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.granted; ") if [ "${noop_object_locks}" != "0" ]; then echo "No-op or unrelated DDL retained the extension object lock" >&2 exit 1 fi release_session_a PGAPPNAME=rls-multireindex-session \ PGOPTIONS="-c pg_textsearch.allow_rls=off" \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "REINDEX DATABASE ${TEST_DB};" \ > "${SESSION_A_OUTPUT}" 2>&1 if grep -q "you don't own a lock" "${SESSION_A_OUTPUT}"; then echo "Multi-relation REINDEX lost transaction lock ownership" >&2 cat "${SESSION_A_OUTPUT}" >&2 exit 1 fi psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' CREATE TABLE upgrade_outer_a (id integer); CREATE TABLE upgrade_outer_b (id integer); CREATE TABLE upgrade_nested_a (id integer); CREATE TABLE upgrade_nested_b (id integer); CREATE FUNCTION test_nested_policy_upgrade() RETURNS event_trigger AS $$ BEGIN IF current_setting('rls_upgrade_test.action', true) IS DISTINCT FROM 'outer' THEN RETURN; END IF; PERFORM set_config('rls_upgrade_test.action', 'running', false); IF current_setting('rls_upgrade_test.pause', true) = 'on' THEN PERFORM pg_sleep(5); END IF; PERFORM set_config('pg_textsearch.allow_rls', 'off', false); EXECUTE format( 'ALTER TABLE %I ENABLE ROW LEVEL SECURITY', current_setting('rls_upgrade_test.nested_table') ); END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER test_nested_policy_upgrade_trigger ON ddl_command_start WHEN TAG IN ('ALTER TABLE') EXECUTE FUNCTION test_nested_policy_upgrade(); SQL PGAPPNAME=rls-upgrade-session-a \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = on; SET rls_upgrade_test.action = 'outer'; SET rls_upgrade_test.nested_table = 'upgrade_nested_a'; SET rls_upgrade_test.pause = on; ALTER TABLE upgrade_outer_a ENABLE ROW LEVEL SECURITY;" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-upgrade-session-a' AND a.wait_event_type = 'Timeout' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.mode = 'ShareLock' AND l.granted ); " "session A to hold the policy lock" PGAPPNAME=rls-upgrade-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = on; SET rls_upgrade_test.action = 'outer'; SET rls_upgrade_test.nested_table = 'upgrade_nested_b'; SET rls_upgrade_test.pause = off; ALTER TABLE upgrade_outer_b ENABLE ROW LEVEL SECURITY;" \ > "${SESSION_B_OUTPUT}" 2>&1 & SESSION_B_PID=$! set +e wait "${SESSION_B_PID}" session_b_status=$? SESSION_B_PID= wait "${SESSION_A_PID}" session_a_status=$? SESSION_A_PID= set -e if [ "${session_a_status}" -ne 0 ] || [ "${session_b_status}" -eq 0 ]; then echo "Nested RLS policy lock upgrade did not fail safely" >&2 cat "${SESSION_A_OUTPUT}" "${SESSION_B_OUTPUT}" >&2 exit 1 fi if grep -q "deadlock detected" "${SESSION_A_OUTPUT}" "${SESSION_B_OUTPUT}"; then echo "Nested RLS policy lock upgrade deadlocked" >&2 cat "${SESSION_A_OUTPUT}" "${SESSION_B_OUTPUT}" >&2 exit 1 fi if ! grep -q "could not acquire the pg_textsearch RLS DDL lock" \ "${SESSION_B_OUTPUT}"; then echo "Nested RLS policy lock upgrade did not report lock contention" >&2 cat "${SESSION_B_OUTPUT}" >&2 exit 1 fi psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' DROP EVENT TRIGGER test_nested_policy_upgrade_trigger; DROP FUNCTION test_nested_policy_upgrade(); SQL psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' CREATE TABLE end_trigger_outer (id integer); CREATE TABLE end_trigger_nested (id integer); CREATE FUNCTION test_end_trigger_nested_ddl() RETURNS event_trigger AS $$ BEGIN IF current_setting('rls_end_test.action', true) IS DISTINCT FROM 'outer' THEN RETURN; END IF; PERFORM set_config('rls_end_test.action', 'running', false); PERFORM pg_sleep(5); PERFORM set_config('pg_textsearch.allow_rls', 'off', false); EXECUTE 'ALTER TABLE end_trigger_nested ENABLE ROW LEVEL SECURITY'; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER test_end_trigger_nested_ddl_trigger ON ddl_command_end WHEN TAG IN ('ALTER TABLE') EXECUTE FUNCTION test_end_trigger_nested_ddl(); SQL PGAPPNAME=rls-end-trigger-session-a \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = on; SET rls_end_test.action = 'outer'; ALTER TABLE end_trigger_outer ADD COLUMN extra integer;" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-end-trigger-session-a' AND a.wait_event_type = 'Timeout' AND l.locktype = 'relation' AND l.relation = 'end_trigger_outer'::regclass AND l.mode = 'AccessExclusiveLock' AND l.granted ); " "session A to hold the outer relation lock" PGAPPNAME=rls-end-trigger-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = off; ALTER TABLE end_trigger_outer ENABLE ROW LEVEL SECURITY;" \ > "${SESSION_B_OUTPUT}" 2>&1 & SESSION_B_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS object_lock ON object_lock.pid = a.pid JOIN pg_locks AS relation_lock ON relation_lock.pid = a.pid WHERE a.application_name = 'rls-end-trigger-session-b' AND object_lock.locktype = 'object' AND object_lock.classid = 'pg_extension'::regclass AND object_lock.objid = 0 AND object_lock.mode = 'ExclusiveLock' AND object_lock.granted AND relation_lock.locktype = 'relation' AND relation_lock.relation = 'end_trigger_outer'::regclass AND relation_lock.mode = 'AccessExclusiveLock' AND NOT relation_lock.granted ); " "session B to hold the policy lock while waiting on the outer table" set +e wait "${SESSION_A_PID}" session_a_status=$? SESSION_A_PID= wait "${SESSION_B_PID}" session_b_status=$? SESSION_B_PID= set -e if [ "${session_a_status}" -eq 0 ] || [ "${session_b_status}" -ne 0 ]; then echo "End-trigger nested DDL did not fail safely" >&2 cat "${SESSION_A_OUTPUT}" "${SESSION_B_OUTPUT}" >&2 exit 1 fi if grep -q "deadlock detected" "${SESSION_A_OUTPUT}" "${SESSION_B_OUTPUT}"; then echo "End-trigger nested DDL deadlocked" >&2 cat "${SESSION_A_OUTPUT}" "${SESSION_B_OUTPUT}" >&2 exit 1 fi if ! grep -q "could not acquire the pg_textsearch RLS DDL lock" \ "${SESSION_A_OUTPUT}"; then echo "End-trigger nested DDL did not report lock contention" >&2 cat "${SESSION_A_OUTPUT}" >&2 exit 1 fi psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' DROP EVENT TRIGGER test_end_trigger_nested_ddl_trigger; DROP FUNCTION test_end_trigger_nested_ddl(); SQL start_rls_holder lock_parent PGAPPNAME=rls-lock-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = off; CREATE INDEX lock_child_idx ON lock_child USING bm25(content) WITH (text_config='english');" \ > "${SESSION_B_OUTPUT}" 2>&1 & SESSION_B_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-b' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.mode = 'ExclusiveLock' AND NOT l.granted ); " "session B to wait on the extension object lock" child_locks=$(run_value " SELECT count(*) FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-b' AND l.locktype = 'relation' AND l.relation = 'lock_child'::regclass AND l.mode IN ('ShareLock', 'ShareUpdateExclusiveLock') AND l.granted; ") if [ "${child_locks}" != "0" ]; then echo "Session B locked the child before DDL serialization" >&2 exit 1 fi release_session_a set +e wait "${SESSION_B_PID}" session_b_status=$? set -e SESSION_B_PID= if [ "${session_b_status}" -eq 0 ]; then echo "BM25 index creation unexpectedly succeeded" >&2 exit 1 fi if grep -q "deadlock detected" "${SESSION_B_OUTPUT}"; then echo "RLS/BM25 DDL deadlocked" >&2 cat "${SESSION_B_OUTPUT}" >&2 exit 1 fi if ! grep -q "BM25 indexes are not allowed on row-level security" \ "${SESSION_B_OUTPUT}"; then echo "Missing expected RLS rejection" >&2 cat "${SESSION_B_OUTPUT}" >&2 exit 1 fi psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' CREATE TABLE create_lock_gate (id integer); CREATE TABLE partition_parent (id integer, content text) PARTITION BY RANGE (id); CREATE INDEX partition_parent_idx ON partition_parent USING bm25(content) WITH (text_config='english'); SQL start_rls_holder create_lock_gate PGAPPNAME=rls-lock-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = off; CREATE TABLE partition_child PARTITION OF partition_parent FOR VALUES FROM (0) TO (10);" \ > "${SESSION_B_OUTPUT}" 2>&1 & SESSION_B_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-b' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.mode = 'ExclusiveLock' AND NOT l.granted ); " "partition creation to wait on the extension object lock" parent_locks=$(run_value " SELECT count(*) FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-b' AND l.locktype = 'relation' AND l.relation = 'partition_parent'::regclass AND l.granted; ") if [ "${parent_locks}" != "0" ]; then echo "Partition creation locked its parent before DDL serialization" >&2 PARTITION_ORDER_FAILED=1 fi release_session_a wait "${SESSION_B_PID}" SESSION_B_PID= psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' CREATE TABLE cic_lock_docs (id integer, content text); INSERT INTO cic_lock_docs SELECT g, 'document ' || g FROM generate_series(1, 1000) AS g; SQL rm -f "${SESSION_A_INPUT}" "${SESSION_A_OUTPUT}" mkfifo "${SESSION_A_INPUT}" PGAPPNAME=rls-lock-session-a \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 < "${SESSION_A_INPUT}" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! exec 3> "${SESSION_A_INPUT}" printf '%s\n' \ "BEGIN;" \ "INSERT INTO cic_lock_docs VALUES (1001, 'old writer');" >&3 wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-a' AND a.state = 'idle in transaction' AND l.locktype = 'relation' AND l.relation = 'cic_lock_docs'::regclass AND l.mode = 'RowExclusiveLock' AND l.granted ); " "session A to hold an old writer lock" PGAPPNAME=rls-lock-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 > "${SESSION_B_OUTPUT}" 2>&1 << 'SQL' & SET pg_textsearch.allow_rls = off; CREATE INDEX CONCURRENTLY cic_lock_idx ON cic_lock_docs USING bm25(content) WITH (text_config='english'); SQL SESSION_B_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_index WHERE indexrelid = to_regclass('cic_lock_idx') AND NOT indisready AND NOT indisvalid ); " "concurrent index creation to cross its first internal commit" object_locks=$(run_value " SELECT count(*) FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-lock-session-b' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.mode = 'ExclusiveLock' AND l.granted; ") if [ "${object_locks}" != "1" ]; then echo "Concurrent index creation lost its DDL serialization lock" >&2 exit 1 fi release_session_a wait "${SESSION_B_PID}" SESSION_B_PID= psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 >/dev/null << 'SQL' CREATE TABLE policy_pin_parent (id integer, content text); CREATE TABLE policy_pin_child () INHERITS (policy_pin_parent); CREATE TABLE policy_pin_nested (id integer, content text); CREATE INDEX policy_pin_nested_idx ON policy_pin_nested USING bm25(content) WITH (text_config='english'); CREATE FUNCTION policy_pin_flip() RETURNS event_trigger AS $$ DECLARE action text := current_setting('rls_pin_test.action', true); BEGIN IF action NOT IN ('flip', 'flip_nested') THEN RETURN; END IF; PERFORM set_config('rls_pin_test.action', 'running', true); PERFORM set_config('pg_textsearch.allow_rls', 'on', false); IF action = 'flip_nested' THEN EXECUTE 'ALTER TABLE policy_pin_nested ENABLE ROW LEVEL SECURITY'; IF NOT EXISTS ( SELECT 1 FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'object' AND classid = 'pg_extension'::regclass AND objid = 0 AND mode = 'ShareLock' AND granted ) THEN RAISE EXCEPTION 'nested utility command did not acquire a policy lock'; END IF; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER policy_pin_flip_trigger ON ddl_command_start WHEN TAG IN ('ALTER TABLE', 'CREATE INDEX') EXECUTE FUNCTION policy_pin_flip(); SQL rm -f "${SESSION_A_INPUT}" "${SESSION_A_OUTPUT}" mkfifo "${SESSION_A_INPUT}" PGAPPNAME=rls-policy-pin-session-a \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 < "${SESSION_A_INPUT}" \ > "${SESSION_A_OUTPUT}" 2>&1 & SESSION_A_PID=$! exec 3> "${SESSION_A_INPUT}" printf '%s\n' \ "SET pg_textsearch.allow_rls = off;" \ "SET rls_pin_test.action = 'flip_nested';" \ "BEGIN;" \ "ALTER TABLE policy_pin_parent ENABLE ROW LEVEL SECURITY;" >&3 wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-policy-pin-session-a' AND a.state = 'idle in transaction' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.mode = 'ExclusiveLock' AND l.granted ); " "session A to finish RLS enablement with its pinned policy lock" PGAPPNAME=rls-policy-pin-session-b \ psql -X -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -v ON_ERROR_STOP=1 \ -c "SET pg_textsearch.allow_rls = off; SET rls_pin_test.action = 'flip'; CREATE INDEX policy_pin_child_idx ON policy_pin_child USING bm25(content) WITH (text_config='english');" \ > "${SESSION_B_OUTPUT}" 2>&1 & SESSION_B_PID=$! wait_for_true " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity AS a JOIN pg_locks AS l ON l.pid = a.pid WHERE a.application_name = 'rls-policy-pin-session-b' AND l.locktype = 'object' AND l.classid = 'pg_extension'::regclass AND l.objid = 0 AND l.mode = 'ExclusiveLock' AND NOT l.granted ); " "session B to wait on session A's pinned policy lock" release_session_a set +e wait "${SESSION_B_PID}" session_b_status=$? set -e SESSION_B_PID= if [ "${session_b_status}" -eq 0 ]; then echo "GUC-changing trigger bypassed the pinned RLS policy" >&2 exit 1 fi if ! grep -q "BM25 indexes are not allowed on row-level security" \ "${SESSION_B_OUTPUT}"; then echo "Missing expected pinned-policy rejection" >&2 cat "${SESSION_B_OUTPUT}" >&2 exit 1 fi policy_pin_state=$(run_value " SELECT relrowsecurity, to_regclass('policy_pin_child_idx') IS NULL FROM pg_class WHERE oid = 'policy_pin_parent'::regclass; ") if [ "${policy_pin_state}" != "t|t" ]; then echo "Opposite hierarchy mutations jointly committed" >&2 exit 1 fi if [ "${PARTITION_ORDER_FAILED}" -ne 0 ]; then exit 1 fi echo "RLS DDL locking tests passed" timescale-pg_textsearch-b63fe58/test/scripts/segment.sh000077500000000000000000000415361525762047700234410ustar00rootroot00000000000000#!/bin/bash # # Multi-backend segment tests for pg_textsearch # Tests segment behavior across multiple concurrent sessions and transactions. # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55435 TEST_DB=pg_textsearch_segment_test DATA_DIR="${SCRIPT_DIR}/../tmp_segment_test" LOGFILE="${DATA_DIR}/postgres.log" # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } cleanup() { local exit_code=$? log "Cleaning up segment test environment (exit code: $exit_code)..." jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast &>/dev/null || pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up segment test PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 local lib_name="pg_textsearch" cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 20 shared_buffers = 128MB unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' log_min_messages = warning shared_preload_libraries = '${lib_name}' EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w || error "Failed to start PostgreSQL" createdb -h "${DATA_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "CREATE EXTENSION pg_textsearch;" >/dev/null # Load validation functions psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -f "${SCRIPT_DIR}/../sql/validation.sql" >/dev/null } run_sql() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } run_sql_quiet() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } run_sql_value() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc "$1" 2>/dev/null } assert_count_file() { local label="$1" local output="$2" local expected_reads="$3" local minimum="$4" local maximum="$5" local reads=0 local count while IFS= read -r count; do reads=$((reads + 1)) if [[ ! "${count}" =~ ^[0-9]+$ ]]; then error "${label}: read ${reads} returned non-numeric output: ${count}" fi if [ "${count}" -lt "${minimum}" ] || [ "${count}" -gt "${maximum}" ]; then error "${label}: read ${reads} returned ${count}, expected ${minimum}-${maximum}" fi done < "${output}" if [ "${reads}" -ne "${expected_reads}" ]; then error "${label}: completed ${reads}/${expected_reads} reads" fi } # # Test 1: Multiple backends reading from segment after spill # test_concurrent_segment_reads() { log "Test 1: Multiple backends reading from segment after spill" run_sql_quiet "CREATE TABLE seg_docs (id SERIAL PRIMARY KEY, content TEXT);" # Insert documents and create index for i in $(seq 1 50); do run_sql_quiet "INSERT INTO seg_docs (content) VALUES ('document $i about alpha beta gamma');" done run_sql_quiet "CREATE INDEX seg_idx ON seg_docs USING bm25(content) WITH (text_config='english');" # Force spill to segment run_sql_quiet "SELECT bm25_spill_index('seg_idx');" info "Segment created, starting concurrent readers..." # Start 5 concurrent readers local pids=() for reader_id in 1 2 3 4 5; do local output="/tmp/seg_reader_${reader_id}.out" { for q in $(seq 1 20); do psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc \ "SELECT COUNT(*) FROM ( SELECT 1 FROM seg_docs ORDER BY content <@> to_bm25query('alpha', 'seg_idx') ) ranked;" done } > "$output" 2>&1 & pids+=($!) done # Wait for all readers local failures=0 for i in "${!pids[@]}"; do local reader_id=$((i + 1)) if ! wait "${pids[$i]}" 2>/dev/null; then warn "Reader $reader_id failed" failures=$((failures + 1)) else assert_count_file "Test 1 reader ${reader_id}" \ "/tmp/seg_reader_${reader_id}.out" 20 50 50 fi rm -f "/tmp/seg_reader_${reader_id}.out" done if [ $failures -eq 0 ]; then log "✅ Test 1 passed: All readers completed successfully" else error "❌ Test 1 failed: $failures readers failed" fi run_sql_quiet "DROP TABLE seg_docs CASCADE;" } # # Test 2: One backend writing to memtable while another reads from segment # test_concurrent_memtable_write_segment_read() { log "Test 2: Concurrent memtable writes while reading from segment" run_sql_quiet "CREATE TABLE mixed_docs (id SERIAL PRIMARY KEY, content TEXT);" # Initial data and spill for i in $(seq 1 30); do run_sql_quiet "INSERT INTO mixed_docs (content) VALUES ('initial document $i with delta term');" done run_sql_quiet "CREATE INDEX mixed_idx ON mixed_docs USING bm25(content) WITH (text_config='english');" run_sql_quiet "SELECT bm25_spill_index('mixed_idx');" info "Segment created with 30 docs, starting mixed read/write..." # Writer: insert new documents to memtable local writer_output="/tmp/mixed_writer.out" { for i in $(seq 31 80); do psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "INSERT INTO mixed_docs (content) VALUES ('new document $i with delta epsilon');" >/dev/null sleep 0.01 done } > "$writer_output" 2>&1 & local writer_pid=$! # Reader: query both segment and memtable data local reader_output="/tmp/mixed_reader.out" { for q in $(seq 1 30); do psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc \ "SELECT COUNT(*) FROM ( SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('delta', 'mixed_idx') ) ranked;" sleep 0.02 done } > "$reader_output" 2>&1 & local reader_pid=$! # Wait for both if ! wait $writer_pid 2>/dev/null; then cat "$writer_output" >&2 error "Writer failed during Test 2" fi wait $reader_pid 2>/dev/null || error "Reader crashed" assert_count_file "Test 2 reader" "$reader_output" 30 30 80 # Verify final count is correct local final_count=$(run_sql_value "SELECT COUNT(*) FROM ( SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('delta', 'mixed_idx') ) ranked;") info "Final count: $final_count" if [ "$final_count" -eq 80 ]; then log "✅ Test 2 passed: Reader and writer completed ($final_count docs with 'delta')" else error "❌ Test 2 failed: Expected 80 final matches, got $final_count" fi rm -f "$writer_output" "$reader_output" run_sql_quiet "DROP TABLE mixed_docs CASCADE;" } # # Test 3: Multiple spills with concurrent reads # test_multiple_spills_concurrent_reads() { log "Test 3: Multiple spills with concurrent reads" run_sql_quiet "CREATE TABLE multi_spill (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "CREATE INDEX multi_idx ON multi_spill USING bm25(content) WITH (text_config='english');" # Insert and spill 3 times to create segment chain for batch in 1 2 3; do for i in $(seq 1 20); do run_sql_quiet "INSERT INTO multi_spill (content) VALUES ('batch $batch doc $i zeta term');" done run_sql_quiet "SELECT bm25_spill_index('multi_idx');" info "Batch $batch spilled" done # Start readers while we do another batch local pids=() for reader_id in 1 2 3; do local output="/tmp/multi_reader_${reader_id}.out" { for q in $(seq 1 15); do psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc \ "SELECT COUNT(*) FROM ( SELECT 1 FROM multi_spill ORDER BY content <@> to_bm25query('zeta', 'multi_idx') ) ranked;" sleep 0.03 done } > "$output" 2>&1 & pids+=($!) done # Insert more and spill again for i in $(seq 1 20); do run_sql_quiet "INSERT INTO multi_spill (content) VALUES ('batch 4 doc $i zeta term');" done run_sql_quiet "SELECT bm25_spill_index('multi_idx');" info "Batch 4 spilled during reads" # Wait for readers local failures=0 for i in "${!pids[@]}"; do if ! wait "${pids[$i]}" 2>/dev/null; then failures=$((failures + 1)) else assert_count_file "Test 3 reader $((i + 1))" \ "/tmp/multi_reader_$((i + 1)).out" 15 60 80 fi rm -f "/tmp/multi_reader_$((i+1)).out" done # Final count should be 80 (use simple COUNT first to verify data) local table_count=$(run_sql_value "SELECT COUNT(*) FROM multi_spill;") local final_count=$(run_sql_value "SELECT COUNT(*) FROM ( SELECT 1 FROM multi_spill ORDER BY content <@> to_bm25query('zeta', 'multi_idx') ) ranked;") info "Table has $table_count rows, query found $final_count" if [ "$failures" -eq 0 ] && [ "$table_count" -eq 80 ] && [ "$final_count" -eq 80 ]; then log "✅ Test 3 passed: Multiple spills handled correctly ($table_count docs)" else error "❌ Test 3 failed: failures=$failures table_count=$table_count final_count=$final_count" fi run_sql_quiet "DROP TABLE multi_spill CASCADE;" } # # Test 4: Validated BM25 scoring across segment + memtable # test_bm25_scoring_validated() { log "Test 4: Validated BM25 scoring across segment + memtable" run_sql_quiet "CREATE TABLE score_test (id SERIAL PRIMARY KEY, content TEXT);" # Insert known documents run_sql_quiet "INSERT INTO score_test (content) VALUES ('hello world'), ('goodbye cruel world'), ('hello goodbye'), ('world peace');" run_sql_quiet "CREATE INDEX score_idx ON score_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75);" # Phase 1: Validate in memtable only local valid_memtable=$(run_sql_value "SELECT validate_bm25_scoring('score_test', 'content', 'score_idx', 'hello world', 'english', 1.2, 0.75);") if [ "$valid_memtable" != "t" ]; then error "❌ Test 4 failed: BM25 validation failed in memtable phase" fi info "Phase 1 (memtable only): validated" # Phase 2: Spill to segment run_sql_quiet "SELECT bm25_spill_index('score_idx');" local valid_segment=$(run_sql_value "SELECT validate_bm25_scoring('score_test', 'content', 'score_idx', 'hello world', 'english', 1.2, 0.75);") if [ "$valid_segment" != "t" ]; then error "❌ Test 4 failed: BM25 validation failed after spill (segment only)" fi info "Phase 2 (segment only): validated" # Phase 3: Add more documents to memtable (mixed segment + memtable) run_sql_quiet "INSERT INTO score_test (content) VALUES ('hello again friend'), ('peaceful world harmony'), ('cruel goodbye forever');" local valid_mixed=$(run_sql_value "SELECT validate_bm25_scoring('score_test', 'content', 'score_idx', 'hello world', 'english', 1.2, 0.75);") if [ "$valid_mixed" != "t" ]; then error "❌ Test 4 failed: BM25 validation failed with mixed segment + memtable" fi info "Phase 3 (segment + memtable): validated" # Phase 4: Second spill run_sql_quiet "SELECT bm25_spill_index('score_idx');" local valid_two_segments=$(run_sql_value "SELECT validate_bm25_scoring('score_test', 'content', 'score_idx', 'hello world', 'english', 1.2, 0.75);") if [ "$valid_two_segments" != "t" ]; then error "❌ Test 4 failed: BM25 validation failed with two segments" fi info "Phase 4 (two segments): validated" # Phase 5: More inserts after second spill run_sql_quiet "INSERT INTO score_test (content) VALUES ('hello hello hello'), ('world world'), ('new unique term');" local valid_final=$(run_sql_value "SELECT validate_bm25_scoring('score_test', 'content', 'score_idx', 'hello world', 'english', 1.2, 0.75);") if [ "$valid_final" != "t" ]; then error "❌ Test 4 failed: BM25 validation failed with two segments + memtable" fi info "Phase 5 (two segments + memtable): validated" log "✅ Test 4 passed: All BM25 scores validated against reference implementation" run_sql_quiet "DROP TABLE score_test CASCADE;" } # # Test 5: Document length lookup across segment and memtable # test_doc_length_lookup() { log "Test 5: Document length lookup across segment and memtable" run_sql_quiet "CREATE TABLE doclen_test (id SERIAL PRIMARY KEY, content TEXT);" # Insert documents with varying lengths run_sql_quiet "INSERT INTO doclen_test (content) VALUES ('short');" run_sql_quiet "INSERT INTO doclen_test (content) VALUES ('medium length document with several words');" run_sql_quiet "INSERT INTO doclen_test (content) VALUES ('this is a much longer document that contains many more words than the others to test document length handling properly');" run_sql_quiet "CREATE INDEX doclen_idx ON doclen_test USING bm25(content) WITH (text_config='english');" run_sql_quiet "SELECT bm25_spill_index('doclen_idx');" # Add more to memtable run_sql_quiet "INSERT INTO doclen_test (content) VALUES ('another short one');" run_sql_quiet "INSERT INTO doclen_test (content) VALUES ('and a longer one with more words to test');" # Explicitly exercise standalone scoring and document-length lookup across # both the segment and memtable sources. local results=$(run_sql_value "SELECT COUNT(*) FROM doclen_test WHERE content <@> to_bm25query('document', 'doclen_idx') < -0.01;") # Should find 2 documents containing 'document' if [ "$results" -eq 2 ]; then log "✅ Test 5 passed: Document length lookup works ($results matches)" else error "❌ Test 5 failed: Expected 2 matches, got $results" fi run_sql_quiet "DROP TABLE doclen_test CASCADE;" } # # Test 6: Concurrent reads during ongoing spill # test_read_during_spill() { log "Test 6: Concurrent reads during ongoing spill" run_sql_quiet "CREATE TABLE spill_race (id SERIAL PRIMARY KEY, content TEXT);" # Insert many documents for i in $(seq 1 200); do run_sql_quiet "INSERT INTO spill_race (content) VALUES ('document $i about theta omega');" done run_sql_quiet "CREATE INDEX spill_idx ON spill_race USING bm25(content) WITH (text_config='english');" # Start readers local reader_output="/tmp/spill_reader.out" { for q in $(seq 1 50); do psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc \ "SELECT COUNT(*) FROM ( SELECT 1 FROM spill_race ORDER BY content <@> to_bm25query('theta', 'spill_idx') ) ranked;" sleep 0.01 done } > "$reader_output" 2>&1 & local reader_pid=$! # Trigger spill while reader is running sleep 0.1 run_sql_quiet "SELECT bm25_spill_index('spill_idx');" info "Spill triggered during reads" wait $reader_pid 2>/dev/null || error "Reader crashed during spill" assert_count_file "Test 6 reader" "$reader_output" 50 200 200 log "✅ Test 6 passed: All reads returned correct count during spill" rm -f "$reader_output" run_sql_quiet "DROP TABLE spill_race CASCADE;" } run_segment_tests() { log "Starting multi-backend segment tests for pg_textsearch" test_concurrent_segment_reads test_concurrent_memtable_write_segment_read test_multiple_spills_concurrent_reads test_bm25_scoring_validated test_doc_length_lookup test_read_during_spill log "Final verification..." local tables=$(run_sql_value "SELECT COUNT(*) FROM pg_tables WHERE schemaname = 'public';") info "Tables remaining: $tables" } main() { log "Starting pg_textsearch multi-backend segment testing..." command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" setup_test_db run_segment_tests log "🎉 All multi-backend segment tests completed successfully!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/segment_io_limits_test.c000066400000000000000000000024551525762047700263520ustar00rootroot00000000000000#include #include #include "segment/alive_bitset.h" #include "segment/io.h" typedef void (*SegmentReadFn)(TpSegmentReader *, uint64, void *, uint64); _Static_assert( _Generic(&tp_segment_read, SegmentReadFn: 1, default: 0), "tp_segment_read length must hold 2^32 bytes"); int main(void) { uint64 max = PG_UINT32_MAX; assert(!tp_string_pool_offset_overflows(0, max + 1)); assert(tp_string_pool_offset_overflows(0, max + 2)); assert(!tp_string_pool_offset_overflows(max, 1)); assert(tp_string_pool_offset_overflows(max, 2)); assert(tp_dictionary_offsets_fit(TP_MAX_DICTIONARY_TERMS)); assert(!tp_dictionary_offsets_fit(TP_MAX_DICTIONARY_TERMS + 1)); assert(tp_dictionary_size(TP_MAX_DICTIONARY_TERMS) == max + 1); assert(tp_posting_block_count(0) == 0); assert(tp_posting_block_count(TP_BLOCK_SIZE) == 1); assert(tp_posting_block_count(TP_BLOCK_SIZE + 1) == 2); assert(tp_posting_block_count(PG_UINT32_MAX) == PG_UINT32_MAX / TP_BLOCK_SIZE + 1); assert(tp_alive_bitset_size(PG_UINT32_MAX - 7) == 536870911); assert(tp_alive_bitset_size(PG_UINT32_MAX - 1) == 536870912); assert(tp_alive_bitset_size(PG_UINT32_MAX) == 536870912); assert(tp_document_count_fits(TP_MAX_GROWABLE_CAPACITY)); assert(!tp_document_count_fits((uint64)TP_MAX_GROWABLE_CAPACITY + 1)); return 0; } timescale-pg_textsearch-b63fe58/test/scripts/segment_wal_recovery.sh000077500000000000000000000230701525762047700262130ustar00rootroot00000000000000#!/bin/bash # # Segment WAL recovery test for pg_textsearch. # # Verifies that segment page writes produce WAL records so they # survive a crash between checkpoints. # # Two-part test: # # Part 1 (WAL coverage check): # Measure WAL bytes generated by an explicit spill. Segment # pages are ~8kB each, so a 20+ page segment should generate # at least 100kB of WAL if pages are WAL-logged. Without # GenericXLog the only WAL comes from the metapage linkage # (~100 bytes). # # Part 2 (crash recovery): # Trigger auto-spill, SIGKILL, recover, query. With WAL- # logged segment pages, the data survives. Without, the # docid recovery path may mask the issue for simple cases, # but a second spill after recovery would read corrupt # segments during compaction. # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55442 TEST_DB=segment_wal_recovery_test DATA_DIR="${SCRIPT_DIR}/../tmp_segment_wal_test" LOGFILE="${DATA_DIR}/postgres.log" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$($PG_CONFIG --bindir)" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' PASS_COUNT=0 FAIL_COUNT=0 log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } pass() { PASS_COUNT=$((PASS_COUNT + 1)) log "PASS: $1" } fail() { FAIL_COUNT=$((FAIL_COUNT + 1)) echo -e "${RED}[$(date '+%H:%M:%S')] FAIL: $1${NC}" } cleanup() { local exit_code=$? log "Cleaning up segment WAL test (exit code: $exit_code)..." jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m fast \ &>/dev/null || "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m immediate \ &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM run_sql() { "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" 2>&1 } run_sql_quiet() { "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } run_sql_value() { "${PGBINDIR}/psql" -h "${DATA_DIR}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -tAc "$1" 2>/dev/null } simulate_crash() { log "Simulating crash (SIGKILL)..." local pid pid=$(head -1 "${DATA_DIR}/postmaster.pid") if [ -n "$pid" ] && kill -0 "$pid" 2>/dev/null; then kill -9 "$pid" sleep 2 rm -f "${DATA_DIR}/postmaster.pid" log "Server killed (PID $pid)" else error "Could not find postmaster to kill" fi } restart_server() { log "Restarting server (recovery will run)..." "${PGBINDIR}/pg_ctl" start -D "${DATA_DIR}" \ -l "${LOGFILE}" -w \ || error "Server failed to restart after crash" run_sql_quiet "SELECT 1;" \ || error "Cannot connect after restart" log "Server restarted" } # ------------------------------------------------------- # Setup # ------------------------------------------------------- log "==========================================" log "Segment WAL recovery test" log "==========================================" rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" "${PGBINDIR}/initdb" -D "${DATA_DIR}" \ --auth-local=trust --auth-host=trust >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 20 shared_buffers = 128MB unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' log_min_messages = warning shared_preload_libraries = 'pg_textsearch' # Tight page threshold so auto-spill triggers during inserts. pg_textsearch.memtable_pages_threshold = 1 pg_textsearch.bulk_load_threshold = 0 pg_textsearch.segments_per_level = 2 EOF "${PGBINDIR}/pg_ctl" start -D "${DATA_DIR}" -l "${LOGFILE}" -w \ || error "Failed to start PostgreSQL" "${PGBINDIR}/createdb" -h "${DATA_DIR}" -p "${TEST_PORT}" \ "${TEST_DB}" run_sql_quiet "CREATE EXTENSION pg_textsearch VERSION '1.5.1';" # ------------------------------------------------------- # Part 1: WAL coverage check # # Spill a memtable to a segment and measure how much WAL # was generated. Each segment page is ~8kB; a spill that # creates N pages should generate at least N*8kB of WAL # if the pages are WAL-logged via GenericXLog. # ------------------------------------------------------- log "Part 1: WAL coverage check" # Temporarily disable auto-spill so data stays in memtable # until we explicitly spill. run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 0;" run_sql_quiet "SELECT pg_reload_conf();" sleep 0.2 run_sql_quiet "CREATE TABLE wal_check ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_wal_check ON wal_check USING bm25(content) WITH (text_config='english');" # Load data into memtable (no spill — 256MB limit). run_sql_quiet "INSERT INTO wal_check (content) SELECT 'walcheck_' || i || ' topic_' || (i % 50) || ' ' || repeat('pad_', 10) FROM generate_series(1, 3000) i;" # Reset WAL stats, record LSN, spill, record LSN. run_sql_quiet "SELECT pg_stat_reset_shared('wal');" lsn_before="" lsn_before=$(run_sql_value "SELECT pg_current_wal_insert_lsn();") run_sql_quiet "SELECT bm25_spill_index('idx_wal_check');" lsn_after="" lsn_after=$(run_sql_value "SELECT pg_current_wal_insert_lsn();") wal_bytes="" wal_bytes=$(run_sql_value "SELECT wal_bytes FROM pg_stat_wal;") # How many segment pages were written? seg_pages="" seg_pages=$(run_sql_value \ "SELECT bm25_summarize_index('idx_wal_check');" \ | grep -oE 'pages=[0-9]+' | head -1 \ | grep -oE '[0-9]+' || echo 0) info "Spill: $seg_pages segment pages, WAL: $wal_bytes bytes" info "LSN: $lsn_before -> $lsn_after" # Each page is 8192 bytes. If WAL-logged, we expect at least # seg_pages * 4096 bytes (conservative — deltas are smaller than # full pages). Without GenericXLog, WAL is < 1kB (just metapage). min_expected_wal=$((seg_pages * 4096)) if [ "$wal_bytes" -ge "$min_expected_wal" ]; then pass "Part 1: spill generated ${wal_bytes} bytes WAL (>= ${min_expected_wal} for ${seg_pages} pages)" else fail "Part 1: spill generated only ${wal_bytes} bytes WAL (expected >= ${min_expected_wal} for ${seg_pages} pages)" fi run_sql_quiet "DROP TABLE wal_check CASCADE;" # Restore tight threshold for Part 2. run_sql_quiet "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = 1;" run_sql_quiet "SELECT pg_reload_conf();" sleep 0.2 # ------------------------------------------------------- # Part 2: Crash recovery with spilled segments # # Trigger auto-spill, crash, recover, # and verify a second spill + compaction cycle works (which # requires reading the pre-crash segments). # ------------------------------------------------------- log "Part 2: Crash recovery with segment reuse" run_sql_quiet "CREATE TABLE recovery_test ( id serial PRIMARY KEY, content text);" run_sql_quiet "CREATE INDEX idx_recovery ON recovery_test USING bm25(content) WITH (text_config='english');" # Insert enough to trigger auto-spill. run_sql_quiet "INSERT INTO recovery_test (content) SELECT 'batch1_' || i || ' topic_' || (i % 40) || ' ' || repeat('pad_', 10) FROM generate_series(1, 3000) i;" # Verify spill happened. seg_before="" seg_before=$(run_sql_value \ "SELECT bm25_summarize_index('idx_recovery');" \ | grep -oE 'Total: [0-9]+ segments' \ | grep -oE '[0-9]+' || echo 0) if [ "${seg_before:-0}" -eq 0 ]; then error "Part 2 setup: auto-spill did not occur" fi log "Pre-crash: $seg_before segments" # Verify queryable. pre_ct="" pre_ct=$(run_sql_value \ "SELECT count(*) FROM recovery_test WHERE content <@> 'batch1'::bm25query < 0;") log "Pre-crash query: $pre_ct rows" # Crash. simulate_crash restart_server # Verify data survived recovery. post_ct="" post_ct=$(run_sql_value \ "SELECT count(*) FROM recovery_test WHERE content <@> 'batch1'::bm25query < 0;" \ || echo "QUERY_FAILED") if [ "$post_ct" = "QUERY_FAILED" ]; then fail "Part 2a: query failed after recovery" elif [ "$post_ct" -eq "$pre_ct" ]; then pass "Part 2a: query returns $post_ct rows after recovery (matches)" else fail "Part 2a: post-crash $post_ct != pre-crash $pre_ct" fi # Insert a second batch to trigger another spill + compaction. # With segments_per_level=2, two L0 segments trigger compaction, # which reads the pre-crash segment. If it's corrupt, this fails. run_sql_quiet "INSERT INTO recovery_test (content) SELECT 'batch2_' || i || ' topic_' || (i % 40) || ' ' || repeat('pad_', 10) FROM generate_series(1, 3000) i;" \ || true # Don't abort on failure — we want to report it. batch2_ct="" batch2_ct=$(run_sql_value \ "SELECT count(*) FROM recovery_test WHERE content <@> 'batch2'::bm25query < 0;" \ || echo "QUERY_FAILED") if [ "$batch2_ct" = "QUERY_FAILED" ]; then fail "Part 2b: query failed after post-recovery spill+compact" elif [ "$batch2_ct" -eq 3000 ]; then pass "Part 2b: post-recovery spill+compact works ($batch2_ct rows)" else fail "Part 2b: batch2 count=$batch2_ct (expected 3000)" fi # ------------------------------------------------------- # Summary # ------------------------------------------------------- log "==========================================" if [ "$FAIL_COUNT" -gt 0 ]; then error "$FAIL_COUNT test(s) FAILED, $PASS_COUNT passed" else log "All tests passed ($PASS_COUNT assertions)" fi timescale-pg_textsearch-b63fe58/test/scripts/shutdown_spill.sh000077500000000000000000000300711525762047700250450ustar00rootroot00000000000000#!/bin/bash # # Clean-shutdown spill test (regression test for issue #333, scenario A). # # On a clean pg_ctl stop, backends that are still alive receive SIGTERM. # The before_shmem_exit hook spills any un-spilled memtable contents to # segments and drains the docid-page chain. Without this, the first # query after the next server start has to walk the entire chain and # re-tokenize every referenced heap tuple. # # This test only exercises the path where a backend is still alive at # shutdown time (hook fires). The short-connection path (no backend # alive at shutdown) is handled by the VACUUM-based fix; see # vacuum_extended.sql Test 7. # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55441 TEST_DB=shutdown_spill_test TEST_HOST=127.0.0.1 DATA_DIR="${SCRIPT_DIR}/../tmp_shutdown_spill_test" LOGFILE="${DATA_DIR}/postgres.log" SLEEPER_PID= PGBINDIR="$(pg_config --bindir)" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } wait_for_child_exit() { local pid=$1 local attempts=$2 for _ in $(seq 1 "${attempts}"); do if ! kill -0 "${pid}" 2>/dev/null; then # The bounded poll established that the child exited; this wait # only reaps its already-available status. wait "${pid}" 2>/dev/null || true return 0 fi sleep 0.1 done return 1 } stop_sleeper() { if [ -z "${SLEEPER_PID}" ]; then return fi if ! wait_for_child_exit "${SLEEPER_PID}" 50; then warn "Sleeper PID ${SLEEPER_PID} did not exit in 5s; sending SIGTERM" kill -TERM "${SLEEPER_PID}" 2>/dev/null || true if ! wait_for_child_exit "${SLEEPER_PID}" 50; then warn "Sleeper PID ${SLEEPER_PID} ignored SIGTERM for 5s; \ sending SIGKILL (state: $(ps -o pid=,stat=,cmd= -p "${SLEEPER_PID}" \ 2>/dev/null || echo unavailable))" kill -9 "${SLEEPER_PID}" 2>/dev/null || true if ! wait_for_child_exit "${SLEEPER_PID}" 50; then warn "Sleeper PID ${SLEEPER_PID} still exists 5s after \ SIGKILL (state: $(ps -o pid=,stat=,cmd= -p "${SLEEPER_PID}" \ 2>/dev/null || echo unavailable)); continuing bounded cleanup" fi fi fi SLEEPER_PID= } cleanup() { local exit_code=$? log "Cleaning up (exit code: $exit_code)..." stop_sleeper exec 3>&- 2>/dev/null || true jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m fast -w -t 30 \ >/dev/null 2>&1 || \ "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m immediate -w -t 30 \ >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup() { rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" "${PGBINDIR}/initdb" -D "${DATA_DIR}" \ --auth-local=trust --auth-host=trust >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} shared_buffers = 128MB max_connections = 20 shared_preload_libraries = 'pg_textsearch' listen_addresses = '${TEST_HOST}' unix_socket_directories = '' EOF "${PGBINDIR}/pg_ctl" start -D "${DATA_DIR}" -l "${LOGFILE}" -w \ >/dev/null "${PGBINDIR}/createdb" -h "${TEST_HOST}" -p "${TEST_PORT}" "${TEST_DB}" "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" \ -c "CREATE EXTENSION pg_textsearch;" >/dev/null } sql() { "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -tA -c "$1" 2>/dev/null } sql_quiet() { "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } main() { log "=== Clean-shutdown spill test (issue #333, scenario A) ===" command -v "${PGBINDIR}/pg_ctl" >/dev/null 2>&1 || \ error "pg_ctl not found" setup # Two tables + indexes so the hook iterates more than one entry. sql_quiet "CREATE TABLE docs (id serial PRIMARY KEY, body text);" sql_quiet "CREATE INDEX docs_idx ON docs USING bm25 (body) WITH (text_config='english');" sql_quiet "CREATE TABLE extras (id serial PRIMARY KEY, body text);" sql_quiet "CREATE INDEX extras_idx ON extras USING bm25 (body) WITH (text_config='english');" # Populate the memtable chains without tripping bulk-load # auto-spill. One INSERT per autocommit statement keeps each # transaction well under the bulk-load threshold, so the rows # accumulate in the on-disk memtable chain instead of # spilling to a segment. log "Inserting 500 rows per table in single-statement transactions..." { for i in $(seq 1 500); do echo "INSERT INTO docs (body) VALUES ('doc ${i} "\ "alpha beta gamma delta epsilon zeta');" echo "INSERT INTO extras (body) VALUES ('extra ${i} "\ "alpha beta gamma delta epsilon zeta');" done } | "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" >/dev/null 2>&1 for idx in docs_idx extras_idx; do local pre pre=$(sql "SELECT ( COALESCE(sum(n_records), 0) = 500 )::int FROM bm25_memtable_chain('${idx}')") if [ "${pre}" != "1" ]; then error "Pre-shutdown: ${idx} chain not populated as expected" fi done log "Pre-shutdown: both chains hold 500 entries (good)" # Long-running backend that has touched BOTH indexes so both # appear in local_state_cache and the shutdown hook's hash-loop # has more than one entry to iterate over. pg_sleep keeps the # backend alive until pg_ctl stop SIGTERMs it. "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " SELECT count(*) FROM ( SELECT 1 FROM docs ORDER BY body <@> to_bm25query('alpha', 'docs_idx') LIMIT 1 ) s; SELECT count(*) FROM ( SELECT 1 FROM extras ORDER BY body <@> to_bm25query('alpha', 'extras_idx') LIMIT 1 ) s; SELECT pg_sleep(60); " >/dev/null 2>&1 & SLEEPER_PID=$! sleep 2 # let the backend attach to both indexes # Clean shutdown. Postmaster sends SIGTERM to the sleeping # backend → die handler fires → FATAL → proc_exit(1) → our # before_shmem_exit hook walks local_state_cache and spills # both indexes. log "Clean-shutting down postgres (hook should fire on the live backend)..." "${PGBINDIR}/pg_ctl" stop -D "${DATA_DIR}" -m fast -w \ >/dev/null stop_sleeper # Restart. "${PGBINDIR}/pg_ctl" start -D "${DATA_DIR}" -l "${LOGFILE}" \ -w -t 30 >/dev/null sleep 1 # Both chains must be drained and both indexes queryable. We # explicitly do not trigger recovery by querying the index # first — bm25_memtable_chain reads the on-disk metapage and # chain pages directly, so we can observe the post-spill # state without any in-process bootstrap. for idx in docs_idx extras_idx; do local post post=$(sql "SELECT ( COALESCE(sum(n_records), 0) = 0 )::int FROM bm25_memtable_chain('${idx}')") if [ "${post}" != "1" ]; then local summary summary=$(sql "SELECT bm25_summarize_index('${idx}')") error "Post-restart: ${idx} chain not drained. Summary: ${summary}" fi done log "Post-restart: both chains are empty (spill happened for each)" for pair in "docs:docs_idx" "extras:extras_idx"; do local tbl="${pair%:*}" local idx="${pair#*:}" local count count=$(sql "SELECT count(*) FROM ( SELECT 1 FROM ${tbl} ORDER BY body <@> to_bm25query('alpha', '${idx}') LIMIT 1000 ) s") if [ "${count}" != "500" ]; then error "Post-restart: ${idx} returned ${count} (expected 500)" fi done log "Post-restart: both indexes return all 500 rows (good)" # A targeted backend termination must not wait for relation or # per-index locks from before_shmem_exit. Hold AccessExclusiveLock # with an uncommitted DROP, then terminate a backend whose local # cache contains an index with a spillable chain. sql_quiet "CREATE TABLE blocked_docs ( id serial PRIMARY KEY, body text NOT NULL);" sql_quiet "CREATE INDEX blocked_docs_idx ON blocked_docs USING bm25(body) WITH (text_config='english', compaction='inline');" { for i in $(seq 1 500); do echo "INSERT INTO blocked_docs (body) VALUES ('blocked ${i} alpha beta gamma delta epsilon');" done } | "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" >/dev/null 2>&1 local blocked_pre blocked_pre=$(sql " SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('blocked_docs_idx');") [ "${blocked_pre}" = "500" ] || error "termination test chain was not populated" local target_fifo="${DATA_DIR}/shutdown_target.fifo" mkfifo "${target_fifo}" PGAPPNAME=pgts-shutdown-target \ "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" <"${target_fifo}" >/dev/null 2>&1 & TARGET_PID=$! exec 3>"${target_fifo}" printf '%s\n' \ "SELECT count(*) FROM ( SELECT 1 FROM blocked_docs ORDER BY body <@> to_bm25query( 'alpha', 'blocked_docs_idx') LIMIT 1 ) ranked;" \ "SELECT pg_sleep(60);" >&3 local target_backend= for _ in $(seq 1 100); do target_backend=$(sql " SELECT pid FROM pg_stat_activity WHERE application_name = 'pgts-shutdown-target' AND wait_event = 'PgSleep';") [ -n "${target_backend}" ] && break sleep 0.1 done [ -n "${target_backend}" ] || error "shutdown target did not attach to the index" PGAPPNAME=pgts-shutdown-blocker \ "${PGBINDIR}/psql" -h "${TEST_HOST}" -p "${TEST_PORT}" \ -d "${TEST_DB}" -c " BEGIN; DROP INDEX blocked_docs_idx; SELECT pg_sleep(60); " >/dev/null 2>&1 & BLOCKER_PID=$! local blocker_backend= for _ in $(seq 1 100); do blocker_backend=$(sql " SELECT a.pid FROM pg_locks l JOIN pg_stat_activity a USING (pid) WHERE a.application_name = 'pgts-shutdown-blocker' AND l.relation = 'blocked_docs_idx'::regclass AND l.mode = 'AccessExclusiveLock' AND l.granted;") [ -n "${blocker_backend}" ] && break sleep 0.1 done if [ -z "${blocker_backend}" ]; then local blocker_locks blocker_locks=$(sql " SELECT COALESCE(string_agg( l.mode || ':' || l.granted::text, ','), 'none') FROM pg_locks l JOIN pg_stat_activity a USING (pid) WHERE a.application_name = 'pgts-shutdown-blocker' AND l.relation = 'blocked_docs_idx'::regclass;") error "relation blocker did not acquire the index lock locks: ${blocker_locks}" fi local terminated terminated=$(sql " SELECT pg_terminate_backend(${target_backend}, 3000);") if [ "${terminated}" != "t" ]; then error "terminated backend blocked in shutdown spill maintenance" fi exec 3>&- wait "${TARGET_PID}" 2>/dev/null || true sql "SELECT pg_terminate_backend(${blocker_backend}, 3000);" >/dev/null wait "${BLOCKER_PID}" 2>/dev/null || true local blocked_post blocked_post=$(sql " SELECT COALESCE(sum(n_records), 0) FROM bm25_memtable_chain('blocked_docs_idx');") [ "${blocked_post}" = "500" ] || error "blocked shutdown spill unexpectedly changed the chain" log "Targeted backend termination skipped a blocked shutdown spill" log "=== PASSED ===" } main "$@" timescale-pg_textsearch-b63fe58/test/scripts/standalone_snapshot.sh000077500000000000000000000370741525762047700260500ustar00rootroot00000000000000#!/bin/bash # # Standalone scoring caches IDF across heap rows. Cache-hit rows must not # recapture every published segment root. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "${SCRIPT_DIR}/../.." && pwd)" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG}" --bindir)" export PATH="${PGBINDIR}:${PATH}" TEST_PORT=55462 TEST_DB=standalone_snapshot_test DATA_DIR="${SCRIPT_DIR}/../tmp_standalone_snapshot" SOCKET_DIR="/tmp/pgts_standalone_snapshot" LOGFILE="${DATA_DIR}/postgres.log" cleanup() { local exit_code=$? trap - EXIT INT TERM if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" "${SOCKET_DIR}" exit "${exit_code}" } trap cleanup EXIT INT TERM rm -rf "${DATA_DIR}" "${SOCKET_DIR}" mkdir -p "${DATA_DIR}" "${SOCKET_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >>"${DATA_DIR}/postgresql.conf" </dev/null; then cat "${LOGFILE}" >&2 exit 1 fi createdb -h "${SOCKET_DIR}" -p "${TEST_PORT}" "${TEST_DB}" PSQL=( psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -qAt -v ON_ERROR_STOP=1 ) "${PSQL[@]}" <<'SQL' >/dev/null CREATE EXTENSION pg_textsearch; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE docs ( id bigint GENERATED ALWAYS AS IDENTITY PRIMARY KEY, body text NOT NULL ); CREATE INDEX docs_bm25 ON docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); DO $$ BEGIN FOR batch IN 1..8 LOOP INSERT INTO docs (body) SELECT CASE WHEN batch = 8 AND gs IN (49, 50) THEN ( SELECT string_agg( 'cacheterm' || term_no::text, ' ') FROM generate_series(1, 65) term_no ) ELSE 'haystack standalone document ' || batch || ' ' || gs END FROM generate_series(1, 50) gs; PERFORM bm25_spill_index('docs_bm25'); END LOOP; END $$; SQL graph=$( "${PSQL[@]}" -c \ "SELECT bm25_level_counts('docs_bm25'::regclass)::text;" ) if [ "${graph}" != "{8,0,0,0,0,0,0,0}" ]; then echo "standalone snapshot fixture built graph ${graph}, expected eight L0 segments" \ >&2 exit 1 fi matches=$( "${PSQL[@]}" <<'SQL' SET enable_indexscan = off; SET enable_bitmapscan = off; SET jit = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 1; SELECT string_agg('cacheterm' || term_no::text, ' ') AS query_text FROM generate_series(1, 65) term_no \gset SELECT count(*) FROM docs WHERE body <@> to_bm25query(:'query_text', 'docs_bm25') < 0; SQL ) matches="$(head -1 <<<"${matches}")" if [ "${matches}" != "2" ]; then echo "standalone scoring returned ${matches}/2 rows" >&2 exit 1 fi snapshot_count=$( grep -c \ 'pg_textsearch segment graph snapshot pause at after-unlock' \ "${LOGFILE}" || true ) if [ "${snapshot_count}" -ne 1 ]; then echo "standalone scoring captured ${snapshot_count} graph snapshots; expected 1" \ >&2 exit 1 fi # Capturing exactly once says nothing about what the single root walk # aggregated. A walk that undercounts segment document frequency still # caches every term, emits one marker and returns negative scores, so # compare the standalone scores against index scan scores over the same # eight-segment graph. score_matches=$( "${PSQL[@]}" <<'SQL' SET jit = off; SELECT string_agg('cacheterm' || term_no::text, ' ') AS query_text FROM generate_series(1, 65) term_no \gset CREATE TEMP TABLE idx_scores AS SELECT id, round((body <@> to_bm25query(:'query_text', 'docs_bm25'))::numeric, 6) AS score FROM docs ORDER BY body <@> to_bm25query(:'query_text', 'docs_bm25') LIMIT 2; SET enable_indexscan = off; SET enable_bitmapscan = off; CREATE TEMP TABLE standalone_scores AS SELECT id, round((body <@> to_bm25query(:'query_text', 'docs_bm25'))::numeric, 6) AS score FROM docs WHERE body <@> to_bm25query(:'query_text', 'docs_bm25') < 0; SELECT count(*) FROM idx_scores JOIN standalone_scores USING (id, score); SQL ) score_matches="$(tail -1 <<<"${score_matches}")" if [ "${score_matches}" != "2" ]; then echo "standalone scores matched index scan scores for ${score_matches}/2 rows" \ >&2 exit 1 fi "${PSQL[@]}" <<'SQL' >/dev/null CREATE TABLE inherited_docs ( id bigint NOT NULL, body text NOT NULL ); CREATE TABLE inherited_docs_child () INHERITS (inherited_docs); INSERT INTO inherited_docs_child (id, body) VALUES (1, 'inheritcache alpha'), (2, 'inheritcache beta'); CREATE INDEX inherited_docs_bm25 ON inherited_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); CREATE INDEX inherited_docs_child_bm25 ON inherited_docs_child USING bm25(body) WITH (text_config = 'english', compaction = 'off'); SQL inherit_snapshot_before=$( grep -c \ 'pg_textsearch segment graph snapshot pause at after-unlock' \ "${LOGFILE}" || true ) inherit_matches=$( "${PSQL[@]}" <<'SQL' SET enable_indexscan = off; SET enable_bitmapscan = off; SET jit = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 1; SELECT count(*) FROM inherited_docs WHERE body <@> to_bm25query('inheritcache', 'inherited_docs_bm25') < 0; SQL ) inherit_matches="$(head -1 <<<"${inherit_matches}")" if [ "${inherit_matches}" != "2" ]; then echo "inherited standalone scoring returned ${inherit_matches}/2 rows" >&2 exit 1 fi inherit_snapshot_after=$( grep -c \ 'pg_textsearch segment graph snapshot pause at after-unlock' \ "${LOGFILE}" || true ) inherit_snapshot_count=$((inherit_snapshot_after - inherit_snapshot_before)) if [ "${inherit_snapshot_count}" -ne 2 ]; then echo "inherited scoring captured ${inherit_snapshot_count} snapshots; expected parent and child" \ >&2 exit 1 fi # A committed INSERT must not split an Incremental Sort tie group or # change standalone scores, including a term first scored after the INSERT. "${PSQL[@]}" <<'SQL' CREATE EXTENSION dblink; CREATE TABLE stable_docs (id int PRIMARY KEY, body text); CREATE INDEX stable_docs_idx ON stable_docs USING bm25(body) WITH (text_config = 'simple'); CREATE SEQUENCE stable_row; CREATE FUNCTION stable_insert_at(n bigint) RETURNS boolean LANGUAGE plpgsql VOLATILE AS $$ BEGIN IF nextval('stable_row') = n THEN PERFORM dblink_exec( format('host=%s port=%s dbname=%s user=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database(), current_user), 'INSERT INTO stable_docs VALUES (4001, ''beta'')'); END IF; RETURN true; END $$; SQL for use_cache in off on; do for use_index in off on; do for insert_at in 0 1 500 1500; do "${PSQL[@]}" -v use_cache="${use_cache}" \ -v use_index="${use_index}" -v insert_at="${insert_at}" \ <<'SQL' >/dev/null SET jit = off; SET statement_timeout = '30s'; SET pg_textsearch.memtable_cache_enabled = :'use_cache'; SET client_min_messages = warning; SET enable_indexscan = :'use_index'; SELECT set_config('enable_seqscan', CASE WHEN :'use_index' = 'on' THEN 'off' ELSE 'on' END, false); TRUNCATE stable_docs; INSERT INTO stable_docs SELECT g, 'alpha beta' FROM generate_series(3000, 1, -1) g; SELECT setval('stable_row', 1, false); CREATE TEMP TABLE stable_result AS SELECT array_agg(id ORDER BY id) AS ids FROM ( SELECT id, body <@> to_bm25query('alpha', 'stable_docs_idx') AS score FROM stable_docs WHERE stable_insert_at(:insert_at) ORDER BY score, id LIMIT 10 ) s; DO $$ BEGIN IF (SELECT ids FROM stable_result) IS DISTINCT FROM ARRAY[1,2,3,4,5,6,7,8,9,10] THEN RAISE EXCEPTION 'concurrent INSERT changed top-k: %', (SELECT ids FROM stable_result); END IF; END $$; SQL done done done "${PSQL[@]}" <<'SQL' >/dev/null SET client_min_messages = warning; SET enable_indexscan = off; SET jit = off; SET statement_timeout = '30s'; TRUNCATE stable_docs; INSERT INTO stable_docs SELECT g, 'alpha beta' FROM generate_series(3000, 1, -1) g; SELECT setval('stable_row', 1, false); CREATE TEMP TABLE late_term_result AS SELECT count(DISTINCT body <@> to_bm25query( CASE WHEN id > 2501 THEN 'alpha' ELSE 'beta' END, 'stable_docs_idx')) AS scores FROM stable_docs WHERE stable_insert_at(500); DO $$ BEGIN IF (SELECT scores FROM late_term_result) <> 1 THEN RAISE EXCEPTION 'late term observed different corpus statistics'; END IF; END $$; -- A prepared execution must refresh statistics, not retain its last run. PREPARE stable_score AS SELECT body <@> to_bm25query('alpha', 'stable_docs_idx') AS score FROM stable_docs WHERE id = 1; EXECUTE stable_score \gset before_ INSERT INTO stable_docs VALUES (4002, 'gamma'); EXECUTE stable_score \gset after_ SELECT :'before_score'::float8 <> :'after_score'::float8 AS refreshed \gset \if :refreshed \else DO $$ BEGIN RAISE EXCEPTION 'prepared execution retained stale statistics'; END $$; \endif -- A suspended cursor keeps its generation across FETCH and nested SQL. BEGIN; DECLARE stable_cursor CURSOR FOR SELECT body <@> to_bm25query('alpha', 'stable_docs_idx') AS score FROM stable_docs WHERE id <= 3000; FETCH stable_cursor \gset first_ SELECT dblink_exec( format('host=%s port=%s dbname=%s user=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database(), current_user), 'INSERT INTO stable_docs VALUES (4003, ''gamma'')'); SELECT body <@> to_bm25query('alpha', 'stable_docs_idx') FROM stable_docs WHERE id = 1; FETCH stable_cursor \gset next_ SELECT :'first_score'::float8 = :'next_score'::float8 AS stable \gset \if :stable \else DO $$ BEGIN RAISE EXCEPTION 'cursor statistics changed across FETCH'; END $$; \endif CLOSE stable_cursor; COMMIT; -- An error in nested execution must not leave a dangling snapshot cache. DO $$ BEGIN FOR attempt IN 1..3 LOOP BEGIN PERFORM body <@> to_bm25query('alpha', 'stable_docs_idx') FROM stable_docs WHERE id = 1; RAISE EXCEPTION 'discard this execution'; EXCEPTION WHEN raise_exception THEN NULL; END; IF NOT EXISTS ( SELECT 1 FROM stable_docs ORDER BY body <@> to_bm25query('alpha', 'stable_docs_idx') LIMIT 1 ) THEN RAISE EXCEPTION 'scoring failed after subtransaction rollback'; END IF; END LOOP; END $$; SQL for use_cache in off on; do "${PSQL[@]}" -v use_cache="${use_cache}" <<'SQL' >/dev/null SET client_min_messages = warning; SET pg_textsearch.memtable_cache_enabled = :'use_cache'; CREATE TABLE rewrite_docs (body text); CREATE INDEX rewrite_idx ON rewrite_docs USING bm25(body) WITH (text_config = 'simple'); INSERT INTO rewrite_docs VALUES ('alpha'), ('beta'); -- Simple PL/pgSQL expressions share the outer SELECT's executor context. CREATE FUNCTION rewrite_and_score() RETURNS void LANGUAGE plpgsql VOLATILE AS $$ DECLARE q bm25query := to_bm25query('alpha', 'rewrite_idx'); score float8; n integer; BEGIN score := 'alpha'::text <@> q; FOR n IN 1..3 LOOP REINDEX INDEX rewrite_idx; score := 'alpha'::text <@> q; IF abs(score + ln((2 * n + 1)::float8 / 1.5)) > 0.000001 THEN RAISE EXCEPTION 'stale score after REINDEX: %', score; END IF; INSERT INTO rewrite_docs VALUES ('beta'), ('beta'); END LOOP; TRUNCATE rewrite_docs; INSERT INTO rewrite_docs VALUES ('beta'); score := 'alpha'::text <@> q; IF score <> 0 THEN RAISE EXCEPTION 'stale score after TRUNCATE: %', score; END IF; END $$; SELECT rewrite_and_score(); DROP FUNCTION rewrite_and_score(); -- Equal-sized replacement records reuse the same cache endpoint, but not -- the same relation file. Rollback must reject the replacement's cache. SET enable_seqscan = off; BEGIN; TRUNCATE rewrite_docs; INSERT INTO rewrite_docs VALUES ('zeta'); SELECT body FROM rewrite_docs ORDER BY body <@> to_bm25query('zeta', 'rewrite_idx') LIMIT 1; ROLLBACK; DO $$ DECLARE hits int; BEGIN SELECT count(*) INTO hits FROM ( SELECT 1 FROM rewrite_docs ORDER BY body <@> to_bm25query('beta', 'rewrite_idx') LIMIT 1 ) s; IF hits <> 1 THEN RAISE EXCEPTION 'rollback reused the replacement file cache'; END IF; END $$; DROP TABLE rewrite_docs; SQL done "${PSQL[@]}" <<'SQL' >/dev/null SET client_min_messages = warning; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.memtable_pages_threshold = 0; SET enable_seqscan = off; CREATE TABLE rollback_docs (id int, body text); CREATE INDEX rollback_idx ON rollback_docs USING bm25(body) WITH (text_config = 'simple', compaction = 'off'); -- Equal-sized tails in different relfiles must not share a cache hit. BEGIN; REINDEX INDEX rollback_idx; INSERT INTO rollback_docs SELECT g, 'bravoterm' FROM generate_series(41, 60) g; SELECT count(*) = 20 AS warmed FROM ( SELECT id FROM rollback_docs ORDER BY body <@> to_bm25query('bravoterm', 'rollback_idx') ) q \gset \if :warmed \else DO $$ BEGIN RAISE EXCEPTION 'replacement cache query did not return 20 rows'; END $$; \endif ROLLBACK; INSERT INTO rollback_docs SELECT g, 'alphaterm' FROM generate_series(21, 40) g; SELECT count(*) = 20 AS restored_hits FROM ( SELECT id FROM rollback_docs ORDER BY body <@> to_bm25query('alphaterm', 'rollback_idx') ) q \gset \if :restored_hits \else DO $$ BEGIN RAISE EXCEPTION 'stale cache after REINDEX rollback'; END $$; \endif DROP TABLE rollback_docs; SQL startup_oid=$( "${PSQL[@]}" <<'SQL' SET client_min_messages = warning; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.memtable_pages_threshold = 0; SET enable_seqscan = off; CREATE TABLE startup_docs (id int, body text); CREATE INDEX startup_idx ON startup_docs USING bm25(body) WITH (text_config = 'simple'); INSERT INTO startup_docs SELECT g, 'alpha beta gamma' FROM generate_series(1, 5000) g; INSERT INTO startup_docs VALUES (5001, 'needle'); SET pg_textsearch.memtable_cache_enabled = off; DO $$ DECLARE plan json; buffers int; pages int; BEGIN EXECUTE $q$EXPLAIN (ANALYZE, BUFFERS, FORMAT JSON, TIMING OFF) SELECT id FROM startup_docs ORDER BY body <@> to_bm25query('needle', 'startup_idx') LIMIT 1$q$ INTO plan; buffers := (plan->0->'Plan'->>'Shared Hit Blocks')::int + (plan->0->'Plan'->>'Shared Read Blocks')::int; pages := pg_relation_size('startup_idx') / current_setting('block_size')::int; IF buffers > pages + 16 THEN RAISE EXCEPTION 'memtable startup used % buffers for % pages', buffers, pages; END IF; END $$; SET pg_textsearch.memtable_cache_enabled = on; SET pg_textsearch.log_cache_state = on; SELECT id FROM startup_docs ORDER BY body <@> to_bm25query('needle', 'startup_idx') LIMIT 1; SELECT 'startup_idx'::regclass::oid; SQL ) startup_oid="$(tail -1 <<<"${startup_oid}")" opens=$(grep -c "cache_source: opened (oid=${startup_oid}," \ "${LOGFILE}" || true) if [ "${opens}" -ne 1 ]; then echo "first ranked pass opened ${opens} memtable sources; expected 1" >&2 exit 1 fi echo "Standalone snapshot test passed" timescale-pg_textsearch-b63fe58/test/scripts/standby_conflict_output.sh000066400000000000000000000005031525762047700267260ustar00rootroot00000000000000#!/bin/bash snapshot_reader_conflict_output_is_valid() { local output=$1 if grep -Eqi \ 'invalid (segment|memtable|page|magic)|magic mismatch|index corrupted|data corrupted' \ "${output}"; then return 1 fi grep -Fq "canceling statement due to conflict with recovery" "${output}" } timescale-pg_textsearch-b63fe58/test/scripts/standby_reclaim.sh000077500000000000000000001442301525762047700251320ustar00rootroot00000000000000#!/bin/bash # # Start a ranked cursor on a hot standby's old segment graph after compaction # has finished its unlocked build but before publication. # hot_standby_feedback must keep the displaced source pages parked until the # cursor and snapshot end. A second case disconnects the standby and verifies # that stock recovery-conflict WAL cancels the old reader before page reuse. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PRIMARY_PORT=55438 STANDBY_PORT=55439 TEST_DB=standby_reclaim_test PRIMARY_DIR="${SCRIPT_DIR}/../tmp_standby_reclaim_primary" STANDBY_DIR="${SCRIPT_DIR}/../tmp_standby_reclaim_standby" # Avoid Unix-socket path limits in deeply nested worktrees. REPL_HOST=127.0.0.1 REPL_SOCKET_DIR= # shellcheck source=replication_lib.sh source "${SCRIPT_DIR}/replication_lib.sh" # shellcheck source=standby_conflict_output.sh source "${SCRIPT_DIR}/standby_conflict_output.sh" READER_PID= READER_BACKEND_PID= READER_OPEN=false HELD_FEEDBACK_XMIN= COMPACTOR_PID= COMPACTOR_BACKEND_PID= SPILLER_PID= SPILLER_BACKEND_PID= SNAPSHOT_READER_PID= SNAPSHOT_READER_BACKEND_PID= wait_for_child_exit() { local pid=$1 local attempts=$2 for _ in $(seq 1 "${attempts}"); do if ! kill -0 "${pid}" 2>/dev/null; then # The bounded poll established that the child exited; this wait # only reaps its already-available status. wait "${pid}" 2>/dev/null || true return 0 fi sleep 0.1 done return 1 } reader_close() { if [ "${READER_OPEN}" != "true" ]; then return fi exec 9>&- || true exec 8<&- || true if ! wait_for_child_exit "${READER_PID}" 50; then warn "Ranked reader PID ${READER_PID} did not exit after EOF in 5s; \ sending SIGTERM" kill -TERM "${READER_PID}" 2>/dev/null || true if ! wait_for_child_exit "${READER_PID}" 50; then warn "Ranked reader PID ${READER_PID} ignored SIGTERM for 5s; \ sending SIGKILL (state: $(ps -o pid=,stat=,cmd= -p "${READER_PID}" \ 2>/dev/null || echo unavailable))" kill -9 "${READER_PID}" 2>/dev/null || true if ! wait_for_child_exit "${READER_PID}" 50; then warn "Ranked reader PID ${READER_PID} still exists 5s after \ SIGKILL (state: $(ps -o pid=,stat=,cmd= -p "${READER_PID}" \ 2>/dev/null || echo unavailable)); continuing bounded cleanup" fi fi fi READER_OPEN=false rm -rf "${STANDBY_DIR}/ranked_reader" } cleanup() { local status=$? set +e if [ -n "${COMPACTOR_BACKEND_PID}" ]; then primary_sql_quiet \ "SELECT injection_points_wakeup('${POINT_AFTER_RESTAMP}');" \ >/dev/null 2>&1 || true fi if [ -n "${SPILLER_BACKEND_PID}" ]; then primary_sql_quiet \ "SELECT injection_points_wakeup( '${POINT_SPILL_BEFORE_FINALIZE}');" >/dev/null 2>&1 || true fi if [ -n "${SNAPSHOT_READER_PID}" ] && kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null; then kill -TERM "${SNAPSHOT_READER_PID}" 2>/dev/null || true wait_for_child_exit "${SNAPSHOT_READER_PID}" 50 || true fi if [ -n "${COMPACTOR_PID}" ] && kill -0 "${COMPACTOR_PID}" 2>/dev/null; then kill -TERM "${COMPACTOR_PID}" 2>/dev/null || true wait_for_child_exit "${COMPACTOR_PID}" 50 || true fi if [ -n "${SPILLER_PID}" ] && kill -0 "${SPILLER_PID}" 2>/dev/null; then kill -TERM "${SPILLER_PID}" 2>/dev/null || true wait_for_child_exit "${SPILLER_PID}" 50 || true fi reader_close if [ "${status}" -eq 0 ]; then true else (exit "${status}") fi repl_cleanup } trap cleanup EXIT INT TERM POINT_AFTER_RESTAMP='pg-textsearch-compaction-after-restamp' # Start a force merge that parks at the post-restamp injection point # until release_compaction_pause() wakes it. start_paused_compaction() { local index_name=$1 local output=$2 PGAPPNAME=pgts-reclaim-compactor \ primary_sql " SELECT injection_points_set_local(); SELECT injection_points_attach('${POINT_AFTER_RESTAMP}', 'wait'); SELECT bm25_force_merge('${index_name}');" \ >"${output}" 2>&1 & COMPACTOR_PID=$! } wait_for_compaction_pause() { local output=$1 local waiting for _ in $(seq 1 300); do COMPACTOR_BACKEND_PID=$(primary_sql_quiet " SELECT pid FROM pg_stat_activity WHERE application_name = 'pgts-reclaim-compactor' AND wait_event_type = 'InjectionPoint' AND wait_event = '${POINT_AFTER_RESTAMP}' LIMIT 1;" 2>/dev/null || true) if [[ "${COMPACTOR_BACKEND_PID}" =~ ^[0-9]+$ ]]; then log "Compaction reached the pre-publication pause" return 0 fi COMPACTOR_BACKEND_PID= if ! kill -0 "${COMPACTOR_PID}" 2>/dev/null; then error "Compactor exited before the pre-publication pause: \ $(cat "${output}" 2>/dev/null || echo no output)" fi sleep 0.1 done error "Timed out 30s waiting for pre-publication pause" } release_compaction_pause() { primary_sql_quiet \ "SELECT injection_points_wakeup('${POINT_AFTER_RESTAMP}');" \ >/dev/null || error "Could not wake ${POINT_AFTER_RESTAMP}" COMPACTOR_BACKEND_PID= } wait_for_spill_pause() { local output=$1 for _ in $(seq 1 300); do SPILLER_BACKEND_PID=$(primary_sql_quiet " SELECT pid FROM pg_stat_activity WHERE application_name = 'pgts-retire-horizon-spiller' AND wait_event_type = 'InjectionPoint' AND wait_event = '${POINT_SPILL_BEFORE_FINALIZE}' LIMIT 1;" 2>/dev/null || true) if [[ "${SPILLER_BACKEND_PID}" =~ ^[0-9]+$ ]]; then log "Spill reached the pre-finalize gate" return 0 fi SPILLER_BACKEND_PID= if ! kill -0 "${SPILLER_PID}" 2>/dev/null; then error "Spiller exited before the pre-finalize pause: \ $(cat "${output}" 2>/dev/null || echo no output)" fi sleep 0.1 done error "Timed out 30s waiting for pre-finalize spill gate" } # Killing the client is not enough: the standby backend sits in a debug # pause and only notices the closed socket afterwards, which would stall # the next case's shutdown. Cancel it and wait for the backend to go. stop_snapshot_reader() { local backend=$1 kill -TERM "${SNAPSHOT_READER_PID}" 2>/dev/null || true if [ -n "${backend}" ]; then standby_sql_quiet \ "SELECT pg_cancel_backend(${backend});" >/dev/null 2>&1 || true for _ in $(seq 1 600); do [ "$(standby_sql_quiet " SELECT count(*) FROM pg_stat_activity WHERE pid = ${backend};" 2>/dev/null || echo 0)" = "0" ] && break sleep 0.1 done fi wait_for_child_exit "${SNAPSHOT_READER_PID}" 50 || true SNAPSHOT_READER_PID= } wait_for_snapshot_pause() { local phase=$1 local index_oid=$2 local output=$3 local marker="pg_textsearch segment graph snapshot pause at ${phase} \ for index ${index_oid}" local logfile="${STANDBY_DIR}/log/postgres.log" for _ in $(seq 1 300); do if grep -Fq "${marker}" "${logfile}" 2>/dev/null; then log "Standby query reached segment snapshot phase ${phase}" return 0 fi if ! kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null; then wait "${SNAPSHOT_READER_PID}" 2>/dev/null || true SNAPSHOT_READER_PID= error "Standby snapshot reader exited before its pause: \ $(cat "${output}" 2>/dev/null || echo no output)" fi sleep 0.1 done error "Timed out 30s waiting for standby segment-root snapshot pause" } wait_for_compactor() { local output=$1 for _ in $(seq 1 400); do if ! kill -0 "${COMPACTOR_PID}" 2>/dev/null; then if wait "${COMPACTOR_PID}"; then COMPACTOR_PID= return 0 fi error "Compactor failed: \ $(cat "${output}" 2>/dev/null || echo no output)" fi sleep 0.1 done error "Timed out 40s waiting for compactor PID ${COMPACTOR_PID}" } wait_for_spiller() { local output=$1 for _ in $(seq 1 400); do if ! kill -0 "${SPILLER_PID}" 2>/dev/null; then if wait "${SPILLER_PID}"; then SPILLER_PID= return 0 fi error "Spiller failed: \ $(cat "${output}" 2>/dev/null || echo no output)" fi sleep 0.1 done error "Timed out 40s waiting for spiller PID ${SPILLER_PID}" } wait_for_snapshot_reader() { local output=$1 for _ in $(seq 1 700); do if ! kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null; then if wait "${SNAPSHOT_READER_PID}"; then SNAPSHOT_READER_PID= return 0 fi error "Standby snapshot reader failed: \ $(cat "${output}" 2>/dev/null || echo no output)" fi sleep 0.1 done error "Timed out 70s waiting for standby snapshot reader PID \ ${SNAPSHOT_READER_PID}" } snapshot_reader_backend_pid() { local output=$1 local deadline=$((SECONDS + 10)) local pid while ((SECONDS < deadline)); do pid=$(sed -n '1p' "${output}" 2>/dev/null || true) if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return fi sleep 0.05 done error "Standby snapshot reader did not report its backend PID" } wait_for_snapshot_reader_conflict() { local output=$1 local deadline=$((SECONDS + 45)) local status while kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null; do ((SECONDS < deadline)) || error "Old-memtable reader was not canceled by conflict WAL" sleep 0.1 done set +e wait "${SNAPSHOT_READER_PID}" status=$? set -e SNAPSHOT_READER_PID= [ "${status}" -ne 0 ] || error "Old-memtable reader completed without a recovery conflict" snapshot_reader_conflict_output_is_valid "${output}" || error "Old-memtable reader lacks scoped conflict proof or reported corruption" } reader_open() { local control_dir="${STANDBY_DIR}/ranked_reader" rm -rf "${control_dir}" mkdir -p "${control_dir}" mkfifo "${control_dir}/in" "${control_dir}/out" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" \ < "${control_dir}/in" > "${control_dir}/out" 2>&1 & READER_PID=$! exec 9>"${control_dir}/in" exec 8<"${control_dir}/out" READER_OPEN=true } reader_query() { local sql=$1 local sentinel="__standby_reclaim_$$_${RANDOM}__" local result="" local line printf '%s\n' "${sql}" >&9 printf "SELECT '%s';\n" "${sentinel}" >&9 while true; do if ! IFS= read -r -t 30 line <&8; then error "Timed out 30s waiting for ranked reader PID ${READER_PID}; \ backend=${READER_BACKEND_PID:-unknown}, partial output=${result:-none}" fi if [ "${line}" = "${sentinel}" ]; then break fi if [ -n "${result}" ]; then result="${result}"$'\n'"${line}" else result="${line}" fi done printf '%s' "${result}" } reader_expect_recovery_conflict() { local output local reader_status local backend_count=1 for _ in $(seq 1 100); do backend_count=$(standby_sql_quiet " SELECT count(*) FROM pg_stat_activity WHERE pid = ${READER_BACKEND_PID};") [ "${backend_count}" = "0" ] && break sleep 0.1 done [ "${backend_count}" = "0" ] || error "Old-graph reader was not canceled by recovery conflict WAL" printf '%s\n' "SELECT 1;" >&9 || true exec 9>&- output=$(timeout 10 cat <&8) exec 8<&- set +e wait "${READER_PID}" reader_status=$? set -e READER_OPEN=false READER_PID= rm -rf "${STANDBY_DIR}/ranked_reader" if [ "${reader_status}" -eq 0 ]; then error "Old-graph reader completed without a recovery conflict" fi if ! grep -Eq 'conflict with recovery|recovery conflict' <<<"${output}" && ! grep -Eq 'conflict with recovery|recovery conflict' \ "${STANDBY_DIR}/log/postgres.log"; then error "Old-graph reader failed without a recovery-conflict message: \ ${output:-no output}" fi } wait_for_feedback_xmin() { local xmin="" for _ in $(seq 1 60); do xmin=$(primary_sql_quiet " SELECT backend_xmin FROM pg_stat_replication WHERE state = 'streaming' AND backend_xmin IS NOT NULL LIMIT 1;") if [ -n "${xmin}" ]; then HELD_FEEDBACK_XMIN="${xmin}" log "Primary sees standby feedback xmin ${xmin}" return 0 fi sleep 0.5 done error "Timed out 30s waiting for hot-standby feedback; replication state: \ $(primary_sql_quiet " SELECT application_name, state, backend_xmin, replay_lsn FROM pg_stat_replication;")" } wait_for_feedback_release() { local xmin="" for _ in $(seq 1 60); do xmin=$(primary_sql_quiet " SELECT coalesce(backend_xmin::text, '') FROM pg_stat_replication WHERE state = 'streaming' LIMIT 1;") if [ -z "${xmin}" ] || [ "${xmin}" -gt "${HELD_FEEDBACK_XMIN}" ]; then log "Primary feedback xmin advanced from \ ${HELD_FEEDBACK_XMIN} to ${xmin:-none}" return 0 fi sleep 0.5 done error "Timed out 30s waiting for standby xmin to advance beyond \ ${HELD_FEEDBACK_XMIN}; backend_xmin=${xmin}" } assert_ranked_plan() { local port=$1 local plan plan=$(node_sql_quiet "${port}" " SET enable_seqscan = off; EXPLAIN (COSTS off) SELECT id FROM rec ORDER BY body <@> to_bm25query('alpha', 'rec_idx') LIMIT 8000;") grep -Fq "Index Scan using rec_idx on rec" <<<"${plan}" || error "Port ${port} ranked query did not use rec_idx: ${plan}" } test_disconnected_standby_conflict() { local graph local parked log "Case: reclaim WAL conflicts with disconnected standby readers..." cat >> "${STANDBY_DIR}/postgresql.conf" </dev/null primary_sql " CREATE TABLE conflict_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO conflict_rec SELECT g, 'delta epsilon disconnected standby document ' || g FROM generate_series(1, 4000) g; CREATE INDEX conflict_idx ON conflict_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO conflict_rec SELECT g, 'delta epsilon disconnected standby document ' || g FROM generate_series(4001, 8000) g;" >/dev/null [ "$(primary_sql_quiet \ "SELECT bm25_spill_index('conflict_idx') > 0;")" = "t" ] || error "Disconnected-standby case did not create a second L0 segment" primary_sql "CREATE TABLE conflict_flush_marker (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before disconnected cursor setup" graph=$(standby_sql_quiet \ "SELECT bm25_level_counts('conflict_idx'::regclass)::text;") [ "${graph}" = "{2,0,0,0,0,0,0,0}" ] || error "Disconnected-standby old graph is ${graph}" reader_open READER_BACKEND_PID=$(reader_query "SELECT pg_backend_pid();") reader_query "BEGIN ISOLATION LEVEL REPEATABLE READ;" >/dev/null reader_query "SET enable_seqscan = off;" >/dev/null reader_query " DECLARE held_ranked NO SCROLL CURSOR FOR SELECT id FROM conflict_rec ORDER BY body <@> to_bm25query('delta', 'conflict_idx') LIMIT 8000;" >/dev/null [ -n "$(reader_query "FETCH FORWARD 1 FROM held_ranked;")" ] || error "Disconnected-standby cursor returned no first document" cat >> "${PRIMARY_DIR}/postgresql.conf" </dev/null [ "$(primary_sql_quiet \ "SELECT count(*) FROM pg_stat_replication;")" = "0" ] || error "Standby remained connected after disabling WAL senders" primary_sql "SELECT bm25_force_merge('conflict_idx');" >/dev/null for _ in $(seq 1 32); do primary_sql_quiet "SELECT txid_current();" >/dev/null done primary_sql "VACUUM conflict_rec;" >/dev/null parked=$(primary_sql_quiet \ "SELECT bm25_pending_free_pages('conflict_idx');") [ "${parked}" = "0" ] || error "Disconnected-standby reclaim left ${parked} pages parked" cat >> "${PRIMARY_DIR}/postgresql.conf" </dev/null wait_for_standby_catchup 30 || error "Standby did not replay disconnected-standby reclaim" reader_expect_recovery_conflict log "PASS: stock WAL replay canceled the disconnected old-graph reader" } test_atomic_segment_graph_snapshot() { local actual_count actual_rows expected expected_rows graph index_oid local output="${STANDBY_DIR}/segment_snapshot_reader.out" local pre_publish_lsn replay_blocked result spilled target_lsn if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping atomic segment-root snapshot case: no injection points" return fi log "Case: standby scoring uses one atomic segment-root snapshot..." primary_sql " CREATE TABLE snapshot_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO snapshot_rec SELECT g, 'atomic snapshot alpha document ' || g FROM generate_series(1, 1000) g; CREATE INDEX snapshot_idx ON snapshot_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off');" >/dev/null for batch in $(seq 1 7); do primary_sql " INSERT INTO snapshot_rec SELECT g, 'atomic snapshot alpha document ' || g FROM generate_series( ${batch} * 1000 + 1, (${batch} + 1) * 1000) g;" >/dev/null spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('snapshot_idx') > 0;") [ "${spilled}" = "t" ] || error "Snapshot case batch ${batch} did not spill" done graph=$(primary_sql_quiet \ "SELECT bm25_level_counts('snapshot_idx'::regclass)::text;") [ "${graph}" = "{8,0,0,0,0,0,0,0}" ] || error "Expected eight-L0 snapshot graph, got ${graph}" primary_sql "CREATE TABLE snapshot_flush_before (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before snapshot compaction" index_oid=$(primary_sql_quiet \ "SELECT 'snapshot_idx'::regclass::oid;") start_paused_compaction snapshot_idx \ "${PRIMARY_DIR}/snapshot_compactor.out" wait_for_compaction_pause "${PRIMARY_DIR}/snapshot_compactor.out" primary_sql " INSERT INTO snapshot_rec SELECT g, 'atomic snapshot alpha document ' || g FROM generate_series(8001, 9000) g;" >/dev/null spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('snapshot_idx') > 0;") [ "${spilled}" = "t" ] || error "Snapshot case did not prepend the ninth L0 segment" graph=$(primary_sql_quiet \ "SELECT bm25_level_counts('snapshot_idx'::regclass)::text;") [ "${graph}" = "{9,0,0,0,0,0,0,0}" ] || error "Expected ninth prepended L0 segment, got ${graph}" primary_sql "CREATE TABLE snapshot_flush_after (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up to the nine-L0 graph" rm -f "${output}" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" >"${output}" 2>&1 <<'SQL' & SET enable_seqscan = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_before_unlock_ms = 15000; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 15000; SELECT id FROM snapshot_rec ORDER BY body <@> to_bm25query('alpha', 'snapshot_idx') LIMIT 9000; SQL SNAPSHOT_READER_PID=$! wait_for_snapshot_pause before-unlock "${index_oid}" "${output}" pre_publish_lsn=$(primary_sql_quiet \ "SELECT pg_current_wal_flush_lsn();") release_compaction_pause wait_for_compactor "${PRIMARY_DIR}/snapshot_compactor.out" primary_sql "CREATE TABLE snapshot_flush_publish (id integer);" >/dev/null target_lsn=$(primary_sql_quiet "SELECT pg_current_wal_flush_lsn();") replay_blocked=f for _ in $(seq 1 50); do replay_blocked=$(primary_sql_quiet " SELECT coalesce(bool_or( replay_lsn >= '${pre_publish_lsn}'::pg_lsn AND replay_lsn < '${target_lsn}'::pg_lsn), false) FROM pg_stat_replication;") [ "${replay_blocked}" = "t" ] && break sleep 0.1 done [ "${replay_blocked}" = "t" ] || error "Standby replay was not blocked by the metapage share lock" kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null || error "Standby snapshot reader left its pre-unlock pause early" log "Standby replay remained blocked during the pre-unlock pause" wait_for_snapshot_pause after-unlock "${index_oid}" "${output}" wait_for_standby_catchup 30 || error "Standby did not replay publication during snapshot scoring" kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null || error "Standby snapshot reader resumed before publication replay" wait_for_snapshot_reader "${output}" result=$(cat "${output}") if grep -Ev '^[0-9]+$' <<<"${result}" >/dev/null; then error "Standby mixed graph returned a non-numeric document ID" fi expected=$(primary_sql_quiet \ "SELECT id FROM snapshot_rec ORDER BY id;") expected_rows="$(printf '%s\n' "${expected}" | wc -l | tr -d ' ')" actual_count="$(printf '%s\n' "${result}" | sort -n | uniq | wc -l)" actual_rows="$(printf '%s\n' "${result}" | wc -l)" [ "${actual_rows}" = "${expected_rows}" ] || error "standby mixed graph returned ${actual_rows}/${expected_rows} rows" [ "${actual_count}" = "${expected_rows}" ] || error "standby mixed graph returned duplicate document IDs" [ "$(printf '%s\n' "${result}" | sort -n)" = "${expected}" ] || error "standby mixed graph IDs differ from the heap-derived set" log "PASS: standby scoring returned all ${expected_rows} IDs exactly once" } test_atomic_ranked_memtable_generation() { local actual_count actual_rows expected index_oid output result spilled log "Case: standby ranked scoring uses one segment+memtable generation..." primary_sql " CREATE TABLE ranked_generation_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO ranked_generation_rec SELECT g, 'ranked generation alpha base ' || g FROM generate_series(1, 1000) g; CREATE INDEX ranked_generation_idx ON ranked_generation_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO ranked_generation_rec SELECT g, 'ranked generation alpha memtable ' || g FROM generate_series(1001, 1500) g; CREATE TABLE ranked_generation_flush_before (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before ranked generation race" index_oid=$(primary_sql_quiet \ "SELECT 'ranked_generation_idx'::regclass::oid;") output="${STANDBY_DIR}/ranked_generation_reader.out" rm -f "${output}" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" >"${output}" 2>&1 <<'SQL' & SET enable_seqscan = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 15000; SELECT id FROM ranked_generation_rec ORDER BY body <@> to_bm25query('alpha', 'ranked_generation_idx') LIMIT 1500; SQL SNAPSHOT_READER_PID=$! wait_for_snapshot_pause after-unlock "${index_oid}" "${output}" spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('ranked_generation_idx') > 0;") [ "${spilled}" = "t" ] || error "Ranked generation case did not spill the captured memtable" primary_sql "CREATE TABLE ranked_generation_flush_after (id integer);" \ >/dev/null wait_for_standby_catchup 30 || error "Standby did not replay ranked generation spill" kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null || error "Ranked generation reader resumed before spill replay" wait_for_snapshot_reader "${output}" result=$(cat "${output}") if grep -Ev '^[0-9]+$' <<<"${result}" >/dev/null; then error "Ranked generation race returned a non-numeric document ID" fi expected=$(seq 1 1500) actual_rows="$(printf '%s\n' "${result}" | wc -l | tr -d ' ')" actual_count="$(printf '%s\n' "${result}" | sort -n | uniq | wc -l)" [ "${actual_rows}" = "1500" ] || error "Ranked generation race returned ${actual_rows}/1500 rows" [ "${actual_count}" = "1500" ] || error "Ranked generation race returned duplicate document IDs" [ "$(printf '%s\n' "${result}" | sort -n)" = "${expected}" ] || error "Ranked generation race omitted or invented document IDs" log "PASS: ranked standby race returned IDs 1..1500 exactly once" } test_atomic_standalone_memtable_generation() { local baseline index_oid output result spilled log "Case: standby standalone scoring uses one segment+memtable generation..." primary_sql " CREATE TABLE standalone_generation_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO standalone_generation_rec SELECT g, 'standalone generation base ' || g FROM generate_series(1, 1000) g; CREATE INDEX standalone_generation_idx ON standalone_generation_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO standalone_generation_rec SELECT g, 'raceword standalone memtable ' || g FROM generate_series(1001, 1500) g; CREATE TABLE standalone_generation_flush_before (id integer);" \ >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before standalone generation race" index_oid=$(primary_sql_quiet \ "SELECT 'standalone_generation_idx'::regclass::oid;") output="${STANDBY_DIR}/standalone_generation_reader.out" rm -f "${output}" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" >"${output}" 2>&1 <<'SQL' & SET pg_textsearch.debug_segment_graph_snapshot_pause_before_lock_ms = 15000; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 15000; SELECT round(( body <@> to_bm25query( 'raceword', 'standalone_generation_idx') )::numeric, 8) FROM standalone_generation_rec WHERE id = 1001; SQL SNAPSHOT_READER_PID=$! wait_for_snapshot_pause before-lock "${index_oid}" "${output}" spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('standalone_generation_idx') > 0;") [ "${spilled}" = "t" ] || error "Standalone generation case did not spill the first memtable" primary_sql " INSERT INTO standalone_generation_rec SELECT g, 'laterword replacement memtable ' || g FROM generate_series(1501, 1800) g; CREATE TABLE standalone_generation_flush_middle (id integer);" \ >/dev/null wait_for_standby_catchup 30 || error "Standby did not replay the first standalone spill" baseline=$(standby_sql_quiet " SELECT round(( body <@> to_bm25query( 'raceword', 'standalone_generation_idx') )::numeric, 8) FROM standalone_generation_rec WHERE id = 1001;") [ -n "${baseline}" ] && [ "${baseline}" != "0.00000000" ] || error "Standalone generation baseline score is ${baseline:-empty}" wait_for_snapshot_pause after-unlock "${index_oid}" "${output}" spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('standalone_generation_idx') > 0;") [ "${spilled}" = "t" ] || error "Standalone generation case did not spill the captured endpoint" primary_sql "CREATE TABLE standalone_generation_flush_after (id integer);" \ >/dev/null wait_for_standby_catchup 30 || error "Standby did not replay the second standalone spill" kill -0 "${SNAPSHOT_READER_PID}" 2>/dev/null || error "Standalone generation reader resumed before spill replay" wait_for_snapshot_reader "${output}" result=$(cat "${output}") [ "${result}" = "${baseline}" ] || error "Standalone generation score ${result}, expected ${baseline}" log "PASS: standalone standby race preserved score ${baseline}" } POINT_SPILL_BEFORE_FINALIZE='pg-textsearch-spill-before-finalize' test_memtable_retire_horizon_covers_prepublish_reader() { local dead_fxid index_oid output reader_xmin spill_output if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping retire-horizon case: no injection points" return fi log "Case: memtable retire horizon covers pre-publication readers..." primary_sql " CREATE TABLE retire_horizon_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO retire_horizon_rec SELECT g, 'retire horizon committed segment ' || g FROM generate_series(1, 1000) g; CREATE INDEX retire_horizon_idx ON retire_horizon_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO retire_horizon_rec SELECT g, 'retire horizon old memtable ' || g FROM generate_series(1001, 1500) g; CREATE TABLE retire_horizon_flush_before (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before retire-horizon race" index_oid=$(primary_sql_quiet \ "SELECT 'retire_horizon_idx'::regclass::oid;") spill_output="${PRIMARY_DIR}/retire_horizon_spill.out" rm -f "${spill_output}" PGAPPNAME=pgts-retire-horizon-spiller \ psql -X -tAq -v ON_ERROR_STOP=1 -p "${PRIMARY_PORT}" \ -d "${TEST_DB}" >"${spill_output}" 2>&1 < 0; SQL SPILLER_PID=$! wait_for_spill_pause "${spill_output}" primary_sql " SELECT txid_current(); CREATE TABLE retire_horizon_flush_between (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not replay the intervening transaction" output="${STANDBY_DIR}/retire_horizon_reader.out" rm -f "${output}" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" >"${output}" 2>&1 <<'SQL' & SELECT pg_backend_pid(); SET enable_seqscan = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 30000; SELECT id FROM retire_horizon_rec ORDER BY body <@> to_bm25query('retire', 'retire_horizon_idx') LIMIT 1500; SQL SNAPSHOT_READER_PID=$! SNAPSHOT_READER_BACKEND_PID=$(snapshot_reader_backend_pid "${output}") wait_for_snapshot_pause after-unlock "${index_oid}" "${output}" reader_xmin=$(standby_sql_quiet " SELECT backend_xmin FROM pg_stat_activity WHERE pid = ${SNAPSHOT_READER_BACKEND_PID};") [ -n "${reader_xmin}" ] || error "Pre-publication reader has no active snapshot xmin" primary_sql_quiet \ "SELECT injection_points_wakeup('${POINT_SPILL_BEFORE_FINALIZE}');" \ >/dev/null || error "Could not wake the pre-finalize spill point" wait_for_spiller "${spill_output}" SPILLER_BACKEND_PID= dead_fxid=$(primary_sql_quiet " SELECT min(dead_fxid) FROM bm25_memtable_dead_pages('retire_horizon_idx');") [ -n "${dead_fxid}" ] || error "Retire-horizon spill produced no DEAD pages" [ "${dead_fxid}" -ge "${reader_xmin}" ] || error "DEAD horizon ${dead_fxid} does not cover newer old-chain \ reader xmin ${reader_xmin}" stop_snapshot_reader "${SNAPSHOT_READER_BACKEND_PID}" log "PASS: DEAD horizon ${dead_fxid} covers reader xmin ${reader_xmin}" } test_disconnected_memtable_reuse_conflict() { local conflict_end_lsn conflict_records conflict_start_lsn dead_after local dead_before dead_fxid expected_ids final_ids graph index_oid output local pending reader_xmin reused spilled log "Case: reclaimed memtable reuse cancels disconnected old readers..." cat >> "${STANDBY_DIR}/postgresql.conf" </dev/null primary_sql " CREATE TABLE memreuse_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO memreuse_rec SELECT g, 'memreuse committed segment ' || g FROM generate_series(1, 1000) g; CREATE INDEX memreuse_idx ON memreuse_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO memreuse_rec SELECT g, 'memreuse retired memtable ' || g || ' ' || repeat(md5(g::text), 8) FROM generate_series(1001, 1500) g; CREATE TABLE memreuse_flush_before (id integer);" >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before memtable reuse conflict" index_oid=$(primary_sql_quiet "SELECT 'memreuse_idx'::regclass::oid;") output="${STANDBY_DIR}/memreuse_reader.out" rm -f "${output}" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" >"${output}" 2>&1 <<'SQL' & SELECT pg_backend_pid(); SET enable_seqscan = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 30000; SELECT id FROM memreuse_rec ORDER BY body <@> to_bm25query('memreuse', 'memreuse_idx') LIMIT 1500; SQL SNAPSHOT_READER_PID=$! SNAPSHOT_READER_BACKEND_PID=$(snapshot_reader_backend_pid "${output}") wait_for_snapshot_pause after-unlock "${index_oid}" "${output}" reader_xmin=$(standby_sql_quiet " SELECT backend_xmin FROM pg_stat_activity WHERE pid = ${SNAPSHOT_READER_BACKEND_PID};") [ -n "${reader_xmin}" ] || error "Old-memtable reader has no active snapshot xmin" cat >> "${PRIMARY_DIR}/postgresql.conf" </dev/null [ "$(primary_sql_quiet "SELECT count(*) FROM pg_stat_replication;")" = "0" ] || error "Standby remained connected before memtable reclaim" spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('memreuse_idx') > 0;") [ "${spilled}" = "t" ] || error "Memtable reuse case did not spill the captured chain" dead_before=$(primary_sql_quiet \ "SELECT count(*) FROM bm25_memtable_dead_pages('memreuse_idx');") [ "${dead_before}" -gt 0 ] || error "Memtable reuse case produced no DEAD pages" primary_sql " DROP TABLE IF EXISTS memreuse_dead_blocks; CREATE TABLE memreuse_dead_blocks AS SELECT blkno FROM bm25_memtable_dead_pages('memreuse_idx'); ALTER TABLE memreuse_dead_blocks ADD PRIMARY KEY (blkno);" >/dev/null dead_fxid=$(primary_sql_quiet " SELECT min(dead_fxid) FROM bm25_memtable_dead_pages('memreuse_idx');") log "Old reader xmin=${reader_xmin}; DEAD horizon=${dead_fxid}" pending=$(primary_sql_quiet \ "SELECT bm25_pending_free_pages('memreuse_idx');") [ "${pending}" = "0" ] || error "Memtable reuse case unexpectedly created segment tombstones" for _ in $(seq 1 64); do primary_sql_quiet "SELECT txid_current();" >/dev/null done conflict_start_lsn=$(primary_sql_quiet \ "SELECT pg_current_wal_insert_lsn();") primary_sql "VACUUM memreuse_rec;" >/dev/null dead_after=$(primary_sql_quiet \ "SELECT count(*) FROM bm25_memtable_dead_pages('memreuse_idx');") [ "${dead_after}" = "0" ] || error "VACUUM left ${dead_after}/${dead_before} DEAD memtable pages" primary_sql " SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO memreuse_rec SELECT g, 'memreuse replacement chain ' || g || ' ' || repeat(md5(g::text), 8) FROM generate_series(1501, 2000) g;" >/dev/null reused=$(primary_sql_quiet " SELECT count(*) FROM bm25_memtable_chain('memreuse_idx') chain JOIN memreuse_dead_blocks dead USING (blkno);") [ "${reused}" -gt 0 ] || error "Replacement memtable did not reuse a reclaimed DEAD page" conflict_end_lsn=$(primary_sql_quiet \ "SELECT pg_current_wal_insert_lsn();") conflict_records=$(pg_waldump -p "${PRIMARY_DIR}/pg_wal" \ -s "${conflict_start_lsn}" -e "${conflict_end_lsn}" -r Btree | grep -c "REUSE_PAGE" || true) [ "${conflict_records}" -ge "${dead_before}" ] || error "WAL contains ${conflict_records}/${dead_before} memtable reuse conflicts" log "WAL contains ${conflict_records} memtable reuse conflict records" cat >> "${PRIMARY_DIR}/postgresql.conf" </dev/null wait_for_snapshot_reader_conflict "${output}" wait_for_standby_catchup 30 || error "Standby did not catch up after memtable reuse conflict" graph=$(standby_sql_quiet \ "SELECT bm25_level_counts('memreuse_idx'::regclass)::text;") [ "${graph}" = "{2,0,0,0,0,0,0,0}" ] || error "Memtable reuse case changed segment graph unexpectedly: ${graph}" expected_ids=$(seq 1 2000 | paste -sd, -) final_ids=$(standby_sql_quiet " SET enable_seqscan = off; SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM memreuse_rec ORDER BY body <@> to_bm25query('memreuse', 'memreuse_idx') LIMIT 2000 ) ranked;" | tail -n 1) [ "${final_ids}" = "${expected_ids}" ] || error "Standby IDs are wrong after memtable conflict and reuse" log "PASS: conflict WAL canceled old memtable reader before ${reused} reused pages" } test_ranked_promotion_pins_recovery_mode() { local actual_count actual_rows expected index_oid output result spilled log "Case: ranked scoring pins recovery mode before its snapshot..." primary_sql " CREATE TABLE promotion_generation_rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO promotion_generation_rec SELECT g, 'promotion generation alpha segment ' || g FROM generate_series(1, 1000) g; CREATE INDEX promotion_generation_idx ON promotion_generation_rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO promotion_generation_rec SELECT g, 'promotion generation alpha memtable ' || g FROM generate_series(1001, 1500) g; CREATE TABLE promotion_generation_flush_before (id integer);" \ >/dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before promotion generation race" index_oid=$(primary_sql_quiet \ "SELECT 'promotion_generation_idx'::regclass::oid;") output="${STANDBY_DIR}/promotion_generation_reader.out" rm -f "${output}" psql -X -tAq -v ON_ERROR_STOP=1 -p "${STANDBY_PORT}" \ -d "${TEST_DB}" >"${output}" 2>&1 <<'SQL' & SELECT pg_backend_pid(); SET enable_seqscan = off; SET pg_textsearch.debug_segment_graph_snapshot_pause_ms = 15000; SELECT id FROM promotion_generation_rec ORDER BY body <@> to_bm25query('alpha', 'promotion_generation_idx') LIMIT 1500; SQL SNAPSHOT_READER_PID=$! wait_for_snapshot_pause after-unlock "${index_oid}" "${output}" spilled=$(primary_sql_quiet \ "SELECT bm25_spill_index('promotion_generation_idx') > 0;") [ "${spilled}" = "t" ] || error "Promotion generation case did not spill the captured memtable" primary_sql "CREATE TABLE promotion_generation_flush_after (id integer);" \ >/dev/null wait_for_standby_catchup 30 || error "Standby did not replay promotion generation spill" pg_ctl promote -D "${STANDBY_DIR}" -w >/dev/null [ "$(standby_sql_quiet "SELECT pg_is_in_recovery();")" = "f" ] || error "Standby did not promote during ranked snapshot pause" wait_for_snapshot_reader "${output}" result=$(tail -n +2 "${output}") if grep -Ev '^[0-9]+$' <<<"${result}" >/dev/null; then error "Promotion generation race returned a non-numeric ID" fi expected=$(seq 1 1500) actual_rows="$(printf '%s\n' "${result}" | wc -l | tr -d ' ')" actual_count="$(printf '%s\n' "${result}" | sort -n | uniq | wc -l)" [ "${actual_rows}" = "1500" ] && [ "${actual_count}" = "1500" ] || error "Promotion generation race returned ${actual_rows} rows, ${actual_count} unique" [ "$(printf '%s\n' "${result}" | sort -n)" = "${expected}" ] || error "Promotion generation race omitted or duplicated IDs" log "PASS: promotion race returned IDs 1..1500 exactly once" } test_late_ranked_cursor_reclaim() { local plan first_id remaining local parked_before_vacuum parked_after_vacuum drained local expected_ids primary_ids standby_ids local held_ids held_ids_sorted held_id_count duplicate_ids if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping late ranked cursor reclaim: no injection points" return fi start_paused_compaction rec_idx "${PRIMARY_DIR}/compactor.out" wait_for_compaction_pause "${PRIMARY_DIR}/compactor.out" # Advance the primary and standby XID horizons while the old graph is # still published. The cursor opened below must therefore be protected # by the publication-time reclaim stamp, not the earlier build start. for xid_id in $(seq 1 32); do primary_sql_quiet " INSERT INTO reclaim_xid_advance VALUES (${xid_id});" >/dev/null done wait_for_standby_catchup 30 || error "Standby did not catch up before late ranked cursor setup" reader_open READER_BACKEND_PID=$(reader_query "SELECT pg_backend_pid();") reader_query "BEGIN ISOLATION LEVEL REPEATABLE READ;" >/dev/null reader_query "SET enable_seqscan = off;" >/dev/null plan=$(reader_query " EXPLAIN (COSTS off) SELECT id FROM rec ORDER BY body <@> to_bm25query('alpha', 'rec_idx') LIMIT 8000;") grep -Fq "Index Scan using rec_idx on rec" <<<"${plan}" || error "Held standby cursor would not use rec_idx: ${plan}" reader_query " DECLARE held_ranked NO SCROLL CURSOR FOR SELECT id FROM rec ORDER BY body <@> to_bm25query('alpha', 'rec_idx') LIMIT 8000;" >/dev/null first_id=$(reader_query "FETCH FORWARD 1 FROM held_ranked;") [ -n "${first_id}" ] || error "Held ranked cursor returned no first document" graph=$(standby_sql_quiet \ "SELECT bm25_level_counts('rec_idx'::regclass)::text;") [ "${graph}" = "{2,0,0,0,0,0,0,0}" ] || error "Late ranked cursor did not start on the old graph: ${graph}" wait_for_feedback_xmin log "Late ranked cursor PID ${READER_BACKEND_PID} is open on the old graph" release_compaction_pause wait_for_compactor "${PRIMARY_DIR}/compactor.out" parked_before_vacuum=$(primary_sql_quiet \ "SELECT bm25_pending_free_pages('rec_idx');") case "${parked_before_vacuum}" in ''|*[!0-9]*) error "Invalid parked count '${parked_before_vacuum}'" ;; esac [ "${parked_before_vacuum}" -gt 0 ] || error "Compaction published without parking displaced pages" # Assign and commit an xid so synchronous commit flushes the publication # WAL, then make a reclaim attempt while feedback pins the old snapshot. primary_sql "CREATE TABLE reclaim_flush_marker (id integer);" >/dev/null primary_sql "VACUUM rec;" >/dev/null parked_after_vacuum=$(primary_sql_quiet \ "SELECT bm25_pending_free_pages('rec_idx');") [ "${parked_after_vacuum}" = "${parked_before_vacuum}" ] || error "VACUUM reclaimed old-graph pages while ranked cursor was open: \ before=${parked_before_vacuum} after=${parked_after_vacuum}" # PostgreSQL 18 can hold publication redo behind the cursor's buffer # pin. Fetch the old graph to completion before requiring replay to # advance; waiting for catch-up first would deadlock until # max_standby_streaming_delay cancels the cursor. remaining=$(reader_query "FETCH ALL FROM held_ranked;") held_ids="${first_id}"$'\n'"${remaining}" if grep -Ev '^[0-9]+$' <<<"${held_ids}" >/dev/null; then error "Old-layout ranked cursor returned a non-numeric document ID" fi held_id_count=$(printf '%s\n' "${held_ids}" | wc -l | tr -d ' ') held_ids_sorted=$(printf '%s\n' "${held_ids}" | LC_ALL=C sort -n | paste -sd, -) expected_ids=$(seq 1 8000 | paste -sd, -) if [ "${held_ids_sorted}" != "${expected_ids}" ]; then duplicate_ids=$(printf '%s\n' "${held_ids}" | LC_ALL=C sort -n | uniq -d | head -10 | paste -sd, -) error "Old-layout ranked cursor returned the wrong exact ID set: \ count=${held_id_count}, duplicate sample=${duplicate_ids:-none}" fi log "PASS: old-layout ranked cursor returned exact IDs 1..8000 \ after primary publication" reader_query "CLOSE held_ranked; COMMIT;" >/dev/null reader_close wait_for_feedback_release wait_for_standby_catchup 30 || error "Standby did not replay publication after the old cursor closed" drained="${parked_after_vacuum}" for _ in $(seq 1 20); do primary_sql_quiet "SELECT txid_current();" >/dev/null primary_sql_quiet "SELECT txid_current();" >/dev/null primary_sql "VACUUM rec;" >/dev/null drained=$(primary_sql_quiet \ "SELECT bm25_pending_free_pages('rec_idx');") [ "${drained}" = "0" ] && break sleep 0.5 done [ "${drained}" = "0" ] || error "Timed out 10s draining parked pages after snapshot release; \ remaining=${drained}, backend_xmin=$(primary_sql_quiet " SELECT coalesce(backend_xmin::text, '') FROM pg_stat_replication LIMIT 1;")" wait_for_standby_catchup 30 || error "Standby did not catch up after safe reclaim" assert_ranked_plan "${PRIMARY_PORT}" assert_ranked_plan "${STANDBY_PORT}" expected_ids=$(primary_sql_quiet \ "SELECT string_agg(id::text, ',' ORDER BY id) FROM rec;") primary_ids=$(primary_sql_quiet " SET enable_seqscan = off; SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM rec ORDER BY body <@> to_bm25query('alpha', 'rec_idx') LIMIT 8000 ) ranked;" | tail -n 1) standby_ids=$(standby_sql_quiet " SET enable_seqscan = off; SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM rec ORDER BY body <@> to_bm25query('alpha', 'rec_idx') LIMIT 8000 ) ranked;" | tail -n 1) [ "${primary_ids}" = "${expected_ids}" ] || error "Primary ranked IDs differ from exact heap IDs after reclaim" [ "${standby_ids}" = "${expected_ids}" ] || error "Standby ranked IDs differ from exact heap IDs after reclaim" log "PASS: VACUUM reclaimed ${parked_before_vacuum} pages only after \ the standby cursor ended" } main() { local spilled graph feedback_setting check_required_tools command -v pg_waldump >/dev/null 2>&1 || error "pg_waldump not found" setup_primary cat >> "${PRIMARY_DIR}/postgresql.conf" </dev/null [ "$(primary_sql_quiet "SHOW wal_keep_size;")" = "128MB" ] || error "Primary did not retain the disconnected standby WAL window" primary_sql " CREATE TABLE rec ( id integer PRIMARY KEY, body text NOT NULL ) WITH (autovacuum_enabled = false); INSERT INTO rec SELECT g, 'alpha beta gamma standby document ' || g FROM generate_series(1, 4000) g; CREATE TABLE reclaim_xid_advance (id integer PRIMARY KEY); CREATE INDEX rec_idx ON rec USING bm25(body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO rec SELECT g, 'alpha beta gamma standby document ' || g FROM generate_series(4001, 8000) g; " >/dev/null spilled=$(primary_sql_quiet "SELECT bm25_spill_index('rec_idx') > 0;") [ "${spilled}" = "t" ] || error "Second L0 segment was not spilled (got '${spilled}')" graph=$(primary_sql_quiet \ "SELECT bm25_level_counts('rec_idx'::regclass)::text;") [ "${graph}" = "{2,0,0,0,0,0,0,0}" ] || error "Expected two-L0 old graph, got ${graph}" setup_standby cat >> "${STANDBY_DIR}/postgresql.conf" </dev/null wait_for_standby_catchup 30 || error "Standby did not catch up before ranked cursor setup" feedback_setting=$(standby_sql_quiet "SHOW hot_standby_feedback;") [ "${feedback_setting}" = "on" ] || error "hot_standby_feedback is ${feedback_setting}, expected on" assert_ranked_plan "${STANDBY_PORT}" test_late_ranked_cursor_reclaim test_disconnected_standby_conflict test_atomic_segment_graph_snapshot test_atomic_ranked_memtable_generation test_atomic_standalone_memtable_generation test_memtable_retire_horizon_covers_prepublish_reader test_disconnected_memtable_reuse_conflict test_ranked_promotion_pins_recovery_mode log "All standby reclaim overlap checks passed" } main "$@" timescale-pg_textsearch-b63fe58/test/scripts/stress.sh000077500000000000000000000676531525762047700233320ustar00rootroot00000000000000#!/bin/bash # # Long-running stress tests for pg_textsearch # # Designed for nightly CI with memory leak detection. Each test is time-based # to ensure sustained load over the configured duration. # # Default nightly configuration targets: # - 30 minutes total runtime # - 100,000+ documents inserted # - 100+ segments created # - 5+ concurrent database connections # set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT="${TEST_PORT:-55436}" TEST_DB=pg_textsearch_stress_test DATA_DIR="${SCRIPT_DIR}/../tmp_stress_test" LOGFILE="${DATA_DIR}/postgres.log" # Test parameters - can be overridden via environment STRESS_DURATION_MINUTES="${STRESS_DURATION_MINUTES:-5}" SPILL_THRESHOLD="${SPILL_THRESHOLD:-2}" # Low threshold for more segments DOCS_PER_BATCH="${DOCS_PER_BATCH:-100}" CONCURRENT_READERS="${CONCURRENT_READERS:-6}" # Readers per test CONCURRENT_WRITERS="${CONCURRENT_WRITERS:-3}" # Writers for concurrent tests VALIDATION_INTERVAL="${VALIDATION_INTERVAL:-30}" # seconds between validations # Calculated durations for each test (in seconds) # Test 1: 40% of time, Test 2: 25%, Test 3: 15%, Test 4: 10%, Test 5: 10% TOTAL_SECONDS=$((STRESS_DURATION_MINUTES * 60)) TEST1_DURATION=$((TOTAL_SECONDS * 40 / 100)) TEST2_DURATION=$((TOTAL_SECONDS * 25 / 100)) TEST3_DURATION=$((TOTAL_SECONDS * 15 / 100)) TEST4_DURATION=$((TOTAL_SECONDS * 10 / 100)) TEST5_DURATION=$((TOTAL_SECONDS * 10 / 100)) # Metrics tracking TOTAL_DOCS_INSERTED=0 TOTAL_SEGMENTS_CREATED=0 TOTAL_QUERIES_EXECUTED=0 TOTAL_ERRORS=0 # Colors for output RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' BLUE='\033[0;34m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } info() { echo -e "${BLUE}[$(date '+%H:%M:%S')] $1${NC}"; } cleanup() { local exit_code=$? log "Cleaning up stress test environment (exit code: $exit_code)..." # Kill any background processes jobs -p | xargs -r kill 2>/dev/null || true if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast &>/dev/null || pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi rm -rf "${DATA_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up stress test PostgreSQL instance..." rm -rf "${DATA_DIR}" mkdir -p "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} max_connections = 30 shared_buffers = 256MB unix_socket_directories = '${DATA_DIR}' listen_addresses = 'localhost' log_min_messages = warning log_statement = 'none' EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w || error "Failed to start PostgreSQL" createdb -h "${DATA_DIR}" -p "${TEST_PORT}" "${TEST_DB}" psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "CREATE EXTENSION pg_textsearch;" >/dev/null # Set low spill threshold psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "ALTER SYSTEM SET pg_textsearch.memtable_pages_threshold = ${SPILL_THRESHOLD};" >/dev/null pg_ctl reload -D "${DATA_DIR}" >/dev/null log "PostgreSQL started with memtable_pages_threshold = ${SPILL_THRESHOLD}" } run_sql() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" 2>&1 } run_sql_quiet() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "$1" >/dev/null 2>&1 } run_sql_value() { psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc "$1" 2>/dev/null } get_segment_count() { local idx_name=$1 local summary=$(run_sql_value "SELECT bm25_summarize_index('${idx_name}');" 2>/dev/null || echo "") # Format is "Total: N segments, ..." - extract the number echo "$summary" | grep -o 'Total: [0-9]* segments' | grep -o '[0-9]*' || echo "0" } # # Generate random text content for documents # generate_doc_content() { local doc_id=$1 local words=("alpha" "beta" "gamma" "delta" "epsilon" "zeta" "eta" "theta" "iota" "kappa" "lambda" "mu" "nu" "xi" "omicron" "pi" "rho" "sigma" "tau" "upsilon" "phi" "chi" "psi" "omega" "database" "search" "query" "index" "document" "text" "ranking" "score" "postgres" "extension" "memory" "segment" "posting" "term") local content="document ${doc_id}" local num_words=$((RANDOM % 30 + 10)) for ((i=0; i to_bm25query('${term}', 'stress_idx') as score FROM stress_docs ORDER BY content <@> to_bm25query('${term}', 'stress_idx') LIMIT 50 ) t WHERE score < 0;" 2>/dev/null) queries_run=$((queries_run + 1)) if [ -z "$result" ]; then errors=$((errors + 1)) fi done # Report progress every 20 batches if [ $((batch_num % 20)) -eq 0 ]; then local segment_count=$(get_segment_count 'stress_idx') if [ "$segment_count" != "$last_segment_count" ]; then last_segment_count=$segment_count fi local elapsed=$(($(date +%s) - start_time)) local rate=$((docs_inserted * 60 / (elapsed + 1))) info "Progress: ${docs_inserted} docs, ${segment_count} segments, ${rate} docs/min" fi # Periodic validation local now=$(date +%s) if [ $((now - last_validation)) -ge $VALIDATION_INTERVAL ]; then # Verify document count matches local table_count=$(run_sql_value "SELECT COUNT(*) FROM stress_docs;") if [ "$table_count" != "$docs_inserted" ]; then warn "Document count mismatch: expected $docs_inserted, got $table_count" errors=$((errors + 1)) fi # Verify top results have valid negative scores local valid_scores=$(run_sql_value " SELECT COUNT(*) FROM ( SELECT content <@> to_bm25query('database', 'stress_idx') as score FROM stress_docs ORDER BY content <@> to_bm25query('database', 'stress_idx') LIMIT 20 ) t WHERE score < 0;") if [ -z "$valid_scores" ] || [ "$valid_scores" -eq 0 ]; then warn "Score validation failed: no valid negative scores" errors=$((errors + 1)) fi last_validation=$now fi sleep 0.02 done # Final statistics local final_segments=$(get_segment_count 'stress_idx') local final_count=$(run_sql_value "SELECT COUNT(*) FROM stress_docs;") info "Final: ${final_count} documents, ${final_segments} segments, ${queries_run} queries, ${errors} errors" # Update global metrics TOTAL_DOCS_INSERTED=$((TOTAL_DOCS_INSERTED + docs_inserted)) TOTAL_SEGMENTS_CREATED=$((TOTAL_SEGMENTS_CREATED + final_segments)) TOTAL_QUERIES_EXECUTED=$((TOTAL_QUERIES_EXECUTED + queries_run)) TOTAL_ERRORS=$((TOTAL_ERRORS + errors)) if [ $errors -eq 0 ]; then log "Test 1 PASSED" else warn "Test 1 completed with $errors errors" fi run_sql_quiet "DROP TABLE stress_docs CASCADE;" } # # Test 2: Concurrent readers and writers # # Target: Multiple readers and writers operating simultaneously # Duration: 25% of total time # Expected: 6 readers + 3 writers = 9 concurrent connections, heavy load # test_concurrent_stress() { local duration=$TEST2_DURATION local total_connections=$((CONCURRENT_READERS + CONCURRENT_WRITERS)) log "Test 2: Concurrent stress (${duration}s, ${CONCURRENT_READERS} readers + ${CONCURRENT_WRITERS} writers = ${total_connections} connections)" run_sql_quiet "CREATE TABLE conc_docs (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "CREATE INDEX conc_idx ON conc_docs USING bm25(content) WITH (text_config='english');" # Initial data load (500 docs) info "Loading 500 initial documents..." for ((batch=1; batch<=10; batch++)); do local insert_sql="INSERT INTO conc_docs (content) VALUES" local first=true for ((i=1; i<=50; i++)); do local content=$(generate_doc_content $((batch * 50 + i))) if [ "$first" = true ]; then insert_sql="${insert_sql} ('${content}')" first=false else insert_sql="${insert_sql}, ('${content}')" fi done run_sql_quiet "${insert_sql};" done run_sql_quiet "SELECT bm25_spill_index('conc_idx');" local start_time=$(date +%s) local end_time=$((start_time + duration)) # Start concurrent readers local reader_pids=() for ((r=1; r<=CONCURRENT_READERS; r++)); do { local queries=0 local rows_fetched=0 local errors=0 local search_terms=("alpha" "beta" "gamma" "database" "search" "document" "postgres" "index") while [ $(date +%s) -lt $end_time ]; do local term="${search_terms[$((RANDOM % ${#search_terms[@]}))]}" # Heavy query: fetch actual rows with scores local result=$(psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -tAc " SELECT COUNT(*) FROM ( SELECT id, content, content <@> to_bm25query('${term}', 'conc_idx') as score FROM conc_docs ORDER BY content <@> to_bm25query('${term}', 'conc_idx') LIMIT 100 ) t;" 2>/dev/null) if [ -n "$result" ]; then rows_fetched=$((rows_fetched + result)) else errors=$((errors + 1)) fi queries=$((queries + 1)) sleep 0.03 done echo "Reader $r: $queries queries, $rows_fetched rows, $errors errors" >&2 } & reader_pids+=($!) done # Start concurrent writers local writer_pids=() for ((w=1; w<=CONCURRENT_WRITERS; w++)); do { local docs=0 local spills=0 local errors=0 local base_id=$((w * 100000)) while [ $(date +%s) -lt $end_time ]; do local insert_sql="INSERT INTO conc_docs (content) VALUES" local first=true for ((i=1; i<=20; i++)); do local content="writer${w} doc$((base_id + docs + i)) alpha beta gamma database search query index" if [ "$first" = true ]; then insert_sql="${insert_sql} ('${content}')" first=false else insert_sql="${insert_sql}, ('${content}')" fi done if psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "${insert_sql};" >/dev/null 2>&1; then docs=$((docs + 20)) else errors=$((errors + 1)) fi # Periodic spill from this writer if [ $((docs % 500)) -eq 0 ] && [ $docs -gt 0 ]; then psql -h "${DATA_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" -c "SELECT bm25_spill_index('conc_idx');" >/dev/null 2>&1 spills=$((spills + 1)) fi sleep 0.02 done echo "Writer $w: $docs docs, $spills spills, $errors errors" >&2 } & writer_pids+=($!) done # Wait for all processes local failures=0 for pid in "${reader_pids[@]}" "${writer_pids[@]}"; do if ! wait $pid 2>/dev/null; then failures=$((failures + 1)) fi done local final_segments=$(get_segment_count 'conc_idx') local final_docs=$(run_sql_value "SELECT COUNT(*) FROM conc_docs;") info "Final: ${final_docs} documents, ${final_segments} segments" # Update global metrics TOTAL_DOCS_INSERTED=$((TOTAL_DOCS_INSERTED + final_docs)) TOTAL_SEGMENTS_CREATED=$((TOTAL_SEGMENTS_CREATED + final_segments)) TOTAL_ERRORS=$((TOTAL_ERRORS + failures)) if [ $failures -eq 0 ]; then log "Test 2 PASSED" else warn "Test 2: $failures process failures" fi run_sql_quiet "DROP TABLE conc_docs CASCADE;" } # # Test 3: Multiple indexes with interleaved operations (TIME-BASED) # # Target: Operate on multiple indexes simultaneously to test resource isolation # Duration: 15% of total time # Expected: 3 indexes, ~2000 docs each, 10+ segments each # test_multiple_indexes() { local duration=$TEST3_DURATION log "Test 3: Multiple indexes with interleaved operations (${duration} seconds)" # Create 3 tables with indexes for t in 1 2 3; do run_sql_quiet "CREATE TABLE multi_${t} (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "CREATE INDEX multi_idx_${t} ON multi_${t} USING bm25(content) WITH (text_config='english');" done local start_time=$(date +%s) local end_time=$((start_time + duration)) local iteration=0 local errors=0 local docs_per_table=0 while [ $(date +%s) -lt $end_time ]; do iteration=$((iteration + 1)) # Insert batch to all tables for t in 1 2 3; do local insert_sql="INSERT INTO multi_${t} (content) VALUES" local first=true for ((i=1; i<=20; i++)); do local content="iteration ${iteration} table ${t} doc ${i} alpha beta gamma database search" if [ "$first" = true ]; then insert_sql="${insert_sql} ('${content}')" first=false else insert_sql="${insert_sql}, ('${content}')" fi done run_sql_quiet "${insert_sql};" done docs_per_table=$((docs_per_table + 20)) # Query from all tables for t in 1 2 3; do local count=$(run_sql_value " SELECT COUNT(*) FROM ( SELECT * FROM multi_${t} ORDER BY content <@> to_bm25query('alpha', 'multi_idx_${t}') LIMIT 100 ) sub;") TOTAL_QUERIES_EXECUTED=$((TOTAL_QUERIES_EXECUTED + 1)) if [ -z "$count" ] || [ "$count" -eq 0 ]; then errors=$((errors + 1)) fi done # Rotating spills across indexes local spill_target=$(( (iteration % 3) + 1 )) if [ $((iteration % 10)) -eq 0 ]; then run_sql_quiet "SELECT bm25_spill_index('multi_idx_${spill_target}');" fi sleep 0.02 done # Final validation local total_segments=0 for t in 1 2 3; do local count=$(run_sql_value "SELECT COUNT(*) FROM multi_${t};") local segments=$(get_segment_count "multi_idx_${t}") total_segments=$((total_segments + segments)) info "Table $t: ${count} docs, ${segments} segments" if [ "$count" != "$docs_per_table" ]; then warn "Table $t count mismatch: expected $docs_per_table, got $count" errors=$((errors + 1)) fi done TOTAL_DOCS_INSERTED=$((TOTAL_DOCS_INSERTED + docs_per_table * 3)) TOTAL_SEGMENTS_CREATED=$((TOTAL_SEGMENTS_CREATED + total_segments)) TOTAL_ERRORS=$((TOTAL_ERRORS + errors)) if [ $errors -eq 0 ]; then log "Test 3 PASSED" else warn "Test 3 completed with $errors errors" fi for t in 1 2 3; do run_sql_quiet "DROP TABLE multi_${t} CASCADE;" done } # # Test 4: Segment creation stress test (TIME-BASED) # # Target: Create as many segments as possible to test segment handling # Duration: 10% of total time # Expected: 50+ segments with small batches and frequent spills # test_segment_stress() { local duration=$TEST4_DURATION log "Test 4: Segment creation stress (${duration} seconds)" run_sql_quiet "CREATE TABLE segment_test (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "CREATE INDEX segment_idx ON segment_test USING bm25(content) WITH (text_config='english');" local start_time=$(date +%s) local end_time=$((start_time + duration)) local segment_num=0 local total_docs=0 local docs_per_segment=30 # Small batches = more segments while [ $(date +%s) -lt $end_time ]; do segment_num=$((segment_num + 1)) # Insert small batch local insert_sql="INSERT INTO segment_test (content) VALUES" local first=true for ((i=1; i<=docs_per_segment; i++)); do local content="segment ${segment_num} document ${i} with unique terms seg${segment_num}word${i}" if [ "$first" = true ]; then insert_sql="${insert_sql} ('${content}')" first=false else insert_sql="${insert_sql}, ('${content}')" fi done run_sql_quiet "${insert_sql};" total_docs=$((total_docs + docs_per_segment)) # Force spill after each batch to create many segments run_sql_quiet "SELECT bm25_spill_index('segment_idx');" # Verify searchability periodically if [ $((segment_num % 20)) -eq 0 ]; then local actual_segments=$(get_segment_count 'segment_idx') info "Created ${actual_segments} segments, ${total_docs} docs" # Query across all segments local search_count=$(run_sql_value " SELECT COUNT(*) FROM ( SELECT * FROM segment_test ORDER BY content <@> to_bm25query('document', 'segment_idx') LIMIT 1000 ) sub;") TOTAL_QUERIES_EXECUTED=$((TOTAL_QUERIES_EXECUTED + 1)) fi sleep 0.01 done local final_segments=$(get_segment_count 'segment_idx') local final_docs=$(run_sql_value "SELECT COUNT(*) FROM segment_test;") # Verify all documents are searchable local searchable=$(run_sql_value " SELECT COUNT(*) FROM ( SELECT * FROM segment_test ORDER BY content <@> to_bm25query('document', 'segment_idx') LIMIT 100000 ) sub;") info "Final: ${final_docs} documents across ${final_segments} segments" TOTAL_DOCS_INSERTED=$((TOTAL_DOCS_INSERTED + total_docs)) TOTAL_SEGMENTS_CREATED=$((TOTAL_SEGMENTS_CREATED + final_segments)) if [ "$searchable" = "$final_docs" ]; then log "Test 4 PASSED: All ${final_docs} documents searchable across ${final_segments} segments" else warn "Test 4: Expected $final_docs searchable, found $searchable" TOTAL_ERRORS=$((TOTAL_ERRORS + 1)) fi run_sql_quiet "DROP TABLE segment_test CASCADE;" } # # Test 5: Resource cleanup stress (TIME-BASED) # # Target: Repeatedly create and drop indexes to test resource cleanup # Duration: 10% of total time # Expected: 20+ create/drop cycles, no resource leaks # # Checks for: # - DSA/shared memory leaks (via reference index DSA size) # - File descriptor leaks (via /proc or lsof) # test_resource_cleanup() { local duration=$TEST5_DURATION log "Test 5: Resource cleanup stress (${duration} seconds)" # Create a reference index to monitor DSA usage run_sql_quiet "CREATE TABLE ref_table (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "INSERT INTO ref_table (content) VALUES ('reference document');" run_sql_quiet "CREATE INDEX ref_idx ON ref_table USING bm25(content) WITH (text_config='english');" # Record baseline DSA size local baseline_dsa=$(run_sql_value " SELECT (regexp_matches(bm25_summarize_index('ref_idx'), 'DSA total size: ([0-9]+)'))[1]::bigint;" 2>/dev/null || echo "0") info "Baseline DSA: ${baseline_dsa} bytes" # Get postgres backend PID for FD and lock tracking local backend_pid=$(run_sql_value "SELECT pg_backend_pid();") local baseline_fds=0 if [ -d "/proc/${backend_pid}/fd" ]; then baseline_fds=$(ls -1 /proc/${backend_pid}/fd 2>/dev/null | wc -l) info "Baseline FDs: ${baseline_fds}" fi # Record baseline advisory locks (should be 0) local baseline_advisory_locks=$(run_sql_value " SELECT COUNT(*) FROM pg_locks WHERE pid = ${backend_pid} AND locktype = 'advisory';") info "Baseline advisory locks: ${baseline_advisory_locks}" local start_time=$(date +%s) local end_time=$((start_time + duration)) local cycle=0 local docs_per_cycle=200 while [ $(date +%s) -lt $end_time ]; do cycle=$((cycle + 1)) # Create table and index run_sql_quiet "CREATE TABLE cleanup_${cycle} (id SERIAL PRIMARY KEY, content TEXT);" run_sql_quiet "INSERT INTO cleanup_${cycle} (content) SELECT 'document ' || g || ' alpha beta gamma database search query' FROM generate_series(1, ${docs_per_cycle}) g;" run_sql_quiet "CREATE INDEX cleanup_idx_${cycle} ON cleanup_${cycle} USING bm25(content) WITH (text_config='english');" run_sql_quiet "SELECT bm25_spill_index('cleanup_idx_${cycle}');" # Query multiple times for ((q=1; q<=5; q++)); do run_sql_quiet " SELECT COUNT(*) FROM ( SELECT * FROM cleanup_${cycle} ORDER BY content <@> to_bm25query('alpha', 'cleanup_idx_${cycle}') LIMIT 100 ) sub;" TOTAL_QUERIES_EXECUTED=$((TOTAL_QUERIES_EXECUTED + 1)) done # Drop table (cascades to index) run_sql_quiet "DROP TABLE cleanup_${cycle} CASCADE;" if [ $((cycle % 10)) -eq 0 ]; then info "Completed ${cycle} create/drop cycles" fi sleep 0.02 done TOTAL_DOCS_INSERTED=$((TOTAL_DOCS_INSERTED + cycle * docs_per_cycle)) # Check for DSA leaks - reference index DSA should not have grown significantly local final_dsa=$(run_sql_value " SELECT (regexp_matches(bm25_summarize_index('ref_idx'), 'DSA total size: ([0-9]+)'))[1]::bigint;" 2>/dev/null || echo "0") local dsa_growth=0 if [ "$baseline_dsa" -gt 0 ] && [ "$final_dsa" -gt 0 ]; then dsa_growth=$(( (final_dsa - baseline_dsa) * 100 / baseline_dsa )) fi info "Final DSA: ${final_dsa} bytes (${dsa_growth}% growth)" # Check for FD leaks local final_fds=0 if [ -d "/proc/${backend_pid}/fd" ]; then final_fds=$(ls -1 /proc/${backend_pid}/fd 2>/dev/null | wc -l) local fd_growth=$((final_fds - baseline_fds)) info "Final FDs: ${final_fds} (${fd_growth} growth)" if [ $fd_growth -gt 10 ]; then warn "Possible FD leak: ${fd_growth} new file descriptors" TOTAL_ERRORS=$((TOTAL_ERRORS + 1)) fi fi # DSA growth > 50% after cleanup would indicate a leak if [ $dsa_growth -gt 50 ]; then warn "Possible DSA leak: ${dsa_growth}% growth in reference index" TOTAL_ERRORS=$((TOTAL_ERRORS + 1)) fi # Check for advisory lock leaks local final_advisory_locks=$(run_sql_value " SELECT COUNT(*) FROM pg_locks WHERE pid = ${backend_pid} AND locktype = 'advisory';") local lock_growth=$((final_advisory_locks - baseline_advisory_locks)) info "Final advisory locks: ${final_advisory_locks} (${lock_growth} growth)" if [ $lock_growth -gt 0 ]; then warn "Possible lock leak: ${lock_growth} advisory locks not released" TOTAL_ERRORS=$((TOTAL_ERRORS + 1)) fi run_sql_quiet "DROP TABLE ref_table CASCADE;" log "Test 5 PASSED: Completed ${cycle} create/drop cycles" } print_summary() { echo "" echo "============================================================" echo " STRESS TEST SUMMARY" echo "============================================================" echo "" echo "Configuration:" echo " Duration: ${STRESS_DURATION_MINUTES} minutes" echo " Spill threshold: ${SPILL_THRESHOLD} pages" echo " Docs per batch: ${DOCS_PER_BATCH}" echo " Concurrent readers: ${CONCURRENT_READERS}" echo " Concurrent writers: ${CONCURRENT_WRITERS}" echo "" echo "Results:" echo " Total documents: ${TOTAL_DOCS_INSERTED}" echo " Total segments: ${TOTAL_SEGMENTS_CREATED}" echo " Total queries: ${TOTAL_QUERIES_EXECUTED}" echo " Total errors: ${TOTAL_ERRORS}" echo "" echo "Time allocation:" echo " Test 1 (insert/query): ${TEST1_DURATION}s (40%)" echo " Test 2 (concurrent): ${TEST2_DURATION}s (25%)" echo " Test 3 (multi-index): ${TEST3_DURATION}s (15%)" echo " Test 4 (segment stress): ${TEST4_DURATION}s (10%)" echo " Test 5 (cleanup): ${TEST5_DURATION}s (10%)" echo "" if [ $TOTAL_ERRORS -eq 0 ]; then echo -e "${GREEN}STATUS: ALL TESTS PASSED${NC}" else echo -e "${YELLOW}STATUS: COMPLETED WITH ${TOTAL_ERRORS} ERRORS${NC}" fi echo "============================================================" } run_stress_tests() { log "Starting pg_textsearch stress tests" test_extended_insert_query test_concurrent_stress test_multiple_indexes test_segment_stress test_resource_cleanup print_summary } main() { log "pg_textsearch stress test suite" echo "" echo "Configuration:" echo " Duration: ${STRESS_DURATION_MINUTES} minutes (${TOTAL_SECONDS} seconds)" echo " Spill threshold: ${SPILL_THRESHOLD} pages" echo " Docs per batch: ${DOCS_PER_BATCH}" echo " Concurrent readers: ${CONCURRENT_READERS}" echo " Concurrent writers: ${CONCURRENT_WRITERS}" echo "" command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" setup_test_db run_stress_tests if [ $TOTAL_ERRORS -eq 0 ]; then log "Stress tests completed successfully!" exit 0 else warn "Stress tests completed with ${TOTAL_ERRORS} errors" exit 1 fi } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/scripts/upgrade_matrix.sh000077500000000000000000000470671525762047700250170ustar00rootroot00000000000000#!/bin/bash # # Upgrade data-integrity matrix for pg_textsearch. # # For each old release, this harness builds several distinct on-disk # index states (single segment, two segments, many segments, and an # index with unspilled L0 memtable data), upgrades to the current # binary, and checks recall against heap ground truth. # # Ground-truth oracle: a rare sentinel token is planted in a known # subset of rows, and recall is measured as how many of the heap's # sentinel rows the BM25 index scan returns in its top-K. Recall is # captured under the old binary (pre-upgrade), the new binary # (post-upgrade), and again after REINDEX. # # Hard assertions (fail the build): # * The old binary has perfect recall (oracle sanity). # * Segment-backed shapes preserve 100% recall across the upgrade. # * REINDEX restores 100% recall for every shape. # * Legacy (metapage v5, <= 0.5.0) indexes ERROR cleanly without a # REINDEX (no crash, no silent wrong answer) and REINDEX recovers. # # The "memtable_unspilled" shape loses the unspilled documents when # upgrading from a pre-1.3 (metapage v6) release; REINDEX recovers. # By default this is surfaced as a non-fatal ::warning:: annotation so # CI stays green; set STRICT_UNSPILLED=1 to make it a hard assertion. # # Usage: # PG_CONFIG=/usr/lib/postgresql/17/bin/pg_config \ # test/scripts/upgrade_matrix.sh [old_version ...] # # Environment: # PG_CONFIG pg_config to use (default: first on PATH) # OLD_VERSIONS space-separated versions (default: representative # set covering every metapage/segment format) # PG_RUNAS OS user to run cluster ops as (default: current; # auto-set to "postgres" when running as root, since # initdb/postgres refuse to run as root) # TEST_PORT port for the throwaway cluster (default 55440) # STRICT_UNSPILLED 1 = hard-fail on unspilled data loss (default 0) # set -uo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/../.." && pwd)" PG_CONFIG="${PG_CONFIG:-pg_config}" BINDIR="$("$PG_CONFIG" --bindir)" SHAREDIR="$("$PG_CONFIG" --sharedir)" export PATH="$BINDIR:$PATH" TEST_PORT="${TEST_PORT:-55440}" BASE_DIR="${BASE_DIR:-$REPO_ROOT/tmp_upgrade_matrix}" DATA_DIR="$BASE_DIR/pgdata" SOCK_DIR="$BASE_DIR/sock" STRICT_UNSPILLED="${STRICT_UNSPILLED:-0}" CURRENT_VERSION="$(sed -n "s/^default_version = '\([^']*\)'/\1/p" "$REPO_ROOT/pg_textsearch.control")" # Representative default matrix: one release per distinct on-disk # format combination. 0.5.0 = metapage v5 (legacy/REINDEX tier); # 0.5.1 = metapage v6 + segment v3; 1.0.0 = v6 + segment v4; # 1.2.0 = v6 + segment v5; 1.3.0 = metapage v7 (native on-disk L0); # 1.4.0 = metapage v8 before indexes recorded zero-lexeme documents; # 1.5.0 = previous stable release. OLD_VERSIONS="${OLD_VERSIONS:-0.5.0 0.5.1 1.0.0 1.2.0 1.3.0 1.4.0 1.5.0}" if [ "$#" -gt 0 ]; then OLD_VERSIONS="$*"; fi # Cluster ops must not run as root. When invoked as root (e.g. inside # a container), drop to PG_RUNAS (default postgres) for pg_ctl/initdb/ # psql; build/install still run as root. if [ "$(id -u)" -eq 0 ]; then PG_RUNAS="${PG_RUNAS:-postgres}" else PG_RUNAS="${PG_RUNAS:-$(id -un)}" fi RED='\033[0;31m'; GREEN='\033[0;32m'; YELLOW='\033[1;33m'; NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $*${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARN: $*${NC}"; } err() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $*${NC}"; } # Installing into the PG sharedir needs root; use sudo when we are not. if [ "$(id -u)" -eq 0 ]; then SUDO=""; else SUDO="sudo"; fi FAILURES=0 fail() { err "$*"; FAILURES=$((FAILURES + 1)); } annotate_warn() { echo "::warning::$*"; warn "$*"; } # Run a command as the cluster OS user without nested-quote pain. as_pg() { if [ "$PG_RUNAS" = "$(id -un)" ]; then "$@" else # Own the data/sock dirs so the runas user can use them. su "$PG_RUNAS" -c "PATH='$PATH' PGPORT='$TEST_PORT' PGHOST='$SOCK_DIR' $(printf '%q ' "$@")" fi } # psql helpers: SQL always travels through a world-readable temp FILE, # never an inline -c string, to survive su-quoting. scalar() echoes a # single value; runsql() runs quietly; run_capture() saves stderr. _mktemp_sql() { local f; f="$(mktemp "${TMPDIR:-/tmp}/um.XXXXXX.sql")"; printf '%s\n' "$1" >"$f"; chmod 644 "$f"; echo "$f"; } scalar() { local f; f="$(_mktemp_sql "$1")"; as_pg psql -h "$SOCK_DIR" -p "$TEST_PORT" -d upg -tAqf "$f" 2>/dev/null; rm -f "$f"; } runsql() { local f; f="$(_mktemp_sql "$1")"; as_pg psql -h "$SOCK_DIR" -p "$TEST_PORT" -d upg -v ON_ERROR_STOP=1 -q -f "$f" >/dev/null; local rc=$?; rm -f "$f"; return $rc; } run_capture() { local f; f="$(_mktemp_sql "$1")"; as_pg psql -h "$SOCK_DIR" -p "$TEST_PORT" -d upg -tAqf "$f" >"$2" 2>"$3"; rm -f "$f"; } cleanup() { as_pg pg_ctl -D "$DATA_DIR" stop -m immediate >/dev/null 2>&1 || true rm -rf "$BASE_DIR" } trap cleanup EXIT INT TERM # ------------------------------------------------------------------ # # Build / install helpers # ------------------------------------------------------------------ # build_install_old() { # $1 = version local v="$1" dir="/tmp/pg_textsearch-$v" rm -rf "$dir" "/tmp/ptsrc-$v.tar.gz" local url="https://github.com/timescale/pg_textsearch/releases/download/v$v/pg_textsearch-$v.tar.gz" curl -fsSL -o "/tmp/ptsrc-$v.tar.gz" "$url" || return 2 tar -xzf "/tmp/ptsrc-$v.tar.gz" -C /tmp || return 2 ( cd "$dir" && make -s PG_CONFIG="$PG_CONFIG" clean >/dev/null 2>&1; make -s PG_CONFIG="$PG_CONFIG" >/dev/null 2>&1 && $SUDO make -s PG_CONFIG="$PG_CONFIG" install >/dev/null 2>&1 ) || return 3 } build_install_current() { $SUDO rm -f "$SHAREDIR/extension/pg_textsearch--"*.sql ( cd "$REPO_ROOT" && make -s PG_CONFIG="$PG_CONFIG" >/dev/null 2>&1 && $SUDO make -s PG_CONFIG="$PG_CONFIG" install >/dev/null 2>&1 ) || return 3 } # ------------------------------------------------------------------ # # Cluster lifecycle # ------------------------------------------------------------------ # fresh_cluster() { as_pg pg_ctl -D "$DATA_DIR" stop -m immediate >/dev/null 2>&1 || true rm -rf "$BASE_DIR"; mkdir -p "$DATA_DIR" "$SOCK_DIR" if [ "$PG_RUNAS" != "$(id -un)" ]; then chown -R "$PG_RUNAS" "$BASE_DIR"; fi as_pg initdb -D "$DATA_DIR" -A trust >/dev/null 2>&1 || { err "initdb failed"; return 1; } { echo "shared_preload_libraries = 'pg_textsearch'" echo "unix_socket_directories = '$SOCK_DIR'" echo "port = $TEST_PORT" echo "logging_collector = off" } >>"$DATA_DIR/postgresql.conf" } start_pg() { as_pg pg_ctl -D "$DATA_DIR" -l "$DATA_DIR/logfile" start -w >/dev/null 2>&1; } stop_pg() { as_pg pg_ctl -D "$DATA_DIR" stop -w >/dev/null 2>&1; } server_up(){ as_pg pg_ctl -D "$DATA_DIR" status >/dev/null 2>&1; } createdb_upg() { as_pg createdb -h "$SOCK_DIR" -p "$TEST_PORT" upg >/dev/null 2>&1; } log_marker() { as_pg wc -l <"$DATA_DIR/logfile" 2>/dev/null | tr -d ' '; } log_since() { as_pg tail -n +"$(( ${1:-0} + 1 ))" "$DATA_DIR/logfile" 2>/dev/null; } # ------------------------------------------------------------------ # # Corpus + oracle # ------------------------------------------------------------------ # # Insert N rows starting at offset OFF; every 12th row carries the rare # sentinel token 'qwxsentinel'. gen() { # $1=n $2=off runsql "INSERT INTO d(c) SELECT CASE WHEN (g % 12) = 0 THEN 'alpha beta gamma qwxsentinel doc ' || g ELSE 'alpha beta gamma common filler doc ' || g END FROM generate_series($2, $2 + $1 - 1) g;" } mk_index() { runsql "CREATE INDEX i ON d USING bm25(c) WITH (text_config='english');"; } spill() { runsql "SELECT bm25_spill_index('i');"; } build_shape() { # $1 = shape runsql "CREATE TABLE d(id serial primary key, c text);" case "$1" in single_seg) gen 600 1; mk_index ;; two_seg) gen 300 1; mk_index; gen 300 301; spill ;; multi_seg) gen 120 1; mk_index gen 120 121; spill; gen 120 241; spill gen 120 361; spill; gen 120 481; spill ;; memtable_unspilled) gen 300 1; mk_index; gen 300 301 ;; # batch 2 stays in L0 esac } TRUTH_Q="SELECT count(*) FROM d WHERE c LIKE '%qwxsentinel%';" # Recall = number of genuine sentinel rows appearing in the BM25 top-K # (K = sentinel count). Non-matching rows score 0 and sort AFTER the # negative-scored matches, so a healthy index yields recall == truth. RECALL_Q="SET enable_seqscan=off; SELECT count(*) FROM ( SELECT id FROM d ORDER BY c <@> to_bm25query('qwxsentinel','i') LIMIT (SELECT count(*) FROM d WHERE c LIKE '%qwxsentinel%') ) s WHERE s.id IN (SELECT id FROM d WHERE c LIKE '%qwxsentinel%');" # ------------------------------------------------------------------ # # Per-scenario runners # ------------------------------------------------------------------ # is_legacy() { [[ "$1" =~ ^0\.[0-4]\. ]] || [[ "$1" == 0.5.0 ]]; } run_compat_shape() { # $1=version $2=shape local v="$1" st="$2" fresh_cluster || { fail "$v/$st: cluster init failed"; return; } start_pg || { fail "$v/$st: old server failed to start"; return; } createdb_upg runsql "CREATE EXTENSION pg_textsearch;" build_shape "$st" local truth pre post reidx forensic truth="$(scalar "$TRUTH_Q")" pre="$(scalar "$RECALL_Q")" stop_pg build_install_current || { fail "current build/install failed"; return; } start_pg || { fail "$v/$st: NEW server failed to start (corruption?)"; return; } runsql "ALTER EXTENSION pg_textsearch UPDATE;" || { fail "$v/$st: ALTER EXTENSION UPDATE failed" return } local installed_version loaded_version installed_version="$(scalar "SELECT extversion FROM pg_extension WHERE extname = 'pg_textsearch';")" loaded_version="$(scalar "SHOW pg_textsearch.library_version;")" if [ "$installed_version" != "$CURRENT_VERSION" ] || [ "$loaded_version" != "$CURRENT_VERSION" ]; then fail "$v/$st: expected $CURRENT_VERSION, SQL=$installed_version library=$loaded_version" return fi local mark; mark="$(log_marker)" runsql "INSERT INTO d(c) VALUES ('alpha beta gamma writeprobe token');" post="$(scalar "$RECALL_Q")" forensic="$(log_since "$mark" | grep -c 'that may have held unspilled documents')" runsql "REINDEX INDEX i;" reidx="$(scalar "$RECALL_Q")" stop_pg log " [$v/$st] truth=$truth pre=$pre post=$post reindex=$reidx forensic=$forensic" # Oracle sanity: the OLD binary must have perfect recall. [ "$pre" = "$truth" ] || fail "$v/$st: OLD-binary recall $pre != truth $truth (oracle broken)" if [ "$st" = "memtable_unspilled" ]; then if [ "$post" != "$truth" ]; then if [ "$STRICT_UNSPILLED" = "1" ]; then fail "$v/$st: unspilled data loss: post=$post truth=$truth" else annotate_warn "upgrading v$v lost unspilled memtable data (recall $post/$truth); REINDEX recovers ($reidx)" fi else log " [$v/$st] no unspilled loss for v$v (consider STRICT_UNSPILLED=1)" fi else # Segment-backed shapes must survive the upgrade losslessly. [ "$post" = "$truth" ] || fail "$v/$st: segment data loss on upgrade: post=$post truth=$truth" fi # Safety net: REINDEX must always restore full recall. [ "$reidx" = "$truth" ] || fail "$v/$st: REINDEX did not restore recall ($reidx/$truth)" } run_legacy() { # $1=version local v="$1" fresh_cluster || { fail "$v(legacy): cluster init failed"; return; } start_pg || { fail "$v(legacy): old server failed to start"; return; } createdb_upg runsql "CREATE EXTENSION pg_textsearch;" build_shape single_seg local truth pre; truth="$(scalar "$TRUTH_Q")"; pre="$(scalar "$RECALL_Q")" stop_pg build_install_current || { fail "current build/install failed"; return; } start_pg || { fail "$v(legacy): NEW server failed to start"; return; } # Query WITHOUT reindex must ERROR cleanly and NOT crash the server. local out err_f; out="$(mktemp)"; err_f="$(mktemp)" run_capture "$RECALL_Q" "$out" "$err_f" if server_up; then :; else fail "$v(legacy): server CRASHED on query without REINDEX"; fi if grep -qiE 'incompatible|REINDEX' "$err_f"; then log " [$v/legacy] clean ERROR without REINDEX (as designed)" else fail "$v(legacy): expected incompatible-version ERROR, got stdout=[$(cat "$out")] stderr=[$(head -1 "$err_f")]" fi rm -f "$out" "$err_f" runsql "REINDEX INDEX i;" local reidx; reidx="$(scalar "$RECALL_Q")" stop_pg log " [$v/legacy] truth=$truth pre=$pre post-REINDEX=$reidx" [ "$pre" = "$truth" ] || fail "$v(legacy): OLD-binary recall $pre != truth $truth" [ "$reidx" = "$truth" ] || fail "$v(legacy): REINDEX did not restore recall ($reidx/$truth)" } run_boolean_completeness_upgrade() { fresh_cluster || { fail "1.4.0/boolean: cluster init failed"; return; } start_pg || { fail "1.4.0/boolean: old server failed to start"; return; } createdb_upg runsql "CREATE EXTENSION pg_textsearch; CREATE TABLE d(id serial primary key, c text NOT NULL); INSERT INTO d(c) VALUES ('alpha'), (''), ('the'); CREATE INDEX i ON d USING bm25(c) WITH (text_config='english');" stop_pg build_install_current || { fail "current build/install failed"; return; } start_pg || { fail "1.4.0/boolean: NEW server failed to start"; return; } runsql "ALTER EXTENSION pg_textsearch UPDATE TO '1.5.1';" local out err_f out="$(mktemp)"; err_f="$(mktemp)" run_capture "SET enable_seqscan=off; SELECT count(*) FROM d WHERE c @@ to_tsquery('english', '!missing');" "$out" "$err_f" if grep -qi 'REINDEX' "$err_f" && [ "$(tr -d '[:space:]' <"$out")" = "1" ]; then log " [1.4.0/boolean] incomplete index warned and continued" else fail "1.4.0/boolean: expected warning and count 1, got stdout=[$(cat "$out")] stderr=[$(head -1 "$err_f")]" fi rm -f "$out" "$err_f" out="$(mktemp)"; err_f="$(mktemp)" run_capture "SET enable_seqscan=off; SELECT count(*) FROM d WHERE c @@ to_tsquery('english', 'alpha & !missing');" "$out" "$err_f" if grep -qi 'REINDEX' "$err_f"; then fail "1.4.0/boolean: positively anchored query emitted upgrade warning" fi [ "$(tr -d '[:space:]' <"$out")" = "1" ] || fail "1.4.0/boolean: anchored query count != 1" rm -f "$out" "$err_f" out="$(mktemp)"; err_f="$(mktemp)" run_capture "SET enable_seqscan=off; SELECT count(*) FROM d WHERE c @@ to_tsquery('english', 'alpha | !missing');" "$out" "$err_f" if grep -qi 'REINDEX' "$err_f" && [ "$(tr -d '[:space:]' <"$out")" = "1" ]; then log " [1.4.0/boolean] mixed negative query warned and continued" else fail "1.4.0/boolean: expected mixed negative warning and count 1, got stdout=[$(cat "$out")] stderr=[$(head -1 "$err_f")]" fi rm -f "$out" "$err_f" runsql "REINDEX INDEX i;" out="$(mktemp)"; err_f="$(mktemp)" run_capture "SET enable_seqscan=off; SELECT count(*) FROM d WHERE c @@ to_tsquery('english', '!missing');" "$out" "$err_f" if grep -qi 'REINDEX' "$err_f"; then fail "1.4.0/boolean: rebuilt index still emitted upgrade warning" fi local count count="$(tr -d '[:space:]' <"$out")" rm -f "$out" "$err_f" stop_pg log " [1.4.0/boolean] post-REINDEX count=$count" [ "$count" = "3" ] || fail "1.4.0/boolean: post-REINDEX count $count != 3" } run_v8_tombstone_compaction_upgrade() { fresh_cluster || { fail "1.4.0/v8-tombstone: cluster init failed"; return; } echo "max_prepared_transactions = 1" >>"$DATA_DIR/postgresql.conf" start_pg || { fail "1.4.0/v8-tombstone: old server failed to start"; return; } createdb_upg runsql "CREATE EXTENSION pg_textsearch; CREATE TABLE d(id serial primary key, c text) WITH (autovacuum_enabled = false); CREATE INDEX i ON d USING bm25(c) WITH (text_config='english');" runsql "BEGIN; SELECT txid_current(); PREPARE TRANSACTION 'v8_tombstone_horizon';" runsql " SET pg_textsearch.segments_per_level = 64; DO \$\$ DECLARE batch integer; BEGIN FOR batch IN 0..7 LOOP INSERT INTO d(c) SELECT CASE WHEN (g % 12) = 0 THEN 'alpha beta gamma qwxsentinel doc ' || g ELSE 'alpha beta gamma common filler doc ' || g END FROM generate_series(batch * 120 + 1, batch * 120 + 120) g; PERFORM bm25_spill_index('i'); END LOOP; PERFORM bm25_force_merge('i'); FOR batch IN 8..9 LOOP INSERT INTO d(c) SELECT CASE WHEN (g % 12) = 0 THEN 'alpha beta gamma qwxsentinel doc ' || g ELSE 'alpha beta gamma common filler doc ' || g END FROM generate_series(batch * 120 + 1, batch * 120 + 120) g; PERFORM bm25_spill_index('i'); END LOOP; END \$\$;" local truth pre old_pending truth="$(scalar "$TRUTH_Q")" pre="$(scalar "$RECALL_Q")" old_pending="$(scalar "SELECT bm25_pending_free_pages('i');")" stop_pg build_install_current || { fail "current build/install failed"; return; } start_pg || { fail "1.4.0/v8-tombstone: NEW server failed to start" return } runsql "ALTER EXTENSION pg_textsearch UPDATE TO '1.5.1';" local before compact_result after post out err_f before="$(scalar "SELECT bm25_pending_free_pages('i');")" out="$BASE_DIR/v8-compact.out" err_f="$BASE_DIR/v8-compact.err" run_capture "SET pg_textsearch.segments_per_level = 2; SELECT bm25_compact_step('i'::regclass);" "$out" "$err_f" compact_result="$(tr -d '[:space:]' <"$out")" after="$(scalar "SELECT bm25_pending_free_pages('i');")" post="$(scalar "$RECALL_Q")" runsql "ROLLBACK PREPARED 'v8_tombstone_horizon';" stop_pg log " [1.4.0/v8-tombstone] truth=$truth pre=$pre old_pending=$old_pending before=$before compact=$compact_result after=$after post=$post" [ "$pre" = "$truth" ] || fail "1.4.0/v8-tombstone: OLD-binary recall $pre != truth $truth" [ "$old_pending" -gt 0 ] || fail "1.4.0/v8-tombstone: old force-merge did not park pages" [ "$before" = "$old_pending" ] || fail "1.4.0/v8-tombstone: binary upgrade changed pending pages ($old_pending -> $before)" [ "$compact_result" = "t" ] || fail "1.4.0/v8-tombstone: compaction did not publish: $(head -1 "$err_f")" [ "$post" = "$truth" ] || fail "1.4.0/v8-tombstone: post-compaction recall $post != truth $truth" [ "$after" -gt "$before" ] || fail "1.4.0/v8-tombstone: compaction did not add pending pages ($before -> $after)" } # ------------------------------------------------------------------ # # Main # ------------------------------------------------------------------ # log "Upgrade matrix: PG bindir=$BINDIR runas=$PG_RUNAS" log "Versions: $OLD_VERSIONS" for v in $OLD_VERSIONS; do log "=== old version v$v ===" if ! build_install_old "$v"; then fail "$v: download/build of old release failed"; continue fi if is_legacy "$v"; then run_legacy "$v" else if [ "$v" = "1.4.0" ]; then run_v8_tombstone_compaction_upgrade build_install_old "$v" || { fail "$v: could not reinstall old binary after V8 tombstone test" continue } run_boolean_completeness_upgrade build_install_old "$v" || { fail "$v: could not reinstall old binary after Boolean upgrade test" continue } fi shapes="single_seg two_seg multi_seg memtable_unspilled" last_shape="${shapes##* }" for st in $shapes; do run_compat_shape "$v" "$st" # Each scenario installs the current binary over the old one, so # restore the old binary before the next shape. Skip after the # last shape — nothing else uses this old binary. [ "$st" = "$last_shape" ] && break build_install_old "$v" || { fail "$v: could not reinstall old binary"; break; } done fi done echo if [ "$FAILURES" -eq 0 ]; then log "UPGRADE MATRIX PASSED (hard assertions); see ::warning:: lines for known issues" exit 0 else err "UPGRADE MATRIX FAILED: $FAILURES hard assertion(s)" exit 1 fi timescale-pg_textsearch-b63fe58/test/scripts/vacuum_concurrent_merge.sh000077500000000000000000001223111525762047700267070ustar00rootroot00000000000000#!/bin/bash # # Regression test for the VACUUM-vs-merge race (issue #411): the index # bulk-delete (tp_bulkdelete) and cleanup (tp_vacuumcleanup) paths walked # segments by block number while reading the metapage snapshot outside the # per-index LWLock, so a concurrent spill/merge could free and recycle those # blocks between identifying a segment and reopening it. This produced # "invalid segment header" errors ("while cleaning up index" / "while # vacuuming index") and, when a recycled block still passed the header magic # check, an out-of-bounds alive-bitset write (TRAP: failed # Assert("doc_id < bitset->num_docs"), or a SIGSEGV in release builds). # # A deterministic first case pauses force merge after source selection and # proves the VACUUM backend waits on the exact private index maintenance # lock. The stress case then has writers spill many small segments, a merger # force-merge them, a deleter create dead tuples, and a vacuumer run VACUUM # concurrently (autovacuum is also aggressive). Before the fix this fails # within seconds; after it completes cleanly with no segment-header errors # and no crash. # # It also covers the tombstone-drain-vs-truncate race (issue #465): # bm25_force_merge truncates the index via tp_truncate_dead_pages # while VACUUM drains the deferred-free tombstone chain, stranding the # drain's page frees past EOF ("could not read blocks N..N"). # # TEST_SIZE_MULTIPLIER scales the loop counts (default 1.0) for # constrained runners. Prefer full scale: these are timing races, and # a shorter run finds them less often. # set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" PG_CONFIG="${PG_CONFIG:-pg_config}" PGBINDIR="$("${PG_CONFIG}" --bindir)" export PATH="${PGBINDIR}:${PATH}" TEST_PORT=55442 TEST_DB=vacuum_concurrent_merge_test DATA_DIR="${SCRIPT_DIR}/../tmp_vacuum_concurrent_merge" SOCKET_DIR="${SCRIPT_DIR}/.vcm_sock" LOGFILE="${DATA_DIR}/postgres.log" ERR_DIR="${DATA_DIR}/client_logs" KEEP_DIR="${SCRIPT_DIR}/../tmp_vacuum_concurrent_merge_logs" TEST_SIZE_MULTIPLIER=${TEST_SIZE_MULTIPLIER:-1.0} POINT_AFTER_SELECT='pg-textsearch-compaction-after-select' POINT_SOURCE_ESTIMATE='pg-textsearch-compaction-source-estimate' POINT_VACUUM_RECLAIM='pg-textsearch-vacuum-memtable-reclaim' POINT_MEMTABLE_EXTEND='pg-textsearch-memtable-extend' POINT_EXCLUSIVE_WAITER='pg-textsearch-index-lock-exclusive-waiter' # Injection points only exist on a server built with # --enable-injection-points; the rest of this script runs either way. HAS_INJECTION_POINTS=0 PRELOAD_LIBRARIES='pg_textsearch' if [ -f "$("${PG_CONFIG}" --pkglibdir)/injection_points.so" ]; then HAS_INJECTION_POINTS=1 PRELOAD_LIBRARIES='pg_textsearch,injection_points' fi STRESS_TIMEOUT_SECONDS=120 # Scale a loop count by TEST_SIZE_MULTIPLIER (minimum 1). scaled_count() { awk "BEGIN {n = int($1 * ${TEST_SIZE_MULTIPLIER} + 0.5); print (n < 1 ? 1 : n)}" } RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' # Diagnostics go to stderr: helpers are called from inside command # substitutions, whose stdout is captured rather than shown. log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}" >&2; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}" >&2; } sql() { psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 "$@" } diagnose() { exec 3>&1 1>&2 warn "pg_stat_activity:" sql -F '|' -c " SELECT pid, application_name, state, coalesce(wait_event_type, ''), coalesce(wait_event, ''), left(query, 100) FROM pg_stat_activity WHERE datname = current_database() ORDER BY pid;" 2>&1 || true warn "pg_locks for test indexes:" sql -F '|' -c " SELECT pid, locktype, CASE WHEN locktype = 'relation' THEN relation::regclass::text WHEN locktype = 'object' THEN classid::regclass::text || ':' || objid || ':' || objsubid END, mode, granted FROM pg_locks WHERE (locktype = 'relation' AND relation IN ('docs_bm25'::regclass, 'coord_bm25'::regclass, 'identify_bm25'::regclass, 'reclaim_bm25'::regclass, 'yield_bm25'::regclass)) OR (locktype = 'object' AND classid = 'pg_am'::regclass AND objid IN ('docs_bm25'::regclass, 'coord_bm25'::regclass, 'identify_bm25'::regclass, 'reclaim_bm25'::regclass, 'yield_bm25'::regclass)) ORDER BY pid, locktype, mode;" 2>&1 || true warn "server log tail:" tail -n 80 "${LOGFILE}" 2>/dev/null || true warn "client log tails:" for file in "${ERR_DIR}"/*.log; do [ -e "${file}" ] || continue echo "==> ${file} <==" tail -n 20 "${file}" || true done exec 1>&3 3>&- } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}" >&2 diagnose exit 1 } cleanup() { local exit_code=$? local pid trap - EXIT INT TERM log "Cleaning up (exit code: $exit_code)..." for pid in $(jobs -p); do kill "${pid}" 2>/dev/null || true done if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m immediate &>/dev/null || true fi # Preserve the logs on failure: the data dir is too large to # upload, and without the server log a CI failure is undebuggable. if [ "$exit_code" -ne 0 ] && [ -d "${DATA_DIR}" ]; then rm -rf "${KEEP_DIR}" mkdir -p "${KEEP_DIR}" cp "${LOGFILE}" "${KEEP_DIR}/" 2>/dev/null || true cp -r "${ERR_DIR}" "${KEEP_DIR}/" 2>/dev/null || true warn "Preserved logs in ${KEEP_DIR}" fi rm -rf "${DATA_DIR}" "${SOCKET_DIR}" exit $exit_code } trap cleanup EXIT INT TERM setup_test_db() { log "Setting up PostgreSQL instance..." rm -rf "${DATA_DIR}" "${SOCKET_DIR}" mkdir -p "${DATA_DIR}" "${SOCKET_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust >/dev/null 2>&1 mkdir -p "${ERR_DIR}" cat >> "${DATA_DIR}/postgresql.conf" << EOF port = ${TEST_PORT} unix_socket_directories = '${SOCKET_DIR}' listen_addresses = 'localhost' shared_preload_libraries = '${PRELOAD_LIBRARIES}' logging_collector = on log_directory = '.' log_filename = 'postgres.log' autovacuum = on autovacuum_naptime = 1s EOF pg_ctl start -D "${DATA_DIR}" -l "${LOGFILE}" -w -o "-p ${TEST_PORT}" || \ error "Failed to start PostgreSQL" createdb -h "${SOCKET_DIR}" -p ${TEST_PORT} ${TEST_DB} sql -c "CREATE EXTENSION pg_textsearch;" >/dev/null if [ "${HAS_INJECTION_POINTS}" -eq 1 ]; then sql -c "CREATE EXTENSION injection_points;" >/dev/null fi log "Test database ready" } PSQL="psql -h ${SOCKET_DIR} -p ${TEST_PORT} -d ${TEST_DB} -qAt -v ON_ERROR_STOP=1" seed_data() { log "Creating bm25 indexes and seeding data..." $PSQL <<'SQL' >/dev/null SET client_min_messages=warning; CREATE TABLE docs (id bigserial PRIMARY KEY, body text NOT NULL); CREATE INDEX docs_bm25 ON docs USING bm25(body) WITH (text_config='english'); -- Force frequent autovacuum bulk-delete on this table. ALTER TABLE docs SET (autovacuum_vacuum_scale_factor=0, autovacuum_vacuum_threshold=200); INSERT INTO docs (body) SELECT 'seed postgres bm25 ' || md5(gs::text) FROM generate_series(1, 2000) gs; SELECT bm25_spill_index('docs_bm25'); CREATE TABLE coord_docs (id bigserial PRIMARY KEY, body text NOT NULL); CREATE INDEX coord_bm25 ON coord_docs USING bm25(body) WITH (text_config='english', compaction='off'); ALTER TABLE coord_docs SET (autovacuum_enabled=false); INSERT INTO coord_docs(body) SELECT 'coordcase batch1 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('coord_bm25'); INSERT INTO coord_docs(body) SELECT 'coordcase batch2 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('coord_bm25'); INSERT INTO coord_docs(body) SELECT 'coordcase batch3 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('coord_bm25'); INSERT INTO coord_docs(body) SELECT 'coordcase batch4 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('coord_bm25'); CREATE TABLE identify_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX identify_bm25 ON identify_docs USING bm25(body) WITH (text_config='english', compaction='off'); ALTER TABLE identify_docs SET (autovacuum_enabled=false); INSERT INTO identify_docs(body) SELECT 'identify vacuum document ' || gs || ' ' || repeat(md5(gs::text), 8) FROM generate_series(1, 20000) gs; SELECT bm25_spill_index('identify_bm25'); DELETE FROM identify_docs WHERE id <= 1000; CREATE TABLE yield_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX yield_bm25 ON yield_docs USING bm25(body) WITH (text_config='english', compaction='off'); ALTER TABLE yield_docs SET (autovacuum_enabled=false); INSERT INTO yield_docs(body) SELECT 'yieldcase batch1 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('yield_bm25'); INSERT INTO yield_docs(body) SELECT 'yieldcase batch2 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('yield_bm25'); INSERT INTO yield_docs(body) SELECT 'yieldcase batch3 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('yield_bm25'); INSERT INTO yield_docs(body) SELECT 'yieldcase batch4 document ' || gs FROM generate_series(1, 8) gs; SELECT bm25_spill_index('yield_bm25'); DELETE FROM yield_docs WHERE id <= 4; CREATE TABLE reclaim_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); CREATE INDEX reclaim_bm25 ON reclaim_docs USING bm25(body) WITH (text_config='english', compaction='off'); ALTER TABLE reclaim_docs SET (autovacuum_enabled=false); INSERT INTO reclaim_docs(body) SELECT 'retired reclaim document ' || gs || ' ' || repeat(md5(gs::text), 8) FROM generate_series(1, 6000) gs; SELECT bm25_spill_index('reclaim_bm25'); INSERT INTO reclaim_docs(body) VALUES ('reclaimrace live memtable document'); CREATE TABLE snapshot_lock_docs ( id bigserial PRIMARY KEY, body text NOT NULL ); INSERT INTO snapshot_lock_docs(body) SELECT 'snapshotlock committed document ' || gs FROM generate_series(1, 10) gs; CREATE INDEX snapshot_lock_bm25 ON snapshot_lock_docs USING bm25(body) WITH (text_config='english', compaction='off'); ALTER TABLE snapshot_lock_docs SET (autovacuum_enabled=false); SQL } backend_pid() { local app_name=$1 local deadline=$((SECONDS + 10)) local pid while ((SECONDS < deadline)); do pid=$(sql -c " SELECT pid FROM pg_stat_activity WHERE application_name = '${app_name}' ORDER BY backend_start DESC LIMIT 1;" 2>/dev/null || true) if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return fi sleep 0.05 done error "backend ${app_name} did not appear within 10 seconds" } client_backend_pid() { local output_file=$1 local label=$2 local deadline=$((SECONDS + 10)) local pid while ((SECONDS < deadline)); do pid=$(sed -n '1p' "${output_file}" 2>/dev/null || true) if [[ "${pid}" =~ ^[0-9]+$ ]]; then echo "${pid}" return fi sleep 0.05 done error "${label} did not report its backend PID within 10 seconds" } # A backend parked in an injection point reports the point name as a # custom wait event, so its state is observable directly. injection_waiting() { local backend=$1 local point=$2 sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = ${backend} AND wait_event_type = 'InjectionPoint' AND wait_event = '${point}' );" 2>/dev/null || true } wait_for_injection() { local backend=$1 local point=$2 local deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do if [ "$(injection_waiting "${backend}" "${point}")" = "t" ]; then log "Backend ${backend} is waiting at ${point}" return fi sleep 0.05 done error "backend ${backend} did not reach ${point}" } assert_still_paused() { local backend=$1 local point=$2 local client_pid=$3 local label=$4 [ "$(injection_waiting "${backend}" "${point}")" = "t" ] || error "${label} left ${point} before the proof completed" kill -0 "${client_pid}" 2>/dev/null || error "${label} client exited before the proof completed" } # Release one backend parked at a point and wait for it to leave. release_injection() { local backend=$1 local point=$2 local deadline sql -c "SELECT injection_points_wakeup('${point}');" >/dev/null 2>&1 || true deadline=$((SECONDS + 10)) while ((SECONDS < deadline)); do [ "$(injection_waiting "${backend}" "${point}")" = "t" ] || return 0 sleep 0.05 done error "backend ${backend} stayed parked at ${point} after wakeup" } # Keep waking a backend that re-enters the same point until its client # finishes. drain_injection() { local backend=$1 local point=$2 local client_pid=$3 while kill -0 "${client_pid}" 2>/dev/null; do if [ "$(injection_waiting "${backend}" "${point}")" = "t" ]; then sql -c "SELECT injection_points_wakeup('${point}');" \ >/dev/null 2>&1 || true fi sleep 0.02 done } # Attach a point for the connecting backend only, so concurrent # sessions are unaffected. Emitted as psql -c arguments. attach_local_wait() { printf "SELECT injection_points_set_local(); SELECT injection_points_attach('%s', 'wait');" "$1" } wait_for_exit() { local pid=$1 local timeout_seconds=$2 local label=$3 local deadline=$((SECONDS + timeout_seconds)) while kill -0 "${pid}" 2>/dev/null; do ((SECONDS < deadline)) || error "${label} did not finish within ${timeout_seconds} seconds" sleep 0.05 done } wait_success() { local pid=$1 local timeout_seconds=$2 local label=$3 local output_file=$4 wait_for_exit "${pid}" "${timeout_seconds}" "${label}" if ! wait "${pid}"; then tail -n 40 "${output_file}" 2>/dev/null || true error "${label} failed" fi } assert_no_segment_errors() { if grep -REIl \ "invalid segment header|could not read blocks?.*read only [0-9]+ of|not a valid memtable page|invalid magic|magic mismatch" \ "${ERR_DIR}" "${LOGFILE}" >/dev/null 2>&1; then warn "Found an index storage error:" grep -REIn \ "invalid segment header|could not read blocks?.*read only [0-9]+ of|not a valid memtable page|invalid magic|magic mismatch" \ "${ERR_DIR}" "${LOGFILE}" | sed -n '1,5p' error "TEST FAILED: concurrent maintenance reported an index storage error" fi } test_memtable_snapshot_lock_order() { local writer_output="${ERR_DIR}/snapshot_lock_writer.log" local reader_output="${ERR_DIR}/snapshot_lock_reader.log" local writer_pid reader_pid writer_backend reader_backend local result expected state_proof=f deadline if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping memtable snapshot lock order: no injection points" return fi # PR3's unlocked VACUUM cleanup uses the same graph snapshot helper while # inserts may extend the memtable. A ranked reader exposes the helper's # exact snapshot and wait state, which VACUUM itself does not return. log "Case: read snapshot acquires memtable tail before metapage..." PGAPPNAME=pgts-snapshot-lock-writer \ PGOPTIONS="-c statement_timeout=30000 -c lock_timeout=15000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "$(attach_local_wait "${POINT_MEMTABLE_EXTEND}")" \ -c "INSERT INTO snapshot_lock_docs(body) SELECT 'snapshotlock uncommitted extension ' || gs FROM generate_series(1, 1000) gs;" \ >"${writer_output}" 2>&1 & writer_pid=$! writer_backend=$(client_backend_pid \ "${writer_output}" "memtable extension writer") wait_for_injection "${writer_backend}" "${POINT_MEMTABLE_EXTEND}" PGAPPNAME=pgts-snapshot-lock-reader \ PGOPTIONS="-c statement_timeout=15000 -c lock_timeout=14000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "SET enable_seqscan = off; SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM snapshot_lock_docs ORDER BY body <@> to_bm25query( 'snapshotlock', 'snapshot_lock_bm25') LIMIT 2000 ) ranked;" \ >"${reader_output}" 2>&1 & reader_pid=$! reader_backend=$(client_backend_pid \ "${reader_output}" "snapshot lock-order reader") deadline=$((SECONDS + 5)) while ((SECONDS < deadline)); do state_proof=$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity reader WHERE reader.pid = ${reader_backend} AND reader.state = 'active' AND reader.wait_event_type = 'LWLock' AND reader.wait_event = 'BufferContent' AND reader.query LIKE '%snapshot_lock_docs%' );" 2>/dev/null || true) [ "${state_proof}" = "t" ] && break sleep 0.05 done [ "${state_proof}" = "t" ] || error "reader did not wait on BufferContent while writer held old tail" assert_still_paused "${writer_backend}" "${POINT_MEMTABLE_EXTEND}" \ "${writer_pid}" "tail-extension writer" kill -0 "${reader_pid}" 2>/dev/null || error "snapshot reader exited before the explicit gate release" drain_injection "${writer_backend}" "${POINT_MEMTABLE_EXTEND}" \ "${writer_pid}" wait_success "${reader_pid}" 10 "snapshot lock-order reader" \ "${reader_output}" wait_success "${writer_pid}" 10 "memtable extension writer" \ "${writer_output}" result=$(tail -n 1 "${reader_output}") expected=$(seq 1 10 | paste -sd, -) [ "${result}" = "${expected}" ] || error "snapshot lock-order reader returned ${result}, expected ${expected}" log "Tail extension and exact ranked snapshot completed without deadlock" } test_vacuum_reclaim_does_not_gate_readers() { local vacuum_output="${ERR_DIR}/reclaim_vacuum.log" local spill_output="${ERR_DIR}/reclaim_spill.log" local reader_output="${ERR_DIR}/reclaim_reader.log" local compactor_output="${ERR_DIR}/reclaim_compactor.log" local vacuum_pid spill_pid spill_drain_pid reader_pid compactor_pid local vacuum_backend spill_backend compactor_backend local oid reader_result lock_proof=f deadline if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping full-fork VACUUM reclaim case: no injection points" return fi log "Case: full-fork VACUUM reclaim does not hold the per-index lock..." oid=$(sql -c "SELECT 'reclaim_bm25'::regclass::oid;") for _ in $(seq 1 32); do sql -c "SELECT txid_current();" >/dev/null done PGAPPNAME=pgts-reclaim-vacuum \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=30000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "$(attach_local_wait "${POINT_VACUUM_RECLAIM}")" \ -c "VACUUM reclaim_docs;" \ >"${vacuum_output}" 2>&1 & vacuum_pid=$! vacuum_backend=$(client_backend_pid "${vacuum_output}" "reclaim VACUUM") wait_for_injection "${vacuum_backend}" "${POINT_VACUUM_RECLAIM}" PGAPPNAME=pgts-reclaim-spill \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=30000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "$(attach_local_wait "${POINT_EXCLUSIVE_WAITER}")" \ -c "SELECT bm25_spill_index('reclaim_bm25');" \ >"${spill_output}" 2>&1 & spill_pid=$! spill_backend=$(client_backend_pid "${spill_output}" "reclaim spill") wait_for_injection "${spill_backend}" "${POINT_EXCLUSIVE_WAITER}" drain_injection "${spill_backend}" "${POINT_EXCLUSIVE_WAITER}" \ "${spill_pid}" & spill_drain_pid=$! PGAPPNAME=pgts-reclaim-reader \ PGOPTIONS="-c statement_timeout=5000 -c lock_timeout=4000" \ sql -c " SELECT count(*) FROM ( SELECT id FROM reclaim_docs ORDER BY body <@> to_bm25query( 'reclaimrace', 'reclaim_bm25') LIMIT 10 ) ranked;" \ >"${reader_output}" 2>&1 & reader_pid=$! wait_success "${spill_pid}" 5 "exclusive spill during reclaim" \ "${spill_output}" wait "${spill_drain_pid}" wait_success "${reader_pid}" 5 "later ranked reader during reclaim" \ "${reader_output}" reader_result=$(tail -n 1 "${reader_output}") [ "${reader_result}" = "1" ] || error "later reclaim reader returned ${reader_result}" assert_still_paused "${vacuum_backend}" "${POINT_VACUUM_RECLAIM}" \ "${vacuum_pid}" "reclaim VACUUM" PGAPPNAME=pgts-reclaim-compactor \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=30000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "SELECT bm25_force_merge('reclaim_bm25');" \ >"${compactor_output}" 2>&1 & compactor_pid=$! compactor_backend=$(client_backend_pid \ "${compactor_output}" "reclaim force merge") deadline=$((SECONDS + 3)) while ((SECONDS < deadline)); do lock_proof=$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity activity JOIN pg_locks pending ON pending.pid = activity.pid WHERE activity.pid = ${compactor_backend} AND activity.state = 'active' AND ${vacuum_backend} = ANY (pg_blocking_pids(activity.pid)) AND pending.locktype = 'object' AND pending.classid = 'pg_am'::regclass AND pending.objid = ${oid} AND pending.objsubid = 3 AND pending.mode = 'ExclusiveLock' AND NOT pending.granted );" 2>/dev/null || true) [ "${lock_proof}" = "t" ] && break sleep 0.05 done [ "${lock_proof}" = "t" ] || error "force merge was not serialized by reclaim maintenance" assert_still_paused "${vacuum_backend}" "${POINT_VACUUM_RECLAIM}" \ "${vacuum_pid}" "reclaim VACUUM" release_injection "${vacuum_backend}" "${POINT_VACUUM_RECLAIM}" wait_success "${vacuum_pid}" 20 "reclaim VACUUM" "${vacuum_output}" wait_success "${compactor_pid}" 15 "post-reclaim force merge" \ "${compactor_output}" log "Exclusive spill and ranked reader completed while reclaim stayed paused" } test_vacuum_identification_does_not_gate_readers() { local vacuum_output="${ERR_DIR}/identify_vacuum.log" local spill_output="${ERR_DIR}/identify_spill.log" local reader_output="${ERR_DIR}/identify_reader.log" local vacuum_pid local spill_pid local spill_drain_pid local reader_pid local vacuum_backend local spill_backend local oid local reader_result if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping VACUUM identification case: no injection points" return fi log "Case: VACUUM identification does not gate later readers..." oid=$(sql -c "SELECT 'identify_bm25'::regclass::oid;") PGAPPNAME=pgts-identify-vacuum \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=30000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "$(attach_local_wait "${POINT_SOURCE_ESTIMATE}")" \ -c "VACUUM identify_docs;" \ >"${vacuum_output}" 2>&1 & vacuum_pid=$! vacuum_backend=$(client_backend_pid \ "${vacuum_output}" "identification VACUUM client") wait_for_injection "${vacuum_backend}" "${POINT_SOURCE_ESTIMATE}" PGAPPNAME=pgts-identify-spill \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=30000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "INSERT INTO identify_docs(body) VALUES ('queued spill reader visibility');" \ -c "$(attach_local_wait "${POINT_EXCLUSIVE_WAITER}")" \ -c "SELECT bm25_spill_index('identify_bm25');" \ >"${spill_output}" 2>&1 & spill_pid=$! spill_backend=$(client_backend_pid "${spill_output}" "identification spill") wait_for_injection "${spill_backend}" "${POINT_EXCLUSIVE_WAITER}" drain_injection "${spill_backend}" "${POINT_EXCLUSIVE_WAITER}" \ "${spill_pid}" & spill_drain_pid=$! assert_still_paused "${vacuum_backend}" "${POINT_SOURCE_ESTIMATE}" \ "${vacuum_pid}" "identification VACUUM" PGAPPNAME=pgts-identify-reader \ PGOPTIONS="-c statement_timeout=2000 -c lock_timeout=1500" \ sql -c " SELECT count(*) FROM ( SELECT id FROM identify_docs ORDER BY body <@> to_bm25query( 'identify', 'identify_bm25') LIMIT 1 ) ranked;" \ >"${reader_output}" 2>&1 & reader_pid=$! wait_success "${reader_pid}" 3 "later identification reader" \ "${reader_output}" reader_result=$(tail -n 1 "${reader_output}") [ "${reader_result}" = "1" ] || error "later identification reader returned ${reader_result}" assert_still_paused "${vacuum_backend}" "${POINT_SOURCE_ESTIMATE}" \ "${vacuum_pid}" "identification VACUUM" log "Later reader completed while VACUUM identification was paused" release_injection "${vacuum_backend}" "${POINT_SOURCE_ESTIMATE}" wait_success "${vacuum_pid}" 15 "identification VACUUM" \ "${vacuum_output}" wait_success "${spill_pid}" 15 "identification spill" "${spill_output}" wait "${spill_drain_pid}" } test_vacuum_waits_for_force_merge() { local compactor_output="${ERR_DIR}/deterministic_compactor.log" local vacuum_output="${ERR_DIR}/deterministic_vacuum.log" local compactor_pid local vacuum_pid local compactor_backend local vacuum_backend local oid local lock_proof=f local deadline local plan local ranked_ids local replacement_ids local resurrected_ids local reused_count local expected_ids="9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32" if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping selected-source VACUUM case: no injection points" return fi log "Case: VACUUM waits for a force merge with selected sources..." oid=$(sql -c "SELECT 'coord_bm25'::regclass::oid;") # Leave one committed, removable dead set so VACUUM must call # ambulkdelete even though the force merge's snapshot will keep the # post-selection deletes below from becoming removable immediately. sql -c " CREATE TABLE coord_deleted_slots ( old_id bigint PRIMARY KEY, old_ctid tid NOT NULL ); INSERT INTO coord_deleted_slots SELECT id, ctid FROM coord_docs WHERE id BETWEEN 1 AND 4;" >/dev/null sql -c "DELETE FROM coord_docs WHERE id BETWEEN 1 AND 4;" >/dev/null PGAPPNAME=pgts-vacuum-compactor \ sql -c " SET statement_timeout = '90s'; SELECT injection_points_set_local(); SELECT injection_points_attach('${POINT_AFTER_SELECT}', 'wait'); SELECT bm25_force_merge('coord_bm25');" \ >"${compactor_output}" 2>&1 & compactor_pid=$! compactor_backend=$(backend_pid pgts-vacuum-compactor) wait_for_injection "${compactor_backend}" "${POINT_AFTER_SELECT}" sql -c "DELETE FROM coord_docs WHERE id BETWEEN 5 AND 8;" >/dev/null PGAPPNAME=pgts-blocked-vacuum \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=15000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" -c "VACUUM coord_docs;" \ >"${vacuum_output}" 2>&1 & vacuum_pid=$! vacuum_backend=$(client_backend_pid \ "${vacuum_output}" "VACUUM client") deadline=$((SECONDS + 3)) while ((SECONDS < deadline)); do lock_proof=$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity activity JOIN pg_locks pending ON pending.pid = activity.pid WHERE activity.pid = ${vacuum_backend} AND activity.state = 'active' AND ${compactor_backend} = ANY (pg_blocking_pids(activity.pid)) AND pending.locktype = 'object' AND pending.classid = 'pg_am'::regclass AND pending.objid = ${oid} AND pending.objsubid = 3 AND pending.mode = 'ExclusiveLock' AND NOT pending.granted );" 2>/dev/null || true) if [ "${lock_proof}" = "t" ]; then break fi sleep 0.05 done [ "${lock_proof}" = "t" ] || error "VACUUM was not blocked by backend ${compactor_backend} on the coord_bm25 maintenance lock" kill -0 "${vacuum_pid}" 2>/dev/null || error "VACUUM exited before the selected-source compaction resumed" kill -0 "${compactor_pid}" 2>/dev/null || error "force merge exited before VACUUM blocking was proved" assert_still_paused "${compactor_backend}" "${POINT_AFTER_SELECT}" \ "${compactor_pid}" "force merge" release_injection "${compactor_backend}" "${POINT_AFTER_SELECT}" wait_success "${compactor_pid}" 15 "force merge" "${compactor_output}" wait_success "${vacuum_pid}" 15 "VACUUM" "${vacuum_output}" sql -c " INSERT INTO coord_docs(body) SELECT 'replacement unrelated row ' || gs FROM generate_series(1, 64) gs;" >/dev/null reused_count=$(sql -c " SELECT count(*) FROM coord_deleted_slots slots JOIN coord_docs replacements ON replacements.ctid = slots.old_ctid WHERE replacements.body LIKE 'replacement unrelated row %';") [ "${reused_count}" = "4" ] || error "replacement rows reused ${reused_count}/4 deleted CTID slots" replacement_ids=$(sql -c " SELECT string_agg(replacements.id::text, ',' ORDER BY replacements.id) FROM coord_deleted_slots slots JOIN coord_docs replacements ON replacements.ctid = slots.old_ctid WHERE replacements.body LIKE 'replacement unrelated row %';") [ -n "${replacement_ids}" ] || error "could not identify replacement IDs in reused CTID slots" plan=$(sql -c " SET enable_seqscan = off; EXPLAIN (COSTS off) SELECT id FROM coord_docs ORDER BY body <@> to_bm25query('coordcase', 'coord_bm25') LIMIT 10000;") grep -Fq "Index Scan using coord_bm25 on coord_docs" <<<"${plan}" || error "post-VACUUM assertion did not use coord_bm25" ranked_ids=$(sql -c " SET enable_seqscan = off; SELECT string_agg(id::text, ',' ORDER BY id) FROM ( SELECT id FROM coord_docs ORDER BY body <@> to_bm25query('coordcase', 'coord_bm25') LIMIT 10000 ) ranked;") [ "${ranked_ids}" = "${expected_ids}" ] || error "forced BM25 Index Scan returned IDs ${ranked_ids}, expected ${expected_ids}" resurrected_ids=$(sql -c " SET enable_seqscan = off; WITH ranked AS MATERIALIZED ( SELECT id FROM coord_docs ORDER BY body <@> to_bm25query('coordcase', 'coord_bm25') LIMIT 10000 ) SELECT coalesce(string_agg(ranked.id::text, ',' ORDER BY ranked.id), '') FROM ranked JOIN coord_docs replacements USING (id) JOIN coord_deleted_slots slots ON slots.old_ctid = replacements.ctid WHERE replacements.body LIKE 'replacement unrelated row %';") [ -z "${resurrected_ids}" ] || error "old-term scan resurrected replacement IDs ${resurrected_ids} from deleted CTIDs ${replacement_ids}" assert_no_segment_errors log "VACUUM waited for force merge and removed all known deleted documents" } test_compact_yields_to_vacuum() { local compactor_output="${ERR_DIR}/yield_compactor.log" local vacuum_output="${ERR_DIR}/yield_vacuum.log" local compactor_pid local vacuum_pid local compactor_backend local vacuum_backend local oid local lock_proof=f local deadline local levels local vacuum_drain_pid local compactor_drain_pid if [ "${HAS_INJECTION_POINTS}" -ne 1 ]; then log "Skipping compaction yield case: no injection points" return fi log "Case: explicit compaction yields maintenance between passes..." oid=$(sql -c "SELECT 'yield_bm25'::regclass::oid;") levels=$(sql -c "SELECT bm25_level_counts('yield_bm25'::regclass);") [ "${levels}" = "{4,0,0,0,0,0,0,0}" ] || error "yield_bm25 starts with unexpected levels ${levels}" PGAPPNAME=pgts-yield-compactor \ sql -c " SET statement_timeout = '90s'; SET pg_textsearch.segments_per_level = 2; SELECT injection_points_set_local(); SELECT injection_points_attach('${POINT_AFTER_SELECT}', 'wait'); SELECT bm25_compact('yield_bm25'::regclass);" \ >"${compactor_output}" 2>&1 & compactor_pid=$! compactor_backend=$(backend_pid pgts-yield-compactor) wait_for_injection "${compactor_backend}" "${POINT_AFTER_SELECT}" PGAPPNAME=pgts-yield-vacuum \ PGOPTIONS="-c statement_timeout=90000 -c lock_timeout=30000" \ psql -h "${SOCKET_DIR}" -p "${TEST_PORT}" -d "${TEST_DB}" \ -qAt -v ON_ERROR_STOP=1 \ -c "SELECT pg_backend_pid();" \ -c "$(attach_local_wait "${POINT_SOURCE_ESTIMATE}")" \ -c "VACUUM yield_docs;" \ >"${vacuum_output}" 2>&1 & vacuum_pid=$! vacuum_backend=$(client_backend_pid "${vacuum_output}" "yield VACUUM") deadline=$((SECONDS + 3)) while ((SECONDS < deadline)); do lock_proof=$(sql -c " SELECT EXISTS ( SELECT 1 FROM pg_stat_activity activity JOIN pg_locks pending ON pending.pid = activity.pid WHERE activity.pid = ${vacuum_backend} AND activity.state = 'active' AND ${compactor_backend} = ANY (pg_blocking_pids(activity.pid)) AND pending.locktype = 'object' AND pending.classid = 'pg_am'::regclass AND pending.objid = ${oid} AND pending.objsubid = 3 AND pending.mode = 'ExclusiveLock' AND NOT pending.granted );" 2>/dev/null || true) if [ "${lock_proof}" = "t" ]; then break fi sleep 0.05 done [ "${lock_proof}" = "t" ] || error "VACUUM did not queue behind the first compaction pass" # Releasing the first pass lets it publish and drop maintenance. The # queued VACUUM must win the handoff: it reaches its own pause while # the compactor is still shut out of a second selection. release_injection "${compactor_backend}" "${POINT_AFTER_SELECT}" wait_for_injection "${vacuum_backend}" "${POINT_SOURCE_ESTIMATE}" [ "$(injection_waiting "${compactor_backend}" "${POINT_AFTER_SELECT}")" \ = "f" ] || error "second compaction pass selected sources before queued VACUUM acquired maintenance" # Both backends re-enter their points once per pass and contend for # the same maintenance lock, so drain them concurrently. drain_injection "${vacuum_backend}" "${POINT_SOURCE_ESTIMATE}" \ "${vacuum_pid}" & vacuum_drain_pid=$! drain_injection "${compactor_backend}" "${POINT_AFTER_SELECT}" \ "${compactor_pid}" & compactor_drain_pid=$! wait_success "${vacuum_pid}" 30 "yield VACUUM" "${vacuum_output}" wait_success "${compactor_pid}" 30 "yielding compactor" \ "${compactor_output}" wait "${vacuum_drain_pid}" "${compactor_drain_pid}" 2>/dev/null || true levels=$(sql -c "SELECT bm25_level_counts('yield_bm25'::regclass);") [ "${levels}" = "{0,1,0,0,0,0,0,0}" ] || error "explicit compaction left unexpected levels ${levels}" log "Queued VACUUM acquired maintenance between compaction passes" } # Spill the memtable aggressively so many small segments accrue for the # merger to compact. writer() { for i in $(seq 1 $(scaled_count 120)); do $PSQL -c "SET pg_textsearch.memtable_pages_threshold=4; SET statement_timeout='60s'; SET lock_timeout='30s'; INSERT INTO docs (body) SELECT 'w postgres bm25 ' || md5(random()::text) FROM generate_series(1, 200) gs; SELECT bm25_spill_index('docs_bm25')" \ >>"${ERR_DIR}/writer.log" 2>&1 || return 10 done } # Concurrent spill + force-merge: the LW_EXCLUSIVE segment recycler that # races VACUUM. merger() { local output for i in $(seq 1 $(scaled_count 250)); do if ! output=$($PSQL -c \ "SET statement_timeout='60s'; SET lock_timeout='30s'; DO \$merge\$ DECLARE spilled bigint; BEGIN spilled := bm25_spill_index('docs_bm25'); IF spilled > 0 THEN RAISE NOTICE 'concurrent spill completed'; END IF; PERFORM bm25_force_merge('docs_bm25'); RAISE NOTICE 'concurrent force merge completed'; END \$merge\$" 2>&1); then printf '%s\n' "$output" >>"${ERR_DIR}/merger.log" return 30 fi printf '%s\n' "$output" >>"${ERR_DIR}/merger.log" done } # Delete the oldest rows by id (monotonic -> no row-lock deadlocks) to # create dead tuples for VACUUM bulk-delete to reclaim. deleter() { for i in $(seq 1 $(scaled_count 250)); do $PSQL -c "SET statement_timeout='60s'; SET lock_timeout='30s'; DELETE FROM docs WHERE id IN (SELECT id FROM docs ORDER BY id ASC LIMIT 120)" \ >>"${ERR_DIR}/deleter.log" 2>&1 || return 40 sleep 0.1 done } # Explicit VACUUM in a loop deterministically exercises the bulk-delete and # cleanup paths concurrently with the merger. vacuumer() { for i in $(seq 1 $(scaled_count 200)); do PGOPTIONS="-c statement_timeout=60000 -c lock_timeout=30000" \ $PSQL -c "VACUUM docs" \ >>"${ERR_DIR}/vacuumer.log" 2>&1 || return 50 sleep 0.05 done } run_test() { log "Running concurrent writer + merger + deleter + vacuumer..." writer & w_pid=$! merger & m_pid=$! deleter & d_pid=$! vacuumer & v_pid=$! wait_success "${w_pid}" "${STRESS_TIMEOUT_SECONDS}" \ "stress writer" "${ERR_DIR}/writer.log" wait_success "${m_pid}" "${STRESS_TIMEOUT_SECONDS}" \ "stress merger" "${ERR_DIR}/merger.log" wait_success "${d_pid}" "${STRESS_TIMEOUT_SECONDS}" \ "stress deleter" "${ERR_DIR}/deleter.log" wait_success "${v_pid}" "${STRESS_TIMEOUT_SECONDS}" \ "stress vacuumer" "${ERR_DIR}/vacuumer.log" # The bug surfaces both server-side (autovacuum) and client-side # (explicit VACUUM). Check both the server log and client logs. assert_no_segment_errors # An out-of-bounds alive-bitset write crashes the backend. if grep -qE "TRAP: failed Assert|was terminated by signal|server closed the connection unexpectedly" \ "${LOGFILE}" "${ERR_DIR}"/*.log 2>/dev/null; then warn "Found a crash signature:" grep -hE "TRAP: failed Assert|was terminated by signal|alive_bitset" \ "${LOGFILE}" 2>/dev/null | sed -n '1,5p' error "TEST FAILED: issue #411 reproduced (backend crash)" fi if ! grep -Rq "concurrent spill completed" "${ERR_DIR}"; then error "TEST FAILED: no concurrent spill completed" fi if ! grep -Rq "concurrent force merge completed" "${ERR_DIR}"; then error "TEST FAILED: no concurrent force merge completed" fi log "TEST PASSED: VACUUM survived concurrent spill/merge" } # Main setup_test_db seed_data test_memtable_snapshot_lock_order test_vacuum_reclaim_does_not_gate_readers test_vacuum_identification_does_not_gate_readers test_vacuum_waits_for_force_merge test_compact_yields_to_vacuum run_test log "All tests passed!" exit 0 timescale-pg_textsearch-b63fe58/test/scripts/wal_audit.sh000077500000000000000000000240131525762047700237370ustar00rootroot00000000000000#!/bin/bash # # wal_audit.sh — single-node primary checks of what we put on the # WAL stream. Three tests: # # 3. UNLOGGED bm25 index emits zero pg_textsearch records. # Vestigial check: after issue #374 deleted the custom rmgr, # there is no pg_textsearch resource manager registered, so # this is trivially true. Kept to catch regressions if a # future change reintroduces a custom rmgr without honoring # RelationNeedsWAL(rel). # # 4. LOGGED bm25 inserts and spills emit ONLY Generic (page-delta) # WAL records — no custom rmgr 149. Verifies the on-disk # memtable + GenericXLog design (issue #374): every page # mutation goes through GenericXLog so stock PostgreSQL replay # (including the single-page WAL-redo helper) can reconstruct # every page without loading pg_textsearch.so. # # 5. Crash + restart: the index is queryable post-recovery. The # on-disk memtable chain pages are themselves the durable # record — no docid-page bootstrap is needed. # # Tests use pg_walinspect (contrib in PG15+). All work happens in # the regression's own database; no replication setup needed. set -e SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" TEST_PORT=55462 TEST_DB=wal_audit DATA_DIR="${SCRIPT_DIR}/../tmp_wal_audit" RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' NC='\033[0m' log() { echo -e "${GREEN}[$(date '+%H:%M:%S')] $1${NC}"; } warn() { echo -e "${YELLOW}[$(date '+%H:%M:%S')] WARNING: $1${NC}"; } error() { echo -e "${RED}[$(date '+%H:%M:%S')] ERROR: $1${NC}"; exit 1; } cleanup() { if [ -f "${DATA_DIR}/postmaster.pid" ]; then pg_ctl stop -D "${DATA_DIR}" -m fast >/dev/null 2>&1 || \ pg_ctl stop -D "${DATA_DIR}" -m immediate >/dev/null 2>&1 || true fi rm -rf "${DATA_DIR}" } trap cleanup EXIT INT TERM setup_node() { log "Initializing primary on port ${TEST_PORT}..." rm -rf "${DATA_DIR}" initdb -D "${DATA_DIR}" --auth-local=trust --auth-host=trust \ >/dev/null 2>&1 cat >> "${DATA_DIR}/postgresql.conf" </dev/null createdb -h /tmp -p "${TEST_PORT}" "${TEST_DB}" psql -h /tmp -p "${TEST_PORT}" -d "${TEST_DB}" -c \ "CREATE EXTENSION pg_textsearch; \ CREATE EXTENSION pg_walinspect;" >/dev/null } sql() { psql -h /tmp -p "${TEST_PORT}" -d "${TEST_DB}" -tA -c "$1" 2>&1 } # Count pg_textsearch WAL records between two LSNs. Argument is an # optional record_type filter. count_records() { local start_lsn=$1 local end_lsn=$2 local filter=${3:-} local where="resource_manager = 'pg_textsearch'" [ -n "$filter" ] && where="${where} AND record_type = '${filter}'" sql " SELECT count(*) FROM pg_get_wal_records_info('${start_lsn}', '${end_lsn}') WHERE ${where};" } # --------------------------------------------------------------- # Test 3: UNLOGGED bm25 index emits zero pg_textsearch records. # --------------------------------------------------------------- test_unlogged_emits_no_records() { log "=== Test 3: UNLOGGED bm25 emits zero pg_textsearch WAL ===" local lsn_before lsn_before=$(sql "SELECT pg_current_wal_lsn();") sql " CREATE UNLOGGED TABLE u_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL); CREATE INDEX u_docs_idx ON u_docs USING bm25(content) WITH (text_config='english'); INSERT INTO u_docs (content) SELECT 'unlogged ' || md5(g::text) FROM generate_series(1, 200) g; SELECT bm25_spill_index('u_docs_idx'); " >/dev/null local lsn_after lsn_after=$(sql "SELECT pg_current_wal_lsn();") local count count=$(count_records "${lsn_before}" "${lsn_after}") log " pg_textsearch records emitted for UNLOGGED ops: ${count}" if [ "${count}" != "0" ]; then error "Test 3: UNLOGGED bm25 emitted ${count} pg_textsearch \ records (expected 0). RelationNeedsWAL guard is broken." fi log "Test 3 PASSED: UNLOGGED emits zero pg_textsearch records" } # --------------------------------------------------------------- # Test 4: LOGGED inserts/spill emit ONLY Generic records — no # custom rmgr 149. This is the core walredo-compatibility check. # --------------------------------------------------------------- test_logged_records_described_by_name() { log "=== Test 4: logged ops use only Generic WAL records ===" sql " CREATE TABLE l_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL); CREATE INDEX l_docs_idx ON l_docs USING bm25(content) WITH (text_config='english'); " >/dev/null local lsn_before lsn_before=$(sql "SELECT pg_current_wal_lsn();") sql " INSERT INTO l_docs (content) SELECT 'logged ' || md5(g::text) FROM generate_series(1, 50) g; SELECT bm25_spill_index('l_docs_idx'); " >/dev/null local lsn_after lsn_after=$(sql "SELECT pg_current_wal_lsn();") # After issue #374 no custom resource manager is registered. # We expect ZERO pg_textsearch-named records and a positive # count of Generic records covering the metapage, chain # pages, and segment header. local custom_count custom_count=$(count_records "${lsn_before}" "${lsn_after}") log " custom pg_textsearch records: ${custom_count} (expect 0)" if [ "${custom_count}" != "0" ]; then error "Test 4: expected 0 pg_textsearch records (custom \ rmgr was deleted in issue #374), got '${custom_count}'. A custom \ WAL record path was reintroduced." fi local generic_count generic_count=$(sql " SELECT count(*) FROM pg_get_wal_records_info('${lsn_before}', '${lsn_after}') WHERE resource_manager = 'Generic';") log " Generic records: ${generic_count} (expect >0)" if [ "${generic_count}" -lt 1 ]; then error "Test 4: expected ≥1 Generic record for the on-disk \ memtable mutations, got '${generic_count}'. The GenericXLog write \ path is silent." fi log "Test 4 PASSED: logged ops emit only Generic records" } # --------------------------------------------------------------- # Test 5: Crash + restart — the index is queryable post-recovery. # The on-disk memtable chain pages + segment pages are themselves # the durable record; stock PostgreSQL replay reconstructs them. # --------------------------------------------------------------- test_crash_replay() { log "=== Test 5: crash + restart preserves the index ===" sql " CREATE TABLE c_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL); CREATE INDEX c_docs_idx ON c_docs USING bm25(content) WITH (text_config='english'); INSERT INTO c_docs (content) SELECT 'precrash ' || md5(g::text) FROM generate_series(1, 30) g; CHECKPOINT; " >/dev/null local lsn_after_checkpoint lsn_after_checkpoint=$(sql "SELECT pg_current_wal_lsn();") # Insert again; these records ride the WAL only (no checkpoint # has flushed the dirty buffers), so on crash the post-CP docs # are recoverable only via WAL replay. sql " INSERT INTO c_docs (content) SELECT 'postcheckpoint ' || md5(g::text) FROM generate_series(31, 60) g; " >/dev/null local lsn_pre_crash lsn_pre_crash=$(sql "SELECT pg_current_wal_lsn();") # Crash the postmaster — kill -9 the process group leader. local pid pid=$(head -n 1 "${DATA_DIR}/postmaster.pid") log " killing postmaster (pid ${pid})" kill -9 "${pid}" 2>/dev/null || true # Make sure all postmaster children are gone before restart. sleep 2 pkill -9 -f "postgres .*${DATA_DIR}" 2>/dev/null || true sleep 1 log " restarting..." pg_ctl start -D "${DATA_DIR}" -l "${DATA_DIR}/postgres.log" -w \ >/dev/null # Recovery completed. The post-CP records were Generic # (page-delta) records. Verify there are no pg_textsearch # custom records in that LSN window — they must not exist # after issue #374. local custom_count custom_count=$(count_records "${lsn_after_checkpoint}" \ "${lsn_pre_crash}") log " custom pg_textsearch records across crash window: ${custom_count} (expect 0)" if [ "${custom_count}" != "0" ]; then error "Test 5: expected 0 custom records across the \ crash-recovery LSN window (issue #374 deleted the custom rmgr), \ got '${custom_count}'" fi # Index must be functional post-recovery and contain all 60 # docs (30 precrash + 30 postcheckpoint). local total_docs total_docs=$(sql " SELECT count(*) FROM ( SELECT 1 FROM c_docs ORDER BY content <@> to_bm25query('precrash OR postcheckpoint', 'c_docs_idx') ) sub;") log " post-recovery doc count: ${total_docs:-} (expect 60)" if [ "${total_docs:-0}" -lt 60 ]; then error "Test 5: expected ≥60 docs queryable post-recovery, \ got '${total_docs}'" fi log "Test 5 PASSED: WAL recovery preserves the index" } main() { command -v pg_ctl >/dev/null 2>&1 || error "pg_ctl not found" command -v psql >/dev/null 2>&1 || error "psql not found" command -v initdb >/dev/null 2>&1 || error "initdb not found" setup_node # Subshell each test so one failure doesn't short-circuit the # rest — verify-coverage workflow needs every test to report. local failures="" ( test_unlogged_emits_no_records ) || \ failures="${failures} test_unlogged_emits_no_records" ( test_logged_records_described_by_name ) || \ failures="${failures} test_logged_records_described_by_name" ( test_crash_replay ) || \ failures="${failures} test_crash_replay" if [ -n "${failures}" ]; then error "wal_audit failures:${failures}" fi log "All wal-audit tests passed!" exit 0 } if [ "${BASH_SOURCE[0]}" == "${0}" ]; then main "$@" fi timescale-pg_textsearch-b63fe58/test/sql/000077500000000000000000000000001525762047700205375ustar00rootroot00000000000000timescale-pg_textsearch-b63fe58/test/sql/abort.sql000066400000000000000000000141141525762047700223700ustar00rootroot00000000000000-- -- Test transaction abort scenarios -- -- Verify that pg_textsearch hooks handle aborted transactions gracefully. -- The post_parse_analyze_hook fires on every SQL statement including -- ROLLBACK. When a transaction is in an aborted state, catalog lookups -- are forbidden, so the hook must skip its work. -- CREATE EXTENSION pg_textsearch; -- Scenario 1: Basic abort with no pg_textsearch objects -- This is the original bug report (#247): ROLLBACK after an error in a -- transaction that has nothing to do with BM25 should not crash. BEGIN; CREATE TABLE abort_test1 (id serial PRIMARY KEY); SELECT 1/0; ROLLBACK; -- Scenario 2: Abort with a BM25 index present (created before the xact) CREATE TABLE abort_test2 (id serial PRIMARY KEY, body text); CREATE INDEX abort_test2_idx ON abort_test2 USING bm25 (body) WITH (text_config = 'english'); BEGIN; INSERT INTO abort_test2 (body) VALUES ('hello world'); SELECT 1/0; ROLLBACK; -- Verify the index is still usable after the abort INSERT INTO abort_test2 (body) VALUES ('hello world'); SELECT id, ROUND((body <@> to_bm25query('hello', 'abort_test2_idx'))::numeric, 4) AS score FROM abort_test2 ORDER BY body <@> to_bm25query('hello', 'abort_test2_idx') LIMIT 5; -- Scenario 3: Abort during CREATE INDEX after charging its runtime cache SELECT bm25_cache_global_estimated_bytes() AS abort_create_baseline \gset BEGIN; CREATE TABLE abort_test3 (id serial PRIMARY KEY, body text); INSERT INTO abort_test3 (body) VALUES ('test document'); CREATE INDEX abort_test3_idx ON abort_test3 USING bm25 (body) WITH (text_config = 'english'); -- INSERT only appends the durable chain. A cold build must read it to charge -- the derived shared-memory cache before abort cleanup is exercised. INSERT INTO abort_test3 (body) VALUES ('charged cache tail'); SELECT result = 'OK' AND estimated_bytes > 0 AS create_abort_cache_charged FROM bm25_cache_cold_build('abort_test3_idx') \gset \echo create_abort_cache_charged=:create_abort_cache_charged SELECT 1/0; ROLLBACK; -- Table and index should not exist after rollback SELECT count(*) FROM pg_class WHERE relname = 'abort_test3'; SELECT bm25_cache_global_estimated_bytes() = :abort_create_baseline AS create_abort_accounting_restored \gset \echo create_abort_accounting_restored=:create_abort_accounting_restored -- Scenario 4: Abort with DDL (DROP TABLE that has a BM25 index) CREATE TABLE abort_test4 (id serial PRIMARY KEY, body text); CREATE INDEX abort_test4_idx ON abort_test4 USING bm25 (body) WITH (text_config = 'english'); INSERT INTO abort_test4 (body) VALUES ('still here'); BEGIN; DROP TABLE abort_test4; SELECT 1/0; ROLLBACK; -- Table should still exist after the aborted DROP SET client_min_messages = warning; SELECT count(*) FROM abort_test4; RESET client_min_messages; -- Scenario 5: Multiple errors in sequence BEGIN; SELECT 1/0; SELECT 2/0; ROLLBACK; -- Scenario 6: SAVEPOINT and error within subtransaction BEGIN; INSERT INTO abort_test2 (body) VALUES ('before savepoint'); SAVEPOINT sp1; SELECT 1/0; ROLLBACK TO sp1; INSERT INTO abort_test2 (body) VALUES ('after rollback to savepoint'); COMMIT; SELECT id, body FROM abort_test2 ORDER BY body <@> to_bm25query('savepoint', 'abort_test2_idx'); -- Scenario 7: Error in a BM25 query BEGIN; INSERT INTO abort_test2 (body) VALUES ('scenario seven'); -- Force an error via division by zero in the same statement SELECT (body <@> to_bm25query('scenario', 'abort_test2_idx'))::int / 0 FROM abort_test2; ROLLBACK; -- Scenario 8: Abort after BM25 query succeeds, then later stmt fails BEGIN; SELECT id FROM abort_test2 ORDER BY body <@> to_bm25query('hello', 'abort_test2_idx') LIMIT 3; SELECT 1/0; ROLLBACK; -- Scenario 9: SAVEPOINT rollback of CREATE INDEX with a charged cache -- Exercises SubXactCallback cleanup of cache dshash tables and accounting. BEGIN; CREATE TABLE abort_subxact1 (id serial PRIMARY KEY, body text); SELECT bm25_cache_global_estimated_bytes() AS abort_subxact_create_baseline \gset SAVEPOINT sp1; CREATE INDEX abort_subxact1_idx ON abort_subxact1 USING bm25 (body) WITH (text_config = 'english'); -- Append durable records, then read them into the shared-memory cache. INSERT INTO abort_subxact1 (body) VALUES ('memtable data one'); INSERT INTO abort_subxact1 (body) VALUES ('memtable data two'); SELECT result = 'OK' AND estimated_bytes > 0 AS subxact_create_abort_cache_charged FROM bm25_cache_cold_build('abort_subxact1_idx') \gset \echo subxact_create_abort_cache_charged=:subxact_create_abort_cache_charged ROLLBACK TO sp1; SELECT bm25_cache_global_estimated_bytes() = :abort_subxact_create_baseline AS subxact_create_abort_accounting_restored \gset \echo subxact_create_abort_accounting_restored=:subxact_create_abort_accounting_restored -- Table should still be usable without the index INSERT INTO abort_subxact1 (body) VALUES ('after rollback'); COMMIT; SELECT count(*) FROM abort_subxact1; DROP TABLE abort_subxact1; -- Scenario 10: Nested savepoints with RELEASE (promotes subxact state) BEGIN; CREATE TABLE abort_subxact2 (id serial PRIMARY KEY, body text); SAVEPOINT sp1; SAVEPOINT sp2; CREATE INDEX abort_subxact2_idx ON abort_subxact2 USING bm25 (body) WITH (text_config = 'english'); RELEASE SAVEPOINT sp2; -- Index alive, state promoted from sp2 to sp1 INSERT INTO abort_subxact2 (body) VALUES ('after release'); ROLLBACK TO sp1; -- Index (promoted to sp1) should be cleaned up COMMIT; SELECT count(*) FROM abort_subxact2; DROP TABLE abort_subxact2; -- Scenario 11: SAVEPOINT abort followed by full transaction abort -- Validates no double-free between subxact and xact cleanup BEGIN; CREATE TABLE abort_subxact3 (id serial PRIMARY KEY, body text); SAVEPOINT sp1; CREATE INDEX abort_subxact3_idx ON abort_subxact3 USING bm25 (body) WITH (text_config = 'english'); INSERT INTO abort_subxact3 (body) VALUES ('will be rolled back'); ROLLBACK TO sp1; -- Subxact cleanup already ran; now abort the whole transaction ROLLBACK; -- Table should not exist after full rollback SELECT count(*) FROM pg_class WHERE relname = 'abort_subxact3'; -- Cleanup DROP TABLE abort_test2; DROP TABLE abort_test4; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/aerodocs.sql000066400000000000000000000323521525762047700230640ustar00rootroot00000000000000-- Aerodynamics Documents pg_textsearch Test - Real Cranfield Dataset -- This test validates pg_textsearch implementation using a substantial subset of the Cranfield collection -- Contains real aerodynamics documents from the Cranfield dataset for realistic testing -- Tests both dataset loading and pg_textsearch scoring functionality with the <@> operator -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all -- Enable score logging for testing SET pg_textsearch.log_scores = true; \set ON_ERROR_STOP on \echo 'Testing pg_textsearch with sample aerodynamics documents...' -- Create tables for aerodynamics documents CREATE TABLE aerodocs_documents ( doc_id INTEGER PRIMARY KEY, title TEXT, author TEXT, bibliography TEXT, content TEXT, full_text TEXT GENERATED ALWAYS AS ( COALESCE(title, '') || ' ' || COALESCE(author, '') || ' ' || COALESCE(content, '') ) STORED ); CREATE TABLE cranfield_queries ( query_id INTEGER PRIMARY KEY, query_text TEXT NOT NULL ); -- Expected rankings table removed - we test against actual output \echo 'Loading subset of Cranfield documents for testing...' INSERT INTO aerodocs_documents (doc_id, title, author, content) VALUES (1, 'experimental investigation of the aerodynamics of a wing in a slipstream .', 'brenckman,m.', 'experimental investigation of the aerodynamics of a wing in a slipstream . an experimental study of a wing in a propeller slipstream was made in order to determine the spanwise distribution of the lift increase due to slipstream at different angles of attack of the wing and at different free stream to slipstream velocity ratios . the results were intended in part as an evaluation basis for different theoretical treatments of this problem . the comparative span loading curves, together with supporting evidence, showed that a substantial part of the lift increment produced by the slipstream was due to a /destalling/ or boundary-layer-control effect . the integrated remaining lift increment, after subtracting this destalling lift, was found to agree well with a potential flow theory . an empirical evaluation of the destalling effects was made for the specific configuration of the experiment .'), (2, 'simple shear flow past a flat plate in an incompressible fluid of small viscosity .', 'ting-yili', 'in the study of high-speed viscous flow past a two-dimensional body it is usually necessary to consider a curved shock wave emitting from the nose or leading edge of the body . consequently, there exists an inviscid rotational flow region between the shock wave and the boundary layer . such a situation arises, for instance, in the study of the hypersonic viscous flow past a flat plate . the situation is somewhat different from prandtl''s classical boundary-layer problem . in prandtl''s original problem the inviscid free stream outside the boundary layer is irrotational while in a hypersonic boundary-layer problem the inviscid free stream must be considered as rotational . the possible effects of vorticity have been recently discussed by ferri and libby . in the present paper, the simple shear flow past a flat plate in a fluid of small viscosity is investigated . it can be shown that this problem can again be treated by the boundary-layer approximation, the only novel feature being that the free stream has a constant vorticity . the discussion here is restricted to two-dimensional incompressible steady flow .'), (3, 'the boundary layer in simple shear flow past a flat plate .', 'm. b. glauert', 'the boundary-layer equations are presented for steady incompressible flow with no pressure gradient .'), (4, 'approximate solutions of the incompressible laminar boundary layer equations for a plate in shear flow .', 'yen,k.t.', 'the two-dimensional steady boundary-layer problem for a flat plate in a shear flow of incompressible fluid is considered . solutions for the boundary- layer thickness, skin friction, and the velocity distribution in the boundary layer are obtained by the karman-pohlhausen technique . comparison with the boundary layer of a uniform flow has also been made to show the effect of vorticity .'), (5, 'one-dimensional transient heat conduction into a double-layer slab subjected to a linear heat input for a small time internal .', 'wasserman,b.', 'analytic solutions are presented for the transient heat conduction in composite slabs exposed at one surface to a triangular heat rate . this type of heating rate may occur, for example, during aerodynamic heating .'), (6, 'one-dimensional transient heat flow in a multilayer slab .', 'campbell,w.f.', 'in a recent contribution to the readers forum wassermann gave analytic solutions for the temperature in a double layer slab, with a triangular heat rate input at one face, insulated at the other, and with no thermal resistance at the interface . his solutions were for the three particular cases.. i propose here to give the general solution to this problem, to indicate briefly how it is obtained using the method of reference 2, and to point out that the solutions given by wassermann are incomplete for times longer than the duration of the heat input .'), (7, 'the effect of controlled three-dimensional roughness on boundary layer transition at supersonic speeds .', 'van driest,e.r. and mccauley,w.d.', 'experiments were performed in the 12-in. supersonic wind tunnel of the jet propulsion laboratory of the california institute of technology to investigate the effect of three-dimensional roughness elements (spheres) on boundary-layer transition on a tained at local mach numbers of 1.90, 2.71, and 3.67 by varying trip size, position, spacing, and reynolds number per inch . the results indicate that (1) transition from laminar to turbulent flow induced by three-dimensional roughness elements begins when the double row of spiral vortices trailing each element contaminates and breaks down the surrounding field of vorticity, (2) transition appears rather suddenly, becoming more violent with increasing roughness height relative to the boundary-layer thickness, (3) after the breakdown of the vorticity field, the strength of the spiral vortices may still persist in the sublayer of the ensuing turbulent flow, (4) lateral spacing of roughness elements has little effect upon the initial breakdown (contamination) of the laminar flow, and (5) the trip reynolds number where u and v are the velocity and kinematic viscosity at the outer edge of the boundary layer and k is roughness height, such that transition occurs at the roughness position, varies as the position reynolds number to the one-fourth power, viz., where x is trip position .'), (8, 'measurements of the effect of two-dimensional and three-dimensional roughness elements on boundary layer transition .', 'klebanoff,p.s.', 'in his study of the effect of roughness on transition, h. l. dryden found, on the basis of available data, that the effect of a two-dimensional roughness element such as a /trip wire/ could be represented reasonably well in terms of a functional relation between and, where is the reynolds number of transition based on distance from the leading edge, is the height of the roughness element, and is the boundary-layer displacement thickness at the position of the element . at his suggestion some additional data were obtained, primarily to extend the range to higher values of, during the course of an investigation of transition on a flat plate conducted at the national bureau of standards . after the results on the two- dimensional roughness elements were obtained, it appeared to be desirable to see whether a row of three-dimensional roughness elements would behave in the same way .'), (9, 'transition studies and skin friction measurements on an insulated flat plate at a mach number of 5.8 .', 'korkegi,r.h.', 'an investigation of transition and skin friction on an insulated flat plate, 5 by 26 in., was made in the galcit 5 by 5 in. hypersonic wind tunnel at a nominal mach number of 5.8 . the phosphorescent lacquer technique was used for transition detection and was found to be in good agreement with total-head rake measurements along the plate surface and pitot boundary- layer surveys . it was found that the boundary layer was laminar at reynolds numbers of at least 5 x 10 . transverse contamination caused by the turbulent boundary layer on the tunnel sidewall originated far downstream of the flat plate leading edge at reynolds numbers of 1.5 to 2 x 10, and spread at a uniform angle of 5 compared to 9 degree in low-speed flow . the effect of two-dimensional and local disturbances was investigated . the technique of air injection into the boundary layer as a means of hastening transition was extensively used . although the onset of transition occurred at reynolds numbers as low as 10, a fully developed turbulent boundary layer was not obtained at reynolds numbers much below 2 x 10 regardless of the amount of air injected . a qualitative discussion of these results is given with emphasis on the possibility of a greater stability of the laminar boundary layer in hypersonic flow than at lower speeds . direct skin-friction measurements were made by means of the floating element technique, over a range of reynolds numbers verified as being laminar over the complete range . with air injection, turbulent shear was obtained only for reynolds numbers greater than 2 x 10, this value being in good agreement with earlier results of this investigation . the turbulent skin-friction coefficient was found to be approximately 0.40 of that for incompressible flow for a constant value of r, and 0.46 for an effective reynolds number between 5 and 6 x 10 .'), (10, 'the theory of the impact tube at low pressure .', 'chambre,p.l. and schaaf,s.a.', 'a theoretical analysis has been made for an impact tube of the relation between free-stream mach number and the impact and free-stream pressures and densities for extremely low pressures . it is shown that the results differ appreciably from the corresponding continuum relations .'); -- Insert sample queries INSERT INTO cranfield_queries (query_id, query_text) VALUES (1, 'aerodynamic flow analysis'), (2, 'boundary layer turbulent'), (3, 'wing design optimization'); -- Expected rankings are now captured in the test's expected output file -- Create pg_textsearch index for Cranfield documents with text_config CREATE INDEX cranfield_tapir_idx ON aerodocs_documents USING bm25(full_text) WITH (text_config='english', k1=1.2, b=0.75); \echo 'pg_textsearch index created. Running validation tests...' -- Disable sequential scans to ensure index usage SET enable_seqscan = off; -- Test 0: Simple single-term validation \echo 'Test 0: Single-term query validation' SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'aerodynamic')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'aerodynamic' ASC LIMIT 3; -- Validate single-term BM25 scoring SELECT validate_bm25_scoring('aerodocs_documents', 'full_text', 'cranfield_tapir_idx', 'aerodynamic', 'english', 1.2, 0.75) as aerodynamic_valid; -- Test 1: Basic search functionality \echo 'Test 1: Basic search with <@> operator' SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'aerodynamic flow')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'aerodynamic flow' ASC LIMIT 5; -- Test 2: Multi-term search \echo 'Test 2: Multi-term search' SELECT doc_id, LEFT(title, 60) as title_preview, ROUND((full_text <@> 'boundary layer turbulent')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'boundary layer turbulent' ASC LIMIT 5; -- Test 3: Top-10 results for first query \echo 'Test 3: Top-10 results for first query' WITH pgts_results AS ( SELECT doc_id, ROUND((full_text <@> 'aerodynamic flow analysis')::numeric, 4) as pgts_score, ROW_NUMBER() OVER (ORDER BY full_text <@> 'aerodynamic flow analysis' ASC) as pgts_rank FROM aerodocs_documents ) SELECT br.doc_id, br.pgts_rank, br.pgts_score as score, LEFT(d.title, 50) as title_preview FROM pgts_results br JOIN aerodocs_documents d ON br.doc_id = d.doc_id WHERE br.pgts_rank <= 10 ORDER BY br.pgts_rank; -- Test 4: Performance test with multiple queries \echo 'Test 4: Performance test with top-k search' SELECT q.query_id, LEFT(q.query_text, 40) || '...' as query_preview, COUNT(d.doc_id) as total_docs, COUNT(CASE WHEN d.full_text <@> to_bm25query(q.query_text, 'cranfield_tapir_idx') < 0 THEN 1 END) as matching_docs FROM cranfield_queries q CROSS JOIN aerodocs_documents d WHERE q.query_id <= 5 -- Test first 5 queries for performance GROUP BY q.query_id, q.query_text ORDER BY q.query_id; -- Test 5: Index usage verification \echo 'Test 5: Verify index usage with EXPLAIN' SET jit = off; EXPLAIN (COSTS OFF) SELECT doc_id, ROUND((full_text <@> 'supersonic aircraft design')::numeric, 4) as score FROM aerodocs_documents ORDER BY full_text <@> 'supersonic aircraft design' ASC LIMIT 10; -- Reset settings SET enable_seqscan = on; \echo 'Cranfield pg_textsearch validation completed successfully.' \echo 'All tests demonstrate proper pg_textsearch functionality with the <@> operator.' -- Clean up DROP TABLE aerodocs_documents CASCADE; DROP TABLE cranfield_queries CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/basic.sql000066400000000000000000000037001525762047700223410ustar00rootroot00000000000000-- Basic functionality tests for pg_textsearch extension -- Test extension creation CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Verify library version GUC is set SELECT current_setting('pg_textsearch.library_version') IS NOT NULL AS has_version; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Test bm25vector type exists SELECT pg_typeof('my_index:{database:2,system:1}'::bm25vector); -- Test bm25vector input/output SELECT 'my_index:{database:2,system:1}'::bm25vector; -- Test empty bm25vector SELECT 'my_index:{}'::bm25vector; -- Test bm25query type SELECT pg_typeof('search terms'::bm25query); -- Test bm25query input/output SELECT 'search terms'::bm25query; -- Test to_bm25query functions SELECT to_bm25query('hello world'); SELECT to_bm25query('test query', 'my_index'); -- Test bm25 access method exists SELECT amname FROM pg_am WHERE amname = 'bm25'; -- Test creating a bm25 index with text_config CREATE TABLE test_docs (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX test_tapir_idx ON test_docs USING bm25(content) WITH (text_config='english'); -- Verify index was created SELECT indexrelid::regclass FROM pg_index WHERE indrelid = 'test_docs'::regclass AND indexrelid::regclass::text LIKE '%tapir%'; -- Test bm25query with new operators INSERT INTO test_docs (content) VALUES ('hello world example'), ('database system design'), ('the quick brown fox'), ('jumped over lazy dog'), ('sphinx of black quartz'); -- Test text <@> bm25query operator with explicit index for score projection SELECT content, content <@> to_bm25query('hello', 'test_tapir_idx') as score FROM test_docs ORDER BY score LIMIT 1; -- Test non-existent index error \set VERBOSITY terse \set ON_ERROR_STOP off SELECT content FROM test_docs ORDER BY content <@> to_bm25query('hello', 'wrong_index') LIMIT 1; \set ON_ERROR_STOP on \set VERBOSITY default -- Clean up DROP TABLE test_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/binary_io.sql000066400000000000000000000047541525762047700232450ustar00rootroot00000000000000-- Test binary I/O for bm25query type (COPY BINARY) CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create a table and index for testing CREATE TABLE binary_io_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO binary_io_docs (content) VALUES ('hello world test'), ('database search query'); CREATE INDEX binary_io_idx ON binary_io_docs USING bm25(content) WITH (text_config='english'); -- Create a table with bm25query column CREATE TABLE query_export (id SERIAL, q bm25query); -- Insert some queries INSERT INTO query_export (q) VALUES (to_bm25query('hello world', 'binary_io_idx')), (to_bm25query('database search', 'binary_io_idx')); -- Show original data before COPY SELECT id, q::text AS original_query FROM query_export ORDER BY id; -- Test COPY TO BINARY (exercises tpquery_send) -- Use \copy (client-side) to avoid /tmp permission issues \copy query_export TO '/tmp/query_export.bin' WITH (FORMAT binary) -- Test COPY FROM BINARY (exercises tpquery_recv) CREATE TABLE query_import (id int, q bm25query); \copy query_import FROM '/tmp/query_export.bin' WITH (FORMAT binary) -- Verify data was imported correctly SELECT COUNT(*) AS imported_count FROM query_import; -- Verify the queries are equivalent SELECT e.id, e.q::text AS exported, i.q::text AS imported, e.q = i.q AS queries_match FROM query_export e JOIN query_import i ON e.id = i.id ORDER BY e.id; -- ============================================================================= -- Test bm25vector binary I/O (exercises vector.c recv/send functions) -- ============================================================================= -- Create table with bm25vector column for binary copy test CREATE TABLE vector_export (id serial, v bm25vector); INSERT INTO vector_export (v) VALUES ('binary_io_idx:{hello:1,world:2}'::bm25vector), ('binary_io_idx:{database:3,search:1}'::bm25vector); -- Export via COPY BINARY (exercises bm25vector_send) \copy vector_export TO '/tmp/vector_export.bin' WITH (FORMAT binary) -- Import via COPY BINARY (exercises bm25vector_recv) CREATE TABLE vector_import (id int, v bm25vector); \copy vector_import FROM '/tmp/vector_export.bin' WITH (FORMAT binary) -- Verify round-trip SELECT COUNT(*) AS vector_imported_count FROM vector_import; -- Clean up \! rm -f /tmp/query_export.bin /tmp/vector_export.bin DROP TABLE query_export; DROP TABLE query_import; DROP TABLE vector_export; DROP TABLE vector_import; DROP TABLE binary_io_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/bmw.sql000066400000000000000000001042531525762047700220520ustar00rootroot00000000000000-- BMW (Block-Max WAND) Algorithm Tests -- Tests the top-k optimization for both single-term and multi-term queries -- Setup: Create extension CREATE EXTENSION pg_textsearch; -- Force index scan so ORDER BY uses the BM25 index (small tables -- would otherwise prefer seq scan, returning non-matching rows) SET enable_seqscan = off; \set ECHO none \i test/sql/validation.sql \set ECHO all -- ============================================================ -- SECTION 1: Basic BMW correctness -- ============================================================ CREATE TABLE bmw_basic (id SERIAL PRIMARY KEY, content TEXT); -- Insert documents with known term frequencies INSERT INTO bmw_basic (content) VALUES ('apple apple apple'), -- id=1, tf=3 ('apple apple'), -- id=2, tf=2 ('apple'), -- id=3, tf=1 ('apple apple apple apple'), -- id=4, tf=4 ('banana'), -- id=5, no apple ('apple banana'), -- id=6, tf=1 ('apple apple banana banana'); -- id=7, tf=2 CREATE INDEX bmw_basic_idx ON bmw_basic USING bm25(content) WITH (text_config='english'); -- Test: Single-term BMW should return docs in score order SELECT id, content <@> 'apple'::bm25query as score FROM bmw_basic ORDER BY score LIMIT 5; DROP TABLE bmw_basic; -- ============================================================ -- SECTION 2: Empty and edge result counts -- ============================================================ CREATE TABLE bmw_empty (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_empty (content) VALUES ('cat dog bird'), ('fish whale shark'), ('lion tiger bear'); CREATE INDEX bmw_empty_idx ON bmw_empty USING bm25(content) WITH (text_config='english'); -- Test: Query term not in corpus (empty results) SELECT id FROM bmw_empty ORDER BY content <@> 'elephant'::bm25query LIMIT 10; -- Test: Only 1 document matches SELECT id FROM bmw_empty ORDER BY content <@> 'whale'::bm25query LIMIT 10; -- Test: Fewer results than LIMIT (3 docs match, LIMIT 10) SELECT id FROM bmw_empty ORDER BY content <@> 'cat fish lion'::bm25query LIMIT 10; DROP TABLE bmw_empty; -- ============================================================ -- SECTION 3: Tie-breaking with identical scores -- ============================================================ CREATE TABLE bmw_ties (id SERIAL PRIMARY KEY, content TEXT); -- Create documents with IDENTICAL scores (same tf, same doc length) INSERT INTO bmw_ties (content) VALUES ('unique term here'), -- id=1 ('exact same length'), -- id=2, same length as id=1 ('exact same length'), -- id=3, same length ('exact same length'), -- id=4, same length ('exact same length'); -- id=5, same length CREATE INDEX bmw_ties_idx ON bmw_ties USING bm25(content) WITH (text_config='english'); -- Test: Documents with identical scores should be ordered by CTID (id) -- All "exact same length" docs have tf=1 and same doc_len SELECT id, content <@> 'exact'::bm25query as score FROM bmw_ties ORDER BY content <@> 'exact'::bm25query LIMIT 10; DROP TABLE bmw_ties; -- ============================================================ -- SECTION 4: LIMIT edge cases -- ============================================================ CREATE TABLE bmw_limits (id SERIAL PRIMARY KEY, content TEXT); -- Create 20 documents with term "test" INSERT INTO bmw_limits (content) SELECT 'test document number ' || i FROM generate_series(1, 20) i; CREATE INDEX bmw_limits_idx ON bmw_limits USING bm25(content) WITH (text_config='english'); -- Test: LIMIT 1 (minimum) SELECT id FROM bmw_limits ORDER BY content <@> 'test'::bm25query LIMIT 1; -- Test: LIMIT equals exact document count SELECT COUNT(*) FROM ( SELECT id FROM bmw_limits ORDER BY content <@> 'test'::bm25query LIMIT 20 ) x; -- Test: LIMIT exceeds document count SELECT COUNT(*) FROM ( SELECT id FROM bmw_limits ORDER BY content <@> 'test'::bm25query LIMIT 100 ) x; DROP TABLE bmw_limits; -- ============================================================ -- SECTION 5: Multi-term query edge cases -- ============================================================ CREATE TABLE bmw_multi (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_multi (content) VALUES ('alpha beta gamma'), -- id=1, has all 3 ('alpha beta'), -- id=2, has 2 ('alpha'), -- id=3, has 1 ('beta gamma'), -- id=4, has 2 ('delta epsilon'), -- id=5, has none ('alpha alpha alpha beta'), -- id=6, high tf for alpha ('gamma gamma gamma gamma'); -- id=7, high tf for gamma CREATE INDEX bmw_multi_idx ON bmw_multi USING bm25(content) WITH (text_config='english'); -- Test: 2-term query SELECT id, content <@> 'alpha beta'::bm25query as score FROM bmw_multi ORDER BY score LIMIT 5; -- Test: 3-term query SELECT id, content <@> 'alpha beta gamma'::bm25query as score FROM bmw_multi ORDER BY score LIMIT 5; -- Test: Query with non-existent term (partial match) SELECT id, content <@> 'alpha nonexistent'::bm25query as score FROM bmw_multi ORDER BY score LIMIT 5; -- Test: Query where all terms are non-existent SELECT id FROM bmw_multi ORDER BY content <@> 'xxx yyy zzz'::bm25query LIMIT 5; DROP TABLE bmw_multi; -- ============================================================ -- SECTION 6: 8-term boundary (max for multi-term BMW) -- ============================================================ CREATE TABLE bmw_8term (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_8term (content) VALUES ('one two three four five six seven eight nine ten'), ('one two three four'), ('five six seven eight'), ('one three five seven nine'), ('two four six eight ten'); CREATE INDEX bmw_8term_idx ON bmw_8term USING bm25(content) WITH (text_config='english'); -- Test: Exactly 8 terms (should use multi-term BMW) SELECT id, content <@> 'one two three four five six seven eight'::bm25query as score FROM bmw_8term ORDER BY score LIMIT 5; -- Test: 9+ terms (should fall back to exhaustive, but still work) SELECT id, content <@> 'one two three four five six seven eight nine'::bm25query as score FROM bmw_8term ORDER BY score LIMIT 5; DROP TABLE bmw_8term; -- ============================================================ -- SECTION 7: Document length extremes -- ============================================================ CREATE TABLE bmw_lengths (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_lengths (content) VALUES ('x'), -- id=1, very short ('word'), -- id=2, short (repeat('word ', 100)), -- id=3, medium (100 words) (repeat('word ', 1000)), -- id=4, long (1000 words) ('word ' || repeat('filler ', 500)); -- id=5, word + filler CREATE INDEX bmw_lengths_idx ON bmw_lengths USING bm25(content) WITH (text_config='english'); -- Test: Query should rank by BM25 (considers doc length) -- Short docs with term should score higher than long docs SELECT id, length(content) as doc_chars, content <@> 'word'::bm25query as score FROM bmw_lengths ORDER BY content <@> 'word'::bm25query LIMIT 5; DROP TABLE bmw_lengths; -- ============================================================ -- SECTION 8: High term frequency -- ============================================================ CREATE TABLE bmw_highfreq (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_highfreq (content) VALUES (repeat('keyword ', 1)), -- id=1, tf=1 (repeat('keyword ', 10)), -- id=2, tf=10 (repeat('keyword ', 50)), -- id=3, tf=50 (repeat('keyword ', 100)), -- id=4, tf=100 (repeat('keyword ', 200)); -- id=5, tf=200 CREATE INDEX bmw_highfreq_idx ON bmw_highfreq USING bm25(content) WITH (text_config='english'); -- Test: High TF documents should rank higher (with saturation) SELECT id, content <@> 'keyword'::bm25query as score FROM bmw_highfreq ORDER BY content <@> 'keyword'::bm25query LIMIT 5; -- Verify: Order should be 5,4,3,2,1 (highest tf first) -- But scores should show diminishing returns due to BM25 saturation DROP TABLE bmw_highfreq; -- ============================================================ -- SECTION 9: Segment + Memtable combination -- ============================================================ CREATE TABLE bmw_hybrid (id SERIAL PRIMARY KEY, content TEXT); -- Insert batch 1 (will go to segment via CREATE INDEX) INSERT INTO bmw_hybrid (content) SELECT 'segment document ' || i || ' with searchterm' FROM generate_series(1, 500) i; CREATE INDEX bmw_hybrid_idx ON bmw_hybrid USING bm25(content) WITH (text_config='english', compaction='manual'); -- Batch 2: insert and spill to create a second segment INSERT INTO bmw_hybrid (content) SELECT 'more segment data ' || i || ' searchterm' FROM generate_series(501, 1000) i; SELECT bm25_spill_index('bmw_hybrid_idx'); -- Insert recent data (stays in memtable) INSERT INTO bmw_hybrid (content) VALUES ('fresh memtable searchterm searchterm searchterm'), -- High relevance ('another memtable searchterm searchterm'); -- Test: BMW should find results from both segment and memtable -- The memtable docs with higher tf should rank at top SELECT id, content <@> 'searchterm'::bm25query as score FROM bmw_hybrid ORDER BY content <@> 'searchterm'::bm25query LIMIT 10; DROP TABLE bmw_hybrid; -- ============================================================ -- SECTION 10: Validate BMW equals exhaustive (LARGE SCALE) -- ============================================================ -- CRITICAL: Must use >128 docs to test block skipping! -- Block size is 128 docs, so we need multiple blocks for skip testing. CREATE TABLE bmw_validate (id SERIAL PRIMARY KEY, content TEXT); -- Create 1000 diverse documents - spans ~8 blocks -- Varying term frequencies to create score diversity INSERT INTO bmw_validate (content) SELECT CASE (i % 10) WHEN 0 THEN 'apple apple apple banana cherry' -- high apple tf WHEN 1 THEN 'apple apple banana' WHEN 2 THEN 'cherry cherry cherry' WHEN 3 THEN 'apple cherry' WHEN 4 THEN 'banana banana cherry cherry' WHEN 5 THEN 'apple' WHEN 6 THEN 'banana apple cherry' WHEN 7 THEN 'cherry banana' WHEN 8 THEN 'apple apple apple apple' -- highest apple tf WHEN 9 THEN 'banana' END || ' document number ' || i FROM generate_series(1, 1000) i; -- Force to segment with low memory limit CREATE INDEX bmw_validate_idx ON bmw_validate USING bm25(content) WITH (text_config='english'); -- Test: BMW (LIMIT) should match exhaustive (subquery LIMIT) -- Single-term WITH bmw AS ( SELECT id, content <@> 'apple'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'apple'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple'::bm25query ) x LIMIT 20 ) SELECT 'single-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; -- Multi-term WITH bmw AS ( SELECT id, content <@> 'apple banana'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'apple banana'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana'::bm25query ) x LIMIT 20 ) SELECT 'multi-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; -- Three-term WITH bmw AS ( SELECT id, content <@> 'apple banana cherry'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana cherry'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'apple banana cherry'::bm25query as score FROM bmw_validate ORDER BY content <@> 'apple banana cherry'::bm25query ) x LIMIT 20 ) SELECT 'three-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; DROP TABLE bmw_validate; -- ============================================================ -- SECTION 11: Score monotonicity with multiple blocks -- ============================================================ -- Use 500 docs to span multiple blocks (~4 blocks) CREATE TABLE bmw_monotonic (id SERIAL PRIMARY KEY, content TEXT); -- Create varying scores by varying term frequency INSERT INTO bmw_monotonic (content) SELECT CASE WHEN i % 50 = 0 THEN repeat('target ', 5) -- high tf every 50 docs WHEN i % 10 = 0 THEN repeat('target ', 3) -- medium tf every 10 docs ELSE 'target other words here' -- low tf END || ' doc' || i FROM generate_series(1, 500) i; CREATE INDEX bmw_monotonic_idx ON bmw_monotonic USING bm25(content) WITH (text_config='english'); -- Test: Scores must be monotonically descending (more negative) -- This tests that BMW correctly orders results from multiple blocks WITH ranked AS ( SELECT id, content <@> 'target'::bm25query as score, LAG(content <@> 'target'::bm25query) OVER (ORDER BY content <@> 'target'::bm25query) as prev_score FROM bmw_monotonic ORDER BY content <@> 'target'::bm25query LIMIT 50 ) SELECT 'monotonic-multiblock' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM ranked WHERE prev_score IS NOT NULL AND score < prev_score; DROP TABLE bmw_monotonic; -- ============================================================ -- SECTION 12: Block boundary stress test -- ============================================================ -- Test with 640 docs = exactly 5 blocks of 128 docs each -- Place high-scoring docs at specific positions to test block skipping CREATE TABLE bmw_blocks (id SERIAL PRIMARY KEY, content TEXT); -- Insert 640 base documents (5 blocks worth) INSERT INTO bmw_blocks (content) SELECT 'baseline searchword document ' || i FROM generate_series(1, 640) i; -- Insert high-scoring docs at specific block positions: -- These should all appear in top-10 regardless of which block they're in INSERT INTO bmw_blocks (content) VALUES ('searchword searchword searchword searchword searchword'), -- id=641, tf=5 ('searchword searchword searchword searchword'), -- id=642, tf=4 ('searchword searchword searchword'); -- id=643, tf=3 CREATE INDEX bmw_blocks_idx ON bmw_blocks USING bm25(content) WITH (text_config='english'); -- Test: High-tf docs (641,642,643) must be in top results -- This validates block-max scoring finds them across multiple blocks SELECT id, content <@> 'searchword'::bm25query as score FROM bmw_blocks ORDER BY content <@> 'searchword'::bm25query LIMIT 10; -- Validation: top 3 must be our high-tf docs WITH top3 AS ( SELECT id FROM bmw_blocks ORDER BY content <@> 'searchword'::bm25query LIMIT 3 ) SELECT 'block-boundary' as test, CASE WHEN COUNT(*) = 3 AND 641 IN (SELECT id FROM top3) AND 642 IN (SELECT id FROM top3) AND 643 IN (SELECT id FROM top3) THEN 'PASS' ELSE 'FAIL' END as result FROM top3; DROP TABLE bmw_blocks; -- ============================================================ -- SECTION 13: Repeated query terms -- ============================================================ CREATE TABLE bmw_repeat (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_repeat (content) VALUES ('cat cat cat'), ('cat dog'), ('dog dog dog'); CREATE INDEX bmw_repeat_idx ON bmw_repeat USING bm25(content) WITH (text_config='english'); -- Test: Repeated terms in query should boost relevance SELECT id, content <@> 'cat cat'::bm25query as score FROM bmw_repeat ORDER BY score LIMIT 5; DROP TABLE bmw_repeat; -- ============================================================ -- SECTION 14: Unicode and special characters -- ============================================================ CREATE TABLE bmw_unicode (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_unicode (content) VALUES ('cafe with accent'), ('naïve approach'), ('北京 beijing china'), ('emoji test'), ('simple ascii text'); CREATE INDEX bmw_unicode_idx ON bmw_unicode USING bm25(content) WITH (text_config='simple'); -- simple config for unicode -- Test: Should handle various character sets SELECT id, content <@> 'beijing'::bm25query as score FROM bmw_unicode ORDER BY score LIMIT 5; SELECT id, content <@> 'simple'::bm25query as score FROM bmw_unicode ORDER BY score LIMIT 5; DROP TABLE bmw_unicode; -- ============================================================ -- SECTION 15: Top-K scattered across blocks -- ============================================================ -- This is the CRITICAL test for BMW: best docs are spread across -- different blocks, forcing the algorithm to check multiple blocks. CREATE TABLE bmw_scattered (id SERIAL PRIMARY KEY, content TEXT); -- Create 1000 docs where high-scoring docs are deliberately scattered: -- - Every 100th doc (100,200,300...) has tf=4 (highest) -- - Every 50th doc has tf=3 -- - Every 10th doc has tf=2 -- - Others have tf=1 -- This ensures top-10 must come from DIFFERENT blocks INSERT INTO bmw_scattered (content) SELECT CASE WHEN i % 100 = 0 THEN 'scattered scattered scattered scattered' WHEN i % 50 = 0 THEN 'scattered scattered scattered' WHEN i % 10 = 0 THEN 'scattered scattered' ELSE 'scattered filler words here' END || ' doc' || i FROM generate_series(1, 1000) i; CREATE INDEX bmw_scattered_idx ON bmw_scattered USING bm25(content) WITH (text_config='english'); -- The top-10 should include docs 100,200,300,400,500,600,700,800,900,1000 -- These are spread across 8 different blocks (1000/128 = ~8 blocks) SELECT id, content <@> 'scattered'::bm25query as score FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query LIMIT 10; -- Validation: Verify top 10 includes the high-tf docs from different blocks WITH top10 AS ( SELECT id FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query LIMIT 10 ) SELECT 'scattered-blocks' as test, CASE WHEN (SELECT COUNT(*) FROM top10 WHERE id IN (100,200,300,400,500,600,700,800,900,1000)) >= 8 THEN 'PASS' ELSE 'FAIL' END as result; -- Also validate with exhaustive comparison WITH bmw AS ( SELECT id, content <@> 'scattered'::bm25query as score FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'scattered'::bm25query as score FROM bmw_scattered ORDER BY content <@> 'scattered'::bm25query ) x LIMIT 10 ) SELECT 'scattered-equals-exhaustive' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; DROP TABLE bmw_scattered; -- ============================================================ -- SECTION 16: Multi-term with scattered high-scorers -- ============================================================ -- Tests that multi-term BMW also works with scattered results CREATE TABLE bmw_multi_scattered (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO bmw_multi_scattered (content) SELECT CASE WHEN i % 100 = 0 THEN 'alpha alpha alpha beta beta gamma' WHEN i % 50 = 0 THEN 'alpha alpha beta beta' WHEN i % 10 = 0 THEN 'alpha beta' ELSE 'alpha filler content' END || ' doc' || i FROM generate_series(1, 1000) i; CREATE INDEX bmw_multi_scattered_idx ON bmw_multi_scattered USING bm25(content) WITH (text_config='english'); -- Multi-term query - top results should come from scattered positions WITH bmw AS ( SELECT id, content <@> 'alpha beta gamma'::bm25query as score FROM bmw_multi_scattered ORDER BY content <@> 'alpha beta gamma'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'alpha beta gamma'::bm25query as score FROM bmw_multi_scattered ORDER BY content <@> 'alpha beta gamma'::bm25query ) x LIMIT 20 ) SELECT 'multiterm-scattered' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; DROP TABLE bmw_multi_scattered; -- ============================================================ -- SECTION 17: Multiple segments -- ============================================================ CREATE TABLE bmw_multiseg (id SERIAL PRIMARY KEY, content TEXT); -- Batch 1 -> segment 1 (via CREATE INDEX) INSERT INTO bmw_multiseg (content) SELECT 'first segment batch ' || i || ' targetword' FROM generate_series(1, 300) i; CREATE INDEX bmw_multiseg_idx ON bmw_multiseg USING bm25(content) WITH (text_config='english'); -- Batch 2 -> segment 2 INSERT INTO bmw_multiseg (content) SELECT 'second segment batch ' || i || ' targetword targetword' FROM generate_series(301, 600) i; SELECT bm25_spill_index('bmw_multiseg_idx'); -- Batch 3 -> memtable (not spilled) INSERT INTO bmw_multiseg (content) SELECT 'third batch ' || i || ' targetword targetword targetword' FROM generate_series(601, 700) i; -- Test: Should find best results across all segments + memtable -- Third batch has highest tf, should rank first SELECT id, content <@> 'targetword'::bm25query as score FROM bmw_multiseg ORDER BY content <@> 'targetword'::bm25query LIMIT 10; -- Verify: Results should include docs from batch 3 (id > 600) at top -- Also verify total count is correct (use to_bm25query for count) SELECT COUNT(*) FROM ( SELECT 1 FROM bmw_multiseg ORDER BY content <@> to_bm25query('targetword', 'bmw_multiseg_idx') ) sub; DROP TABLE bmw_multiseg; -- ============================================================ -- SECTION 18: Partial last block -- ============================================================ -- Test with 150 docs = 1 full block (128) + 1 partial block (22) -- This tests off-by-one errors in block iteration CREATE TABLE bmw_partial (id SERIAL PRIMARY KEY, content TEXT); -- 150 docs with varying tf to create score diversity INSERT INTO bmw_partial (content) SELECT CASE WHEN i <= 5 THEN 'partial partial partial partial' -- tf=4, in block 0 WHEN i = 130 THEN 'partial partial partial' -- tf=3, in partial block 1 WHEN i = 145 THEN 'partial partial partial partial partial' -- tf=5, in partial block 1 ELSE 'partial filler doc' || i END FROM generate_series(1, 150) i; CREATE INDEX bmw_partial_idx ON bmw_partial USING bm25(content) WITH (text_config='english'); -- Test: Should find high-tf docs from BOTH blocks -- Doc 145 (tf=5) should be #1, docs 1-5 (tf=4) should follow SELECT validate_bm25_scoring('bmw_partial', 'content', 'bmw_partial_idx', 'partial', 'english', 1.2, 0.75) as partial_block_valid; -- Verify doc 145 is top result (highest tf in partial block) SELECT 'partial-block-top' as test, CASE WHEN (SELECT id FROM bmw_partial ORDER BY content <@> to_bm25query('partial', 'bmw_partial_idx') LIMIT 1) = 145 THEN 'PASS' ELSE 'FAIL' END as result; DROP TABLE bmw_partial; -- ============================================================ -- SECTION 19: Term only in memtable (not in segments) -- ============================================================ -- Tests the case where segment iterators return "not found" -- but memtable has data for the term CREATE TABLE bmw_memonly (id SERIAL PRIMARY KEY, content TEXT); -- Insert docs that will go to segment (common term) INSERT INTO bmw_memonly (content) SELECT 'segment content common word doc' || i FROM generate_series(1, 200) i; CREATE INDEX bmw_memonly_idx ON bmw_memonly USING bm25(content) WITH (text_config='english'); -- Force spill SELECT bm25_spill_index('bmw_memonly_idx'); -- Now insert docs with a NEW term that only exists in memtable INSERT INTO bmw_memonly (content) VALUES ('uniquememterm uniquememterm uniquememterm'), -- tf=3 ('uniquememterm uniquememterm'), -- tf=2 ('uniquememterm'); -- tf=1 -- Test: Query for memtable-only term should work SELECT id, content <@> 'uniquememterm'::bm25query as score FROM bmw_memonly ORDER BY content <@> 'uniquememterm'::bm25query LIMIT 5; -- Verify: Should return exactly 3 docs, in tf order (201, 202, 203) SELECT 'memtable-only-term' as test, CASE WHEN (SELECT COUNT(*) FROM ( SELECT 1 FROM bmw_memonly ORDER BY content <@> to_bm25query('uniquememterm', 'bmw_memonly_idx') ) sub) = 3 THEN 'PASS' ELSE 'FAIL' END as result; -- Test: Multi-term with one term in segment, one in memtable only SELECT id, content <@> 'common uniquememterm'::bm25query as score FROM bmw_memonly ORDER BY content <@> 'common uniquememterm'::bm25query LIMIT 5; DROP TABLE bmw_memonly; -- ============================================================ -- SECTION 20: Sparse posting lists (few postings per block) -- ============================================================ -- When a term has very few postings spread across many doc IDs, -- each "block" may have only 1-2 postings. This tests block-max -- effectiveness when block_max ≈ actual score. CREATE TABLE bmw_sparse (id SERIAL PRIMARY KEY, content TEXT); -- Create 500 docs where only every 50th doc has the target term -- This creates ~10 postings spread across what would be 4 blocks INSERT INTO bmw_sparse (content) SELECT CASE WHEN i % 50 = 0 THEN 'sparse rare term here doc' || i ELSE 'common filler words only doc' || i END FROM generate_series(1, 500) i; CREATE INDEX bmw_sparse_idx ON bmw_sparse USING bm25(content) WITH (text_config='english'); -- Test: Should find all 10 sparse docs SELECT 'sparse-count' as test, CASE WHEN (SELECT COUNT(*) FROM ( SELECT 1 FROM bmw_sparse ORDER BY content <@> to_bm25query('sparse', 'bmw_sparse_idx') ) sub) = 10 THEN 'PASS' ELSE 'FAIL' END as result; -- Test: BMW should match exhaustive for sparse terms WITH bmw AS ( SELECT id, content <@> 'sparse'::bm25query as score FROM bmw_sparse ORDER BY content <@> 'sparse'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'sparse'::bm25query as score FROM bmw_sparse ORDER BY content <@> 'sparse'::bm25query ) x LIMIT 10 ) SELECT 'sparse-equals-exhaustive' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; DROP TABLE bmw_sparse; -- ============================================================ -- SECTION 21: Multi-term with different block structures -- ============================================================ -- Term A has many postings (many blocks), term B has few (1 block) -- Tests WAND traversal when iterators have very different lengths CREATE TABLE bmw_asymmetric (id SERIAL PRIMARY KEY, content TEXT); -- 500 docs: "common" in all, "rare" in only 5 specific docs INSERT INTO bmw_asymmetric (content) SELECT CASE WHEN i IN (50, 150, 250, 350, 450) THEN 'common rare both terms doc' || i ELSE 'common only single term doc' || i END FROM generate_series(1, 500) i; CREATE INDEX bmw_asymmetric_idx ON bmw_asymmetric USING bm25(content) WITH (text_config='english'); -- Test: "common rare" should find only the 5 docs with both terms at top -- (they have higher combined score than single-term docs) SELECT validate_bm25_scoring('bmw_asymmetric', 'content', 'bmw_asymmetric_idx', 'common rare', 'english', 1.2, 0.75) as asymmetric_valid; -- Verify top 5 are the dual-term docs (they should have best scores) WITH top5 AS ( SELECT id FROM bmw_asymmetric ORDER BY content <@> 'common rare'::bm25query LIMIT 5 ) SELECT 'asymmetric-top5' as test, CASE WHEN (SELECT COUNT(*) FROM top5 WHERE id IN (50, 150, 250, 350, 450)) = 5 THEN 'PASS' ELSE 'FAIL' END as result; DROP TABLE bmw_asymmetric; -- ============================================================ -- SECTION 22: Threshold exactly equals block_max -- ============================================================ -- Create scenario where block_max score equals the threshold exactly -- Tests boundary condition in score < threshold vs score <= threshold CREATE TABLE bmw_threshold (id SERIAL PRIMARY KEY, content TEXT); -- Create docs with IDENTICAL scores to force threshold edge case -- All docs have tf=1 and same length -> identical BM25 scores INSERT INTO bmw_threshold (content) SELECT 'threshold test word' FROM generate_series(1, 200); CREATE INDEX bmw_threshold_idx ON bmw_threshold USING bm25(content) WITH (text_config='english'); -- Test: With 200 identical-score docs, LIMIT 10 should return 10 -- and they should be ordered by CTID (id 1-10) SELECT 'threshold-boundary' as test, CASE WHEN (SELECT COUNT(*) FROM ( SELECT id FROM bmw_threshold ORDER BY content <@> to_bm25query('threshold', 'bmw_threshold_idx') LIMIT 10 ) x) = 10 THEN 'PASS' ELSE 'FAIL' END as result; -- Verify CTID ordering for tied scores (should be ids 1-10) SELECT 'threshold-tiebreak' as test, CASE WHEN (SELECT array_agg(id ORDER BY id) FROM ( SELECT id FROM bmw_threshold ORDER BY content <@> to_bm25query('threshold', 'bmw_threshold_idx') LIMIT 10 ) x) = ARRAY[1,2,3,4,5,6,7,8,9,10] THEN 'PASS' ELSE 'FAIL' END as result; DROP TABLE bmw_threshold; -- ============================================================ -- SECTION 23: All iterators exhaust simultaneously -- ============================================================ -- Create a scenario where multiple term iterators all finish -- at exactly the same document CREATE TABLE bmw_simul (id SERIAL PRIMARY KEY, content TEXT); -- 100 docs where all have both terms - iterators move in lockstep INSERT INTO bmw_simul (content) SELECT 'alpha beta both terms doc' || i FROM generate_series(1, 100) i; -- Add one doc with only alpha at end INSERT INTO bmw_simul (content) VALUES ('alpha only final'); CREATE INDEX bmw_simul_idx ON bmw_simul USING bm25(content) WITH (text_config='english'); -- Test: Should handle simultaneous exhaustion correctly WITH bmw AS ( SELECT id, content <@> 'alpha beta'::bm25query as score FROM bmw_simul ORDER BY content <@> 'alpha beta'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'alpha beta'::bm25query as score FROM bmw_simul ORDER BY content <@> 'alpha beta'::bm25query ) x LIMIT 20 ) SELECT 'simultaneous-exhaust' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; DROP TABLE bmw_simul; -- ============================================================ -- SECTION 24: Document length variance within blocks -- ============================================================ -- When a block contains a mix of very short and very long docs, -- the block-max upper bound must use the shortest doc's length -- (lowest fieldnorm) to produce a valid BM25 upper bound. -- This test catches bugs where the longest doc's fieldnorm is -- used instead, making the upper bound too tight and causing -- BMW to incorrectly skip blocks with high-scoring short docs. CREATE TABLE bmw_length_variance (id SERIAL PRIMARY KEY, content TEXT); -- Create 500 docs with high length variance within each 128-doc block. -- Short docs (3 words) are interleaved with long docs (100+ words). -- A rare term "unicorn" appears only in a few short docs, scattered -- across blocks. These short docs should score highest for "unicorn" -- because BM25 penalizes longer documents. INSERT INTO bmw_length_variance (content) SELECT CASE -- Rare term in short docs at positions that span multiple blocks WHEN i IN (10, 140, 270, 400) THEN 'unicorn magic sparkle doc' || i -- Long docs with padding (pushes block fieldnorm high) WHEN i % 3 = 0 THEN 'common words here doc' || i || ' ' || repeat('padding ', 100) -- Medium docs WHEN i % 3 = 1 THEN 'common words here with more content doc' || i -- Short docs without the rare term ELSE 'common words here doc' || i END FROM generate_series(1, 500) i; CREATE INDEX bmw_length_variance_idx ON bmw_length_variance USING bm25(content) WITH (text_config='english'); -- Single-term: BMW must match exhaustive for "unicorn" WITH bmw AS ( SELECT id, content <@> 'unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'unicorn'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'unicorn'::bm25query ) x LIMIT 10 ) SELECT 'length-variance-single-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; -- Multi-term: "common unicorn" -- the rare "unicorn" term has high IDF, -- so short docs with "unicorn" should dominate. BMW must not skip their -- blocks just because long docs in the same block deflate the upper bound. -- Verify the unicorn docs are top-4 for "common unicorn" WITH top4 AS ( SELECT id FROM bmw_length_variance ORDER BY content <@> 'common unicorn'::bm25query LIMIT 4 ) SELECT 'length-variance-top4' as test, CASE WHEN (SELECT COUNT(*) FROM top4 WHERE id IN (10, 140, 270, 400)) = 4 THEN 'PASS' ELSE 'FAIL' END as result; -- BMW vs exhaustive with enough LIMIT to avoid tie boundary WITH bmw AS ( SELECT id, content <@> 'common unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'common unicorn'::bm25query LIMIT 20 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'common unicorn'::bm25query as score FROM bmw_length_variance ORDER BY content <@> 'common unicorn'::bm25query ) x LIMIT 20 ) SELECT 'length-variance-bmw-exhaustive' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; DROP TABLE bmw_length_variance; -- ============================================================ -- Cleanup -- ============================================================ DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/bmw_skip_advance.sql000066400000000000000000000105671525762047700245650ustar00rootroot00000000000000-- Coverage / regression test for the BMW block_max_skip_advance path -- (issue #355). -- -- The fix in src/scoring/bmw.c changed block_max_skip_advance() to -- pick the pivot term whose current block ends *soonest* as the -- term to seek forward, instead of the highest-max_score term. The -- old selection could pick a term whose cur_doc_id was already past -- min_block_end + 1, making seek_term_to_doc a no-op and spinning -- the outer WAND loop forever on certain MS MARCO data -- distributions. -- -- A deterministic SQL workload that triggers the *old* infinite loop -- has been elusive to construct synthetically: real-world repro -- requires the ~8.8M-passage MS MARCO corpus with concurrent-insert -- segment topology. Internal HorizonDB tests at Microsoft are the -- real validation signal for the bug. -- -- This test is a coverage-focused smoke test: it builds a workload -- that reliably enters block_max_skip_advance many times (verified -- via pg_textsearch.log_bmw_stats: seeks_performed > 0), exercising -- the patched code path. statement_timeout ensures any future -- regression that reintroduces a no-op-seek hang is caught quickly -- rather than burning the whole test run. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; SET statement_timeout = '30s'; CREATE TABLE bmw_skip_advance (id serial PRIMARY KEY, content text); -- Anchor docs: short with high TF on two query terms, raise the -- top-k threshold so most subsequent blocks fail block_max upper -- bound and force the WAND loop into block_max_skip_advance. INSERT INTO bmw_skip_advance (content) SELECT 'alpha alpha alpha alpha alpha beta beta beta beta beta' FROM generate_series(1, 20); -- Bulk: each doc omits exactly one of the four query terms in -- rotation. The four per-term posting lists end up misaligned, so -- per-term block boundaries land at different doc_ids -- which is -- exactly the data condition for the WAND skip-advance path. INSERT INTO bmw_skip_advance (content) SELECT CASE i % 4 WHEN 0 THEN 'beta gamma delta ' WHEN 1 THEN 'alpha gamma delta ' WHEN 2 THEN 'alpha beta delta ' ELSE 'alpha beta gamma ' END || array_to_string(ARRAY( SELECT 'filler' || g FROM generate_series(1, 80) g), ' ') FROM generate_series(1, 5000) i; CREATE INDEX bmw_skip_advance_idx ON bmw_skip_advance USING bm25(content) WITH (text_config = 'simple'); -- This query exercises block_max_skip_advance heavily. Asserting it -- terminates and returns 10 rows is the regression signal -- any -- future change that reintroduces the no-op-seek would cause this to -- hang and statement_timeout would fire. SELECT count(*) AS got_results FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'alpha beta gamma delta', 'bmw_skip_advance_idx') LIMIT 10 ) sub; -- Same query under BMW stats logging, to exercise the -- seeks_performed / blocks_skipped accounting in the patched -- function. (The stats LOG line itself is filtered out of pg_regress -- diff comparison via :v variable; we just want the code path -- exercised.) SET pg_textsearch.log_bmw_stats = on; SELECT count(*) AS got_results FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'alpha beta gamma delta', 'bmw_skip_advance_idx') LIMIT 10 ) sub; SET pg_textsearch.log_bmw_stats = off; -- Now exercise the *memtable* scoring paths (single-term and -- multi-term), where the CHECK_FOR_INTERRUPTS calls added by this -- patch sit. Above this point everything was in segments after -- CREATE INDEX. Inserting fresh docs here without forcing a spill -- leaves them in the memtable, so the next queries traverse both -- segments (via BMW skip-advance) and the memtable. INSERT INTO bmw_skip_advance (content) SELECT 'alpha beta gamma delta extra_memtable_token' FROM generate_series(1, 100); -- Single-term query against memtable (score_memtable_single_term). SELECT count(*) FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'extra_memtable_token', 'bmw_skip_advance_idx') LIMIT 5 ) sub; -- Multi-term query against memtable (score_memtable_multi_term). SELECT count(*) FROM ( SELECT id FROM bmw_skip_advance ORDER BY content <@> to_bm25query( 'alpha extra_memtable_token', 'bmw_skip_advance_idx') LIMIT 5 ) sub; DROP TABLE bmw_skip_advance; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/boolean_queries.sql000066400000000000000000000345141525762047700244430ustar00rootroot00000000000000CREATE EXTENSION pg_textsearch; SET default_text_search_config = 'pg_catalog.english'; CREATE TABLE boolean_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_docs VALUES (1, 'refund approved'), (2, 'refund fraud'), (3, 'fraud refund investigation'), (4, 'billing refund request'), (5, 'billing question'), (8, ''), (9, 'the'); SET client_min_messages = WARNING; CREATE INDEX boolean_docs_body_idx ON boolean_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; INSERT INTO boolean_docs VALUES (6, 'refund fraud followup'), (7, 'refund suspected fraud'); SET enable_seqscan = off; SET pg_textsearch.memtable_cache_enabled = off; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!fraud') ORDER BY id; RESET pg_textsearch.memtable_cache_enabled; EXPLAIN (COSTS OFF) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & fraud'); CREATE FUNCTION pg_temp.first_plan_child(query text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE plan json; BEGIN EXECUTE 'EXPLAIN (FORMAT JSON, COSTS OFF) ' || query INTO plan; RETURN plan->0->'Plan'->'Plans'->0->>'Node Type'; END $$; CREATE TEMP TABLE boolean_other (id integer); INSERT INTO boolean_other SELECT generate_series(1, 9); \pset format unaligned SET enable_sort = off; SELECT pg_temp.first_plan_child($query$ SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund') ORDER BY body <@> to_bm25query('refund', 'boolean_docs_body_idx') LIMIT 1 $query$) = 'Sort' AS unsupported_path_falls_back; SET enable_nestloop = off; SELECT pg_temp.first_plan_child($query$ SELECT d.id FROM boolean_docs d JOIN boolean_other other ON other.id = d.id WHERE d.body @@ to_tsquery('english', 'refund') ORDER BY d.body <@> to_bm25query('refund', 'boolean_docs_body_idx') LIMIT 1 $query$) = 'Sort' AS unsupported_join_path_falls_back; SELECT ( SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'fraud') ORDER BY body <@> to_bm25query('refund', 'boolean_docs_body_idx') LIMIT 1 ) = 2 AS combined_query_ranks_matches; RESET enable_nestloop; RESET enable_sort; \pset format aligned SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund | fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & !fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'billing & (refund | !question)') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund <-> fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund | !fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!(refund <-> fraud)') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund:*') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund:A') ORDER BY id; SET plan_cache_mode = force_generic_plan; PREPARE boolean_scan(tsquery) AS SELECT id FROM boolean_docs WHERE body @@ $1 ORDER BY id; EXPLAIN (COSTS OFF) EXECUTE boolean_scan(to_tsquery('english', 'refund & !fraud')); EXECUTE boolean_scan(to_tsquery('english', 'refund & fraud')); EXECUTE boolean_scan(to_tsquery('english', '!fraud')); EXECUTE boolean_scan(to_tsquery('english', 'refund <-> fraud')); EXECUTE boolean_scan(to_tsquery('english', 'refund:*')); SET default_text_search_config = 'pg_catalog.simple'; EXECUTE boolean_scan(NULL); EXECUTE boolean_scan(to_tsquery('simple', 'refund & fraud')); DEALLOCATE boolean_scan; RESET plan_cache_mode; RESET enable_seqscan; EXPLAIN (COSTS OFF) SELECT id FROM boolean_docs WHERE body @@ to_tsquery('simple', 'refund & fraud'); SET enable_seqscan = off; SET default_text_search_config = 'pg_catalog.english'; SET client_min_messages = WARNING; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'the') ORDER BY id; RESET client_min_messages; UPDATE boolean_docs SET body = 'billing fraud question' WHERE id = 5; DELETE FROM boolean_docs WHERE id = 1; VACUUM boolean_docs; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', '!fraud') ORDER BY id; SELECT id FROM boolean_docs WHERE body @@ to_tsquery('english', 'refund & !fraud') ORDER BY id; RESET enable_seqscan; RESET default_text_search_config; DROP TABLE boolean_docs; CREATE TABLE boolean_empty_docs ( id integer PRIMARY KEY, body text NOT NULL ); SET client_min_messages = WARNING; CREATE INDEX boolean_empty_docs_body_idx ON boolean_empty_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; INSERT INTO boolean_empty_docs VALUES (1, ''), (2, 'the'); SET enable_seqscan = off; SET pg_textsearch.memtable_cache_enabled = off; SELECT id FROM boolean_empty_docs WHERE body @@ to_tsquery('english', '!missing') ORDER BY id; RESET pg_textsearch.memtable_cache_enabled; SELECT bm25_spill_index('boolean_empty_docs_body_idx') IS NOT NULL AS spilled_empty_docs; INSERT INTO boolean_empty_docs VALUES (3, ''), (4, 'the'); SELECT bm25_spill_index('boolean_empty_docs_body_idx') IS NOT NULL AS spilled_more_empty_docs; SELECT bm25_force_merge('boolean_empty_docs_body_idx'); SELECT id FROM boolean_empty_docs WHERE body @@ to_tsquery('english', '!missing') ORDER BY id; RESET enable_seqscan; DROP TABLE boolean_empty_docs; CREATE TABLE boolean_segment_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_segment_docs VALUES (1, 'alpha beta'), (2, 'alpha'), (3, 'beta'); SET client_min_messages = WARNING; CREATE INDEX boolean_segment_docs_body_idx ON boolean_segment_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; INSERT INTO boolean_segment_docs VALUES (4, 'gamma delta'), (5, 'gamma'), (6, 'delta'), (7, 'alpha gamma'); SELECT bm25_spill_index('boolean_segment_docs_body_idx') IS NOT NULL AS spilled_segment_docs; INSERT INTO boolean_segment_docs VALUES (8, 'alpha delta'), (9, 'beta gamma'), (10, 'epsilon'); DELETE FROM boolean_segment_docs WHERE id = 2; VACUUM boolean_segment_docs; SET enable_seqscan = off; SELECT array_agg(id ORDER BY id) AS nested_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', '(alpha & beta) | (gamma & delta)'); SELECT array_agg(id ORDER BY id) AS anchored_not_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha & !gamma'); SELECT array_agg(id ORDER BY id) AS duplicate_term_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha | alpha'); SELECT count(*) AS missing_and_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha & missing'); SELECT array_agg(id ORDER BY id) AS mixed_source_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha | epsilon'); SELECT array_agg(id ORDER BY id) AS pure_negative_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', '!missing'); SELECT array_agg(id ORDER BY id) AS phrase_matches FROM boolean_segment_docs WHERE body @@ to_tsquery('english', 'alpha <-> beta'); RESET enable_seqscan; DROP TABLE boolean_segment_docs; SET default_text_search_config = 'pg_catalog.simple'; CREATE TABLE boolean_prefix_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_prefix_docs VALUES (1, 'cat'), (2, 'cater'), (3, 'dog'); SET client_min_messages = WARNING; SET pg_textsearch.compress_segments = off; CREATE INDEX boolean_prefix_docs_body_idx ON boolean_prefix_docs USING bm25(body) WITH (text_config = 'simple'); RESET client_min_messages; INSERT INTO boolean_prefix_docs VALUES (4, 'cattle'), (5, 'catalog'), (6, 'catch'); \pset format unaligned SELECT bm25_spill_index('boolean_prefix_docs_body_idx') IS NOT NULL AS spilled_prefix_docs; RESET pg_textsearch.compress_segments; INSERT INTO boolean_prefix_docs VALUES (7, 'catfish'), (8, 'category'), (9, 'bird'), (10, 'cat cater'); SET enable_seqscan = off; SELECT array_agg(id ORDER BY id) AS prefix_matches FROM boolean_prefix_docs WHERE body @@ to_tsquery('simple', 'cat:*'); \pset format aligned RESET enable_seqscan; DROP TABLE boolean_prefix_docs; CREATE TABLE boolean_broad_prefix_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_broad_prefix_docs SELECT id, CASE WHEN id <= 257 THEN format('broadprefix%s', lpad(id::text, 4, '0')) ELSE 'unrelated' END FROM generate_series(1, 258) AS id; SET client_min_messages = WARNING; CREATE INDEX boolean_broad_prefix_docs_body_idx ON boolean_broad_prefix_docs USING bm25(body) WITH (text_config = 'simple'); RESET client_min_messages; SET enable_seqscan = off; \pset format unaligned SELECT count(*) AS broad_prefix_matches FROM boolean_broad_prefix_docs WHERE body @@ to_tsquery('simple', 'broadprefix:*'); \pset format aligned RESET enable_seqscan; DROP TABLE boolean_broad_prefix_docs; SET default_text_search_config = 'pg_catalog.english'; CREATE TABLE boolean_seek_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_seek_docs SELECT id, concat_ws(' ', CASE WHEN id % 2 = 0 THEN 'common' END, CASE WHEN id % 997 = 0 OR id = 5001 THEN 'anchor' END) FROM generate_series(1, 5001) AS id; SET client_min_messages = WARNING; CREATE INDEX boolean_seek_docs_body_idx ON boolean_seek_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; ANALYZE boolean_seek_docs; \pset format unaligned SELECT pg_temp.first_plan_child($query$ SELECT id FROM boolean_seek_docs WHERE body @@ to_tsquery('english', 'common') LIMIT 1 $query$) = 'Seq Scan' AS boolean_limit_avoids_materializing_index; \pset format aligned SET enable_seqscan = off; \pset format unaligned SELECT array_agg(id ORDER BY id) AS galloping_seek_matches FROM boolean_seek_docs WHERE body @@ to_tsquery('english', 'anchor & common'); \pset format aligned RESET enable_seqscan; DROP TABLE boolean_seek_docs; CREATE TABLE boolean_memtable_stream_docs ( id integer PRIMARY KEY, body text NOT NULL ); SET client_min_messages = WARNING; CREATE INDEX boolean_memtable_stream_docs_body_idx ON boolean_memtable_stream_docs USING bm25(body) WITH (text_config = 'english'); RESET client_min_messages; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO boolean_memtable_stream_docs SELECT id, concat_ws(' ', CASE WHEN id % 2 = 0 THEN 'common' END, CASE WHEN id % 997 = 0 OR id = 5001 THEN 'anchor' END, CASE WHEN id = 5001 THEN 'alternative' END, 'ubiquitous') FROM generate_series(1, 5001) AS id; SET enable_seqscan = off; \pset format unaligned SELECT array_agg(id ORDER BY id) AS memtable_selective_and_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery('english', 'anchor & common'); SELECT array_agg(id ORDER BY id) AS memtable_nested_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery( 'english', 'anchor & (common | alternative)'); SELECT array_agg(id ORDER BY id) AS memtable_anchored_not_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery('english', 'anchor & !common'); SELECT count(*) AS memtable_nested_not_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery( 'english', '(alternative | !common) & ubiquitous'); \pset format aligned SET pg_textsearch.memtable_cache_enabled = off; \pset format unaligned SELECT array_agg(id ORDER BY id) AS chain_selective_and_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery('english', 'anchor & common'); SELECT count(*) AS chain_nested_not_matches FROM boolean_memtable_stream_docs WHERE body @@ to_tsquery( 'english', '(alternative | !common) & ubiquitous'); \pset format aligned RESET pg_textsearch.memtable_cache_enabled; RESET enable_seqscan; RESET pg_textsearch.bulk_load_threshold; RESET pg_textsearch.memtable_pages_threshold; DROP TABLE boolean_memtable_stream_docs; CREATE TABLE boolean_many_term_docs ( id integer PRIMARY KEY, body text NOT NULL ); INSERT INTO boolean_many_term_docs SELECT document, query.body FROM generate_series(1, 1000) AS document CROSS JOIN ( SELECT string_agg(format('term%s', term), ' ') AS body FROM generate_series(1, 201) AS term ) AS query; SET pg_textsearch.compress_segments = off; SET client_min_messages = WARNING; CREATE INDEX boolean_many_term_docs_body_idx ON boolean_many_term_docs USING bm25(body) WITH (text_config = 'simple'); RESET client_min_messages; RESET pg_textsearch.compress_segments; CREATE FUNCTION pg_temp.boolean_and_query(term_count integer) RETURNS tsquery LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT string_agg(format('term%s', term), ' & ')::tsquery FROM generate_series(1, term_count) AS term $$; CREATE FUNCTION pg_temp.boolean_repeated_or_query(term_count integer) RETURNS tsquery LANGUAGE sql IMMUTABLE STRICT AS $$ SELECT string_agg('term1', ' | ')::tsquery FROM generate_series(1, term_count) $$; SET default_text_search_config = 'pg_catalog.simple'; SELECT pg_temp.boolean_and_query(64)::text AS query \gset exact_limit_ SELECT pg_temp.boolean_and_query(201)::text AS query \gset oversized_ SELECT pg_temp.boolean_repeated_or_query(65)::text AS query \gset repeated_ \pset format unaligned SELECT pg_temp.first_plan_child(format( 'SELECT count(*) FROM boolean_many_term_docs WHERE body @@ %L::tsquery', :'oversized_query' )) = 'Seq Scan' AS oversized_boolean_falls_back; SELECT pg_temp.first_plan_child(format( 'SELECT count(*) FROM boolean_many_term_docs WHERE body @@ %L::tsquery', :'repeated_query' )) = 'Seq Scan' AS repeated_boolean_falls_back; SET enable_seqscan = off; SELECT pg_temp.first_plan_child(format( 'SELECT count(*) FROM boolean_many_term_docs WHERE body @@ %L::tsquery', :'exact_limit_query' )) = 'Index Scan' AS exact_operand_limit_uses_index; SELECT count(*) = 1000 AS exact_operand_limit_succeeds FROM boolean_many_term_docs WHERE body @@ :'exact_limit_query'::tsquery; RESET enable_seqscan; \pset format aligned DROP TABLE boolean_many_term_docs; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/build_progress.sql000066400000000000000000000132531525762047700243070ustar00rootroot00000000000000-- Test BM25 build-progress lifecycle. CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Progress notices should be emitted only when an index is actually created. CREATE TABLE progress_noop_docs (content text); CREATE INDEX progress_noop_idx ON progress_noop_docs USING bm25(content) WITH (text_config = 'english'); CREATE INDEX IF NOT EXISTS progress_noop_idx ON progress_noop_docs USING bm25(content) WITH (text_config = 'english'); CREATE TABLE progress_empty_partitioned ( id integer, content text ) PARTITION BY RANGE (id); CREATE INDEX progress_empty_partitioned_idx ON progress_empty_partitioned USING bm25(content) WITH (text_config = 'english'); CREATE TABLE progress_actual_docs (content text); CREATE TABLE progress_collision_docs (id integer); CREATE INDEX progress_actual_idx ON progress_collision_docs(id); CREATE FUNCTION progress_drop_collision() RETURNS event_trigger AS $$ BEGIN IF current_setting('progress_test.drop_collision', true) = 'on' THEN PERFORM set_config('progress_test.drop_collision', 'off', true); EXECUTE 'DROP INDEX progress_actual_idx'; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER progress_drop_collision_trigger ON ddl_command_start WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION progress_drop_collision(); SET progress_test.drop_collision = on; CREATE INDEX IF NOT EXISTS progress_actual_idx ON progress_actual_docs USING bm25(content) WITH (text_config = 'english'); RESET progress_test.drop_collision; SELECT indrelid = 'progress_actual_docs'::regclass AS actual_index_created FROM pg_index WHERE indexrelid = 'progress_actual_idx'::regclass; -- A failed build must not leave progress active for later implicit builds. CREATE TABLE progress_abort_docs ( id integer, content text ) PARTITION BY RANGE (id); CREATE TABLE progress_abort_part PARTITION OF progress_abort_docs FOR VALUES FROM (0) TO (10); CREATE FUNCTION progress_fail_build_end() RETURNS event_trigger AS $$ BEGIN IF current_setting('progress_test.fail_build_end', true) = 'on' THEN PERFORM set_config('progress_test.fail_build_end', 'off', true); RAISE EXCEPTION 'forced progress abort'; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER progress_fail_build_end_trigger ON ddl_command_end WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION progress_fail_build_end(); SET progress_test.fail_build_end = on; \set VERBOSITY terse CREATE INDEX progress_abort_idx ON progress_abort_docs USING bm25(content) WITH (text_config = 'english'); \set VERBOSITY default RESET progress_test.fail_build_end; REINDEX INDEX progress_actual_idx; -- Nested index DDL must not replace an outer partitioned build's progress. CREATE TABLE progress_nested_docs ( id integer, content text ) PARTITION BY RANGE (id); CREATE TABLE progress_nested_part_a PARTITION OF progress_nested_docs FOR VALUES FROM (0) TO (10); CREATE TABLE progress_nested_part_b PARTITION OF progress_nested_docs FOR VALUES FROM (10) TO (20); CREATE TABLE progress_nested_side (content text); CREATE TABLE progress_nested_reindex_side (content text); INSERT INTO progress_nested_docs VALUES (1, 'apple'), (11, 'apple banana'), (12, 'apple banana cherry'); INSERT INTO progress_nested_side VALUES ('kiwi kiwi kiwi kiwi'); INSERT INTO progress_nested_reindex_side VALUES ('plum plum plum plum plum'); CREATE INDEX progress_nested_reindex_idx ON progress_nested_reindex_side USING bm25(content) WITH (text_config = 'english'); CREATE FUNCTION progress_create_nested_index() RETURNS event_trigger AS $$ DECLARE action text := current_setting('progress_test.nested_action', true); BEGIN IF action = 'create' THEN PERFORM set_config('progress_test.nested_action', 'running', true); EXECUTE 'CREATE INDEX progress_nested_side_idx ' 'ON progress_nested_side USING bm25(content) ' 'WITH (text_config = ''english'')'; ELSIF action = 'reindex' THEN PERFORM set_config('progress_test.nested_action', 'running', true); EXECUTE 'REINDEX INDEX progress_nested_reindex_idx'; END IF; END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER progress_create_nested_trigger ON ddl_command_end WHEN TAG IN ('CREATE INDEX') EXECUTE FUNCTION progress_create_nested_index(); SET progress_test.nested_action = 'create'; CREATE INDEX progress_nested_idx ON progress_nested_docs USING bm25(content) WITH (text_config = 'english'); RESET progress_test.nested_action; CREATE TABLE progress_reindex_docs ( id integer, content text ) PARTITION BY RANGE (id); CREATE TABLE progress_reindex_part_a PARTITION OF progress_reindex_docs FOR VALUES FROM (0) TO (10); CREATE TABLE progress_reindex_part_b PARTITION OF progress_reindex_docs FOR VALUES FROM (10) TO (20); INSERT INTO progress_reindex_docs VALUES (1, 'apple'), (11, 'apple banana'), (12, 'apple banana cherry'); SET progress_test.nested_action = reindex; CREATE INDEX progress_reindex_outer_idx ON progress_reindex_docs USING bm25(content) WITH (text_config = 'english'); RESET progress_test.nested_action; DROP EVENT TRIGGER progress_create_nested_trigger; DROP FUNCTION progress_create_nested_index(); DROP EVENT TRIGGER progress_fail_build_end_trigger; DROP FUNCTION progress_fail_build_end(); DROP EVENT TRIGGER progress_drop_collision_trigger; DROP FUNCTION progress_drop_collision(); DROP TABLE progress_noop_docs; DROP TABLE progress_empty_partitioned; DROP TABLE progress_actual_docs; DROP TABLE progress_collision_docs; DROP TABLE progress_abort_docs; DROP TABLE progress_nested_docs; DROP TABLE progress_nested_side; DROP TABLE progress_nested_reindex_side; DROP TABLE progress_reindex_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/bulk_load.sql000066400000000000000000000041641525762047700232210ustar00rootroot00000000000000-- Test bulk load spill threshold -- Exercises state.c bulk_load_spill_check path CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Set a very low bulk load threshold to trigger automatic spill SET pg_textsearch.bulk_load_threshold = 100; -- Create table and index CREATE TABLE bulk_load_test (id serial PRIMARY KEY, content text); CREATE INDEX bulk_load_idx ON bulk_load_test USING bm25(content) WITH (text_config='english'); -- Insert enough distinct terms in one transaction to exceed threshold -- With threshold=100, this should trigger auto-spill at commit BEGIN; INSERT INTO bulk_load_test (content) SELECT 'uniqueterm' || i || ' extra words for length variation' FROM generate_series(1, 200) AS i; COMMIT; -- Verify the data is searchable (spill should have happened) SELECT count(*) FROM ( SELECT 1 FROM bulk_load_test ORDER BY content <@> to_bm25query('uniqueterm1', 'bulk_load_idx') ) sub; -- Check index summary shows segment data SELECT bm25_summarize_index('bulk_load_idx') IS NOT NULL AS has_summary; -- ============================================================================= -- Test 2: Double bulk load spill with pre-existing L0 segment -- Exercises state.c segment chain linking in tp_bulk_load_spill_check -- ============================================================================= -- First, create a segment by spilling the existing memtable SELECT bm25_spill_index('bulk_load_idx'); -- Now insert more data in a single transaction with low threshold -- This triggers bulk_load_spill_check with a pre-existing L0 chain BEGIN; INSERT INTO bulk_load_test (content) SELECT 'secondbatch' || i || ' more content for spill' FROM generate_series(1, 200) AS i; COMMIT; -- Verify both batches are searchable SELECT count(*) FROM ( SELECT 1 FROM bulk_load_test ORDER BY content <@> to_bm25query('uniqueterm1', 'bulk_load_idx') ) sub; SELECT count(*) FROM ( SELECT 1 FROM bulk_load_test ORDER BY content <@> to_bm25query('secondbatch1', 'bulk_load_idx') ) sub; -- Reset threshold SET pg_textsearch.bulk_load_threshold = 100000; -- Clean up DROP TABLE bulk_load_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/cache_apply.sql000066400000000000000000000067351525762047700235430ustar00rootroot00000000000000-- Unit coverage for the in-memory memtable cache apply protocol -- (cache.c; see ARCHITECTURE.md, "Memtable Cache"). -- -- Drives bm25_cache_cold_build / bm25_cache_apply_to_tail directly -- so we can observe (result, records_applied, cursor_seq, -- estimated_bytes) deltas across each step without going through -- the normal reader path. -- -- Records come from real heap inserts so we exercise the -- production write path end-to-end. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE cache_apply_t (id int, body text); CREATE INDEX cache_apply_idx ON cache_apply_t USING bm25 (body) WITH (text_config = 'english'); -- ---------- empty chain ---------- -- Cold build over an empty chain returns OK with 0 records and -- leaves the cursor unseeded (head_blkno was Invalid). SELECT result, records_applied FROM bm25_cache_cold_build('cache_apply_idx'); -- apply_to_tail on an empty/uninitialized cache returns -- NOT_INITIALIZED so the caller knows to cold-build first. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); -- ---------- cold build over real records ---------- INSERT INTO cache_apply_t SELECT g, 'shared term ' || g FROM generate_series(1, 5) g; -- Cold-build picks up every record. records_applied = 5, -- cursor_seq = 5, estimated_bytes > 0. SELECT result, records_applied, cursor_seq, (estimated_bytes > 0) AS bytes_nonzero FROM bm25_cache_cold_build('cache_apply_idx'); -- Re-running cold_build now returns RETRY (cache already built); -- cursor_seq stays at 5 and no new records get applied. SELECT result, records_applied FROM bm25_cache_cold_build('cache_apply_idx'); -- apply_to_tail with no new records is a no-op: OK / 0 records. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); -- ---------- incremental catchup ---------- INSERT INTO cache_apply_t SELECT g, 'newer term ' || g FROM generate_series(6, 10) g; -- apply_to_tail picks up the 5 new records. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); -- A second apply_to_tail is again a no-op. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); -- ---------- spill invalidates the cache ---------- -- Bump the spill generation directly to exercise the DROPPED -- defensive branch in tp_cache_apply_to_tail. Real spill paths -- (tp_spill_finalize) bump the generation AND immediately drop -- the cache's dshash tables via tp_cache_clear, so the next -- apply_to_tail lands on NOT_INITIALIZED instead of DROPPED. -- The helper isolates the stale-cursor-without-cleared-tables -- case so the safety net stays exercised. SELECT (bm25_cache_bump_spill_generation('cache_apply_idx') > 0) AS gen_advanced; -- apply_to_tail sees the stale generation token and DROPs the -- cache. cursor_seq drops back to 0 because tp_cache_clear -- resets it. SELECT result, cursor_seq FROM bm25_cache_apply_to_tail('cache_apply_idx'); -- The cache is now back at the cold state: NOT_INITIALIZED. SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); -- Cold-build re-bootstraps with the current generation, so the -- next apply_to_tail returns OK with 0 new records. SELECT result, (records_applied >= 0) AS records_ok FROM bm25_cache_cold_build('cache_apply_idx'); SELECT result, records_applied FROM bm25_cache_apply_to_tail('cache_apply_idx'); DROP INDEX cache_apply_idx; DROP TABLE cache_apply_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/cache_memory_cap.sql000066400000000000000000000233771525762047700245520ustar00rootroot00000000000000-- Integration coverage for the memory cap and the -- cross-index eviction path -- (see ARCHITECTURE.md, "Memtable Cache"). -- -- We exercise the registry-wide accounting counter and the -- `tp_cache_evict_largest` argmax routine directly through -- internal SQL scaffolds. End-to-end cap-driven eviction -- (where `apply_to_tail` itself triggers eviction because -- `pg_textsearch.memory_limit` was exceeded) cannot be -- exercised from a regression test because `memory_limit` -- is PGC_SIGHUP, so we focus on the deterministic primitives -- that the cap protocol composes from. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_cache_enabled = on; CREATE TABLE cache_cap_a (id int, body text); CREATE TABLE cache_cap_b (id int, body text); CREATE INDEX cache_cap_a_idx ON cache_cap_a USING bm25 (body) WITH (text_config = 'english'); CREATE INDEX cache_cap_b_idx ON cache_cap_b USING bm25 (body) WITH (text_config = 'english'); -- ---------- empty registry: nothing to evict ---------- -- Eviction with no caches populated returns nothing_found. -- `bm25_cache_evict_largest('cache_cap_a_idx')` excludes -- cache_cap_a_idx from the argmax, but cache_cap_b_idx has -- zero estimated_bytes so still no eligible victim. SELECT bm25_cache_evict_largest('cache_cap_a_idx') AS empty_evict; SELECT bm25_cache_global_estimated_bytes() AS empty_global_bytes; -- ---------- populate index A heavily, B lightly ---------- INSERT INTO cache_cap_a SELECT g, 'apple banana cherry doc ' || g FROM generate_series(1, 80) g; INSERT INTO cache_cap_b SELECT g, 'kiwi ' || g FROM generate_series(1, 5) g; -- Drive the apply protocol so the dshash tables are -- populated and per-index estimated_bytes is bumped. -- We check the apply outcome was "applied" (vs ABORT) and -- that bytes is non-zero. SELECT result, (estimated_bytes > 0) AS a_bytes_nonzero FROM bm25_cache_cold_build('cache_cap_a_idx'); SELECT result, (estimated_bytes > 0) AS b_bytes_nonzero FROM bm25_cache_cold_build('cache_cap_b_idx'); -- After both caches built, global counter must equal the -- sum of per-index estimated_bytes (we already proved both -- are > 0; here we just assert that global > 0). SELECT bm25_cache_global_estimated_bytes() > 0 AS post_build_global_nonzero; -- ---------- cross-index eviction ---------- -- evict_largest called with caller = cache_cap_b_idx should -- pick cache_cap_a_idx as victim (a has more bytes than b) -- and evict it. SELECT bm25_cache_evict_largest('cache_cap_b_idx') AS evict_a_from_b; -- After evicting A, the global counter should equal B's -- bytes only. We check by re-opening B's cache source -- without rebuilding (open is idempotent for a populated -- cache: `catchup_cache` returns OK with zero new records). -- Easier observable: global counter dropped strictly. -- Snapshot pre-evict (we recomputed above to be > 0) and -- post-evict, then verify post < pre via a CTE. WITH post AS ( SELECT bm25_cache_global_estimated_bytes() AS bytes ) SELECT bytes > 0 AS post_evict_b_still_present FROM post; -- A second call to evict_largest('cache_cap_b_idx') now -- finds no eligible victim: the only other index -- (cache_cap_a_idx) was just drained to zero, and B is -- excluded. SELECT bm25_cache_evict_largest('cache_cap_b_idx') AS evict_again_no_victim; -- And evicting with caller = cache_cap_a_idx drains B as -- the sole remaining candidate. SELECT bm25_cache_evict_largest('cache_cap_a_idx') AS evict_b_from_a; -- All caches drained: global must be zero. SELECT bm25_cache_global_estimated_bytes() AS final_global_bytes; -- ---------- rebuild + drop equivalence ---------- -- After eviction the caches are empty but the cache_source -- chooser will rebuild them on the next query, so end-to-end -- search still works. SELECT count(*) AS a_hits FROM ( SELECT id FROM cache_cap_a ORDER BY body <@> to_bm25query('apple', 'cache_cap_a_idx') LIMIT 100 ) q; SELECT count(*) AS b_hits FROM ( SELECT id FROM cache_cap_b ORDER BY body <@> to_bm25query('kiwi', 'cache_cap_b_idx') LIMIT 100 ) q; -- Remove the original test indexes before taking a baseline for the -- REINDEX accounting regression. The baseline is intentionally -- relative because the global counter belongs to shared memory. DROP INDEX cache_cap_a_idx; DROP INDEX cache_cap_b_idx; DROP TABLE cache_cap_a; DROP TABLE cache_cap_b; SELECT bm25_cache_global_estimated_bytes() AS reindex_original_baseline \gset -- Keep one populated cache alive across both REINDEX cycles. This -- proves each replaced index releases only its own accounting rather -- than relying on the global counter reaching zero. CREATE TABLE cache_reindex_control (id int, body text); CREATE INDEX cache_reindex_control_idx ON cache_reindex_control USING bm25 (body) WITH (text_config = 'english'); INSERT INTO cache_reindex_control SELECT g, 'control retained ' || g FROM generate_series(1, 10) g; SELECT count(*) AS retained_control_hits FROM ( SELECT 1 FROM cache_reindex_control ORDER BY body <@> to_bm25query('control', 'cache_reindex_control_idx') ) q; SELECT bm25_cache_global_estimated_bytes() - :reindex_original_baseline AS retained_control_bytes \gset SELECT :retained_control_bytes > 0 AS retained_control_charged; -- Exercise the parallel REINDEX completion path separately. The -- analyzed row count and worker settings match parallel_build.sql so -- both the initial build and REINDEX request one parallel worker. SET min_parallel_table_scan_size = 0; SET maintenance_work_mem = '256MB'; SET max_parallel_maintenance_workers = 1; CREATE TABLE cache_reindex_parallel (id int, body text); INSERT INTO cache_reindex_parallel SELECT g, 'parallel database document ' || g FROM generate_series(1, 100000) g; ANALYZE cache_reindex_parallel; CREATE INDEX cache_reindex_parallel_idx ON cache_reindex_parallel USING bm25 (body) WITH (text_config = 'english'); -- Add an unflushed runtime tail after the parallel build, then force -- that tail into the cache so REINDEX replaces a charged entry. INSERT INTO cache_reindex_parallel SELECT g, 'parallel database tail ' || g FROM generate_series(100001, 100025) g; SELECT count(*) AS parallel_reindex_hits FROM ( SELECT 1 FROM cache_reindex_parallel ORDER BY body <@> to_bm25query('database', 'cache_reindex_parallel_idx') ) q; SELECT bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS parallel_runtime_cache_charged; REINDEX INDEX cache_reindex_parallel_idx; -- Finalization itself must drain the old cache. Checking before DROP -- distinguishes in-place REINDEX cleanup from eventual relation teardown. SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS parallel_reindex_drained_before_drop \gset \echo parallel_reindex_drained_before_drop=:parallel_reindex_drained_before_drop DROP TABLE cache_reindex_parallel; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS parallel_reindex_released, bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS parallel_reindex_exceeds_retained; -- Cycle 1: segment spill, unflushed tail, cache-building scan, -- REINDEX, and drop. CREATE TABLE cache_reindex_test (id int, body text); CREATE INDEX cache_reindex_test_idx ON cache_reindex_test USING bm25 (body) WITH (text_config = 'english'); INSERT INTO cache_reindex_test SELECT g, 'databases original ' || g FROM generate_series(1, 50) g; SELECT bm25_spill_index('cache_reindex_test_idx'); INSERT INTO cache_reindex_test SELECT g, 'databases tail ' || g FROM generate_series(51, 75) g; SELECT count(*) FROM ( SELECT 1 FROM cache_reindex_test ORDER BY body <@> to_bm25query('databases', 'cache_reindex_test_idx') ) q; REINDEX INDEX cache_reindex_test_idx; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS serial_reindex_drained_before_drop \gset \echo serial_reindex_drained_before_drop=:serial_reindex_drained_before_drop DROP TABLE cache_reindex_test; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_1_released, bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_1_exceeds_retained; -- Repeat the complete cycle to show that each registry replacement -- leaks another cache charge on the unfixed code. CREATE TABLE cache_reindex_test (id int, body text); CREATE INDEX cache_reindex_test_idx ON cache_reindex_test USING bm25 (body) WITH (text_config = 'english'); INSERT INTO cache_reindex_test SELECT g, 'databases original ' || g FROM generate_series(1, 50) g; SELECT bm25_spill_index('cache_reindex_test_idx'); INSERT INTO cache_reindex_test SELECT g, 'databases tail ' || g FROM generate_series(51, 75) g; SELECT count(*) FROM ( SELECT 1 FROM cache_reindex_test ORDER BY body <@> to_bm25query('databases', 'cache_reindex_test_idx') ) q; REINDEX INDEX cache_reindex_test_idx; DROP TABLE cache_reindex_test; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_2_released, bm25_cache_global_estimated_bytes() > :reindex_original_baseline + :retained_control_bytes AS reindex_cycle_2_exceeds_retained; DROP TABLE cache_reindex_control; SELECT bm25_cache_global_estimated_bytes() = :reindex_original_baseline AS control_drop_restored_baseline, bm25_cache_global_estimated_bytes() > :reindex_original_baseline AS control_drop_exceeds_baseline; RESET pg_textsearch.memtable_cache_enabled; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/cache_source.sql000066400000000000000000000057741525762047700237200ustar00rootroot00000000000000-- Unit coverage for the in-memory memtable cache TpDataSource -- (cache_source.c; see ARCHITECTURE.md, "Memtable Cache"). -- -- Drives bm25_test_cache_source directly so we can observe the -- TpDataSource interface (open / total_docs / get_postings / -- get_doc_freq / get_doc_length / close) backed by the cache -- rather than the chain, without going through the query -- planner. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE cache_source_t (id int, body text); CREATE INDEX cache_source_idx ON cache_source_t USING bm25 (body) WITH (text_config = 'english'); -- ---------- empty memtable ---------- -- An empty memtable yields a NULL cache source, just like the -- chain source does. SELECT bm25_test_cache_source('cache_source_idx', 'empty'); -- ---------- populate and serve ---------- INSERT INTO cache_source_t SELECT g, 'shared term ' || g FROM generate_series(1, 5) g; -- Opening a cache source over a 5-doc memtable reports -- total_docs = 5 and a non-zero total_len. We assert -- structurally to avoid pinning the tokenizer's exact length -- output. SELECT split_part(bm25_test_cache_source('cache_source_idx', 'open'), ',', 1) AS total_docs_clause; SELECT (split_part( split_part(bm25_test_cache_source('cache_source_idx', 'open'), ',', 2), '=', 2)::bigint > 0) AS total_len_positive; -- get_postings via the cache: 'shared' appears in every -- document. We snapshot doc_freq = 5 (all five docs) and -- count = 5 (one CTID per occurrence). Tokenizer stemming may -- shape the lexeme, so we look up the stemmed form 'share'. SELECT bm25_test_cache_source('cache_source_idx', 'lookup:share'); -- A term that doesn't exist returns 'miss:df=0'. SELECT bm25_test_cache_source('cache_source_idx', 'lookup:nonexistentterm'); -- get_doc_length via the cache: doc 1 is at heap block 0, -- offset 1. We don't pin the exact length (depends on tokens -- after stemming), only that the lookup succeeds. SELECT (split_part( split_part(bm25_test_cache_source( 'cache_source_idx', 'doclen:0'), ':', 2), '=', 2)::int >= 0) AS doc_length_lookup_ok; -- ---------- end-to-end equivalence (chain vs cache) ---------- -- With the cache GUC off the read path uses chain_source; with -- it on the read path uses cache_source. Either way the BM25 -- ranking must be identical because the cache is derived state -- over the same chain. SET pg_textsearch.memtable_cache_enabled = off; SELECT id, body FROM cache_source_t ORDER BY body <@> to_bm25query('shared', 'cache_source_idx') LIMIT 3; SET pg_textsearch.memtable_cache_enabled = on; SELECT id, body FROM cache_source_t ORDER BY body <@> to_bm25query('shared', 'cache_source_idx') LIMIT 3; RESET pg_textsearch.memtable_cache_enabled; DROP INDEX cache_source_idx; DROP TABLE cache_source_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/cache_spill.sql000066400000000000000000000076331525762047700235370ustar00rootroot00000000000000-- Integration coverage for the spill -> cache-invalidate -> rebuild -- cycle (cache_source.c + log.c; see ARCHITECTURE.md, -- "Memtable Cache"). -- -- tp_spill_finalize (a) bumps the per-index spill_generation atomic -- and (b) drops the cache's dshash tables via tp_cache_clear, so a -- query that runs after a spill rebuilds a fresh cache from -- whatever chain pages exist post-spill (zero in the typical case, -- since spill resets the chain head/tail to InvalidBlockNumber). -- -- We exercise both with the cache GUC explicitly enabled so the -- chooser routes reads through cache_source. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_cache_enabled = on; CREATE TABLE cache_spill_t (id int, body text); CREATE INDEX cache_spill_idx ON cache_spill_t USING bm25 (body) WITH (text_config = 'english'); -- ---------- populate + open cache, pre-spill ---------- INSERT INTO cache_spill_t SELECT g, 'alpha term ' || g FROM generate_series(1, 10) g; -- Cache source opens against the populated memtable. total_docs -- should be 10; we check structurally. SELECT split_part(bm25_test_cache_source('cache_spill_idx', 'open'), ',', 1) AS open_docs; -- The term 'alpha' is in every doc so doc_freq = 10. SELECT bm25_test_cache_source('cache_spill_idx', 'lookup:alpha'); -- End-to-end query through the chooser returns all 10 rows. SELECT count(*) AS pre_spill_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; -- ---------- force a spill ---------- -- bm25_spill_index drives tp_do_spill -> tp_spill_finalize, which -- bumps spill_generation and clears the cache. SELECT bm25_spill_index('cache_spill_idx') > 0 AS spill_wrote_docs; -- After spill the chain is empty (head/tail reset). The cache -- source constructor returns NULL on empty memtable. SELECT bm25_test_cache_source('cache_spill_idx', 'empty'); -- The query still returns 10 rows: cache_source returns NULL, -- chain source returns NULL, the scoring path falls through to -- segments only and finds all 10 docs in the new segment. SELECT count(*) AS post_spill_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; -- ---------- populate again post-spill, cache rebuilds cold ---------- INSERT INTO cache_spill_t SELECT g, 'beta term ' || g FROM generate_series(11, 15) g; -- New cache source opens over the 5 fresh chain records (the -- alpha docs are in the segment now, not the chain). SELECT split_part(bm25_test_cache_source('cache_spill_idx', 'open'), ',', 1) AS post_spill_open_docs; -- 'beta' is unique to the new chain records. SELECT bm25_test_cache_source('cache_spill_idx', 'lookup:beta'); -- 'alpha' is in the segment but not the new chain; cache lookup -- misses for it. SELECT bm25_test_cache_source('cache_spill_idx', 'lookup:alpha'); -- End-to-end queries see both alpha (10, segment) and beta (5, -- cache) with no duplicates. SELECT count(*) AS alpha_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; SELECT count(*) AS beta_hits FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('beta', 'cache_spill_idx') LIMIT 100 ) q; -- ---------- equivalence with cache disabled ---------- -- Same queries with the chooser falling back to the chain must -- return identical counts. SET pg_textsearch.memtable_cache_enabled = off; SELECT count(*) AS alpha_hits_chain FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('alpha', 'cache_spill_idx') LIMIT 100 ) q; SELECT count(*) AS beta_hits_chain FROM ( SELECT id FROM cache_spill_t ORDER BY body <@> to_bm25query('beta', 'cache_spill_idx') LIMIT 100 ) q; RESET pg_textsearch.memtable_cache_enabled; DROP INDEX cache_spill_idx; DROP TABLE cache_spill_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/catalog_stats.sql000066400000000000000000000162641525762047700241210ustar00rootroot00000000000000-- Test that BM25 indexes report correct catalog statistics -- (pg_class.reltuples, pg_class.relpages, pg_stat_user_indexes) CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================ -- heap_tuples vs index_tuples for partial indexes -- -- IndexBuildResult.heap_tuples updates pg_class.reltuples for -- the HEAP relation. For a partial index, this must be the -- total heap rows scanned, not just rows that passed the -- predicate. Otherwise the planner underestimates table size. -- ============================================================ CREATE TABLE stats_partial ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO stats_partial (content, category) SELECT 'document ' || i || ' about databases', CASE WHEN i <= 10 THEN 'tech' ELSE 'other' END FROM generate_series(1, 100) i; -- Record heap reltuples before index build ANALYZE stats_partial; SELECT reltuples AS heap_reltuples_before FROM pg_class WHERE oid = 'stats_partial'::regclass; -- Build partial index (only 10 of 100 rows match) CREATE INDEX stats_partial_idx ON stats_partial USING bm25 (content) WITH (text_config='english') WHERE category = 'tech'; -- heap reltuples must still reflect ~100, not 10 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_partial'::regclass; -- index reltuples should reflect ~10 (indexed docs) SELECT reltuples BETWEEN 5 AND 15 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_partial_idx'::regclass; DROP TABLE stats_partial CASCADE; -- ============================================================ -- heap_tuples for non-partial index (baseline sanity check) -- ============================================================ CREATE TABLE stats_full ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO stats_full (content) SELECT 'document ' || i || ' about databases' FROM generate_series(1, 100) i; CREATE INDEX stats_full_idx ON stats_full USING bm25 (content) WITH (text_config='english'); -- Both heap and index should reflect ~100 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_full'::regclass; SELECT reltuples >= 90 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_full_idx'::regclass; DROP TABLE stats_full CASCADE; -- ============================================================ -- heap_tuples for expression index -- ============================================================ CREATE TABLE stats_expr ( id SERIAL PRIMARY KEY, data JSONB ); INSERT INTO stats_expr (data) SELECT ('{"content": "document ' || i || ' about databases"}')::jsonb FROM generate_series(1, 100) i; CREATE INDEX stats_expr_idx ON stats_expr USING bm25 ((data->>'content')) WITH (text_config='english'); SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_expr'::regclass; SELECT reltuples >= 90 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_expr_idx'::regclass; DROP TABLE stats_expr CASCADE; -- ============================================================ -- heap_tuples for expression + partial combined -- ============================================================ CREATE TABLE stats_expr_partial ( id SERIAL PRIMARY KEY, data JSONB, active BOOLEAN ); INSERT INTO stats_expr_partial (data, active) SELECT ('{"text": "document ' || i || ' about databases"}')::jsonb, (i <= 20) FROM generate_series(1, 100) i; CREATE INDEX stats_ep_idx ON stats_expr_partial USING bm25 ((data->>'text')) WITH (text_config='english') WHERE active = true; -- heap must reflect ~100 SELECT reltuples >= 90 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_expr_partial'::regclass; -- index should reflect ~20 SELECT reltuples BETWEEN 15 AND 25 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_ep_idx'::regclass; DROP TABLE stats_expr_partial CASCADE; -- ============================================================ -- pg_relation_size: index should have nonzero size after build -- ============================================================ CREATE TABLE stats_size ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO stats_size (content) SELECT 'document ' || i || ' about databases and search' FROM generate_series(1, 50) i; CREATE INDEX stats_size_idx ON stats_size USING bm25 (content) WITH (text_config='english'); SELECT pg_relation_size('stats_size_idx') > 0 AS has_size; DROP TABLE stats_size CASCADE; -- ============================================================ -- Stats after VACUUM: reltuples should decrease after deletes -- ============================================================ -- Isolate physical cleanup from other backends and standby feedback. CREATE TEMP TABLE stats_vacuum ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO stats_vacuum (content) SELECT 'document ' || i || ' about databases' FROM generate_series(1, 100) i; CREATE INDEX stats_vacuum_idx ON stats_vacuum USING bm25 (content) WITH (text_config='english'); -- Delete half the rows and vacuum DELETE FROM stats_vacuum WHERE id > 50; VACUUM stats_vacuum; -- Index reltuples should reflect roughly 50 after vacuum SELECT reltuples BETWEEN 40 AND 60 AS index_tuples_after_vacuum FROM pg_class WHERE oid = 'stats_vacuum_idx'::regclass; -- Second delete + vacuum; reltuples must continue tracking the -- live count (regression test: reporting metap->total_docs - only -- this-round deaths would leave reltuples stuck at the first -- round's value). DELETE FROM stats_vacuum WHERE id > 25; VACUUM stats_vacuum; SELECT reltuples BETWEEN 15 AND 35 AS index_tuples_after_second_vacuum FROM pg_class WHERE oid = 'stats_vacuum_idx'::regclass; -- No-deletes maintenance round; reltuples must stay at the live -- count, not reset to the cumulative insert count. VACUUM stats_vacuum; SELECT reltuples BETWEEN 15 AND 35 AS index_tuples_after_noop_vacuum FROM pg_class WHERE oid = 'stats_vacuum_idx'::regclass; DROP TABLE stats_vacuum CASCADE; -- ============================================================ -- Stats after REINDEX: should reflect current row count -- ============================================================ -- An unrelated snapshot must not keep the deleted rows recently dead. CREATE TEMP TABLE stats_reindex ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO stats_reindex (content, category) SELECT 'document ' || i || ' about databases', CASE WHEN i <= 30 THEN 'tech' ELSE 'other' END FROM generate_series(1, 100) i; CREATE INDEX stats_reindex_idx ON stats_reindex USING bm25 (content) WITH (text_config='english') WHERE category = 'tech'; -- Delete some matching rows DELETE FROM stats_reindex WHERE category = 'tech' AND id > 20; REINDEX INDEX stats_reindex_idx; -- heap reltuples should still be ~90 (100 - 10 deleted) SELECT reltuples >= 80 AS heap_tuples_ok FROM pg_class WHERE oid = 'stats_reindex'::regclass; -- index reltuples should reflect ~20 remaining tech rows SELECT reltuples BETWEEN 15 AND 25 AS index_tuples_ok FROM pg_class WHERE oid = 'stats_reindex_idx'::regclass; DROP TABLE stats_reindex CASCADE; -- ============================================================ -- Cleanup -- ============================================================ DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/chain_source.sql000066400000000000000000000063671525762047700237360ustar00rootroot00000000000000-- Unit-level coverage for the on-disk memtable chain -- TpDataSource implementation (memtable v2, issue #374), driven -- through the bm25_test_chain_source() scaffold. Each case -- writes records through tp_memtable_append() with real TpVector -- payloads, then opens a chain source and verifies the read-back -- through the TpDataSource interface (get_postings, -- get_doc_length, total_docs, total_len). CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE chain_source_t (id int, body text); CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- Empty chain: source create returns NULL. SELECT bm25_test_chain_source('chain_source_idx', 'empty_chain'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- Single doc, single term: count=1, doc_freq=1, freq matches. SELECT bm25_test_chain_source('chain_source_idx', 'one_doc_one_term'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- Multiple docs share one term: count=N, all ctids match. SELECT bm25_test_chain_source( 'chain_source_idx', 'multi_doc_shared_term'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- One doc, three distinct terms: each term round-trips. SELECT bm25_test_chain_source('chain_source_idx', 'multi_term'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- A non-NULL zero-length query-term list requests document -- metadata without materializing any term postings. \pset tuples_only on SELECT bm25_test_chain_source('chain_source_idx', 'documents_only'); \pset tuples_only off DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- Records spill across multiple memtable pages: chain walk -- aggregates all docs into one term entry. SELECT bm25_test_chain_source('chain_source_idx', 'multi_page_chain'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- get_postings for an absent term returns NULL. SELECT bm25_test_chain_source('chain_source_idx', 'term_not_found'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- get_doc_length: present ctids return their length; absent -- ctids return -1. SELECT bm25_test_chain_source('chain_source_idx', 'doc_length_lookup'); DROP INDEX chain_source_idx; CREATE INDEX chain_source_idx ON chain_source_t USING bm25 (body) WITH (text_config = 'english'); -- Fragment round-trip: write a single record whose TpVector -- payload exceeds TP_MEMTABLE_PAGE_MAX_VECTOR_LEN, then read it -- back through the chain source. Verifies that the writer -- splits the payload across a fragment head page + N -- continuation pages and that the reader correctly reassembles -- the bytes before decoding term entries. SELECT bm25_test_chain_source('chain_source_idx', 'fragment_roundtrip'); -- Clean up DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/chinese.sql000066400000000000000000000057631525762047700227110ustar00rootroot00000000000000-- End-to-end Chinese full-text search test. -- -- pg_textsearch tokenizes both documents and queries through the index's -- text_config, so Chinese support is a matter of choosing a Chinese-aware -- text search configuration. This test pairs pg_textsearch with zhparser -- (https://github.com/amutu/zhparser), an SCWS dictionary-based Chinese -- word segmenter, and verifies indexing and ranked retrieval end to end. -- -- Requires the zhparser extension (and its SCWS library) to be installed. -- This test is not part of the default REGRESS schedule; run it via -- `make test-chinese` in an environment where zhparser is available. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE EXTENSION IF NOT EXISTS zhparser; -- Pin zhparser's segmentation GUCs to their defaults so tokenization is -- deterministic regardless of any cluster-level configuration. SET zhparser.multi_short = off; SET zhparser.multi_duality = off; SET zhparser.multi_zmain = off; SET zhparser.multi_zall = off; SET zhparser.seg_with_duality = off; -- CREATE EXTENSION zhparser also registers the zhparser parser. Define a -- configuration that maps zhparser's content token types (nouns, verbs, -- adjectives, idioms, interjections, set phrases). The configuration -- must be schema-qualified so the bm25 index build can resolve it. CREATE TEXT SEARCH CONFIGURATION public.chinese (PARSER = zhparser); ALTER TEXT SEARCH CONFIGURATION public.chinese ADD MAPPING FOR n, v, a, i, e, l WITH simple; -- zhparser segments Chinese into dictionary words (deterministic for a -- given SCWS dictionary): 机器学习 splits into 机器 + 学习. COLLATE "C" -- keeps the ordering byte-stable across locales. SELECT alias, token FROM ts_debug('public.chinese', '机器学习') ORDER BY token COLLATE "C"; CREATE TABLE zh_docs (id int PRIMARY KEY, content text); INSERT INTO zh_docs VALUES (1, 'PostgreSQL 是一个强大的开源关系数据库'), -- database (2, '机器学习是人工智能的一个重要分支'), -- machine learning (3, '全文检索能够快速查找文档内容'), -- full-text search (4, '自然语言处理研究人类语言'), -- natural language (5, '深度学习推动人工智能技术发展'); -- shares 学习 only CREATE INDEX zh_docs_bm25 ON zh_docs USING bm25 (content) WITH (text_config = 'public.chinese'); -- Force the bm25 index scan so we exercise indexed retrieval, not -- standalone scoring. SET enable_seqscan = off; -- Query 机器学习 (words 机器 + 学习): doc 2 contains both, doc 5 contains -- only 学习, so doc 2 outranks doc 5 and no other doc matches. SELECT id, content FROM zh_docs ORDER BY content <@> to_bm25query('机器学习', 'zh_docs_bm25') LIMIT 2; -- Query 数据库: only doc 1 contains that word. SELECT id, content FROM zh_docs ORDER BY content <@> to_bm25query('数据库', 'zh_docs_bm25') LIMIT 1; RESET enable_seqscan; DROP TABLE zh_docs; DROP TEXT SEARCH CONFIGURATION public.chinese; DROP EXTENSION zhparser; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/compaction.sql000066400000000000000000000452551525762047700234270ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_textsearch; \pset format unaligned SET client_min_messages = warning; SET pg_textsearch.segments_per_level = 64; CREATE TABLE compaction_step (id serial PRIMARY KEY, body text); CREATE TABLE compaction_full (id serial PRIMARY KEY, body text); CREATE INDEX compaction_step_idx ON compaction_step USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE INDEX compaction_full_idx ON compaction_full USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE INDEX compaction_btree_idx ON compaction_step(id); CREATE TABLE compaction_partitioned (id integer, body text) PARTITION BY RANGE (id); CREATE TABLE compaction_partitioned_leaf PARTITION OF compaction_partitioned FOR VALUES FROM (0) TO (10); CREATE INDEX compaction_partitioned_idx ON compaction_partitioned USING bm25(body) WITH (text_config = 'english'); -- New indexes expose all levels and have no compaction debt. SELECT array_length( bm25_level_counts('compaction_step_idx'::regclass), 1) = 8 AS has_all_levels; SELECT bm25_needs_compaction('compaction_step_idx'::regclass) AS new_index_needs_compaction; -- Runtime spill publishes under the per-index lock, then applies inline -- compaction under a separately acquired maintenance lock. CREATE TABLE compaction_inline (id serial PRIMARY KEY, body text); CREATE INDEX compaction_inline_idx ON compaction_inline USING bm25(body) WITH (text_config = 'english'); SET pg_textsearch.segments_per_level = 2; BEGIN; DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO compaction_inline (body) VALUES (format('inline spill document %s filler', n)); PERFORM bm25_spill_index('compaction_inline_idx'); END LOOP; END $$; SELECT bm25_level_counts('compaction_inline_idx'::regclass) = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AS inline_policy_compacts_after_spill; SELECT NOT EXISTS ( SELECT 1 FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'object' AND classid = 'pg_am'::regclass AND objid = 'compaction_inline_idx'::regclass AND objsubid = 3 AND mode = 'ExclusiveLock' AND granted ) AS inline_policy_releases_maintenance_lock; COMMIT; DROP TABLE compaction_inline CASCADE; SET pg_textsearch.segments_per_level = 64; -- Non-bm25 relations are rejected. SELECT bm25_level_counts('compaction_btree_idx'::regclass); SELECT bm25_level_counts('compaction_step'::regclass); -- Partitioned parents have no storage; their physical leaves remain usable. SELECT bm25_level_counts('compaction_partitioned_idx'::regclass); SELECT bm25_compact('compaction_partitioned_idx'::regclass); SELECT bm25_compact_step('compaction_partitioned_idx'::regclass); SELECT bm25_needs_compaction('compaction_partitioned_idx'::regclass); SELECT array_length(bm25_level_counts(i.inhrelid), 1) = 8 AS partition_leaf_counts_supported FROM pg_inherits i WHERE i.inhparent = 'compaction_partitioned_idx'::regclass; SELECT bm25_compact(i.inhrelid) FROM pg_inherits i WHERE i.inhparent = 'compaction_partitioned_idx'::regclass; SELECT NOT bm25_compact_step(i.inhrelid) AND NOT bm25_needs_compaction(i.inhrelid) AS partition_leaf_compaction_supported FROM pg_inherits i WHERE i.inhparent = 'compaction_partitioned_idx'::regclass; -- Construct identical four-segment layouts without inline compaction. DO $$ DECLARE n integer; BEGIN FOR n IN 1..4 LOOP INSERT INTO compaction_step (body) SELECT format('step batch %s document %s filler', n, i) FROM generate_series(1, 20) i; INSERT INTO compaction_full (body) SELECT format('full batch %s document %s filler', n, i) FROM generate_series(1, 20) i; PERFORM bm25_spill_index('compaction_step_idx'); PERFORM bm25_spill_index('compaction_full_idx'); END LOOP; END $$; SET pg_textsearch.segments_per_level = 2; SELECT bm25_level_counts('compaction_step_idx'::regclass) = ARRAY[4, 0, 0, 0, 0, 0, 0, 0] AND bm25_needs_compaction('compaction_step_idx'::regclass) AS step_starts_with_debt; UPDATE pg_catalog.pg_class SET reloptions = array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'compaction_step_idx'::regclass; SELECT d.oid AS db_oid, c.oid AS index_oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace) AS spc_oid, pg_relation_filenode(c.oid) AS relfilenumber, c.relowner AS owner_oid FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'compaction_step_idx'::regclass \gset target_ -- Stale physical identities are rejected without touching the live index. SELECT NOT bm25_compact_step_if_current( :target_index_oid::oid, 0::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AND NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, 0::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AND NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, 0::oid, :target_owner_oid::oid) AND NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, 0::oid) AS stale_step_targets_rejected; -- Permanent-index mutators reject read-only transactions. BEGIN READ ONLY; SELECT bm25_compact('compaction_step_idx'::regclass); ROLLBACK; BEGIN READ ONLY; SELECT bm25_compact_step('compaction_step_idx'::regclass); ROLLBACK; -- In this layout one pass happens to merge exactly one batch. A pass is -- not limited to one batch in general; it may span several, and may pull -- in further batches to unblock a destination level. SELECT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AS one_batch_ran; SELECT bm25_level_counts('compaction_step_idx'::regclass) = ARRAY[2, 1, 0, 0, 0, 0, 0, 0] AND bm25_needs_compaction('compaction_step_idx'::regclass) AS one_batch_only; -- Repeated steps converge, then report that no batch ran. CREATE TEMP TABLE compaction_step_result (steps integer); DO $$ DECLARE step_count integer := 0; BEGIN WHILE bm25_needs_compaction('compaction_step_idx'::regclass) LOOP step_count := step_count + 1; IF step_count > 100 THEN RAISE EXCEPTION 'bm25_compact_step did not terminate'; END IF; IF NOT bm25_compact_step('compaction_step_idx'::regclass) THEN RAISE EXCEPTION 'bm25_needs_compaction disagrees with step'; END IF; END LOOP; INSERT INTO compaction_step_result VALUES (step_count); END $$; SELECT steps = 2 AS step_split_cascade FROM compaction_step_result; SELECT NOT bm25_needs_compaction('compaction_step_idx'::regclass) AND NOT bm25_compact_step('compaction_step_idx'::regclass) AS step_converged; -- Whole-index compaction reaches the same terminal layout. SELECT bm25_compact('compaction_full_idx'::regclass); SELECT NOT bm25_needs_compaction('compaction_full_idx'::regclass) AND bm25_level_counts('compaction_step_idx'::regclass) = bm25_level_counts('compaction_full_idx'::regclass) AS full_compaction_converged; SELECT count(*) = 80 AS step_preserves_documents FROM ( SELECT 1 FROM compaction_step ORDER BY body <@> to_bm25query('filler', 'compaction_step_idx') ) ranked; SELECT count(*) = 80 AS full_preserves_documents FROM ( SELECT 1 FROM compaction_full ORDER BY body <@> to_bm25query('filler', 'compaction_full_idx') ) ranked; -- Background workers validate mode and physical identity without writing. CREATE TABLE compaction_background_target (id integer, body text); CREATE INDEX compaction_background_target_idx ON compaction_background_target USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..4 LOOP INSERT INTO compaction_background_target (body) SELECT format('background batch %s document %s', n, i) FROM generate_series(1, 20) i; PERFORM bm25_spill_index('compaction_background_target_idx'); END LOOP; END $$; UPDATE pg_catalog.pg_class SET reloptions = pg_catalog.array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'compaction_background_target_idx'::regclass; SELECT d.oid AS db_oid, c.oid AS index_oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace) AS spc_oid, pg_relation_filenode(c.oid) AS relfilenumber, c.relowner AS owner_oid FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'compaction_background_target_idx'::regclass \gset background_ SELECT bm25_background_target_is_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid) AS background_target_is_current; ALTER INDEX compaction_background_target_idx SET (compaction = 'manual'); SELECT NOT bm25_background_target_is_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid) AS manual_target_is_not_current; SELECT NOT bm25_compact_step_if_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid) AND bm25_level_counts('compaction_background_target_idx'::regclass) = ARRAY[4, 0, 0, 0, 0, 0, 0, 0] AS manual_target_is_not_compacted; -- Inspection is public, but only the index owner may mutate it. DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'compaction_user') THEN EXECUTE 'REASSIGN OWNED BY compaction_user TO CURRENT_USER'; EXECUTE 'DROP OWNED BY compaction_user CASCADE'; DROP ROLE compaction_user; END IF; END $$; CREATE ROLE compaction_user LOGIN; GRANT USAGE ON SCHEMA public TO compaction_user; SET ROLE compaction_user; SELECT array_length( bm25_level_counts('compaction_step_idx'::regclass), 1) = 8 AS nonowner_can_inspect; SELECT bm25_needs_compaction('compaction_step_idx'::regclass) AS nonowner_can_check; SELECT bm25_compact('compaction_step_idx'::regclass); SELECT bm25_compact_step('compaction_step_idx'::regclass); SELECT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid); SELECT bm25_background_target_is_current( :background_index_oid::oid, :background_db_oid::oid, :background_spc_oid::oid, :background_relfilenumber::oid, :background_owner_oid::oid); RESET ROLE; DROP OWNED BY compaction_user CASCADE; DROP ROLE compaction_user; DROP TABLE compaction_background_target CASCADE; -- Local temporary indexes remain mutable in read-only transactions. SET pg_textsearch.segments_per_level = 64; CREATE TEMP TABLE compaction_temp_step (id serial, body text); CREATE TEMP TABLE compaction_temp_full (id serial, body text); CREATE INDEX compaction_temp_step_idx ON compaction_temp_step USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); CREATE INDEX compaction_temp_full_idx ON compaction_temp_full USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO compaction_temp_step (body) VALUES (format('temporary step document %s filler', n)); INSERT INTO compaction_temp_full (body) VALUES (format('temporary full document %s filler', n)); PERFORM bm25_spill_index('compaction_temp_step_idx'); PERFORM bm25_spill_index('compaction_temp_full_idx'); END LOOP; END $$; SET pg_textsearch.segments_per_level = 2; BEGIN READ ONLY; SELECT bm25_compact_step('compaction_temp_step_idx'::regclass) AS temp_step_ran; SELECT bm25_compact('compaction_temp_full_idx'::regclass); COMMIT; SELECT NOT bm25_needs_compaction('compaction_temp_step_idx'::regclass) AND NOT bm25_needs_compaction( 'compaction_temp_full_idx'::regclass) AS temp_indexes_converged; -- A published pass is a physical change, so ROLLBACK does not undo it. -- A caller must not treat a step as transactional work. CREATE TABLE compaction_rollback (id serial, body text); CREATE INDEX compaction_rollback_idx ON compaction_rollback USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); SET pg_textsearch.segments_per_level = 64; DO $$ DECLARE n integer; BEGIN FOR n IN 1..4 LOOP INSERT INTO compaction_rollback (body) VALUES (format('rollback document %s filler', n)); PERFORM bm25_spill_index('compaction_rollback_idx'); END LOOP; END $$; SET pg_textsearch.segments_per_level = 2; CREATE TEMP TABLE compaction_rollback_before AS SELECT bm25_level_counts('compaction_rollback_idx'::regclass) AS counts; BEGIN; SELECT bm25_compact_step('compaction_rollback_idx'::regclass) AS rollback_step_ran; ROLLBACK; SELECT bm25_level_counts('compaction_rollback_idx'::regclass) <> before.counts AS rollback_does_not_undo_a_published_pass FROM compaction_rollback_before before; DROP TABLE compaction_rollback CASCADE; -- A level can sit at the segment threshold with nothing to compact: -- every candidate group already exceeds max_segment_size, and an -- over-budget segment is an uncombinable singleton. This is not -- compaction debt, and bm25_needs_compaction() reports false so a -- scheduler looping on it does not spin. CREATE TABLE compaction_unreducible (id bigint PRIMARY KEY, body text); CREATE INDEX compaction_unreducible_idx ON compaction_unreducible USING bm25(body) WITH (text_config = 'simple'); SET pg_textsearch.max_segment_size = '1MB'; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.segments_per_level = 2; DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..1 LOOP INSERT INTO compaction_unreducible SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 32) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('compaction_unreducible_idx'); END LOOP; END $$; SELECT bm25_level_counts('compaction_unreducible_idx'::regclass) = ARRAY[2, 0, 0, 0, 0, 0, 0, 0] AS unreducible_level_is_at_threshold; SELECT NOT bm25_needs_compaction('compaction_unreducible_idx'::regclass) AS unreducible_level_reports_no_work; SELECT bm25_compact_step('compaction_unreducible_idx'::regclass) AS unreducible_step_declines; SELECT bm25_compact('compaction_unreducible_idx'::regclass); SELECT bm25_level_counts('compaction_unreducible_idx'::regclass) = ARRAY[2, 0, 0, 0, 0, 0, 0, 0] AS unreducible_full_is_a_noop; SELECT count(*) = 4000 AS unreducible_preserves_documents FROM ( SELECT 1 FROM compaction_unreducible ORDER BY body <@> to_bm25query('common', 'compaction_unreducible_idx') LIMIT 5000 ) ranked; RESET pg_textsearch.max_segment_size; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP TABLE compaction_unreducible CASCADE; -- A pass must not copy segments it cannot combine. Spills prepend, so -- a level reads newest-first and its oldest, largest runs sit at the -- tail. Here two fresh small segments head a 2MB run that is already -- over max_segment_size: the pair merges, and the run is handed back -- instead of being rewritten to say the same thing in a new place. -- -- The handback is visible as a level count: a retained segment keeps -- its level, while a rewritten one is promoted. Without the trim this -- ends at [0, 2, ...] with the run copied to a fresh block, and the -- index grows by the full size of the run rather than by the pair. CREATE TABLE compaction_uncombinable_tail (id bigint PRIMARY KEY, body text); CREATE INDEX compaction_uncombinable_tail_idx ON compaction_uncombinable_tail USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.max_segment_size = '1MB'; -- Manual mode preserves the layout until the explicit pass. SET pg_textsearch.segments_per_level = 64; INSERT INTO compaction_uncombinable_tail SELECT gs, 'common ' || repeat(md5(gs::text), 32) FROM generate_series(1, 2000) gs; SELECT bm25_spill_index('compaction_uncombinable_tail_idx') > 0 AS tail_run_spilled; INSERT INTO compaction_uncombinable_tail SELECT 100000 + gs, 'common small ' || gs FROM generate_series(1, 20) gs; SELECT bm25_spill_index('compaction_uncombinable_tail_idx') > 0 AS head_pair_first_spilled; INSERT INTO compaction_uncombinable_tail SELECT 200000 + gs, 'common small ' || gs FROM generate_series(1, 20) gs; SELECT bm25_spill_index('compaction_uncombinable_tail_idx') > 0 AS head_pair_second_spilled; SET pg_textsearch.segments_per_level = 3; SELECT bm25_level_counts('compaction_uncombinable_tail_idx'::regclass) = ARRAY[3, 0, 0, 0, 0, 0, 0, 0] AS tail_level_is_at_threshold; CREATE TEMP TABLE compaction_tail_before AS SELECT pg_relation_size('compaction_uncombinable_tail_idx') AS bytes; SELECT bm25_compact('compaction_uncombinable_tail_idx'::regclass); SELECT bm25_level_counts('compaction_uncombinable_tail_idx'::regclass) = ARRAY[1, 1, 0, 0, 0, 0, 0, 0] AS tail_run_retained_at_its_level; SELECT pg_relation_size('compaction_uncombinable_tail_idx') - before.bytes < 1024 * 1024 AS tail_run_was_not_copied FROM compaction_tail_before before; SELECT count(*) = 2040 AS tail_preserves_documents FROM ( SELECT 1 FROM compaction_uncombinable_tail ORDER BY body <@> to_bm25query( 'common', 'compaction_uncombinable_tail_idx') LIMIT 5000 ) ranked; RESET pg_textsearch.max_segment_size; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP TABLE compaction_uncombinable_tail CASCADE; RESET pg_textsearch.segments_per_level; DROP TABLE compaction_partitioned CASCADE; DROP TABLE compaction_full CASCADE; DROP TABLE compaction_step CASCADE; SELECT NOT bm25_compact_step_if_current( :target_index_oid::oid, :target_db_oid::oid, :target_spc_oid::oid, :target_relfilenumber::oid, :target_owner_oid::oid) AS dropped_step_target_rejected; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/compaction_error_injection.sql000066400000000000000000000063241525762047700266740ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE TABLE compaction_unwind (id serial PRIMARY KEY, body text); CREATE INDEX compaction_unwind_idx ON compaction_unwind USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); SET pg_textsearch.segments_per_level = 2; -- An error raised anywhere in a compaction pass must unwind without -- losing documents or wedging the index: the pass is discarded, the -- published graph still answers queries, and a later pass succeeds. -- A buffer pin or index lock escaping the error path surfaces here as -- a WARNING in the test output. DO $$ DECLARE point text; n integer; docs integer; found integer; raised boolean; BEGIN FOREACH point IN ARRAY ARRAY[ 'pg-textsearch-compaction-after-select', 'pg-textsearch-compaction-alloc-output-data', 'pg-textsearch-compaction-alloc-page-index', 'pg-textsearch-before-compaction-publish', 'pg-textsearch-compaction-after-restamp', 'pg-textsearch-after-compaction-publish'] LOOP -- Accumulate level 0 debt without letting the spill's own -- inline compaction consume it. PERFORM set_config( 'pg_textsearch.segments_per_level', '64', true); FOR n IN 1..6 LOOP INSERT INTO compaction_unwind (body) SELECT format('unwind filler document %s %s', n, gs) FROM generate_series(1, 10) gs; PERFORM bm25_spill_index('compaction_unwind_idx'); END LOOP; PERFORM set_config( 'pg_textsearch.segments_per_level', '2', true); SELECT count(*) INTO docs FROM compaction_unwind; PERFORM injection_points_attach(point, 'error'); raised := false; BEGIN PERFORM bm25_compact_step( 'compaction_unwind_idx'::regclass); EXCEPTION WHEN others THEN raised := true; END; PERFORM injection_points_detach(point); IF NOT raised THEN RAISE EXCEPTION 'injected error at % did not fire', point; END IF; SELECT count(*) INTO found FROM ( SELECT 1 FROM compaction_unwind ORDER BY body <@> to_bm25query('filler', 'compaction_unwind_idx') ) ranked; IF found <> docs THEN RAISE EXCEPTION 'error at % lost documents: % of %', point, found, docs; END IF; PERFORM bm25_compact('compaction_unwind_idx'::regclass); SELECT count(*) INTO found FROM ( SELECT 1 FROM compaction_unwind ORDER BY body <@> to_bm25query('filler', 'compaction_unwind_idx') ) ranked; IF found <> docs THEN RAISE EXCEPTION 'compaction after % lost documents: % of %', point, found, docs; END IF; END LOOP; END $$; SELECT count(*) FROM compaction_unwind; SELECT bm25_needs_compaction('compaction_unwind_idx'::regclass); RESET pg_textsearch.segments_per_level; DROP TABLE compaction_unwind CASCADE; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/compaction_injection.sql000066400000000000000000000076341525762047700254700ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.segments_per_level = 64; CREATE TABLE compaction_terminal (id integer, body text); CREATE INDEX compaction_terminal_idx ON compaction_terminal USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO compaction_terminal SELECT batch * 100 + n, 'common document ' || n FROM generate_series(1, 100) n; PERFORM bm25_spill_index('compaction_terminal_idx'); END LOOP; END $$; -- Tiny outputs no longer climb to L7 naturally. Construct the legacy layout. SELECT pg_textsearch_test_move_level('compaction_terminal_idx', 0, 7); SELECT pg_textsearch_test_attach_segment_limit(2); SET pg_textsearch.segments_per_level = 2; SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[0, 0, 0, 0, 0, 0, 0, 2] AND bm25_needs_compaction('compaction_terminal_idx') AS terminal_debt_constructed; SELECT bm25_compact_step('compaction_terminal_idx') AS terminal_debt_reduced; SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AND NOT bm25_needs_compaction('compaction_terminal_idx') AS terminal_output_placed_by_size; -- Compatible segments on different levels are debt below both thresholds. SELECT pg_textsearch_test_move_level('compaction_terminal_idx', 1, 7); SET pg_textsearch.segments_per_level = 64; INSERT INTO compaction_terminal SELECT 1000 + n, 'common document ' || n FROM generate_series(1, 200) n; SELECT bm25_spill_index('compaction_terminal_idx') > 0 AS lower_spilled; SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[1, 0, 0, 0, 0, 0, 0, 1] AND bm25_needs_compaction('compaction_terminal_idx') AS cross_level_debt_constructed; SELECT bm25_compact_step('compaction_terminal_idx') AS cross_level_pass_ran; SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AND NOT bm25_compact_step('compaction_terminal_idx') AS cross_level_pass_converged; SELECT count(*) = 400 AS cross_level_merge_preserves_matches FROM (SELECT id FROM compaction_terminal ORDER BY body <@> to_bm25query('common', 'compaction_terminal_idx') LIMIT 1000) ranked; -- Force merge also accepts terminal-level inputs. SELECT pg_textsearch_test_move_level('compaction_terminal_idx', 1, 7); SELECT bm25_force_merge('compaction_terminal_idx'); SELECT bm25_level_counts('compaction_terminal_idx') = ARRAY[1, 0, 0, 0, 0, 0, 0, 0] AS force_merge_reclassifies_terminal_segment; SELECT injection_points_detach('pg-textsearch-segment-count-limit'); -- Background spills enqueue work without running the planner in the writer. CREATE TABLE compaction_signal (id integer, body text); CREATE INDEX compaction_signal_idx ON compaction_signal USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO compaction_signal SELECT n, 'common document ' || n FROM generate_series(1, 100) n; SELECT bm25_spill_index('compaction_signal_idx') > 0 AS initial_spill; BEGIN; UPDATE pg_class SET reloptions = array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'compaction_signal_idx'::regclass; INSERT INTO compaction_signal SELECT n, 'common document ' || n FROM generate_series(101, 200) n; SELECT injection_points_attach( 'pg-textsearch-compaction-source-estimate', 'error'); SELECT bm25_spill_index('compaction_signal_idx') > 0 AS background_spill_defers_planning; ROLLBACK; SELECT injection_points_detach('pg-textsearch-compaction-source-estimate'); DROP TABLE compaction_signal; RESET pg_textsearch.segments_per_level; DROP TABLE compaction_terminal CASCADE; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/compaction_policy.sql000066400000000000000000000164571525762047700250100ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_textsearch; \pset format unaligned SET client_min_messages = warning; SET pg_textsearch.segments_per_level = 64; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; -- Comparable spills are debt even below the per-level count threshold. CREATE TEMP TABLE policy_small (id integer, body text); CREATE INDEX policy_small_idx ON policy_small USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DO $$ BEGIN FOR batch IN 1..4 LOOP INSERT INTO policy_small SELECT batch * 100 + n, 'common document ' || n FROM generate_series(1, 100) n; PERFORM bm25_spill_index('policy_small_idx'); END LOOP; IF NOT bm25_needs_compaction('policy_small_idx') THEN RAISE EXCEPTION 'below-threshold small segments were not selected'; END IF; PERFORM bm25_compact('policy_small_idx'); IF (SELECT sum(n) FROM unnest( bm25_level_counts('policy_small_idx')) n) <> 1 OR bm25_needs_compaction('policy_small_idx') THEN RAISE EXCEPTION 'small-segment consolidation did not converge'; END IF; END $$; SELECT count(*) = 400 AS small_merge_preserves_matches FROM (SELECT id FROM policy_small ORDER BY body <@> to_bm25query('common', 'policy_small_idx') LIMIT 1000) ranked; -- A tiny spill must not cause the much larger preceding output to rewrite. CREATE TEMP TABLE policy_unequal (id integer, body text); CREATE INDEX policy_unequal_idx ON policy_unequal USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO policy_unequal SELECT n, 'common ' || md5(n::text) FROM generate_series(1, 2000) n; SELECT bm25_spill_index('policy_unequal_idx') > 0 AS large_spilled; INSERT INTO policy_unequal VALUES (2001, 'common tiny'); SELECT bm25_spill_index('policy_unequal_idx') > 0 AS tiny_spilled; SELECT NOT bm25_needs_compaction('policy_unequal_idx') AND NOT bm25_compact_step('policy_unequal_idx') AS unequal_segments_are_not_repeatedly_rewritten; -- A singleton becomes eligible at 50% dead, not just at zero survivors. CREATE TEMP TABLE policy_deleted (id integer, body text); INSERT INTO policy_deleted SELECT n, 'common document ' || n FROM generate_series(1, 100) n; CREATE INDEX policy_deleted_idx ON policy_deleted USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DELETE FROM policy_deleted WHERE id <= 49; VACUUM policy_deleted; SELECT NOT bm25_needs_compaction('policy_deleted_idx') AS minor_deletion_is_not_debt; DELETE FROM policy_deleted WHERE id = 50; VACUUM policy_deleted; DO $$ BEGIN IF NOT bm25_needs_compaction('policy_deleted_idx') OR NOT bm25_compact_step('policy_deleted_idx') THEN RAISE EXCEPTION 'half-dead singleton was not rewritten'; END IF; IF bm25_needs_compaction('policy_deleted_idx') OR bm25_compact_step('policy_deleted_idx') THEN RAISE EXCEPTION 'clean singleton was rewritten again'; END IF; END $$; SELECT array_agg(id ORDER BY id) = ARRAY( SELECT generate_series(51, 100)) AS rewrite_preserves_survivors FROM (SELECT id FROM policy_deleted ORDER BY body <@> to_bm25query('common', 'policy_deleted_idx') LIMIT 1000) ranked; -- Inline maintenance gets an opportunity after VACUUM, without a new spill. CREATE TEMP TABLE policy_vacuum (id integer, body text); INSERT INTO policy_vacuum SELECT n, 'common document ' || n FROM generate_series(1, 100) n; CREATE INDEX policy_vacuum_idx ON policy_vacuum USING bm25(body) WITH (text_config = 'simple'); DELETE FROM policy_vacuum WHERE id <= 75; VACUUM policy_vacuum; SELECT bm25_pending_free_pages('policy_vacuum_idx') > 0 AND NOT bm25_needs_compaction('policy_vacuum_idx') AND NOT bm25_compact_step('policy_vacuum_idx') AS vacuum_runs_inline_policy; SELECT count(*) = 25 AS vacuum_policy_preserves_survivors FROM (SELECT id FROM policy_vacuum ORDER BY body <@> to_bm25query('common', 'policy_vacuum_idx') LIMIT 1000) ranked; DELETE FROM policy_deleted; VACUUM policy_deleted; SELECT (SELECT sum(n) FROM unnest( bm25_level_counts('policy_deleted_idx')) n) = 0 AS empty_segment_cleanup_preserved; -- Sparse cleanup can splice out an interior segment without merging its head. CREATE TEMP TABLE policy_interior (id integer, body text); CREATE INDEX policy_interior_idx ON policy_interior USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO policy_interior SELECT n, 'common document ' || n FROM generate_series(1, 100) n; SELECT bm25_spill_index('policy_interior_idx') > 0 AS older_spilled; INSERT INTO policy_interior SELECT n, 'common document ' || n FROM generate_series(101, 200) n; SELECT bm25_spill_index('policy_interior_idx') > 0 AS newer_spilled; DELETE FROM policy_interior WHERE id <= 50; VACUUM policy_interior; CREATE TEMP TABLE policy_interior_before AS SELECT (regexp_match(summary, 'L0 Segment 1: block=([0-9]+),'))[1] AS clean_root, (regexp_match(summary, 'L0 Segment 2: block=([0-9]+),'))[1] AS sparse_root FROM (SELECT bm25_summarize_index('policy_interior_idx') AS summary) s; SELECT bm25_compact_step('policy_interior_idx') AS interior_rewrite_ran; SELECT bm25_level_counts('policy_interior_idx') = ARRAY[2, 0, 0, 0, 0, 0, 0, 0] AND bm25_summarize_index('policy_interior_idx') ~ E'total_docs: 150\n' AS interior_rewrite_does_not_copy_clean_head; SELECT (regexp_match(summary, 'L0 Segment 1: block=([0-9]+),'))[1] = before.clean_root AND (regexp_match(summary, 'L0 Segment 2: block=([0-9]+),'))[1] <> before.sparse_root AS only_sparse_root_replaced FROM policy_interior_before before, (SELECT bm25_summarize_index('policy_interior_idx') AS summary) s; SELECT count(*) = 150 AS interior_rewrite_preserves_matches FROM (SELECT id FROM policy_interior ORDER BY body <@> to_bm25query('common', 'policy_interior_idx') LIMIT 1000) ranked; DROP TABLE policy_small, policy_unequal, policy_deleted, policy_vacuum, policy_interior, policy_interior_before; -- The managed worker's physical-target entry point sees the same small debt. CREATE TABLE policy_background (id integer, body text); CREATE INDEX policy_background_idx ON policy_background USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO policy_background SELECT batch * 100 + n, 'common document ' || n FROM generate_series(1, 100) n; PERFORM bm25_spill_index('policy_background_idx'); END LOOP; END $$; UPDATE pg_class SET reloptions = array_replace( reloptions, 'compaction=manual', 'compaction=background') WHERE oid = 'policy_background_idx'::regclass; SELECT bm25_compact_step_if_current( c.oid, d.oid, coalesce(nullif(c.reltablespace, 0), d.dattablespace), pg_relation_filenode(c.oid), c.relowner) AS background_target_compacts_below_threshold FROM pg_class c JOIN pg_database d ON d.datname = current_database() WHERE c.oid = 'policy_background_idx'::regclass; SELECT NOT bm25_needs_compaction('policy_background_idx') AS background_target_converged; DROP TABLE policy_background; RESET pg_textsearch.segments_per_level; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/compaction_request.sql000066400000000000000000000121231525762047700251630ustar00rootroot00000000000000CREATE EXTENSION IF NOT EXISTS pg_textsearch; \pset format unaligned SET client_min_messages = warning; SET pg_textsearch.segments_per_level = 2; SHOW pg_textsearch.background_compaction_schedule; -- Legacy "off" remains accepted as an alias for manual mode. CREATE TABLE legacy_off_docs (id serial PRIMARY KEY, body text); CREATE INDEX legacy_off_docs_idx ON legacy_off_docs USING bm25(body) WITH (text_config = 'english', compaction = 'off'); SELECT reloptions @> ARRAY['compaction=off'] FROM pg_class WHERE oid = 'legacy_off_docs_idx'::regclass; INSERT INTO legacy_off_docs (body) SELECT 'legacy off one ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('legacy_off_docs_idx') IS NOT NULL; INSERT INTO legacy_off_docs (body) SELECT 'legacy off two ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('legacy_off_docs_idx') IS NOT NULL; SELECT bm25_needs_compaction('legacy_off_docs_idx'::regclass) AS legacy_off_debt_remains; -- The compaction policy and schedule are per-index options. CREATE TABLE relopt_docs (id serial PRIMARY KEY, body text); CREATE INDEX relopt_docs_idx ON relopt_docs USING bm25(body) WITH (text_config = 'english'); ALTER INDEX relopt_docs_idx SET (compaction = 'manual'); SELECT reloptions @> ARRAY['compaction=manual'] FROM pg_class WHERE oid = 'relopt_docs_idx'::regclass; ALTER INDEX relopt_docs_idx SET (compaction_schedule = '17 * * * *'); SELECT reloptions @> ARRAY['compaction_schedule=17 * * * *'] FROM pg_class WHERE oid = 'relopt_docs_idx'::regclass; -- The post-ALTER activation hook preserves IF EXISTS no-op semantics. ALTER INDEX IF EXISTS missing_background_idx SET (compaction = 'background'); ALTER INDEX IF EXISTS missing_background_idx RESET (compaction_schedule); -- Managed background mode requires pg_durable, but temporary indexes are -- rejected before admission is attempted. CREATE INDEX CONCURRENTLY relopt_background_idx ON relopt_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); SELECT pg_catalog.to_regclass('relopt_background_idx') IS NULL AS rejected_cic_left_no_relation; CREATE TEMP TABLE relopt_temp_docs (id integer, body text); CREATE INDEX relopt_temp_background_idx ON relopt_temp_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); CREATE INDEX CONCURRENTLY relopt_temp_background_cic_idx ON relopt_temp_docs USING bm25(body) WITH (text_config = 'english', compaction = 'background'); -- Manual mode leaves spill-time compaction debt in place. CREATE TABLE manual_docs (id serial PRIMARY KEY, body text); CREATE INDEX manual_docs_idx ON manual_docs USING bm25(body) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO manual_docs (body) SELECT 'manual one ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('manual_docs_idx') IS NOT NULL; INSERT INTO manual_docs (body) SELECT 'manual two ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('manual_docs_idx') IS NOT NULL; SELECT bm25_needs_compaction('manual_docs_idx'::regclass) AS manual_debt_remains; -- The final serial build batch gets the same inline policy as -- intermediate batches. SET max_parallel_maintenance_workers = 0; SET maintenance_work_mem = '1MB'; CREATE TABLE final_build_docs ( id integer PRIMARY KEY, body text ); INSERT INTO final_build_docs SELECT i, 'token' || i || ' ' || repeat(md5(i::text) || ' ', 8) FROM generate_series(1, 10000) i; CREATE INDEX final_build_docs_idx ON final_build_docs USING bm25(body) WITH (text_config = 'simple'); SELECT NOT bm25_needs_compaction('final_build_docs_idx'::regclass) AS final_build_batch_compacted; -- CREATE INDEX honors manual mode, leaving its build batches uncompacted. CREATE TABLE manual_build_docs ( id integer PRIMARY KEY, body text ); INSERT INTO manual_build_docs SELECT i, 'token' || i || ' ' || repeat(md5(i::text) || ' ', 8) FROM generate_series(1, 10000) i; CREATE INDEX manual_build_docs_idx ON manual_build_docs USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); SELECT bm25_needs_compaction('manual_build_docs_idx'::regclass) AS manual_build_debt_remains; RESET maintenance_work_mem; RESET max_parallel_maintenance_workers; -- Inline mode compacts synchronously. CREATE TABLE inline_docs (id serial PRIMARY KEY, body text); CREATE INDEX inline_docs_idx ON inline_docs USING bm25(body) WITH (text_config = 'english', compaction = 'inline'); INSERT INTO inline_docs (body) SELECT 'inline one ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('inline_docs_idx') IS NOT NULL; INSERT INTO inline_docs (body) SELECT 'inline two ' || i FROM generate_series(1, 20) i; SELECT bm25_spill_index('inline_docs_idx') IS NOT NULL; SELECT NOT bm25_needs_compaction('inline_docs_idx'::regclass) AS inline_compacted; RESET pg_textsearch.segments_per_level; RESET client_min_messages; DROP TABLE inline_docs CASCADE; DROP TABLE manual_build_docs CASCADE; DROP TABLE final_build_docs CASCADE; DROP TABLE manual_docs CASCADE; DROP TABLE relopt_docs CASCADE; DROP TABLE legacy_off_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/compression.sql000066400000000000000000000143441525762047700236270ustar00rootroot00000000000000-- Test compression functionality -- Tests that compression (now default) produces correct results CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Compression is now on by default SHOW pg_textsearch.compress_segments; -- Create test table CREATE TABLE compression_docs ( id serial PRIMARY KEY, content text ); -- Create BM25 index CREATE INDEX compression_idx ON compression_docs USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); -- Insert enough documents to create multiple blocks (block size is 128) -- We need at least 128+ docs with the same term to test block compression INSERT INTO compression_docs (content) SELECT 'the quick brown fox jumps over lazy dog number ' || i FROM generate_series(1, 200) AS i; -- Force segment creation by spilling memtable SELECT bm25_spill_index('compression_idx'); -- Query should work correctly with compressed segments SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('fox', 'compression_idx')) sub; -- Test with multiple query terms SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('quick brown', 'compression_idx')) sub; -- Test that scores are reasonable (not corrupted by compression) -- Scores should be negative (BM25 returns negative for ORDER BY ASC) SELECT id, content <@> to_bm25query('fox', 'compression_idx') AS score FROM compression_docs ORDER BY score LIMIT 5; -- Add more documents and test again INSERT INTO compression_docs (content) SELECT 'different content with unique words like zephyr ' || i FROM generate_series(1, 100) AS i; -- Spill again to create a second segment SELECT bm25_spill_index('compression_idx'); -- Query across both segments SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('fox', 'compression_idx')) sub; -- Query the new content SELECT count(*) FROM (SELECT 1 FROM compression_docs ORDER BY content <@> to_bm25query('zephyr', 'compression_idx')) sub; -- Test BMW query with compressed segments SELECT id FROM compression_docs ORDER BY content <@> to_bm25query('fox', 'compression_idx') LIMIT 5; -- Clean up DROP TABLE compression_docs; -- Test that compression can be disabled SET pg_textsearch.compress_segments = off; SHOW pg_textsearch.compress_segments; -- Create another table with compression off CREATE TABLE uncompressed_docs ( id serial PRIMARY KEY, content text ); CREATE INDEX uncompressed_idx ON uncompressed_docs USING bm25 (content) WITH (text_config = 'english'); INSERT INTO uncompressed_docs (content) SELECT 'testing uncompressed segments ' || i FROM generate_series(1, 150) AS i; SELECT bm25_spill_index('uncompressed_idx'); -- Should work the same SELECT count(*) FROM (SELECT 1 FROM uncompressed_docs ORDER BY content <@> to_bm25query('testing', 'uncompressed_idx')) sub; -- Clean up DROP TABLE uncompressed_docs; -- ============================================================================= -- Test mixed compression: toggle GUC after index has segments -- ============================================================================= -- Reset to default (compression on) SET pg_textsearch.compress_segments = on; -- Create table and index CREATE TABLE mixed_docs ( id serial PRIMARY KEY, content text ); CREATE INDEX mixed_idx ON mixed_docs USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); -- Insert first batch with compression ON INSERT INTO mixed_docs (content) SELECT 'alpha beta gamma document ' || i FROM generate_series(1, 150) AS i; SELECT bm25_spill_index('mixed_idx'); -- Verify compressed segment works SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('alpha', 'mixed_idx')) sub; -- Now turn compression OFF and add more data SET pg_textsearch.compress_segments = off; INSERT INTO mixed_docs (content) SELECT 'alpha delta epsilon document ' || i FROM generate_series(151, 300) AS i; -- This creates an uncompressed segment SELECT bm25_spill_index('mixed_idx'); -- Query should work across mixed compressed/uncompressed segments SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('alpha', 'mixed_idx')) sub; -- Query term only in compressed segment SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('beta', 'mixed_idx')) sub; -- Query term only in uncompressed segment SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('delta', 'mixed_idx')) sub; -- Turn compression back on and add a third segment SET pg_textsearch.compress_segments = on; INSERT INTO mixed_docs (content) SELECT 'alpha zeta theta document ' || i FROM generate_series(301, 450) AS i; SELECT bm25_spill_index('mixed_idx'); -- Now we have: compressed segment, uncompressed segment, compressed segment -- Query should still work SELECT count(*) FROM (SELECT 1 FROM mixed_docs ORDER BY content <@> to_bm25query('alpha', 'mixed_idx')) sub; -- Clean up DROP TABLE mixed_docs; -- ============================================================================= -- Test: start uncompressed, then enable compression -- ============================================================================= SET pg_textsearch.compress_segments = off; CREATE TABLE mixed_docs2 ( id serial PRIMARY KEY, content text ); CREATE INDEX mixed_idx2 ON mixed_docs2 USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); -- First batch uncompressed INSERT INTO mixed_docs2 (content) SELECT 'apple banana cherry ' || i FROM generate_series(1, 150) AS i; SELECT bm25_spill_index('mixed_idx2'); -- Enable compression SET pg_textsearch.compress_segments = on; -- Second batch compressed INSERT INTO mixed_docs2 (content) SELECT 'apple date elderberry ' || i FROM generate_series(151, 300) AS i; SELECT bm25_spill_index('mixed_idx2'); -- Query across uncompressed and compressed segments SELECT count(*) FROM (SELECT 1 FROM mixed_docs2 ORDER BY content <@> to_bm25query('apple', 'mixed_idx2')) sub; SELECT count(*) FROM (SELECT 1 FROM mixed_docs2 ORDER BY content <@> to_bm25query('banana', 'mixed_idx2')) sub; SELECT count(*) FROM (SELECT 1 FROM mixed_docs2 ORDER BY content <@> to_bm25query('date', 'mixed_idx2')) sub; -- Clean up DROP TABLE mixed_docs2; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/concurrent_build.sql000066400000000000000000000051451525762047700246260ustar00rootroot00000000000000-- Test CREATE INDEX CONCURRENTLY with bm25 indexes -- -- CIC requires proper ambulkdelete callback implementation to report all -- indexed TIDs during validation phase. This ensures the validate_index -- function can determine which tuples are already indexed and avoid -- duplicating entries. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -------------------------------------------------------------------------------- -- Test 1: Basic CREATE INDEX CONCURRENTLY -------------------------------------------------------------------------------- CREATE TABLE cic_basic ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO cic_basic (content) VALUES ('the quick brown fox jumps over the lazy dog'), ('postgresql is a powerful database system'), ('full text search with bm25 ranking'); -- Create index concurrently - should complete without error CREATE INDEX CONCURRENTLY cic_basic_idx ON cic_basic USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Verify index is valid (indisvalid = true) SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_basic_idx'::regclass; -- Verify no duplicate entries (this was a bug symptom before the fix) SELECT COUNT(*) = 1 AS no_duplicates FROM ( SELECT 1 FROM cic_basic ORDER BY content <@> to_bm25query('database', 'cic_basic_idx') ) sub; -- Verify index is used in query plan EXPLAIN (COSTS OFF) SELECT id FROM cic_basic ORDER BY content <@> to_bm25query('database', 'cic_basic_idx') LIMIT 3; -------------------------------------------------------------------------------- -- Test 2: DROP INDEX CONCURRENTLY -------------------------------------------------------------------------------- DROP INDEX CONCURRENTLY cic_basic_idx; -- Verify index is gone SELECT COUNT(*) FROM pg_indexes WHERE indexname = 'cic_basic_idx'; -------------------------------------------------------------------------------- -- Test 3: Recreate with regular CREATE INDEX for comparison -------------------------------------------------------------------------------- CREATE INDEX cic_basic_idx ON cic_basic USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Verify index is valid SELECT indisvalid FROM pg_index WHERE indexrelid = 'cic_basic_idx'::regclass; -- Query should work and return expected results SELECT id FROM cic_basic ORDER BY content <@> to_bm25query('database', 'cic_basic_idx'); -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- DROP TABLE cic_basic CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/coverage.sql000066400000000000000000000261741525762047700230650ustar00rootroot00000000000000-- Additional coverage tests for untested code paths -- Tests debug functions, segment dump, and text<@>text operator CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================================= -- Test 1: Basic debug functions with memtable-only data -- ============================================================================= -- Create a simple table and index CREATE TABLE coverage_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO coverage_docs (content) VALUES ('hello world test document'), ('database search query optimization'), ('full text search engine postgresql'); CREATE INDEX coverage_idx ON coverage_docs USING bm25(content) WITH (text_config='english', compaction='manual'); -- Test bm25_summarize_index with memtable data SELECT bm25_summarize_index('coverage_idx') IS NOT NULL AS summarize_memtable; -- Test bm25_dump_index with memtable data SELECT length(bm25_dump_index('coverage_idx')) > 0 AS dump_memtable; -- ============================================================================= -- Test 2: Debug functions with segment data (exercises segment dump code) -- ============================================================================= -- Force segment spill to have data on disk -- Note: With build mode, data is already spilled during CREATE INDEX, -- so memtable is empty here and returns NULL SELECT bm25_spill_index('coverage_idx'); -- Test bm25_summarize_index with segment data -- This exercises tp_summarize_index_to_output with segments SELECT bm25_summarize_index('coverage_idx') ~ E'L0 Segment 1:.*\n Total: 1 segments' AS summarize_segment; -- Test bm25_dump_index with segment data -- This exercises tp_dump_segment_to_output, read_dict_entry, read_term_at_index SELECT length(bm25_dump_index('coverage_idx')) > 0 AS dump_segment; -- ============================================================================= -- Test 3: bm25query equality (tpquery_eq) -- ============================================================================= -- Test equality of identical queries SELECT to_bm25query('hello', 'coverage_idx') = to_bm25query('hello', 'coverage_idx') AS same_query_eq; -- Test inequality of different queries SELECT to_bm25query('hello', 'coverage_idx') = to_bm25query('world', 'coverage_idx') AS diff_query_eq; -- ============================================================================= -- Test 4: text <@> text operator (bm25_text_text_score) -- This is the implicit form without explicit bm25query -- ============================================================================= -- Insert more data to create variation INSERT INTO coverage_docs (content) VALUES ('additional text for testing'), ('more content with different words'); -- Test text <@> text implicit scoring -- This uses find_first_child_index to locate the bm25 index SET enable_seqscan = off; -- The implicit form should find coverage_idx automatically SELECT content, content <@> 'hello'::text AS score FROM coverage_docs ORDER BY content <@> 'hello'::text LIMIT 3; SET enable_seqscan = on; -- ============================================================================= -- Test 5: Multiple segments (exercises segment iteration in dump) -- ============================================================================= -- Add more data and spill again to create multiple segments INSERT INTO coverage_docs (content) SELECT 'document number ' || i || ' with varying content for coverage' FROM generate_series(1, 100) AS i; SELECT bm25_spill_index('coverage_idx'); -- Both debug readers must enumerate every root recorded by the metapage. -- Count the segment headers rather than merely matching one: a LIKE test -- passes with a single segment and would not catch a dump that stops -- after the first root. SELECT bm25_summarize_index('coverage_idx') ~ E'Total: 2 segments' AND array_length( string_to_array(bm25_dump_index('coverage_idx'), '========== Segment at block'), 1) - 1 = 2 AS dump_multi_segment; -- ============================================================================= -- Test 6: Page visualization - moved behind DEBUG_DUMP_INDEX compile flag -- ============================================================================= -- ============================================================================= -- Test 7: Score logging (exercises am/scan.c log_scores path) -- ============================================================================= SET pg_textsearch.log_scores = true; SELECT content <@> to_bm25query('hello', 'coverage_idx') AS score FROM coverage_docs ORDER BY content <@> to_bm25query('hello', 'coverage_idx') LIMIT 1; SET pg_textsearch.log_scores = false; -- ============================================================================= -- Test 8: BMW stats logging (exercises score.c bmw_stats path) -- ============================================================================= SET pg_textsearch.log_bmw_stats = true; SET enable_seqscan = off; -- Single-term query SELECT count(*) FROM ( SELECT 1 FROM coverage_docs ORDER BY content <@> to_bm25query('hello', 'coverage_idx') ) sub; -- Multi-term query SELECT count(*) FROM ( SELECT 1 FROM coverage_docs ORDER BY content <@> to_bm25query('hello world', 'coverage_idx') ) sub; SET enable_seqscan = on; SET pg_textsearch.log_bmw_stats = false; -- ============================================================================= -- Test 9: AM handler property callbacks (exercises handler.c tp_property) -- ============================================================================= -- Test pg_indexam_has_property for our access method SELECT pg_indexam_has_property(am.oid, 'can_order') AS can_order, pg_indexam_has_property(am.oid, 'can_unique') AS can_unique FROM pg_am am WHERE am.amname = 'bm25'; -- Test index-level properties SELECT pg_index_has_property('coverage_idx'::regclass, 'clusterable') AS clusterable, pg_index_has_property('coverage_idx'::regclass, 'index_scan') AS index_scan; -- Test column-level property (distance_orderable triggers tp_property) SELECT pg_index_column_has_property('coverage_idx'::regclass, 1, 'distance_orderable') AS dist_orderable; -- ============================================================================= -- Test 10: AM validate error path (exercises opclass resolution) -- ============================================================================= -- Try creating a bm25 index on integer column (should fail validation) CREATE TABLE validate_test (id serial, num integer); CREATE INDEX validate_test_idx ON validate_test USING bm25(num) WITH (text_config='english'); DROP TABLE validate_test; -- ============================================================================= -- Test 11: File-writing dump - moved behind DEBUG_DUMP_INDEX compile flag -- ============================================================================= -- ============================================================================= -- Test 12: Segment BMW seek (exercises segment/scan.c seek path) -- Uses ORDER BY ... LIMIT to trigger top-k BMW with segment data -- ============================================================================= -- We already have segment data from Test 5; add more for multi-block INSERT INTO coverage_docs (content) SELECT 'seek test document with term alpha ' || i FROM generate_series(1, 200) AS i; SELECT bm25_spill_index('coverage_idx'); -- ORDER BY score LIMIT triggers BMW seek path on segments -- Wrap in outer query with id tiebreaker for deterministic ordering -- (adding secondary sort key to inner query would prevent index use) SELECT id FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('alpha', 'coverage_idx') LIMIT 5 ) sub ORDER BY id; -- Multi-term BMW seek with segment data SELECT id FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('seek alpha', 'coverage_idx') LIMIT 3 ) sub ORDER BY id; -- ============================================================================= -- Test 13: tpquery_in with index name prefix (exercises query.c parsing) -- ============================================================================= -- Cast string with index:query format to bm25query type SELECT 'coverage_idx:hello world'::bm25query IS NOT NULL AS tpquery_in_with_index; -- ============================================================================= -- Test 14: BMW stats logging with segment ORDER BY LIMIT queries -- ============================================================================= SET pg_textsearch.log_bmw_stats = true; -- Single-term with ORDER BY LIMIT (BMW seek on segments) SELECT count(*) FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('alpha', 'coverage_idx') LIMIT 10 ) sub; -- Multi-term with ORDER BY LIMIT SELECT count(*) FROM ( SELECT id FROM coverage_docs ORDER BY content <@> to_bm25query('seek alpha', 'coverage_idx') LIMIT 10 ) sub; SET pg_textsearch.log_bmw_stats = false; -- ============================================================================= -- Test 15: Plain text ORDER BY (exercises scan.c sk_subtype == TEXTOID) -- ============================================================================= SET enable_seqscan = off; -- The implicit text <@> text form in ORDER BY SELECT id FROM ( SELECT id FROM coverage_docs ORDER BY content <@> 'alpha'::text LIMIT 3 ) sub ORDER BY id; SET enable_seqscan = on; -- ============================================================================= -- Test 16: GROUP BY / HAVING with BM25 operator -- ============================================================================= -- Use BM25 operator in HAVING clause SELECT left(content, 20) AS prefix, count(*) AS cnt FROM ( SELECT content FROM coverage_docs ORDER BY content <@> to_bm25query('document', 'coverage_idx') ) sub GROUP BY left(content, 20) HAVING count(*) > 1 ORDER BY cnt DESC LIMIT 3; -- ============================================================================= -- Test 17: Build-mode spill (exercises state.c tp_clear_memtable build path) -- ============================================================================= -- Create table with enough data to trigger mid-build spill CREATE TABLE build_spill_test (id serial PRIMARY KEY, content text); INSERT INTO build_spill_test (content) SELECT 'buildspill unique term' || i || ' extra words for document ' || i FROM generate_series(1, 500) AS i; -- Set very low spill threshold so CREATE INDEX spills during build SET pg_textsearch.memtable_pages_threshold = 1; CREATE INDEX build_spill_idx ON build_spill_test USING bm25(content) WITH (text_config='english'); -- Reset threshold RESET pg_textsearch.memtable_pages_threshold; -- Verify index works after build-mode spill SELECT count(*) FROM ( SELECT 1 FROM build_spill_test ORDER BY content <@> to_bm25query('buildspill', 'build_spill_idx') ) sub; DROP TABLE build_spill_test; -- ============================================================================= -- Cleanup -- ============================================================================= DROP TABLE coverage_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/deletion.sql000066400000000000000000000047441525762047700230740ustar00rootroot00000000000000-- Test handling of row deletions in indexed tables -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table CREATE TABLE deletion_test ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert some documents INSERT INTO deletion_test (content) VALUES ('first document with some content'), ('second document with different content'), ('third document with test keyword'), ('fourth document without the keyword'), ('fifth document with test again'); -- Create index CREATE INDEX deletion_idx ON deletion_test USING bm25 (content) WITH (text_config = 'english'); -- Force index scan (small table would otherwise prefer seq scan) SET enable_seqscan = off; -- Initial search - should find documents with "test" SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; -- Delete one of the documents that contains "test" DELETE FROM deletion_test WHERE id = 3; -- Search again - deleted document should not appear -- (Note: corpus statistics won't be updated, but shouldn't crash) SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; -- Delete a document that doesn't contain "test" DELETE FROM deletion_test WHERE id = 4; -- Search again - should still work SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; -- Delete all remaining documents with "test" DELETE FROM deletion_test WHERE content LIKE '%test%'; -- Search again - should return no results (but not crash) SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; -- Insert new documents after deletions INSERT INTO deletion_test (content) VALUES ('new document with test keyword'), ('another new document'); -- Search again - should find the new document SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; -- Test with empty table after deleting everything DELETE FROM deletion_test; -- Search in empty table - should return no results SELECT id, substring(content, 1, 40) as content_preview FROM deletion_test ORDER BY content <@> to_bm25query('test', 'deletion_idx') LIMIT 5; -- Clean up DROP TABLE deletion_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/dropped.sql000066400000000000000000000065361525762047700227270ustar00rootroot00000000000000-- Test behavior when using a dropped index name in queries -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table CREATE TABLE dropped_idx_test ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO dropped_idx_test (content) VALUES ('first test document'), ('second test document'), ('third test document'), ('fourth test document'), ('fifth test document'); -- Create index CREATE INDEX dropped_idx ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); -- Query should work with index (using explicit index reference) SELECT COUNT(*) AS with_index FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'dropped_idx') ) sub; -- Drop the index DROP INDEX dropped_idx; -- Query should ERROR when index doesn't exist SELECT COUNT(*) AS after_drop FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'dropped_idx') ) sub; -- Also test with a completely non-existent index name SELECT COUNT(*) AS nonexistent FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'totally_fake_index') ) sub; -- Test registry slot cleanup (issue #83) -- Create multiple indexes, drop them, then create new ones -- This verifies that registry slots are properly freed on drop -- Create 5 indexes CREATE INDEX dropped_idx_1 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_2 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_3 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_4 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_5 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); -- Drop all 5 DROP INDEX dropped_idx_1; DROP INDEX dropped_idx_2; DROP INDEX dropped_idx_3; DROP INDEX dropped_idx_4; DROP INDEX dropped_idx_5; -- Create 5 more indexes (should succeed if slots were freed) CREATE INDEX dropped_idx_6 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_7 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_8 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_9 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); CREATE INDEX dropped_idx_10 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); -- Test that the new indexes work SELECT COUNT(*) AS recycled_slots_work FROM ( SELECT 1 FROM dropped_idx_test ORDER BY content <@> to_bm25query('test', 'dropped_idx_6') ) sub; -- Also test CASCADE drop (table drop should clean up index slots) CREATE TABLE cascade_test (id SERIAL, content TEXT); INSERT INTO cascade_test (content) VALUES ('cascade test doc'); CREATE INDEX cascade_idx ON cascade_test USING bm25 (content) WITH (text_config = 'english'); -- Drop table should cascade to drop index and free its registry slot DROP TABLE cascade_test CASCADE; -- Create another index to verify the cascade-dropped slot was freed CREATE INDEX dropped_idx_11 ON dropped_idx_test USING bm25 (content) WITH (text_config = 'english'); -- Clean up DROP TABLE dropped_idx_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/empty.sql000066400000000000000000000026371525762047700224260ustar00rootroot00000000000000-- Test handling of empty and whitespace-only documents -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table with various empty/whitespace content CREATE TABLE empty_docs ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert various empty/whitespace documents INSERT INTO empty_docs (content) VALUES (''), -- completely empty (' '), -- only spaces (' '), -- only tabs (' '), -- only newlines (' '), -- mixed whitespace (NULL); -- NULL content -- Create index on content CREATE INDEX empty_docs_idx ON empty_docs USING bm25 (content) WITH (text_config = 'english'); -- Force index scan (small table would otherwise prefer seq scan) SET enable_seqscan = off; -- Try searching - should return no results without warnings SELECT id, content FROM empty_docs ORDER BY content <@> to_bm25query('test', 'empty_docs_idx') LIMIT 5; -- Add a document with actual content INSERT INTO empty_docs (content) VALUES ('test document with content'); -- Search again - should find only the document with content SELECT id, substring(content, 1, 30) as content_preview FROM empty_docs ORDER BY content <@> to_bm25query('test', 'empty_docs_idx') LIMIT 5; -- Clean up DROP TABLE empty_docs; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/explicit_index.sql000066400000000000000000000066661525762047700243060ustar00rootroot00000000000000-- Test explicit index name in to_bm25query -- Validates fixes for GitHub issues #183 and #194 CREATE EXTENSION IF NOT EXISTS pg_textsearch; ----------------------------------------------------------------------- -- Issue #194: to_bm25query with explicit index ignores index column -- Using an index on a different column should error ----------------------------------------------------------------------- -- Create table with TWO text columns CREATE TABLE explicit_index_test ( id SERIAL PRIMARY KEY, title TEXT, content TEXT ); -- Index on 'content' column only CREATE INDEX content_idx ON explicit_index_test USING bm25 (content) WITH (text_config = 'english'); -- Insert test data INSERT INTO explicit_index_test (title, content) VALUES ('rust c java', 'postgres mysql'), ('python ruby', 'redis mongodb'); -- This should ERROR: using content_idx to search 'title' column -- The index is on 'content' but we're searching 'title' SELECT * FROM explicit_index_test ORDER BY title <@> to_bm25query('postgres', 'content_idx'); -- Positive test: using correct column should work SELECT id, content <@> to_bm25query('postgres', 'content_idx') AS score FROM explicit_index_test ORDER BY score LIMIT 10; -- This should ERROR: left operand is a constant, not a column SELECT 'hello' <@> to_bm25query('postgres', 'content_idx') AS score; -- Clean up DROP TABLE explicit_index_test; ----------------------------------------------------------------------- -- Issue #183: to_bm25query with explicit index name does not force -- planner to use that index ----------------------------------------------------------------------- -- Create table CREATE TABLE multi_index_test ( id SERIAL PRIMARY KEY, category_id INT, content TEXT ); -- Insert test data - enough rows to make planner consider index choices INSERT INTO multi_index_test (category_id, content) SELECT (i % 10), 'Les ressources humaines gèrent les employés et leurs contrats de travail' FROM generate_series(1, 1000) i; -- Create two BM25 indexes with different text configurations on same column CREATE INDEX content_idx_french ON multi_index_test USING bm25 (content) WITH (text_config = 'french'); CREATE INDEX content_idx_simple ON multi_index_test USING bm25 (content) WITH (text_config = 'simple'); -- Analyze for statistics ANALYZE multi_index_test; -- Query explicitly requesting the French index -- The planner should use content_idx_french as specified EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('ressources', 'content_idx_french') AS score FROM multi_index_test WHERE category_id = 1 ORDER BY score LIMIT 10; -- Positive test: without an explicit index, implicit resolution still -- works. All matching rows share the same text, so the BM25 scores are -- deterministic even though the tied tuple ids (and which tied index is -- scanned) are not. SET client_min_messages = error; \pset format unaligned SELECT content <@> 'ressources' AS score FROM multi_index_test WHERE category_id = 1 ORDER BY score LIMIT 3; \pset format aligned RESET client_min_messages; -- Positive test: explicit index matching the scanned index works -- Drop the simple index so french is the only choice DROP INDEX content_idx_simple; EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('ressources', 'content_idx_french') AS score FROM multi_index_test WHERE category_id = 1 ORDER BY score LIMIT 10; -- Clean up DROP TABLE multi_index_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/expression_index.sql000066400000000000000000000240141525762047700246470ustar00rootroot00000000000000-- Expression index tests for pg_textsearch CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================ -- JSONB expression index -- ============================================================ CREATE TABLE expr_jsonb ( id SERIAL PRIMARY KEY, data JSONB ); INSERT INTO expr_jsonb (data) VALUES ('{"content": "postgresql database management system"}'), ('{"content": "machine learning algorithms"}'), ('{"content": "full text search and retrieval"}'); CREATE INDEX expr_jsonb_idx ON expr_jsonb USING bm25 ((data->>'content')) WITH (text_config='english', compaction='manual'); SET enable_seqscan = off; -- Query with explicit index naming SELECT id, ROUND(((data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx') LIMIT 5; -- ============================================================ -- Multi-column concatenation expression index -- ============================================================ CREATE TABLE expr_concat ( id SERIAL PRIMARY KEY, title TEXT, body TEXT ); INSERT INTO expr_concat (title, body) VALUES ('Database Systems', 'An introduction to relational databases'), ('Machine Learning', 'Algorithms for pattern recognition'), ('Search Engines', 'Full text search and information retrieval'); CREATE INDEX expr_concat_idx ON expr_concat USING bm25 ((coalesce(title, '') || ' ' || coalesce(body, ''))) WITH (text_config='english'); SELECT id, ROUND(((coalesce(title, '') || ' ' || coalesce(body, '')) <@> to_bm25query('database', 'expr_concat_idx'))::numeric, 4) AS score FROM expr_concat ORDER BY (coalesce(title, '') || ' ' || coalesce(body, '')) <@> to_bm25query('database', 'expr_concat_idx') LIMIT 5; -- ============================================================ -- Simple transformation expression index -- ============================================================ CREATE TABLE expr_lower ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO expr_lower (content) VALUES ('PostgreSQL Database Management'), ('Machine Learning Algorithms'), ('Full Text Search and Retrieval'); CREATE INDEX expr_lower_idx ON expr_lower USING bm25 ((lower(content))) WITH (text_config='simple'); SELECT id, ROUND(((lower(content)) <@> to_bm25query('database', 'expr_lower_idx'))::numeric, 4) AS score FROM expr_lower ORDER BY (lower(content)) <@> to_bm25query('database', 'expr_lower_idx') LIMIT 5; -- ============================================================ -- Implicit resolution (no explicit index name) -- ============================================================ SELECT id, ROUND(((data->>'content') <@> to_bm25query('database'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database') LIMIT 5; -- ============================================================ -- Verify index scan is used -- ============================================================ EXPLAIN (COSTS OFF) SELECT id FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx') LIMIT 5; -- ============================================================ -- NULL handling: row with missing JSONB key -- ============================================================ INSERT INTO expr_jsonb (data) VALUES ('{"other_field": "no content key"}'), (NULL); -- These should not affect results SELECT id, ROUND(((data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx') LIMIT 5; -- ============================================================ -- Delete a document and verify index still works -- ============================================================ DELETE FROM expr_jsonb WHERE id = 1; SELECT id, ROUND(((data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx') LIMIT 5; -- ============================================================ -- VACUUM: delete rows and vacuum to exercise rebuild -- ============================================================ DELETE FROM expr_jsonb WHERE id <= 2; VACUUM expr_jsonb; -- Verify index still works after vacuum SELECT id, ROUND(((data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('search', 'expr_jsonb_idx') LIMIT 5; -- ============================================================ -- Partitioned table with expression index -- ============================================================ CREATE TABLE expr_part ( id SERIAL, data JSONB, created_at DATE NOT NULL ) PARTITION BY RANGE (created_at); CREATE TABLE expr_part_2024 PARTITION OF expr_part FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); CREATE TABLE expr_part_2025 PARTITION OF expr_part FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); INSERT INTO expr_part (data, created_at) VALUES ('{"text": "database management systems"}', '2024-06-15'), ('{"text": "machine learning algorithms"}', '2024-09-20'), ('{"text": "search engine optimization"}', '2025-03-10'); CREATE INDEX expr_part_idx ON expr_part USING bm25 ((data->>'text')) WITH (text_config='english'); SELECT id, ROUND(((data->>'text') <@> to_bm25query('database', 'expr_part_idx'))::numeric, 4) AS score FROM expr_part ORDER BY (data->>'text') <@> to_bm25query('database', 'expr_part_idx') LIMIT 5; DROP TABLE expr_part CASCADE; -- ============================================================ -- Multiple expression indexes trigger warning -- ============================================================ -- Suppress build NOTICEs: the reported doc count is non-deterministic -- here (recently-dead tuples from the earlier delete/vacuum may be -- indexed) and irrelevant to this multi-index test. SET client_min_messages = warning; CREATE INDEX expr_jsonb_idx2 ON expr_jsonb USING bm25 ((data->>'content')) WITH (text_config='simple'); RESET client_min_messages; -- Implicit resolution warns about multiple matching indexes. SELECT id, ROUND(((data->>'content') <@> to_bm25query('database'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database') LIMIT 3; -- Explicit naming with multiple expression indexes. SELECT id, ROUND(((data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx2'))::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> to_bm25query('database', 'expr_jsonb_idx2') LIMIT 3; DROP INDEX expr_jsonb_idx2; -- ============================================================ -- Implicit text <@> text with expression index -- ============================================================ SELECT id, ROUND(((data->>'content') <@> 'database')::numeric, 4) AS score FROM expr_jsonb ORDER BY (data->>'content') <@> 'database' LIMIT 3; -- ============================================================ -- Non-NULL INSERT/UPDATE values must use the expression's type, -- not heap attribute zero. Run these under ASan as well. -- ============================================================ INSERT INTO expr_lower (content) VALUES ('Insertedneedle'); SELECT id FROM expr_lower ORDER BY lower(content) <@> to_bm25query('insertedneedle', 'expr_lower_idx') LIMIT 5; UPDATE expr_lower SET content = 'Updatedneedle' WHERE id = 4; SELECT id FROM expr_lower ORDER BY lower(content) <@> to_bm25query('updatedneedle', 'expr_lower_idx') LIMIT 5; -- Array-valued expressions need flattening in both build and insert. CREATE TABLE expr_array (id int, title text, body text); INSERT INTO expr_array VALUES (1, 'buildneedle', 'common'); CREATE INDEX expr_array_idx ON expr_array USING bm25 ((ARRAY[title, body])) WITH (text_config='simple'); SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('buildneedle', 'expr_array_idx') LIMIT 5; INSERT INTO expr_array VALUES (2, 'insertneedle', 'common'); SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('insertneedle', 'expr_array_idx') LIMIT 5; UPDATE expr_array SET body = 'common updateneedle' WHERE id = 1; SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('updateneedle', 'expr_array_idx') LIMIT 5; -- Ignore recently-dead rows in the reported REINDEX build count. SET client_min_messages = warning; REINDEX INDEX expr_array_idx; RESET client_min_messages; SELECT array_agg(id ORDER BY id) AS rebuilt_ids FROM ( SELECT id FROM expr_array ORDER BY ARRAY[title, body] <@> to_bm25query('common', 'expr_array_idx') LIMIT 5 ) hits; DROP TABLE expr_array; -- ============================================================ -- Error cases -- ============================================================ \set ON_ERROR_STOP off -- Non-text expression type CREATE INDEX expr_bad_idx ON expr_lower USING bm25 ((length(content))) WITH (text_config='simple'); -- Constant left operand with bm25query SELECT 'hello' <@> to_bm25query('world', 'expr_jsonb_idx'); -- Constant left operand with text SELECT 'hello' <@> 'world'; \set ON_ERROR_STOP on -- ============================================================ -- Cleanup -- ============================================================ DROP TABLE expr_jsonb CASCADE; DROP TABLE expr_concat CASCADE; DROP TABLE expr_lower CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/filtered_seed.sql000066400000000000000000000640521525762047700240650ustar00rootroot00000000000000-- Test: selectivity-seeded top-K for filtered BM25 search -- -- A filtered top-k query -- WHERE ORDER BY LIMIT k -- -- is planned as a BM25 top-k index scan with applied as a -- Filter above it. pg_textsearch.filtered_seed seeds the scan's -- internal top-K from the planner's estimated filter selectivity so a -- single scoring pass usually surfaces enough matching rows, avoiding -- the executor's backoff re-drives. The seed only changes scan depth, -- never which rows win, so results must be IDENTICAL to the un-seeded -- plan. -- -- This test asserts correctness, not latency: -- * parity - seeded result == un-seeded result (same code path, -- so immune to score ties) -- * oracle - the filtered index scan returns exactly the set of -- documents that match the filter AND contain a query -- term, checked against an independent plain-SQL regex -- (k large enough that the full match set is returned, -- so immune to ties and to score values) -- * no-op - a query with no filter is unaffected by seeding -- * robustness - the result is independent of filtered_seed_margin -- * GUC - defaults and range enforcement SET log_duration = off; SET client_min_messages = WARNING; -- suppress index-build NOTICE chatter CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = false; ------------------------------------------------------------------------ -- Setup: 1000 docs, a 50-bucket facet column, controlled vocabulary. -- 'common' in every doc; 'alpha' in even ids; 'beta' every 3rd; -- 'gamma' every 7th; a unique 'doc' token per row. -- facet_id = id % 50 => 20 docs per bucket (2% selectivity each). ------------------------------------------------------------------------ CREATE TABLE fs_docs ( id int PRIMARY KEY, facet_id int, body text ); INSERT INTO fs_docs SELECT g, g % 50, concat_ws(' ', 'common', CASE WHEN g % 2 = 0 THEN 'alpha' END, CASE WHEN g % 3 = 0 THEN 'beta' END, CASE WHEN g % 7 = 0 THEN 'gamma' END, 'doc' || g) FROM generate_series(1, 1000) g; CREATE INDEX fs_docs_idx ON fs_docs USING bm25(body) WITH (text_config='english'); -- No index on facet_id: the facet is applied as a Filter above the -- BM25 index scan, which is exactly the path filtered_seed optimizes. -- (With a facet index the planner could instead scan the facet and -- sort by score, bypassing the seed; that plan is equally correct but -- not what this test exercises.) ANALYZE fs_docs; ------------------------------------------------------------------------ -- Parity: seeded (default on) vs un-seeded top-k are identical. -- Compared as sets (array_agg ORDER BY id); same index-scan code path -- on both sides, so any score ties break identically. ------------------------------------------------------------------------ -- Confirm the plan under test: a BM25 index scan with the facet applied -- as a Filter -- the path filtered_seed seeds. If this ever becomes a -- facet-index scan + sort, the checks below stop exercising the seed. EXPLAIN (COSTS OFF) SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10; CREATE FUNCTION fs_check(qry text, pred text, k int) RETURNS text LANGUAGE plpgsql AS $$ DECLARE q text; seeded int[]; unseeded int[]; BEGIN IF pred IS NULL THEN q := format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (SELECT id FROM fs_docs ' ' ORDER BY body <@> to_bm25query(%L, ''fs_docs_idx'') ' ' LIMIT %s) s', qry, k); ELSE q := format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (SELECT id FROM fs_docs WHERE %s ' ' ORDER BY body <@> to_bm25query(%L, ''fs_docs_idx'') ' ' LIMIT %s) s', pred, qry, k); END IF; SET LOCAL pg_textsearch.filtered_seed = on; EXECUTE q INTO seeded; SET LOCAL pg_textsearch.filtered_seed = off; EXECUTE q INTO unseeded; IF seeded IS NOT DISTINCT FROM unseeded THEN RETURN format('PASS (%s rows)', coalesce(array_length(seeded, 1), 0)); END IF; RETURN format('FAIL seeded=%s unseeded=%s', seeded, unseeded); END; $$; -- Independent oracle: the filtered index scan must return exactly the -- documents matching the filter AND containing the query term. k is -- large enough (>= facet bucket size) that the whole match set is -- returned, so this is immune to ranking, ties, and score values. CREATE FUNCTION fs_oracle_check(qry text, term text, pred text, k int) RETURNS text LANGUAGE plpgsql AS $$ DECLARE idx int[]; ora int[]; BEGIN EXECUTE format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (SELECT id FROM fs_docs WHERE %s ' ' ORDER BY body <@> to_bm25query(%L, ''fs_docs_idx'') ' ' LIMIT %s) s', pred, qry, k) INTO idx; EXECUTE format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM fs_docs WHERE (%s) AND body ~ %L', pred, '\y' || term || '\y') INTO ora; IF idx IS NOT DISTINCT FROM ora THEN RETURN format('PASS (%s rows)', coalesce(array_length(idx, 1), 0)); END IF; RETURN format('FAIL index=%s oracle=%s', idx, ora); END; $$; -- Realistic filtered top-k (k < facet size, partial matches, ties). SELECT n, qry, pred, fs_check(qry, pred, k) AS result FROM (VALUES (1, 'alpha', 'facet_id = 6', 10), (2, 'beta', 'facet_id < 5', 10), (3, 'common', 'facet_id = 13', 5), (4, 'alpha beta', 'facet_id < 10', 8), (5, 'gamma', 'facet_id IN (0, 7, 14)', 6), (6, 'gamma', 'facet_id >= 45', 7) ) v(n, qry, pred, k) ORDER BY n; -- Cross-check the returned set against a plain-SQL regex oracle. SELECT n, qry, pred, fs_oracle_check(qry, term, pred, k) AS result FROM (VALUES (1, 'alpha', 'alpha', 'facet_id = 6', 1000), (2, 'gamma', 'gamma', 'facet_id = 9', 1000), (3, 'beta', 'beta', 'facet_id = 2', 1000), (4, 'common', 'common', 'facet_id = 40', 1000) ) v(n, qry, term, pred, k) ORDER BY n; ------------------------------------------------------------------------ -- No-op: a query with no filter is unaffected by seeding. ------------------------------------------------------------------------ SELECT n, qry, fs_check(qry, NULL, k) AS result FROM (VALUES (1, 'alpha', 10), (2, 'common', 5) ) v(n, qry, k) ORDER BY n; ------------------------------------------------------------------------ -- Robustness: the result is independent of the margin (margin only -- affects how deep the seeded pass scores, never the final top-k). ------------------------------------------------------------------------ SET pg_textsearch.filtered_seed_margin = 1.0; SELECT 'margin=1.0' AS margin, fs_check('alpha', 'facet_id = 6', 10) AS result; SET pg_textsearch.filtered_seed_margin = 1000.0; SELECT 'margin=1000' AS margin, fs_check('alpha', 'facet_id = 6', 10) AS result; RESET pg_textsearch.filtered_seed_margin; ------------------------------------------------------------------------ -- GUC contract: defaults and range enforcement. ------------------------------------------------------------------------ SHOW pg_textsearch.filtered_seed; SHOW pg_textsearch.filtered_seed_margin; -- Out of range (min 1.0, max 1000.0) -- both should error. SET pg_textsearch.filtered_seed_margin = 0.5; SET pg_textsearch.filtered_seed_margin = 2000; ------------------------------------------------------------------------ -- Scan identity (issue #435): the seed is bound per index scan, so -- several BM25 scans of one index in one statement each get the seed -- derived from their own Filter and their own Limit. Results are -- identical either way -- Filter + Limit + backoff still decide the -- top-k -- so these are parity checks; the depth assertions that -- actually prove per-scan seeding are in the section after this one. ------------------------------------------------------------------------ -- Parity for an arbitrary query shape: run it with seeding on and off -- and compare the results as a multiset. q must yield a column named -- id. CREATE FUNCTION fs_check_q(q text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE wrapped text; seeded int[]; unseeded int[]; BEGIN wrapped := format( 'SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) ' 'FROM (%s) s', q); SET LOCAL pg_textsearch.filtered_seed = on; EXECUTE wrapped INTO seeded; SET LOCAL pg_textsearch.filtered_seed = off; EXECUTE wrapped INTO unseeded; IF seeded IS NOT DISTINCT FROM unseeded THEN RETURN format('PASS (%s rows)', coalesce(array_length(seeded, 1), 0)); END IF; RETURN format('FAIL seeded=%s unseeded=%s', seeded, unseeded); END; $$; SELECT n, what, fs_check_q(q) AS result FROM (VALUES (1, 'UNION ALL, two facets, same LIMIT', $q$ (SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 12 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) $q$), (2, 'UNION ALL, two different LIMITs', $q$ (SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id < 10 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50) $q$), (3, 'self-join, two facets', $q$ SELECT a.id AS id FROM (SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 5) a CROSS JOIN (SELECT id FROM fs_docs WHERE facet_id = 12 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 5) b WHERE b.id > a.id $q$), (4, 'CTE referenced twice', $q$ WITH t AS ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) SELECT id FROM t UNION ALL SELECT id FROM t $q$), (5, 'CTE NOT MATERIALIZED referenced twice', $q$ WITH t AS NOT MATERIALIZED ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) SELECT id FROM t UNION ALL SELECT id FROM t $q$), (6, 'unfiltered UNION ALL, two LIMITs', $q$ (SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50) $q$), (7, 'LIMIT with OFFSET', $q$ SELECT id FROM fs_docs WHERE facet_id < 10 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10 OFFSET 20 $q$), (8, 'Sort above the BM25 subquery', $q$ SELECT id FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx')) s ORDER BY id LIMIT 10 $q$), (9, 'BM25 scan in a correlated subquery', $q$ SELECT f.facet_id AS id FROM (SELECT DISTINCT facet_id FROM fs_docs WHERE facet_id < 4) f WHERE EXISTS ( SELECT 1 FROM fs_docs d WHERE d.facet_id = f.facet_id ORDER BY d.body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 3) $q$) ) v(n, what, q) ORDER BY n; -- Each arm of a two-arm UNION ALL must return what that arm returns -- standalone. Before #435 the two scans shared one per-index limit -- slot, so one arm could run at the other's depth (or at the default). CREATE FUNCTION fs_arms_check(q1 text, q2 text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE agg text := 'coalesce(array_agg(id ORDER BY id), ARRAY[]::int[])'; solo1 int[]; solo2 int[]; both1 int[] := ARRAY[]::int[]; both2 int[] := ARRAY[]::int[]; r record; BEGIN EXECUTE format('SELECT %s FROM (%s) s', agg, q1) INTO solo1; EXECUTE format('SELECT %s FROM (%s) s', agg, q2) INTO solo2; -- One statement, two scans of fs_docs_idx. The GROUP BY sits -- above the UNION ALL so neither arm can be pruned away. FOR r IN EXECUTE format( 'SELECT arm, %s AS ids FROM (' ' SELECT 1 AS arm, id FROM (%s) a ' ' UNION ALL ' ' SELECT 2 AS arm, id FROM (%s) b) u ' 'GROUP BY arm', agg, q1, q2) LOOP IF r.arm = 1 THEN both1 := r.ids; ELSE both2 := r.ids; END IF; END LOOP; IF both1 IS NOT DISTINCT FROM solo1 AND both2 IS NOT DISTINCT FROM solo2 THEN RETURN format('PASS (%s + %s rows)', coalesce(array_length(solo1, 1), 0), coalesce(array_length(solo2, 1), 0)); END IF; RETURN format('FAIL arm1 both=%s solo=%s / arm2 both=%s solo=%s', both1, solo1, both2, solo2); END; $$; SELECT n, what, fs_arms_check(q1, q2) AS result FROM (VALUES (1, 'two facets, same LIMIT', $q$SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$, $q$SELECT id FROM fs_docs WHERE facet_id = 12 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$), (2, 'two facets, two LIMITs', $q$SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$, $q$SELECT id FROM fs_docs WHERE facet_id < 10 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50$q$), (3, 'no filter, two LIMITs', $q$SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10$q$, $q$SELECT id FROM fs_docs ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 50$q$) ) v(n, what, q1, q2) ORDER BY n; -- Cached plans: possible custom-to-generic plan changes and changes -- to the margin between executions must preserve results, whether the -- scan receives a query hint or falls back to normal backoff. PREPARE fs_union(int, int) AS SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) FROM ( (SELECT id FROM fs_docs WHERE facet_id = $1 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = $2 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10)) s; EXECUTE fs_union(6, 12); EXECUTE fs_union(6, 12); EXECUTE fs_union(6, 12); EXECUTE fs_union(6, 12); EXECUTE fs_union(6, 12); EXECUTE fs_union(6, 12); EXECUTE fs_union(6, 12); SET pg_textsearch.filtered_seed_margin = 50.0; EXECUTE fs_union(6, 12); RESET pg_textsearch.filtered_seed_margin; EXECUTE fs_union(6, 12); DEALLOCATE fs_union; -- An unresolved LIMIT $1 cannot be carried in a planner query hint. -- Both parameter types therefore use normal backoff on a generic plan. PREPARE fs_lim(bigint) AS SELECT coalesce(array_agg(id ORDER BY id), ARRAY[]::int[]) FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT $1) s; EXECUTE fs_lim(10); EXECUTE fs_lim(10); EXECUTE fs_lim(10); EXECUTE fs_lim(10); EXECUTE fs_lim(10); EXECUTE fs_lim(10); EXECUTE fs_lim(10); DEALLOCATE fs_lim; -- Agg above a limited BM25 subquery. SELECT count(*) AS agg_rows FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('alpha', 'fs_docs_idx') LIMIT 10) s; ------------------------------------------------------------------------ -- Depth: the acceptance test for issue #435. -- -- Scan depth is invisible in results, so the parity checks above pass -- with or without the fix. bm25_debug_scoring_passes counts scoring -- passes: a scan seeded for its own filter costs exactly one, and each -- executor backoff re-drive adds another. -- -- Both arms below filter down to fewer than k rows, so the executor -- always exhausts the batch and backs off -- the pass count is then a -- function of the seed alone, not of score ties. With a correct -- per-scan seed each arm costs one pass; when one arm steals the -- other's seed and the loser falls back to pg_textsearch.default_limit, -- the loser pays a chain of doublings instead. ------------------------------------------------------------------------ CREATE FUNCTION fs_passes(q text) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE n bigint; BEGIN PERFORM bm25_debug_scoring_passes(true); EXECUTE q; SELECT bm25_debug_scoring_passes() INTO n; RETURN n; END; $$; -- A small default_limit lengthens the backoff chain an unseeded arm -- pays, so the two cases are far apart rather than adjacent. SET pg_textsearch.default_limit = 100; -- Confirm the plan under test: two BM25 index scans of one index, each -- under its own Limit and with its own Filter. Counted rather than -- printed, so subquery aliases and node order cannot make this brittle. CREATE FUNCTION fs_plan_shape(q text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE r record; scans int := 0; limits int := 0; filters int := 0; BEGIN FOR r IN EXECUTE 'EXPLAIN (COSTS OFF) ' || q LOOP IF r."QUERY PLAN" LIKE '%Index Scan using fs_docs_idx%' THEN scans := scans + 1; ELSIF r."QUERY PLAN" LIKE '%Limit%' THEN limits := limits + 1; ELSIF r."QUERY PLAN" LIKE '%Filter:%' THEN filters := filters + 1; END IF; END LOOP; RETURN format('%s bm25 scans, %s limits, %s filters', scans, limits, filters); END; $$; SELECT fs_plan_shape($q$ (SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 13 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) $q$) AS plan_shape; -- Control: one scan, one Limit, one Filter -- one pass. This holds -- with or without the fix; it calibrates the counter. SELECT fs_passes($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 $q$) AS single_scan_passes; -- The issue #435 example: two scans of one index, each with its own -- Filter. Must be one pass per arm. SELECT fs_passes($q$ (SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 13 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) $q$) AS union_all_passes; -- Three arms, to show the count tracks the number of scans rather -- than happening to be small. SELECT fs_passes($q$ (SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 13 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) UNION ALL (SELECT id FROM fs_docs WHERE facet_id = 21 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) $q$) AS three_arm_passes; -- The planner walker finds the Limit inside a correlated SubPlan. -- Every rescan reads the hint from its query value, restoring the seed -- on each of the three evaluations. SELECT fs_passes($q$ SELECT sum(c) FROM ( SELECT (SELECT count(*) FROM ( SELECT id FROM fs_docs WHERE facet_id = v.fid AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) x) AS c FROM (VALUES (6), (13), (21)) v(fid)) y $q$) AS correlated_subplan_passes; -- Materialized CTEs are reached through their InitPlan. Referencing -- the CTE twice still executes its single seeded scan only once. SELECT fs_passes($q$ WITH t AS MATERIALIZED ( SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) SELECT id FROM t UNION ALL SELECT id FROM t $q$) AS materialized_cte_passes; -- An uncorrelated scalar subquery exercises the other InitPlan case. SELECT fs_passes($q$ SELECT (SELECT count(*) FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10) s) $q$) AS initplan_passes; -- Only direct Limit -> IndexScan pairs are seeded. LockRows is an -- intentional boundary, even though it preserves ordering. With -- fewer than 10 matches, the unseeded scan exhausts all 1000 candidates -- through backoff (100, 200, 400, 800, 1600): five scoring passes. EXPLAIN (COSTS OFF) SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 FOR UPDATE; SELECT fs_passes($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 FOR UPDATE $q$) AS lockrows_fallback_passes; SELECT fs_check_q($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 FOR UPDATE $q$) AS lockrows_fallback_parity; -- Typed literals resolved to an index must retain variable-length metadata -- when the planner appends a hint and PostgreSQL copies the query value. SELECT fs_passes($q$ SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> 'common'::bm25query LIMIT 10 $q$) AS typed_literal_passes; -- Planning a saved query must not depend on the last parsed statement. -- Force generic plans rather than relying on the custom-plan cost heuristic. SET plan_cache_mode = force_generic_plan; PREPARE fs_const_hint AS SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> 'common'::bm25query LIMIT 10; SELECT 1 AS unrelated_statement; SELECT fs_passes('EXECUTE fs_const_hint') AS prepared_constant_passes; SELECT fs_passes('EXECUTE fs_const_hint') AS cached_constant_passes; DISCARD PLANS; SELECT 1 AS unrelated_statement; SELECT fs_passes('EXECUTE fs_const_hint') AS replanned_constant_passes; DEALLOCATE fs_const_hint; -- A runtime query operand cannot carry a hint. The cached plan must use -- default-limit backoff and read the new query value on each execution. PREPARE fs_query(bm25query) AS SELECT array_agg(id ORDER BY id) FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> $1 LIMIT 10) s; SELECT fs_plan_shape( 'EXECUTE fs_query(to_bm25query(''common'', ''fs_docs_idx''))' ) AS runtime_query_shape; SELECT fs_passes( 'EXECUTE fs_query(to_bm25query(''common'', ''fs_docs_idx''))' ) AS runtime_query_passes; EXECUTE fs_query(to_bm25query('common', 'fs_docs_idx')); EXECUTE fs_query(to_bm25query('beta', 'fs_docs_idx')); DEALLOCATE fs_query; RESET plan_cache_mode; -- A generic plan's LIMIT is a PARAM_EXTERN and cannot be folded into the -- planner-carried query hint, so it uses normal backoff. -- The parser coerces a LIMIT to bigint, while an int parameter arrives -- wrapped in an int8() coercion. Both cases intentionally fall back. PREPARE fs_dp_big(bigint) AS SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT $1; PREPARE fs_dp_int(int) AS SELECT id FROM fs_docs WHERE facet_id = 6 AND id % 7 = 3 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT $1; -- Runs stmt n times first, so the plan has flipped from custom to -- generic before the measured execution. CREATE FUNCTION fs_generic_passes(stmt text, n int) RETURNS bigint LANGUAGE plpgsql AS $$ DECLARE p bigint; BEGIN FOR i IN 1..n LOOP EXECUTE stmt; END LOOP; PERFORM bm25_debug_scoring_passes(true); EXECUTE stmt; SELECT bm25_debug_scoring_passes() INTO p; RETURN p; END; $$; SELECT fs_generic_passes('EXECUTE fs_dp_big(10)', 6) AS bigint_param_passes; SELECT fs_generic_passes('EXECUTE fs_dp_int(10)', 6) AS int_param_passes; DEALLOCATE fs_dp_big; DEALLOCATE fs_dp_int; RESET pg_textsearch.default_limit; ------------------------------------------------------------------------ -- Saturating LIMIT: offset + count at or above INT_MAX yields no k. -- -- The seed becomes so->limit, which sizes a palloc of k -- ItemPointerDatas, so a k of INT_MAX asks for 12 GB and the query -- dies with "invalid memory alloc request size". LIMIT 2147483647 is -- the usual generated-SQL spelling of "no limit", so these must fall -- back to the default rather than bind a k that cannot be served. ------------------------------------------------------------------------ -- 20 docs per facet bucket; each query must return the whole bucket. SELECT count(*) AS int_max_limit FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 2147483647) s; SELECT count(*) AS count_over_int_max FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 9999999999) s; -- k is offset + count, so a large OFFSET saturates it just as a large -- count does. Nothing survives the offset; the point is that it runs. SELECT count(*) AS int_max_offset FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 10 OFFSET 3000000000) s; -- A k that binds normally, so the cases above are shown to be about -- saturation and not about large limits in general. SELECT count(*) AS large_bound_limit FROM ( SELECT id FROM fs_docs WHERE facet_id = 6 ORDER BY body <@> to_bm25query('common', 'fs_docs_idx') LIMIT 1000000) s; ------------------------------------------------------------------------ -- Cleanup ------------------------------------------------------------------------ DROP FUNCTION fs_check(text, text, int); DROP FUNCTION fs_oracle_check(text, text, text, int); DROP FUNCTION fs_check_q(text); DROP FUNCTION fs_arms_check(text, text); DROP FUNCTION fs_plan_shape(text); DROP FUNCTION fs_passes(text); DROP FUNCTION fs_generic_passes(text, int); DROP TABLE fs_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/force_merge.sql000066400000000000000000000530401525762047700235370ustar00rootroot00000000000000-- Test case: force_merge -- Tests bm25_force_merge() bounded best-effort compaction. -- -- This test exercises: -- 1. Multiple segments across levels -- 2. Force merge into size-bounded segments -- 3. Queries return correct results after merge -- 4. Inserts work normally after force merge CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Use 2 segments per level so we get multi-level segments quickly SET pg_textsearch.segments_per_level = 2; CREATE TABLE force_merge_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX force_merge_idx ON force_merge_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Empty indexes are already a valid zero-segment force-merge result. DO $$ DECLARE summary text; BEGIN PERFORM bm25_force_merge('force_merge_idx'); summary := bm25_summarize_index('force_merge_idx'); IF summary !~ E'Segments:\n \\(none\\)' THEN RAISE EXCEPTION 'empty force merge created a segment: %', summary; END IF; END $$; -------------------------------------------------------------------------------- -- Build up segments across multiple levels -------------------------------------------------------------------------------- -- Batch 1 -> spill to L0 INSERT INTO force_merge_test (content) VALUES ('hello world database'), ('goodbye cruel world'), ('hello goodbye friend'), ('world peace harmony'); SELECT bm25_spill_index('force_merge_idx') IS NOT NULL AS spill1; -- Batch 2 -> spill to L0, triggers L0->L1 merge INSERT INTO force_merge_test (content) VALUES ('database indexing query'), ('search engine optimization'), ('database world news'), ('goodbye database friend'); SELECT bm25_spill_index('force_merge_idx') IS NOT NULL AS spill2; -- Batch 3 -> new L0 segment (so we have L0 + L1) INSERT INTO force_merge_test (content) VALUES ('hello search results'), ('optimization database tuning'); SELECT bm25_spill_index('force_merge_idx') IS NOT NULL AS spill3; -- Verify counts before force merge SELECT COUNT(*) AS hello_before FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('hello', 'force_merge_idx') LIMIT 100 ) t; SELECT COUNT(*) AS database_before FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('database', 'force_merge_idx') LIMIT 100 ) t; -------------------------------------------------------------------------------- -- Force merge all segments into one -------------------------------------------------------------------------------- SELECT bm25_force_merge('force_merge_idx'); DO $$ DECLARE summary text := bm25_summarize_index('force_merge_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 THEN RAISE EXCEPTION 'force merge left multiple segments: %', summary; END IF; END $$; -- Verify same counts after force merge SELECT COUNT(*) AS hello_after FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('hello', 'force_merge_idx') LIMIT 100 ) t; SELECT COUNT(*) AS database_after FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('database', 'force_merge_idx') LIMIT 100 ) t; -- Validate BM25 scoring is correct after force merge SELECT validate_bm25_scoring('force_merge_test', 'content', 'force_merge_idx', 'hello', 'english', 1.2, 0.75) AS hello_scores_valid; SELECT validate_bm25_scoring('force_merge_test', 'content', 'force_merge_idx', 'database', 'english', 1.2, 0.75) AS database_scores_valid; -------------------------------------------------------------------------------- -- Verify inserts work after force merge -------------------------------------------------------------------------------- INSERT INTO force_merge_test (content) VALUES ('hello after merge'), ('new database entry'); SELECT COUNT(*) AS hello_with_new FROM ( SELECT id FROM force_merge_test ORDER BY content <@> to_bm25query('hello', 'force_merge_idx') LIMIT 100 ) t; SELECT validate_bm25_scoring('force_merge_test', 'content', 'force_merge_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid_after_insert; DROP TABLE force_merge_test CASCADE; -------------------------------------------------------------------------------- -- A force-merge spill must not run ordinary threshold compaction first. -- -- Build two L0 segments below the initial threshold, then lower the threshold -- while a memtable remains pending. Force merge must spill the memtable -- without running threshold compaction before its own merge plan. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 3; CREATE TABLE force_spill_cascade (id serial PRIMARY KEY, content text); CREATE INDEX force_spill_cascade_idx ON force_spill_cascade USING bm25(content) WITH (text_config='english', compaction='manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO force_spill_cascade (content) VALUES (format('spill cascade document %s filler', n)); PERFORM bm25_spill_index('force_spill_cascade_idx'); END LOOP; INSERT INTO force_spill_cascade (content) VALUES ('spill cascade pending memtable document filler'); END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_spill_cascade_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 2 OR summary !~ 'L0 Segment 2:' OR summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'force spill cascade layout was not constructed: %', summary; END IF; END $$; SET pg_textsearch.segments_per_level = 2; ALTER INDEX force_spill_cascade_idx SET (compaction = 'inline'); DO $$ BEGIN PERFORM bm25_force_merge('force_spill_cascade_idx'); END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_spill_cascade_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'force spill cascade did not reach one segment: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_spill_cascade ORDER BY content <@> to_bm25query('filler', 'force_spill_cascade_idx') ) ranked) <> 3 THEN RAISE EXCEPTION 'force spill cascade lost documents'; END IF; END $$; DROP TABLE force_spill_cascade CASCADE; -------------------------------------------------------------------------------- -- Force merge is bounded best-effort compaction, not forceMerge(1). -- -- Four layouts: a single L1 segment, repeated merges that stay in L1, -- a mixed L0+L1 pair, and an L1 segment with a pending memtable. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 2; CREATE TABLE force_tier_single (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_single_idx ON force_tier_single USING bm25(content) WITH (text_config='english'); CREATE TABLE force_tier_deep (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_deep_idx ON force_tier_deep USING bm25(content) WITH (text_config='english'); CREATE TABLE force_tier_mixed (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_mixed_idx ON force_tier_mixed USING bm25(content) WITH (text_config='english'); CREATE TABLE force_tier_memtable (id serial PRIMARY KEY, content text); CREATE INDEX force_tier_memtable_idx ON force_tier_memtable USING bm25(content) WITH (text_config='english'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..256 LOOP INSERT INTO force_tier_deep (content) VALUES (format('multiple terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_deep_idx'); IF n <= 128 THEN INSERT INTO force_tier_single (content) VALUES (format('single terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_single_idx'); INSERT INTO force_tier_mixed (content) VALUES (format('mixed terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_mixed_idx'); INSERT INTO force_tier_memtable (content) VALUES (format('memtable terminal %s filler', n)); PERFORM bm25_spill_index('force_tier_memtable_idx'); END IF; END LOOP; -- Runtime inline policy performs one bounded pass per spill. Explicitly -- drain the remaining cascade to construct the size-tiered fixtures. PERFORM bm25_compact('force_tier_deep_idx'::regclass); PERFORM bm25_compact('force_tier_single_idx'::regclass); PERFORM bm25_compact('force_tier_mixed_idx'::regclass); PERFORM bm25_compact('force_tier_memtable_idx'::regclass); ALTER INDEX force_tier_mixed_idx SET (compaction = 'manual'); INSERT INTO force_tier_mixed (content) VALUES ('mixed lower filler'); PERFORM bm25_spill_index('force_tier_mixed_idx'); INSERT INTO force_tier_memtable (content) VALUES ('memtable pending filler'); END $$; DO $$ DECLARE single_summary text := bm25_summarize_index('force_tier_single_idx'); deep_summary text := bm25_summarize_index('force_tier_deep_idx'); mixed_summary text := bm25_summarize_index('force_tier_mixed_idx'); memtable_summary text := bm25_summarize_index('force_tier_memtable_idx'); BEGIN IF regexp_count(single_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR single_summary !~ 'L1 Segment 1:' THEN RAISE EXCEPTION 'single L1 layout was not constructed: %', single_summary; END IF; IF regexp_count(deep_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR deep_summary !~ 'L1 Segment 1:' THEN RAISE EXCEPTION 'deep cascade did not settle at one L1 segment: %', deep_summary; END IF; IF regexp_count(mixed_summary, 'L[0-7] Segment [0-9]+:') <> 2 OR mixed_summary !~ 'L0 Segment 1:' OR mixed_summary !~ 'L1 Segment 1:' THEN RAISE EXCEPTION 'mixed L1 layout was not constructed: %', mixed_summary; END IF; IF regexp_count(memtable_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR memtable_summary !~ 'L1 Segment 1:' OR memtable_summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'memtable L1 layout was not constructed: %', memtable_summary; END IF; END $$; DO $$ DECLARE single_before text := bm25_summarize_index('force_tier_single_idx'); deep_before text := bm25_summarize_index('force_tier_deep_idx'); mixed_before text := bm25_summarize_index('force_tier_mixed_idx'); memtable_before text := bm25_summarize_index('force_tier_memtable_idx'); single_summary text; deep_summary text; mixed_summary text; memtable_summary text; BEGIN PERFORM bm25_force_merge('force_tier_single_idx'); PERFORM bm25_force_merge('force_tier_deep_idx'); PERFORM bm25_force_merge('force_tier_mixed_idx'); PERFORM bm25_force_merge('force_tier_memtable_idx'); single_summary := bm25_summarize_index('force_tier_single_idx'); deep_summary := bm25_summarize_index('force_tier_deep_idx'); mixed_summary := bm25_summarize_index('force_tier_mixed_idx'); memtable_summary := bm25_summarize_index('force_tier_memtable_idx'); IF regexp_count(single_summary, 'L[0-7] Segment [0-9]+:') > regexp_count(single_before, 'L[0-7] Segment [0-9]+:') THEN RAISE EXCEPTION 'force merge increased segment count: before %, after %', single_before, single_summary; END IF; IF regexp_count(deep_summary, 'L[0-7] Segment [0-9]+:') > regexp_count(deep_before, 'L[0-7] Segment [0-9]+:') THEN RAISE EXCEPTION 'force merge increased segment count: before %, after %', deep_before, deep_summary; END IF; -- The mixed fixture is the multi-segment case, so force merge must -- actually combine its L0 and L1 segments. Asserting only that the -- count did not increase would also accept a no-op. IF regexp_count(mixed_summary, 'L[0-7] Segment [0-9]+:') <> 1 OR mixed_summary !~ 'L0 Segment 1:' THEN RAISE EXCEPTION 'force merge did not combine the mixed layout: before %, after %', mixed_before, mixed_summary; END IF; IF regexp_count(memtable_summary, 'L[0-7] Segment [0-9]+:') > regexp_count(memtable_before, 'L[0-7] Segment [0-9]+:') THEN RAISE EXCEPTION 'force merge increased segment count: before %, after %', memtable_before, memtable_summary; END IF; IF memtable_summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'force merge did not clear the memtable: %', memtable_summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_tier_single ORDER BY content <@> to_bm25query('filler', 'force_tier_single_idx') ) ranked) <> 128 OR (SELECT count(*) FROM ( SELECT 1 FROM force_tier_deep ORDER BY content <@> to_bm25query('filler', 'force_tier_deep_idx') ) ranked) <> 256 OR (SELECT count(*) FROM ( SELECT 1 FROM force_tier_mixed ORDER BY content <@> to_bm25query('filler', 'force_tier_mixed_idx') ) ranked) <> 129 OR (SELECT count(*) FROM ( SELECT 1 FROM force_tier_memtable ORDER BY content <@> to_bm25query('filler', 'force_tier_memtable_idx') ) ranked) <> 129 THEN RAISE EXCEPTION 'size-tiered force merge lost documents'; END IF; END $$; -- A termless memtable contributes document accounting but no prospective -- segment, so it can be cleared without rewriting a compacted singleton. DO $$ BEGIN IF bm25_test_chain_source( 'force_tier_single_idx', 'one_doc_no_terms') <> 'OK' THEN RAISE EXCEPTION 'could not construct termless singleton memtable'; END IF; END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_tier_single_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ 'L0 Segment 1:' OR summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'termless singleton layout was not constructed: %', summary; END IF; PERFORM bm25_force_merge('force_tier_single_idx'); summary := bm25_summarize_index('force_tier_single_idx'); IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ 'L0 Segment 1:' OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'termless force merge changed segment layout: %', summary; END IF; IF bm25_dump_index('force_tier_single_idx')::text !~ E'total_docs: 129\n total_len: 519\n' THEN RAISE EXCEPTION 'termless force merge lost document accounting'; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_tier_single ORDER BY content <@> to_bm25query('filler', 'force_tier_single_idx') ) ranked) <> 128 THEN RAISE EXCEPTION 'termless force merge changed query results'; END IF; END $$; DROP TABLE force_tier_single CASCADE; DROP TABLE force_tier_deep CASCADE; DROP TABLE force_tier_mixed CASCADE; DROP TABLE force_tier_memtable CASCADE; -------------------------------------------------------------------------------- -- Termless accounting survives a non-no-op force compaction. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 64; CREATE TABLE force_termless_multi ( id bigint PRIMARY KEY, content text ); CREATE INDEX force_termless_multi_idx ON force_termless_multi USING bm25(content) WITH (text_config='simple'); DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO force_termless_multi VALUES (batch, 'accounted'); PERFORM bm25_spill_index('force_termless_multi_idx'); END LOOP; IF bm25_test_chain_source( 'force_termless_multi_idx', 'one_doc_no_terms') <> 'OK' THEN RAISE EXCEPTION 'could not append termless accounting record'; END IF; END $$; SELECT bm25_force_merge('force_termless_multi_idx'); DO $$ DECLARE dump text := bm25_dump_index('force_termless_multi_idx')::text; summary text := bm25_summarize_index('force_termless_multi_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 1 OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'termless multi-source force merge did not compact: %', summary; END IF; IF dump !~ E'total_docs: 3\n total_len: 9\n' THEN RAISE EXCEPTION 'termless multi-source accounting was lost: %', dump; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_termless_multi ORDER BY content <@> to_bm25query('accounted', 'force_termless_multi_idx') ) ranked) <> 2 THEN RAISE EXCEPTION 'termless multi-source compaction lost documents'; END IF; END $$; DROP TABLE force_termless_multi CASCADE; -------------------------------------------------------------------------------- -- Skip-entry estimates are per term. Two low-frequency vocabularies fit the -- budget only if skip blocks are incorrectly rounded after global summation. -------------------------------------------------------------------------------- SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE force_low_frequency ( id bigint PRIMARY KEY, content text ); CREATE INDEX force_low_frequency_idx ON force_low_frequency USING bm25(content) WITH (text_config='simple'); DO $$ BEGIN FOR batch IN 1..2 LOOP INSERT INTO force_low_frequency SELECT batch, 'lowfrequency ' || string_agg(format('lf%sx%s', batch, term), ' ' ORDER BY term) FROM generate_series(1, 9000) term; PERFORM bm25_spill_index('force_low_frequency_idx'); END LOOP; END $$; SELECT bm25_force_merge('force_low_frequency_idx'); DO $$ DECLARE summary text := bm25_summarize_index('force_low_frequency_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 2 THEN RAISE EXCEPTION 'low-frequency sources were merged over budget: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_low_frequency ORDER BY content <@> to_bm25query('lowfrequency', 'force_low_frequency_idx') ) ranked) <> 2 THEN RAISE EXCEPTION 'low-frequency force merge lost documents'; END IF; END $$; RESET pg_textsearch.max_segment_size; DROP TABLE force_low_frequency CASCADE; -------------------------------------------------------------------------------- -- A bounded force merge produces maximal batches without exceeding the -- configured segment-size budget. -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 64; SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE force_bounded (id bigint PRIMARY KEY, content text); CREATE INDEX force_bounded_idx ON force_bounded USING bm25(content) WITH (text_config='simple'); DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..3 LOOP INSERT INTO force_bounded SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 4) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('force_bounded_idx'); END LOOP; END $$; SELECT bm25_force_merge('force_bounded_idx'); DO $$ DECLARE first_summary text := bm25_summarize_index('force_bounded_idx'); second_summary text; output_pages bigint; parsed_outputs integer := 0; BEGIN IF regexp_count(first_summary, 'L[0-7] Segment [0-9]+:') <> 2 THEN RAISE EXCEPTION 'bounded force merge did not produce two segments: %', first_summary; END IF; FOR output_pages IN SELECT captures[1]::bigint FROM regexp_matches( first_summary, 'pages=([0-9]+), size=', 'g') AS m(captures) LOOP parsed_outputs := parsed_outputs + 1; IF (output_pages + 1) * current_setting('block_size')::bigint > 1024 * 1024 THEN RAISE EXCEPTION 'bounded output exceeds 1 MiB: %', first_summary; END IF; END LOOP; IF parsed_outputs <> 2 THEN RAISE EXCEPTION 'could not parse bounded outputs: %', first_summary; END IF; PERFORM bm25_force_merge('force_bounded_idx'); second_summary := bm25_summarize_index('force_bounded_idx'); IF second_summary <> first_summary THEN RAISE EXCEPTION 'bounded force merge was not idempotent: before %, after %', first_summary, second_summary; END IF; END $$; SELECT count(*) = 8000 AS bounded_docs_preserved FROM ( SELECT 1 FROM force_bounded ORDER BY content <@> to_bm25query('common', 'force_bounded_idx') LIMIT 8000 ) ranked; RESET pg_textsearch.max_segment_size; DROP TABLE force_bounded CASCADE; -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- RESET pg_textsearch.segments_per_level; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/force_merge_injection.sql000066400000000000000000000044411525762047700256020ustar00rootroot00000000000000SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE force_lowered_capacity ( id bigint PRIMARY KEY, content text ); CREATE INDEX force_lowered_capacity_idx ON force_lowered_capacity USING bm25(content) WITH (text_config='simple'); DO $$ BEGIN FOR batch IN 1..9 LOOP INSERT INTO force_lowered_capacity SELECT batch, 'capacitytoken ' || string_agg(format('cap%sx%s', batch, term), ' ' ORDER BY term) FROM generate_series(1, 12000) term; PERFORM bm25_spill_index('force_lowered_capacity_idx'); END LOOP; END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_lowered_capacity_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 9 OR summary !~ 'L0 Segment 9:' THEN RAISE EXCEPTION 'lowered-capacity layout was not constructed: %', summary; END IF; END $$; SELECT pg_textsearch_test_attach_segment_limit(1); DO $$ BEGIN PERFORM bm25_force_merge('force_lowered_capacity_idx'); END $$; INSERT INTO force_lowered_capacity VALUES (10, 'capacitytoken pending spill'); DO $$ BEGIN PERFORM bm25_force_merge('force_lowered_capacity_idx'); END $$; DO $$ DECLARE summary text := bm25_summarize_index('force_lowered_capacity_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 9 OR summary !~ E'Memtable:\n terms: 0\n documents: 0' THEN RAISE EXCEPTION 'lowered-capacity force merge changed topology: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM force_lowered_capacity ORDER BY content <@> to_bm25query('capacitytoken', 'force_lowered_capacity_idx') ) ranked) <> 10 THEN RAISE EXCEPTION 'lowered-capacity force merge lost documents'; END IF; END $$; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); RESET pg_textsearch.max_segment_size; DROP TABLE force_lowered_capacity CASCADE; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/implicit.sql000066400000000000000000000216011525762047700230720ustar00rootroot00000000000000-- Test implicit index resolution via planner hook -- Tests that to_bm25query() without an index name automatically finds the BM25 index CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -- Create test table with BM25 index CREATE TABLE implicit_docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX implicit_docs_idx ON implicit_docs USING bm25(content) WITH (text_config='english'); -- Insert test documents INSERT INTO implicit_docs (content) VALUES ('hello world example'), ('database system design'), ('hello database world'); -- Test 1: Explicit index name works (baseline) -- Note: ORDER BY must use the raw expression, not the ROUND() alias, for index scan SELECT id, content, ROUND((content <@> to_bm25query('hello', 'implicit_docs_idx'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('hello', 'implicit_docs_idx'), id LIMIT 10; -- Test 2: Implicit index resolution (main feature being tested) SELECT id, content, ROUND((content <@> to_bm25query('hello'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('hello'), id LIMIT 10; -- Test 3: Implicit resolution works with different query terms SELECT id, content, ROUND((content <@> to_bm25query('database'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('database'), id LIMIT 10; -- Test 4: Works with ORDER BY only (no WHERE clause) SELECT id, content, ROUND((content <@> to_bm25query('database'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query('database'), id LIMIT 3; -- Test 5: Multiple BM25 indexes on different columns - should pick correct one CREATE TABLE multi_col_docs ( id SERIAL PRIMARY KEY, title TEXT, body TEXT ); CREATE INDEX multi_col_title_idx ON multi_col_docs USING bm25(title) WITH (text_config='english'); CREATE INDEX multi_col_body_idx ON multi_col_docs USING bm25(body) WITH (text_config='english'); INSERT INTO multi_col_docs (title, body) VALUES ('hello title', 'goodbye body content'), ('goodbye title', 'hello body content'), ('neutral title', 'neutral body content'); -- Query on title column should use title index SELECT id, title, ROUND((title <@> to_bm25query('hello'))::numeric, 4) as score FROM multi_col_docs ORDER BY title <@> to_bm25query('hello'), id LIMIT 10; -- Query on body column should use body index SELECT id, body, ROUND((body <@> to_bm25query('hello'))::numeric, 4) as score FROM multi_col_docs ORDER BY body <@> to_bm25query('hello'), id LIMIT 10; -- Test 6: Schema-qualified table still works with implicit resolution CREATE SCHEMA implicit_schema; CREATE TABLE implicit_schema.schema_docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX schema_docs_idx ON implicit_schema.schema_docs USING bm25(content) WITH (text_config='english'); INSERT INTO implicit_schema.schema_docs (content) VALUES ('hello from schema'), ('goodbye from schema'); -- Implicit resolution should work even with schema-qualified tables SELECT id, content, ROUND((content <@> to_bm25query('hello'))::numeric, 4) as score FROM implicit_schema.schema_docs ORDER BY content <@> to_bm25query('hello'), id LIMIT 10; -- Test 7: text <@> text implicit syntax for standalone scoring -- The planner hook transforms text <@> text to text <@> bm25query for non-ORDER BY -- expressions, enabling implicit index resolution without to_bm25query(). SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM implicit_docs ORDER BY content <@> 'hello', id LIMIT 10; -- Test 8: text <@> text works in ORDER BY with implicit index SELECT id, content FROM implicit_docs ORDER BY content <@> 'database' LIMIT 3; -- Test 9: Combining implicit text <@> text scoring with ORDER BY SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM implicit_docs ORDER BY content <@> 'hello', id LIMIT 3; -- Test 10: Prepared statements require explicit index name -- Implicit resolution doesn't work because to_bm25query($1) can't be -- folded to a constant at plan time when the parameter isn't known. PREPARE explicit_search(text) AS SELECT id, content, ROUND((content <@> to_bm25query($1, 'implicit_docs_idx'))::numeric, 4) as score FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx'), id LIMIT 10; EXECUTE explicit_search('hello'); EXECUTE explicit_search('database'); DEALLOCATE explicit_search; -- The score rewrite must use the query being planned, not parse-hook state -- left over from PREPARE. The DO statement itself clears that old state. PREPARE prepared_score(text) AS SELECT id FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx') LIMIT 10; -- The planner-time BM25 check must descend into a FROM subquery. The outer -- SELECT has no BM25 operator of its own. PREPARE prepared_subquery_score(text) AS SELECT id FROM ( SELECT id FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx') LIMIT 2 ) AS ranked; -- An inlined CTE also has no BM25 operator at the outer query level. PREPARE prepared_cte_score(text) AS WITH ranked AS ( SELECT id FROM implicit_docs ORDER BY content <@> to_bm25query($1, 'implicit_docs_idx') LIMIT 2 ) SELECT id FROM ranked; DO $$ DECLARE plan json; mode text; statement text; BEGIN FOREACH mode IN ARRAY ARRAY['force_generic_plan', 'force_custom_plan'] LOOP PERFORM set_config('plan_cache_mode', mode, true); FOREACH statement IN ARRAY ARRAY[ 'prepared_score', 'prepared_subquery_score', 'prepared_cte_score' ] LOOP EXECUTE format( 'EXPLAIN (FORMAT JSON, COSTS OFF, VERBOSE) EXECUTE %I(%L)', statement, 'hello') INTO plan; IF plan::text NOT LIKE '%bm25_get_current_score()%' THEN RAISE EXCEPTION '% (%) did not reuse the index score: %', statement, mode, plan; END IF; END LOOP; END LOOP; END $$; DEALLOCATE prepared_score; DEALLOCATE prepared_subquery_score; DEALLOCATE prepared_cte_score; -- Nested scoring must not force its explicit index onto the outer scan. CREATE TABLE planner_scope_docs (id int PRIMARY KEY, content text); INSERT INTO planner_scope_docs VALUES (1, 'running'), (2, 'runs'), (3, 'run'); CREATE INDEX planner_scope_simple ON planner_scope_docs USING bm25(content) WITH (text_config = 'simple') WHERE id = 1; CREATE INDEX planner_scope_english ON planner_scope_docs USING bm25(content) WITH (text_config = 'english'); DO $$ DECLARE baseline_ids int[]; nested_ids int[]; BEGIN SELECT array_agg(id ORDER BY id) INTO baseline_ids FROM ( SELECT id FROM planner_scope_docs ORDER BY content <@> 'running' LIMIT 3 ) AS ranked; IF baseline_ids IS DISTINCT FROM ARRAY[1, 2, 3] THEN RAISE EXCEPTION 'unexpected outer scan results: %', baseline_ids; END IF; SELECT array_agg(id ORDER BY id) INTO nested_ids FROM ( SELECT id FROM planner_scope_docs WHERE ( SELECT content <@> to_bm25query('running', 'planner_scope_simple') FROM planner_scope_docs WHERE id = 1 ) IS NOT NULL ORDER BY content <@> 'running' LIMIT 3 ) AS ranked; IF nested_ids IS DISTINCT FROM baseline_ids THEN RAISE EXCEPTION 'nested index changed outer results: % vs %', nested_ids, baseline_ids; END IF; END $$; DROP TABLE planner_scope_docs; -- Test 11: Implicit text <@> text in DELETE subquery (issue #213) -- The implicit form should work inside subqueries of DML statements DELETE FROM implicit_docs WHERE id = ( SELECT id FROM implicit_docs ORDER BY content <@> 'database' LIMIT 1 ); -- Verify the top-scoring document for 'database' was deleted SELECT id, content FROM implicit_docs ORDER BY content <@> 'database' LIMIT 3; -- Re-insert the deleted row for subsequent tests INSERT INTO implicit_docs (content) VALUES ('database system design'); -- Test 12: Implicit text <@> text in UPDATE subquery UPDATE implicit_docs SET content = 'updated via subquery' WHERE id = ( SELECT id FROM implicit_docs ORDER BY content <@> 'hello' LIMIT 1 ); -- Verify the update happened to the top-scoring 'hello' document SELECT id, content FROM implicit_docs ORDER BY id; -- Test 13: Implicit form in UPDATE SET subquery UPDATE implicit_docs SET content = ( SELECT content FROM implicit_docs ORDER BY content <@> 'database' LIMIT 1 ) WHERE id = (SELECT max(id) FROM implicit_docs); SELECT id, content FROM implicit_docs ORDER BY id; -- Test 14: Nested subquery (exercises recursive SubLink handling) SELECT id, content FROM implicit_docs WHERE id IN ( SELECT id FROM implicit_docs WHERE id = ( SELECT id FROM implicit_docs ORDER BY content <@> 'database' LIMIT 1 ) ); -- Cleanup DROP TABLE implicit_schema.schema_docs CASCADE; DROP SCHEMA implicit_schema CASCADE; DROP TABLE multi_col_docs CASCADE; DROP TABLE implicit_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/index.sql000066400000000000000000000025071525762047700223730ustar00rootroot00000000000000-- Test pg_textsearch index access method functionality -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Setup test table CREATE TABLE test_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO test_docs (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz'), ('hello world example'), ('postgresql full text search'); -- Test index creation with different text_config options CREATE INDEX docs_english_idx ON test_docs USING bm25(content) WITH (text_config='english'); CREATE INDEX docs_simple_idx ON test_docs USING bm25(content) WITH (text_config='simple', k1=1.5, b=0.8); -- Verify indexes were created \d+ test_docs -- Test basic index operations INSERT INTO test_docs (content) VALUES ('new document for testing'); -- Test that index options are correctly stored and retrievable SELECT i.relname as index_name, CASE WHEN i.reloptions IS NOT NULL THEN 'has_options' ELSE 'no_options' END as options_status FROM pg_class i WHERE i.relname IN ('docs_english_idx', 'docs_simple_idx') ORDER BY i.relname; -- Cleanup DROP INDEX docs_english_idx; DROP INDEX docs_simple_idx; DROP TABLE test_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/inheritance.sql000066400000000000000000000216101525762047700235510ustar00rootroot00000000000000-- Test BM25 index behavior with table inheritance -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================================= -- Test 1: PostgreSQL native table inheritance -- BM25 indexes on parent tables only index direct rows (ONLY semantics), -- not inherited rows from child tables -- ============================================================================= CREATE TABLE parent_docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE child_docs1 ( category TEXT DEFAULT 'category1' ) INHERITS (parent_docs); CREATE TABLE child_docs2 ( category TEXT DEFAULT 'category2' ) INHERITS (parent_docs); -- Insert data into child tables INSERT INTO child_docs1 (content) VALUES ('The quick brown fox jumps over the lazy dog'), ('PostgreSQL inheritance allows tables to share structure'); INSERT INTO child_docs2 (content) VALUES ('Database normalization reduces redundancy'), ('Indexes help improve query performance'); -- Verify parent sees inherited data SELECT COUNT(*) as parent_count FROM parent_docs; SELECT COUNT(*) as parent_only_count FROM ONLY parent_docs; -- BM25 index on parent only indexes direct rows (0 documents) \set VERBOSITY terse CREATE INDEX parent_bm25_idx ON parent_docs USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- BM25 index on child table should work CREATE INDEX child1_bm25_idx ON child_docs1 USING bm25(content) WITH (text_config='english'); -- Verify child index was created successfully with correct document count SELECT bm25_dump_index('child1_bm25_idx')::text ~ 'total_docs: 2' as has_two_docs; -- BM25 search on the child table returns its top hit. SELECT content FROM child_docs1 ORDER BY content <@> to_bm25query('postgresql inheritance', 'child1_bm25_idx') LIMIT 1; -- Cleanup inheritance test DROP TABLE child_docs1 CASCADE; DROP TABLE child_docs2 CASCADE; DROP TABLE parent_docs CASCADE; -- ============================================================================= -- Test 2: Regular tables (should work normally) -- ============================================================================= CREATE TABLE regular_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO regular_docs (content) VALUES ('Regular table without inheritance'), ('Should work perfectly with BM25'); -- BM25 index on regular table should work CREATE INDEX regular_bm25_idx ON regular_docs USING bm25(content) WITH (text_config='english'); -- Verify regular index works SELECT bm25_dump_index('regular_bm25_idx')::text ~ 'total_docs: 2' as has_two_docs; -- Cleanup DROP TABLE regular_docs CASCADE; -- ============================================================================= -- Test 3: Inheritance scoring via find_first_child_bm25_index -- When querying a parent index that has 0 docs, the scoring code -- falls back to the first child's BM25 index for actual scoring. -- ============================================================================= CREATE TABLE inh_parent ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE inh_child ( category TEXT DEFAULT 'cat1' ) INHERITS (inh_parent); -- Insert data only into child INSERT INTO inh_child (content) VALUES ('the quick brown fox jumps over the lazy dog'), ('postgresql full text search engine'), ('database query optimization techniques'), ('information retrieval and ranking'); -- Create BM25 index on parent (will have 0 docs since data is in child) \set VERBOSITY terse CREATE INDEX inh_parent_bm25 ON inh_parent USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- Create BM25 index on child (will have the actual data) CREATE INDEX inh_child_bm25 ON inh_child USING bm25(content) WITH (text_config='english'); -- Query via parent index — should fall back to child index. -- ORDER BY makes this an index scan, which resolves the parent in the -- planner, so it does not reach find_first_child_bm25_index. SELECT content, content <@> to_bm25query('fox', 'inh_parent_bm25') AS score FROM inh_parent ORDER BY content <@> to_bm25query('fox', 'inh_parent_bm25') LIMIT 3; -- Standalone scoring against the storage-less parent. Deliberately no -- ORDER BY: that is what forces the standalone path (the documented -- exception for tests whose target *is* standalone scoring), which is -- the only caller of find_first_child_bm25_index. Scoring every row -- exercises the parent -> first-child hop, which swaps the relation, -- the index state, the segment snapshot and the memtable source -- together; non-matching rows score 0. -- The plan assertion guards against this silently becoming an index -- scan, which is how the query above stopped covering this path. EXPLAIN (COSTS OFF) SELECT content, content <@> to_bm25query('fox', 'inh_parent_bm25') FROM inh_parent; SELECT content, round((content <@> to_bm25query('fox', 'inh_parent_bm25'))::numeric, 4) AS score FROM inh_parent ORDER BY content; -- Cleanup DROP TABLE inh_child CASCADE; DROP TABLE inh_parent CASCADE; -- ============================================================================= -- Test 4: Inheritance with dropped columns (attnum drift) -- When a parent table has dropped columns, child tables created after -- the drop get different physical attnum values for the same logical -- column. BM25 index matching must handle this by comparing column -- names rather than raw attnums. (GitHub issue #288) -- ============================================================================= -- 4a: Index scan path (tests indexes_match_by_attribute in scan.c) CREATE TABLE drift_parent ( id SERIAL PRIMARY KEY, extra_col TEXT, content TEXT ); -- Insert data into parent before dropping column INSERT INTO drift_parent (extra_col, content) VALUES ('filler', 'the quick brown fox jumps over the lazy dog'), ('filler', 'postgresql full text search engine'); -- Drop a column to create an attnum gap in the parent ALTER TABLE drift_parent DROP COLUMN extra_col; -- Create child AFTER the column drop. The child will NOT inherit the -- dropped-column slot, so "content" gets a different attnum than in -- the parent. CREATE TABLE drift_child () INHERITS (drift_parent); INSERT INTO drift_child (content) VALUES ('memory store retrieval and caching layer'), ('database query optimization techniques'); -- Create BM25 indexes on both parent and child \set VERBOSITY terse CREATE INDEX drift_parent_bm25 ON drift_parent USING bm25(content) WITH (text_config='english'); \set VERBOSITY default CREATE INDEX drift_child_bm25 ON drift_child USING bm25(content) WITH (text_config='english'); -- Verify attnum drift exists: parent content attnum != child content attnum SELECT (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_parent_bm25'::regclass) AS parent_attnum, (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_child_bm25'::regclass) AS child_attnum, (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_parent_bm25'::regclass) <> (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift_child_bm25'::regclass) AS attnums_differ; -- Index scan through parent should work despite attnum mismatch. -- Before the fix, this failed with "tpquery index mismatch". SET enable_seqscan = off; SELECT id, content <@> 'fox' AS score FROM drift_parent ORDER BY content <@> 'fox' LIMIT 3; RESET enable_seqscan; -- Cleanup DROP TABLE drift_child CASCADE; DROP TABLE drift_parent CASCADE; -- 4b: Scoring fallback path (tests find_first_child_bm25_index in -- query.c). Parent index has 0 docs so the scorer falls back to -- the first child index. With attnum drift that lookup used to fail. CREATE TABLE drift2_parent ( id SERIAL PRIMARY KEY, extra_col TEXT, content TEXT ); ALTER TABLE drift2_parent DROP COLUMN extra_col; CREATE TABLE drift2_child () INHERITS (drift2_parent); INSERT INTO drift2_child (content) VALUES ('the quick brown fox jumps over the lazy dog'), ('postgresql full text search engine'), ('database query optimization techniques'), ('information retrieval and ranking'); \set VERBOSITY terse CREATE INDEX drift2_parent_bm25 ON drift2_parent USING bm25(content) WITH (text_config='english'); \set VERBOSITY default CREATE INDEX drift2_child_bm25 ON drift2_child USING bm25(content) WITH (text_config='english'); -- Verify attnum drift SELECT (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift2_parent_bm25'::regclass) <> (SELECT indkey[0] FROM pg_index WHERE indexrelid = 'drift2_child_bm25'::regclass) AS attnums_differ; -- Query via parent index — should fall back to child index for scoring SELECT content, content <@> to_bm25query('fox', 'drift2_parent_bm25') AS score FROM drift2_parent ORDER BY content <@> to_bm25query('fox', 'drift2_parent_bm25') LIMIT 3; -- Cleanup DROP TABLE drift2_child CASCADE; DROP TABLE drift2_parent CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/large_documents.sql000066400000000000000000000036471525762047700244450ustar00rootroot00000000000000-- Test indexing of documents whose unique-token volume would otherwise -- exceed Postgres's tsvector 1 MB lexeme dictionary cap. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET client_min_messages = WARNING; -- Sanity: bare to_tsvector still rejects oversized inputs. DO $$ DECLARE big_doc text; BEGIN SELECT string_agg('w' || gs::text, ' ') INTO big_doc FROM generate_series(1, 250000) gs; PERFORM to_tsvector('english', big_doc); RAISE EXCEPTION 'to_tsvector unexpectedly succeeded'; EXCEPTION WHEN program_limit_exceeded THEN RAISE NOTICE 'to_tsvector rejected oversized input as expected'; END $$; -- CREATE INDEX over a >2 MB many-unique-token document must succeed. CREATE TABLE large_docs (id int, content text); INSERT INTO large_docs VALUES (1, 'short doc with hello world'); INSERT INTO large_docs (id, content) SELECT 2, string_agg('w' || gs::text, ' ') FROM generate_series(1, 250000) gs; INSERT INTO large_docs VALUES (3, 'another short doc mentioning hello'); SELECT id, length(content) > 1048576 AS oversized FROM large_docs ORDER BY id; CREATE INDEX large_idx ON large_docs USING bm25 (content text_bm25_ops) WITH (text_config = 'english'); -- Index scan: a token from the large doc must be retrievable. SELECT id FROM large_docs ORDER BY content <@> to_bm25query('w12345', 'large_idx') LIMIT 1; -- Index scan over a token that is in two short docs (sanity). SELECT id FROM large_docs ORDER BY content <@> to_bm25query('hello', 'large_idx') LIMIT 5; -- INSERT (aminsert) of a >2 MB many-unique-token row. INSERT INTO large_docs (id, content) SELECT 4, string_agg('z' || gs::text, ' ') FROM generate_series(1, 250000) gs; SELECT id FROM large_docs ORDER BY content <@> to_bm25query('z67890', 'large_idx') LIMIT 1; -- VACUUM after deleting the large rows must succeed. DELETE FROM large_docs WHERE id IN (2, 4); VACUUM large_docs; DROP TABLE large_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/limits.sql000066400000000000000000000172211525762047700225640ustar00rootroot00000000000000-- Test file for LIMIT detection and optimization in Tapir -- Disable duration logging to avoid timing differences in sanitizer tests SET log_duration = off; -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Create test table with sufficient data for meaningful LIMIT testing CREATE TABLE limit_test ( id SERIAL PRIMARY KEY, title TEXT, content TEXT ); -- Insert test data with varied content for BM25 scoring INSERT INTO limit_test (title, content) VALUES ('Database Systems', 'postgresql database management with advanced indexing and query optimization'), ('Search Algorithms', 'full text search algorithms with relevance scoring and ranking mechanisms'), ('Information Retrieval', 'information retrieval systems using vector space models and term frequency'), ('Machine Learning', 'machine learning algorithms for data analysis and pattern recognition'), ('Data Mining', 'data mining techniques for knowledge discovery in large databases'), ('Text Processing', 'natural language processing and text analysis with stemming algorithms'), ('Database Performance', 'database optimization strategies for improving query performance'), ('Search Engines', 'search engine design with ranking algorithms and user experience'), ('Big Data Analytics', 'big data processing platforms for massive dataset analysis'), ('Distributed Systems', 'distributed computing and scalability in cloud environments'), ('Database Architecture', 'database system architecture with storage and indexing strategies'), ('Query Optimization', 'query optimization techniques for efficient database operations'), ('Text Analytics', 'text mining and document classification using statistical methods'), ('Information Systems', 'information management systems with data organization principles'), ('Database Theory', 'theoretical foundations of database systems and relational algebra'), ('Search Technology', 'search technology innovations with semantic understanding capabilities'), ('Data Structures', 'efficient data structures for database storage and retrieval operations'), ('Algorithm Design', 'algorithm design principles for optimal computational performance'), ('System Performance', 'system performance tuning and resource optimization strategies'), ('Database Security', 'database security measures and access control mechanisms'); -- Create pg_textsearch index CREATE INDEX limit_test_idx ON limit_test USING bm25(content) WITH (text_config='english'); -- Test 1: Basic LIMIT functionality -- Should detect and optimize for LIMIT 5 EXPLAIN (COSTS OFF) SELECT title, content, content <@> 'database' as score FROM limit_test ORDER BY content <@> 'database' LIMIT 5; SELECT title, content, ROUND((content <@> 'database')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'database' LIMIT 5; -- Test 2: Different LIMIT values -- Test LIMIT 1 (should be highly optimized) SELECT title, ROUND((content <@> 'search')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'search' LIMIT 1; -- Test LIMIT 3 SELECT title, ROUND((content <@> 'optimization')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'optimization' LIMIT 3; -- Test LIMIT 10 SELECT title, ROUND((content <@> 'algorithm')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'algorithm' LIMIT 10; -- Test 3: LIMIT with WHERE clause (should prevent pushdown for safety) -- This should NOT use LIMIT pushdown due to additional WHERE clause EXPLAIN (COSTS OFF) SELECT title, ROUND((content <@> 'database system')::numeric, 4) as score FROM limit_test WHERE id > 5 ORDER BY content <@> 'database system' LIMIT 7; SELECT title, ROUND((content <@> 'database system')::numeric, 4) as score FROM limit_test WHERE id > 5 ORDER BY content <@> 'database system' LIMIT 7; -- Test 4: No LIMIT (should use default behavior) -- Note: Query plan varies by PG version - not testing EXPLAIN here -- Test 5: LIMIT with OFFSET SELECT title, ROUND((content <@> 'performance')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'performance' LIMIT 5 OFFSET 2; -- Test 6: Very small LIMIT (edge case) SELECT title, ROUND((content <@> 'text')::numeric, 4) as score FROM limit_test ORDER BY content <@> 'text' LIMIT 1; -- Test 7: Large LIMIT (should still use index optimization) SELECT COUNT(*) > 0 as has_results FROM ( SELECT title, content <@> to_bm25query('xyzabc123', 'limit_test_idx') as score FROM limit_test -- Test for non-matching terms (should return no results) ORDER BY content <@> to_bm25query('xyzabc123', 'limit_test_idx') LIMIT 1000 ) subq; -- Test 8: LIMIT in subquery (requires explicit index name) SELECT * FROM ( SELECT title, ROUND((content <@> to_bm25query('mining', 'limit_test_idx'))::numeric, 4) as score FROM limit_test ORDER BY content <@> to_bm25query('mining', 'limit_test_idx') LIMIT 3 ) sub; -- Test 9: Multiple queries with different LIMIT values to test limit storage/cleanup SELECT 'Query 1' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('database', 'limit_test_idx') LIMIT 2 ) q1; SELECT 'Query 2' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('search', 'limit_test_idx') LIMIT 8 ) q2; SELECT 'Query 3' as query_name, COUNT(*) as results FROM ( SELECT title FROM limit_test ORDER BY content <@> to_bm25query('algorithm', 'limit_test_idx') LIMIT 4 ) q3; -- Test 10: LIMIT pushdown safety verification -- These tests verify that LIMIT pushdown is only used when safe -- Safe case: Simple ORDER BY with pg_textsearch score, no WHERE clause -- This SHOULD allow LIMIT pushdown EXPLAIN (COSTS OFF) SELECT title, content <@> 'simple' as score FROM limit_test ORDER BY content <@> 'simple' LIMIT 3; -- Case: Multiple ORDER BY clauses -- Tapir requires exactly one ORDER BY, so not using the index here is correct behavior -- Query plan varies by PG version (Sort vs Incremental Sort) - not testing EXPLAIN -- Unsafe case: Complex WHERE clause with additional conditions -- This should NOT allow LIMIT pushdown due to WHERE clause -- Query plan varies by PG version - not testing EXPLAIN -- Test 11: Basic LIMIT functionality verification -- Test with safe pushdown case SELECT COUNT(*) as pushdown_safe_count FROM ( SELECT title, content <@> to_bm25query('pushdown_safe', 'limit_test_idx') as score FROM limit_test ORDER BY content <@> to_bm25query('pushdown_safe', 'limit_test_idx') LIMIT 2 ) subq; -- Test with unsafe pushdown case (has WHERE clause) SELECT COUNT(*) as pushdown_unsafe_count FROM ( SELECT title, content <@> to_bm25query('pushdown_unsafe', 'limit_test_idx') as score FROM limit_test WHERE id % 2 = 0 ORDER BY content <@> to_bm25query('pushdown_unsafe', 'limit_test_idx') LIMIT 2 ) subq; -- Test 12: Edge cases for LIMIT pushdown -- Very large LIMIT (should still use index optimization efficiently) SELECT COUNT(*) > 0 as large_limit_has_results FROM ( SELECT title, content <@> to_bm25query('qwertyuiop999', 'limit_test_idx') as score FROM limit_test -- Test with non-matching nonsense term ORDER BY content <@> to_bm25query('qwertyuiop999', 'limit_test_idx') LIMIT 50000 -- Much larger than our dataset ) subq; -- LIMIT 0 edge case SELECT title, content <@> to_bm25query('zero_limit', 'limit_test_idx') as score FROM limit_test ORDER BY content <@> to_bm25query('zero_limit', 'limit_test_idx') LIMIT 0; -- Cleanup DROP TABLE limit_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/lock.sql000066400000000000000000000030301525762047700222040ustar00rootroot00000000000000-- Test lock upgrade from shared to exclusive within a single transaction -- This exercises the tp_acquire_index_lock upgrade path CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table and index CREATE TABLE lock_upgrade_test ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO lock_upgrade_test (content) VALUES ('database systems and concurrent access'), ('full text search indexing'), ('query optimization techniques'); CREATE INDEX lock_upgrade_idx ON lock_upgrade_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Force index scan (small tables default to seq scan) SET enable_seqscan = off; SET enable_bitmapscan = off; -- Test: Single transaction that first reads (shared lock) then writes (exclusive lock) -- This should trigger the lock upgrade path in tp_acquire_index_lock BEGIN; -- First operation: SELECT acquires shared lock via index scan SELECT id, content FROM lock_upgrade_test ORDER BY content <@> 'database' LIMIT 5; -- Second operation: INSERT in same transaction should upgrade to exclusive lock INSERT INTO lock_upgrade_test (content) VALUES ('new document about database concurrency'); -- Verify the insert worked SELECT COUNT(*) FROM lock_upgrade_test; COMMIT; -- Verify final state SELECT COUNT(*) FROM lock_upgrade_test; -- Test that subsequent operations work normally SELECT id, content FROM lock_upgrade_test ORDER BY content <@> 'database concurrency' LIMIT 5; -- Clean up DROP TABLE lock_upgrade_test CASCADE; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/manyterms.sql000066400000000000000000000045631525762047700233070ustar00rootroot00000000000000-- Test hash table automatic resizing with many terms -- This test creates enough unique terms to trigger hash table resize -- and verifies the system continues to work correctly CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Create test table CREATE TABLE manyterms_test (id serial, content text); -- Create index to initialize hash table with small initial size CREATE INDEX manyterms_idx ON manyterms_test USING bm25(content) WITH (text_config='english'); -- Add enough unique terms to exceed initial hash table capacity -- With 64 buckets at 0.75 load factor, >48 unique terms should trigger automatic resize INSERT INTO manyterms_test (content) VALUES ('alpha'), ('beta'), ('gamma'), ('delta'), ('epsilon'), ('zeta'), ('eta'), ('theta'), ('iota'), ('kappa'), ('lambda'), ('mu'), ('nu'), ('xi'), ('omicron'), ('pi'), ('rho'), ('sigma'), ('tau'), ('upsilon'), ('phi'), ('chi'), ('psi'), ('omega'), ('one'), ('two'), ('three'), ('four'), ('five'), ('six'), ('seven'), ('eight'), ('nine'), ('ten'), ('eleven'), ('twelve'), ('thirteen'), ('fourteen'), ('fifteen'), ('sixteen'), ('seventeen'), ('eighteen'), ('nineteen'), ('twenty'), ('twentyone'), ('twentytwo'), ('twentythree'), ('twentyfour'), ('twentyfive'), ('twentysix'), ('twentyseven'), ('twentyeight'), ('twentynine'), ('thirty'), ('thirtyone'), ('thirtytwo'), ('thirtythree'), ('thirtyfour'), ('thirtyfive'), ('thirtysix'), ('thirtyseven'), ('thirtyeight'), ('thirtynine'), ('forty'), ('fortyone'), ('fortytwo'), ('fortythree'), ('fortyfour'), ('fortyfive'), ('fortysix'), ('fortyseven'), ('fortyeight'), ('fortynine'), ('fifty'), ('fiftyone'), ('fiftytwo'), ('fiftythree'), ('fiftyfour'), ('fiftyfive'), ('fiftysix'); -- Verify we can count all documents successfully (no errors from resize) SELECT COUNT(*) as total_docs FROM manyterms_test; -- Test searching for specific terms works correctly after resize SELECT id, content FROM manyterms_test WHERE content = 'alpha' ORDER BY id LIMIT 1; SELECT id, content FROM manyterms_test WHERE content = 'omega' ORDER BY id LIMIT 1; -- Test BM25 scoring still works after hash table resize SELECT id, content, ROUND((content <@> 'fifty')::numeric, 4) as score FROM manyterms_test WHERE content = 'fifty' ORDER BY score LIMIT 1; -- Clean up DROP INDEX manyterms_idx; DROP TABLE manyterms_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memory.sql000066400000000000000000000034121525762047700225700ustar00rootroot00000000000000-- Test bulk insert with auto-spill for pg_textsearch indexes -- Create extension if not exists CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create test table DROP TABLE IF EXISTS memory_test CASCADE; CREATE TABLE memory_test (id serial PRIMARY KEY, content text); -- Suppress auto-spill NOTICE messages during test to avoid non-deterministic output -- (The exact DSA memory values vary between runs) SET client_min_messages = WARNING; -- Create an index CREATE INDEX idx_memory_test ON memory_test USING bm25(content) WITH (text_config='english'); -- Insert some documents INSERT INTO memory_test (content) VALUES ('This is a test document with some content'); INSERT INTO memory_test (content) VALUES ('Another document with different words'); -- Insert many documents with lots of unique terms to exercise auto-spill. -- With 1000 docs * 100 unique terms = 100K terms per transaction, -- this should trigger the bulk_load_threshold (default 100K). DO $$ DECLARE i integer; doc text; BEGIN FOR i IN 1..1000 LOOP -- Generate document with unique terms to increase memory usage doc := 'Document number ' || i || ' with unique terms: '; -- Add lots of unique words per document FOR j IN 1..100 LOOP doc := doc || ' term_' || i || '_' || j; END LOOP; INSERT INTO memory_test (content) VALUES (doc); END LOOP; END $$; -- Re-enable notices for verification output RESET client_min_messages; -- Verify that all documents were inserted and index works SELECT COUNT(*) AS total_docs FROM memory_test; -- Search should find documents SELECT id, left(content, 50) AS content_preview FROM memory_test ORDER BY content <@> 'test' LIMIT 5; -- Clean up DROP TABLE memory_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memtable_append.sql000066400000000000000000000077471525762047700244140ustar00rootroot00000000000000-- Unit-level coverage for the on-disk write path -- tp_memtable_append() (memtable v2, issue #374), driven through -- the bm25_test_memtable_append() scaffold function, plus the -- chain-inspection SRF bm25_memtable_chain(). -- -- These tests use an ordinary pg_textsearch index purely as a -- host for the memtable pages; the assertions are about the -- chain layout, not end-to-end query semantics (which the rest -- of the regression suite covers). CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE memtable_append_t (id int, body text); CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); -- Empty index: head and tail are InvalidBlockNumber, chain is empty. SELECT bm25_test_memtable_append('memtable_append_idx', 'empty_index'); SELECT count(*) AS chain_pages FROM bm25_memtable_chain('memtable_append_idx'); -- One append: head == tail, chain length 1, page has 1 record. SELECT bm25_test_memtable_append('memtable_append_idx', 'single_append'); SELECT n_records, next_block IS NULL AS is_tail FROM bm25_memtable_chain('memtable_append_idx'); -- Drop and recreate so each scenario starts with an empty chain. DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); -- Many small appends all fit on the first page. SELECT bm25_test_memtable_append('memtable_append_idx', 'fits_on_one_page'); SELECT count(*) AS chain_pages, sum(n_records)::int AS total_records FROM bm25_memtable_chain('memtable_append_idx'); DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); -- Larger records force the chain to extend past one page. SELECT bm25_test_memtable_append('memtable_append_idx', 'extends_chain'); SELECT (count(*) >= 2) AS chain_extended, sum(n_records) > 0 AS has_records FROM bm25_memtable_chain('memtable_append_idx'); DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); -- ctid, doc_length, and vector_bytes all round-trip exactly. SELECT bm25_test_memtable_append('memtable_append_idx', 'roundtrip'); DROP INDEX memtable_append_idx; CREATE INDEX memtable_append_idx ON memtable_append_t USING bm25 (body) WITH (text_config = 'english'); -- Oversized record goes through the multi-page (fragment) path: -- a fragment head page followed by N continuation pages and a -- fresh tail page, with a small bookend record on either side -- to verify the head record stays on the original page and the -- new tail can still accept subsequent appends. SELECT bm25_test_memtable_append('memtable_append_idx', 'multi_page_fragment'); -- Page shape: -- small_record_page (n_records=1, flags=0) -- fragment_head (n_records=1, FRAGMENT flag set on the record; -- page flags=0) -- continuation*N (n_records=0, CONTINUATION flag set on the page, -- page flags=0x02) -- new_tail (n_records=1, flags=0) SELECT n_records FROM bm25_memtable_chain('memtable_append_idx'); -- Assert page-level flag bits. The continuation pages must carry -- TP_MEMTABLE_PAGE_FLAG_CONTINUATION (0x02); every other page must -- have flags = 0. Iterated record_count of fragment continuation -- pages is 0 because the fragment head holds the record's metadata -- and continuations carry only opaque tail bytes. SELECT sum(((flags & 2) <> 0)::int)::int AS continuation_pages, sum(CASE WHEN (flags & 2) <> 0 THEN 0 ELSE 1 END)::int AS regular_pages, bool_and((flags & 2) <> 0 OR flags = 0) AS only_known_flags FROM bm25_memtable_chain('memtable_append_idx'); SELECT bool_and( CASE WHEN (flags & 2) <> 0 THEN n_records = 0 ELSE n_records >= 0 END ) AS continuation_pages_have_zero_records FROM bm25_memtable_chain('memtable_append_idx'); -- Clean up DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memtable_page.sql000066400000000000000000000027761525762047700240560ustar00rootroot00000000000000-- Unit-level coverage for the new on-disk memtable page format. -- Exercises tp_memtable_page_{init,append,iterate,can_fit,is_valid} -- via the bm25_test_memtable_page() scaffold function. Each case -- returns 'OK' on success or 'FAIL: ' otherwise. -- -- Memtable v2 page format (see plan / issue #374). CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- An uninitialized (zero-filled) page must be rejected. SELECT bm25_test_memtable_page('reject_uninitialized'); -- After init, the page is valid, has 0 records, an invalid next_block, -- and free_offset pointing past the header. SELECT bm25_test_memtable_page('init_empty'); -- Iteration over an empty page yields no records. SELECT bm25_test_memtable_page('iterate_empty'); -- A single appended record round-trips its ctid, doc_length, and -- vector bytes verbatim. SELECT bm25_test_memtable_page('one_record_roundtrip'); -- Many records of varying sizes appended sequentially are all -- recoverable in insertion order. SELECT bm25_test_memtable_page('many_records_roundtrip'); -- can_fit() returns false when the requested record would not fit. SELECT bm25_test_memtable_page('can_fit_boundary'); -- Appending records until the page is full leaves the page in a -- consistent state (n_records matches what was actually appended, -- and iteration walks them all). SELECT bm25_test_memtable_page('fill_until_full'); -- next_block setter / getter round-trip. SELECT bm25_test_memtable_page('next_block_roundtrip'); -- Clean up DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memtable_reclaim.sql000066400000000000000000000131761525762047700245520ustar00rootroot00000000000000-- Regression: Step 2 FSM reclaim of DEAD memtable orphans. -- After spill + VACUUM, tp_reclaim_dead_memtable_pages returns blocks -- to the index FSM; tp_memtable_alloc_page should reuse them when the -- chain grows again (main fork should not grow by ~one block per dead -- page if reuse works). CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Temporary relations have a session-local reclaim horizon, unaffected -- by other backends or standby feedback. CREATE TEMP TABLE memtable_reclaim_t (id serial PRIMARY KEY, body text); CREATE INDEX memtable_reclaim_idx ON memtable_reclaim_t USING bm25(body) WITH (text_config = 'english'); SELECT count(*)::int AS dead_pages_before_spill FROM bm25_memtable_dead_pages('memtable_reclaim_idx'); INSERT INTO memtable_reclaim_t (body) SELECT 'reclaim ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; CREATE TEMP TABLE spill_stats AS SELECT count(*)::int AS live_pages FROM bm25_memtable_chain('memtable_reclaim_idx'); SELECT live_pages > 1 AS chain_multi_page FROM spill_stats; SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS spill_ok; SELECT count(*)::int AS live_chain_after_spill FROM bm25_memtable_chain('memtable_reclaim_idx'); CREATE TEMP TABLE reclaim_dead AS SELECT count(*)::int AS dead_count FROM bm25_memtable_dead_pages('memtable_reclaim_idx'); SELECT dead_count > 0 AS spilled_dead_pages FROM reclaim_dead; SELECT r.dead_count = s.live_pages AS dead_matches_spilled_chain FROM reclaim_dead r, spill_stats s; CREATE TEMP TABLE reclaim_sizes AS SELECT pg_relation_size('memtable_reclaim_idx'::regclass, 'main')::bigint AS sz_after_spill; -- Horizon: spill xact is committed (autocommit per statement). VACUUM ANALYZE memtable_reclaim_t; -- Idempotent second pass (RecordFreeIndexPage on same blocks is safe). VACUUM ANALYZE memtable_reclaim_t; SELECT count(*) >= 1 AS search_after_vacuum FROM ( SELECT 1 FROM memtable_reclaim_t ORDER BY body <@> to_bm25query('reclaim', 'memtable_reclaim_idx') LIMIT 10 ) sub; -- Rebuild chain without auto-spill so pages come from FSM vs extend. SET pg_textsearch.memtable_pages_threshold = 0; INSERT INTO memtable_reclaim_t (body) SELECT 'reclaim2 ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; SELECT count(*)::int > 0 AS chain_rebuilt FROM bm25_memtable_chain('memtable_reclaim_idx'); -- VACUUM freed the DEAD pages and this rebuild reused them, so the main -- fork does not grow and no DEAD pages remain. SELECT (pg_relation_size('memtable_reclaim_idx'::regclass, 'main') - (SELECT sz_after_spill FROM reclaim_sizes)) = 0 AND (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_reclaim_idx')) = 0 AS single_cycle_growth_valid; -- Multi-cycle: five spill→VACUUM cycles (VACUUM cannot run inside DO). -- With per-cycle FSM reuse, cumulative growth stays bounded by the live -- memtable work per cycle; without reuse it would grow ~N×K pages. CREATE TEMP TABLE multi_cycle_bounds ( sz_start bigint, max_live_pages int, num_cycles int ); INSERT INTO multi_cycle_bounds (sz_start, max_live_pages, num_cycles) SELECT pg_relation_size('memtable_reclaim_idx'::regclass, 'main')::bigint, 0, 5; -- Cycle 1 INSERT INTO memtable_reclaim_t (body) SELECT 'mc1 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_1; VACUUM ANALYZE memtable_reclaim_t; -- Cycle 2 INSERT INTO memtable_reclaim_t (body) SELECT 'mc2 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_2; VACUUM ANALYZE memtable_reclaim_t; -- Cycle 3 INSERT INTO memtable_reclaim_t (body) SELECT 'mc3 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_3; VACUUM ANALYZE memtable_reclaim_t; -- Cycle 4 INSERT INTO memtable_reclaim_t (body) SELECT 'mc4 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_4; VACUUM ANALYZE memtable_reclaim_t; -- Cycle 5 INSERT INTO memtable_reclaim_t (body) SELECT 'mc5 doc ' || g || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) g; UPDATE multi_cycle_bounds SET max_live_pages = GREATEST( max_live_pages, (SELECT count(*)::int FROM bm25_memtable_chain('memtable_reclaim_idx'))); SELECT bm25_spill_index('memtable_reclaim_idx') IS NOT NULL AS mc_spill_5; VACUUM ANALYZE memtable_reclaim_t; SELECT max_live_pages > 1 AS multi_cycle_chain_multi_page FROM multi_cycle_bounds; -- Cumulative growth must stay within one live chain per cycle -- (num_cycles * max_live_pages). SELECT (pg_relation_size('memtable_reclaim_idx'::regclass, 'main') - (SELECT sz_start FROM multi_cycle_bounds)) <= (SELECT num_cycles * max_live_pages FROM multi_cycle_bounds) * current_setting('block_size')::bigint AS multi_cycle_growth_valid; DROP TABLE memtable_reclaim_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memtable_spill.sql000066400000000000000000000072451525762047700242610ustar00rootroot00000000000000-- Tests the on-disk memtable chain lifecycle introduced by the -- page-backed memtable rewrite (issue #374): -- -- * inserts append to chain pages, -- * manual bm25_spill_index() drains the chain into an L0 -- segment, -- * memtable_pages_threshold triggers an auto-spill at -- transaction end, -- * queries return correct results across spills. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE memtable_spill_test (id serial PRIMARY KEY, body text); CREATE INDEX memtable_spill_idx ON memtable_spill_test USING bm25(body) WITH (text_config = 'english'); -- Test 1: a freshly built empty index has no chain pages -- (metapage.memtable_head_blkno = InvalidBlockNumber). SELECT count(*) AS chain_pages_at_create FROM bm25_memtable_chain('memtable_spill_idx'); -- Test 2: inserts populate the chain. Use enough distinct rows -- that more than one chain page is required (records are -- ~40-80 bytes after packing, plus per-doc TpVector overhead). INSERT INTO memtable_spill_test (body) SELECT 'alpha doc ' || i || ' ' || repeat('filler ', 4) FROM generate_series(1, 200) i; SELECT count(*) > 1 AS chain_grew_past_one_page FROM bm25_memtable_chain('memtable_spill_idx'); -- Test 3: queries see memtable contents directly (no segments yet). SELECT count(*) AS alpha_hits_before_spill FROM ( SELECT 1 FROM memtable_spill_test ORDER BY body <@> to_bm25query('alpha', 'memtable_spill_idx') LIMIT 1000 ) sub; -- Test 4: manual spill flushes the chain into an L0 segment and -- resets metapage.memtable_head_blkno/tail_blkno to Invalid. SELECT bm25_spill_index('memtable_spill_idx') IS NOT NULL AS produced_segment; SELECT count(*) AS chain_pages_after_spill FROM bm25_memtable_chain('memtable_spill_idx'); -- Test 5: queries still return the same answers (data is now in -- the segment, not the memtable). SELECT count(*) AS alpha_hits_after_spill FROM ( SELECT 1 FROM memtable_spill_test ORDER BY body <@> to_bm25query('alpha', 'memtable_spill_idx') LIMIT 1000 ) sub; -- Test 6: spilling an empty chain is a no-op and returns NULL. SELECT bm25_spill_index('memtable_spill_idx') AS empty_chain_spill; -- Test 7: subsequent inserts build a fresh chain. Run them in -- a transaction with the auto-spill threshold disabled so the -- chain remains observable at COMMIT. SET pg_textsearch.memtable_pages_threshold = 0; BEGIN; INSERT INTO memtable_spill_test (body) SELECT 'beta ' || i FROM generate_series(1, 30) i; SELECT count(*) > 0 AS chain_rebuilt FROM bm25_memtable_chain('memtable_spill_idx'); COMMIT; -- After commit with threshold = 0 the chain still has the new -- entries; verify they survive across the COMMIT boundary. SELECT count(*) > 0 AS chain_persists_after_commit FROM bm25_memtable_chain('memtable_spill_idx'); -- Test 8: memtable_pages_threshold (in chain pages) drains the -- chain at COMMIT. Spill once to start from empty, then insert -- past the threshold and verify the chain ends empty. SELECT bm25_spill_index('memtable_spill_idx') IS NOT NULL OR true AS reset; SET pg_textsearch.memtable_pages_threshold = 1; INSERT INTO memtable_spill_test (body) SELECT 'gamma ' || i || ' delta ' || (i+1) || ' epsilon ' || (i+2) FROM generate_series(1, 100) i; SELECT count(*) AS chain_pages_after_autospill FROM bm25_memtable_chain('memtable_spill_idx'); RESET pg_textsearch.memtable_pages_threshold; -- Test 9: query the (now-spilled) data to confirm end-to-end -- correctness across the auto-spill path. SELECT count(*) > 0 AS gamma_queryable FROM ( SELECT 1 FROM memtable_spill_test ORDER BY body <@> to_bm25query('gamma', 'memtable_spill_idx') LIMIT 1000 ) sub; DROP TABLE memtable_spill_test; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memtable_spill_dead.sql000066400000000000000000000071551525762047700252360ustar00rootroot00000000000000-- Regression tests for spill step 4: tp_memtable_mark_chain_dead -- stamps every page of the spilled chain (including fragment -- continuation pages) with TP_MEMTABLE_PAGE_FLAG_DEAD before -- tp_spill_finalize unlinks the chain from the metapage. -- Orphans are visible via bm25_memtable_dead_pages(). CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Test 1: heap inserts, manual spill, dead orphan count matches -- the live chain page count that existed immediately before spill. CREATE TABLE memtable_spill_dead_t (id serial PRIMARY KEY, body text); CREATE INDEX memtable_spill_dead_idx ON memtable_spill_dead_t USING bm25(body) WITH (text_config = 'english'); SELECT count(*) AS dead_pages_before_spill FROM bm25_memtable_dead_pages('memtable_spill_dead_idx'); INSERT INTO memtable_spill_dead_t (body) SELECT 'spill dead ' || i || ' ' || repeat('pad ', 6) FROM generate_series(1, 200) i; CREATE TEMP TABLE spill_dead_live_pages AS SELECT count(*)::int AS n FROM bm25_memtable_chain('memtable_spill_dead_idx'); SELECT n > 1 AS chain_multi_page FROM spill_dead_live_pages; SELECT bm25_spill_index('memtable_spill_dead_idx') IS NOT NULL AS spill_ok; SELECT count(*) AS live_chain_pages_after_spill FROM bm25_memtable_chain('memtable_spill_dead_idx'); SELECT (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_spill_dead_idx')) = (SELECT n FROM spill_dead_live_pages) AS dead_matches_spilled_chain; SELECT bool_and((flags & 1) <> 0) AS all_dead_flag_set, bool_and(dead_fxid > 0) AS all_dead_fxid_set FROM bm25_memtable_dead_pages('memtable_spill_dead_idx'); -- Second spill on an empty chain must not add more dead pages. DO $$ BEGIN PERFORM bm25_spill_index('memtable_spill_dead_idx'); END $$; SELECT (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_spill_dead_idx')) = (SELECT n FROM spill_dead_live_pages) AS dead_count_unchanged_after_empty_respill; DROP TABLE memtable_spill_dead_t; -- Test 2: large INSERT forces fragment pages; spill stamps them DEAD. CREATE TABLE memtable_spill_dead_frag_t (id int, body text); CREATE INDEX memtable_spill_dead_frag_idx ON memtable_spill_dead_frag_t USING bm25(body) WITH (text_config = 'english'); INSERT INTO memtable_spill_dead_frag_t (body) SELECT string_agg('tok' || gs::text || ' ', '') FROM generate_series(1, 5000) gs; CREATE TEMP TABLE frag_spill_live_pages AS SELECT count(*)::int AS n FROM bm25_memtable_chain('memtable_spill_dead_frag_idx'); SELECT n > 2 AS frag_chain_has_continuations FROM frag_spill_live_pages; DO $$ BEGIN PERFORM bm25_spill_index('memtable_spill_dead_frag_idx'); END $$; SELECT count(*) AS live_pages_after_frag_spill FROM bm25_memtable_chain('memtable_spill_dead_frag_idx'); SELECT (SELECT count(*)::int FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx')) = (SELECT n FROM frag_spill_live_pages) AS frag_dead_matches_spilled_chain; SELECT bool_and((flags & 1) <> 0) AS frag_all_dead_flag, sum(((flags & 2) <> 0)::int)::int AS frag_dead_continuation_pages, sum(CASE WHEN (flags & 2) <> 0 THEN 0 ELSE 1 END)::int AS frag_dead_regular_pages FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx'); SELECT sum(((flags & 2) <> 0)::int)::int > 0 AS frag_has_dead_continuation_pages FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx'); SELECT bool_and( CASE WHEN (flags & 2) <> 0 THEN n_records = 0 ELSE n_records >= 0 END ) AS frag_dead_continuation_zero_records FROM bm25_memtable_dead_pages('memtable_spill_dead_frag_idx'); DROP TABLE memtable_spill_dead_frag_t; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/memtable_stale_tail_injection.sql000066400000000000000000000103751525762047700273170ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION dblink; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE tail_retry (id int, body text); CREATE INDEX tail_retry_idx ON tail_retry USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO tail_retry VALUES (0, 'retry seed'); SELECT dblink_connect('retry_writer', format( 'host=%s port=%s dbname=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database())); SELECT dblink_exec('retry_writer', 'SET pg_textsearch.memtable_pages_threshold = 0'); SELECT dblink_exec('retry_writer', 'SET pg_textsearch.bulk_load_threshold = 0'); SELECT * FROM dblink('retry_writer', 'SELECT injection_points_set_local()') AS t(result text); -- Force both append paths to encounter a legitimately advanced tail. DO $$ DECLARE point text := 'pg-textsearch-memtable-tail-read'; writer_pid int; terms int; old_tail bigint; new_tail bigint; inserted_id int; deadline timestamptz; BEGIN SELECT pid INTO writer_pid FROM dblink('retry_writer', 'SELECT pg_backend_pid()') AS t(pid int); FOREACH terms IN ARRAY ARRAY[60, 1000] LOOP SELECT blkno INTO STRICT old_tail FROM bm25_memtable_chain('tail_retry_idx') WHERE next_block IS NULL; PERFORM result FROM dblink('retry_writer', format( 'SELECT injection_points_attach(%L, %L)', point, 'wait') ) AS t(result text); PERFORM dblink_send_query('retry_writer', format($q$ INSERT INTO tail_retry SELECT %s, 'retry ' || string_agg(md5(j::text), ' ') FROM generate_series(1, %s) j RETURNING id $q$, terms, terms)); deadline := clock_timestamp() + interval '30 seconds'; LOOP PERFORM pg_stat_clear_snapshot(); EXIT WHEN EXISTS ( SELECT 1 FROM pg_stat_activity WHERE pid = writer_pid AND wait_event_type = 'InjectionPoint' AND wait_event = point ); IF dblink_is_busy('retry_writer') = 0 THEN RAISE EXCEPTION 'writer finished without the tail-read pause'; END IF; IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'timed out waiting for the tail-read pause'; END IF; PERFORM pg_sleep(0.01); END LOOP; -- An oversized append always publishes a new tail. INSERT INTO tail_retry SELECT -terms, 'retry ' || string_agg(md5(j::text), ' ') FROM generate_series(1, 1000) j; SELECT blkno INTO STRICT new_tail FROM bm25_memtable_chain('tail_retry_idx') WHERE next_block IS NULL; IF new_tail = old_tail OR NOT EXISTS ( SELECT 1 FROM bm25_memtable_chain('tail_retry_idx') WHERE blkno = old_tail AND next_block IS NOT NULL ) THEN RAISE EXCEPTION 'concurrent append did not advance the old tail'; END IF; -- Detach before waking so the metapage reread cannot pause again. PERFORM injection_points_detach(point); PERFORM injection_points_wakeup(point); deadline := clock_timestamp() + interval '30 seconds'; WHILE dblink_is_busy('retry_writer') <> 0 LOOP IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'writer did not retry the advanced tail'; END IF; PERFORM pg_sleep(0.01); END LOOP; SELECT id INTO STRICT inserted_id FROM dblink_get_result('retry_writer') AS t(id int); IF inserted_id <> terms THEN RAISE EXCEPTION 'retry inserted the wrong document'; END IF; PERFORM id FROM dblink_get_result('retry_writer') AS t(id int); END LOOP; END $$; SELECT dblink_disconnect('retry_writer'); SELECT array_agg(id ORDER BY id) FROM ( SELECT id FROM tail_retry ORDER BY body <@> to_bm25query('retry', 'tail_retry_idx') LIMIT 10 ) ranked; DROP TABLE tail_retry; RESET pg_textsearch.memtable_pages_threshold; RESET pg_textsearch.bulk_load_threshold; DROP EXTENSION dblink; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/merge.sql000066400000000000000000000234561525762047700223710ustar00rootroot00000000000000-- Test case: merge -- Tests segment merge functionality and queries across multiple segment levels. -- -- This test exercises: -- 1. Multiple L0 segments -- 2. L0 -> L1 merge triggered by segments_per_level = 2 -- 3. Queries across multiple segment levels (L0, L1) CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Enable merge: 2 segments per level triggers L0->L1 merge on 2nd spill SET pg_textsearch.segments_per_level = 2; -- Create table and index CREATE TABLE merge_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX merge_test_idx ON merge_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -------------------------------------------------------------------------------- -- Phase 1: First batch - will become first L0 segment after spill -------------------------------------------------------------------------------- INSERT INTO merge_test (content) VALUES ('hello world database'); INSERT INTO merge_test (content) VALUES ('goodbye cruel world'); INSERT INTO merge_test (content) VALUES ('hello goodbye friend'); INSERT INTO merge_test (content) VALUES ('world peace harmony'); -- Verify data is queryable in memtable SELECT 'Phase 1: memtable only' AS phase; SELECT COUNT(*) AS count_before_spill FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; -- First spill creates segment 1 in L0 SELECT bm25_spill_index('merge_test_idx') IS NOT NULL AS first_spill; -- Verify data is still queryable from L0 segment SELECT 'Phase 1b: after first spill (1 segment in L0)' AS phase; SELECT COUNT(*) AS hello_count_after_first_spill FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; -------------------------------------------------------------------------------- -- Phase 2: Second batch - triggers L0->L1 merge (segments_per_level=2) -------------------------------------------------------------------------------- INSERT INTO merge_test (content) VALUES ('database indexing query'); INSERT INTO merge_test (content) VALUES ('search engine optimization'); INSERT INTO merge_test (content) VALUES ('database world news'); INSERT INTO merge_test (content) VALUES ('goodbye database friend'); -- Second spill creates segment 2 in L0, which triggers merge to L1 -- After merge: L0 is empty (cleared), L1 has merged segment -- Note: Run with client_min_messages=debug1 to see merge DEBUG messages SELECT bm25_spill_index('merge_test_idx') IS NOT NULL AS second_spill; -- After merge: merged segment in L1, L0 is empty SELECT 'Phase 2: after second spill + merge (L0 empty, 1 segment in L1)' AS phase; -- Should find 2 documents with 'hello' (both in L1 merged segment) SELECT COUNT(*) AS hello_count_after_merge FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; -- Should find 4 documents with 'database' (all in L1 merged segment) SELECT COUNT(*) AS database_count_after_merge FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('database', 'merge_test_idx') LIMIT 100 ) t; -- Validate BM25 scoring is correct across L1 data SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid_after_merge; SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'database', 'english', 1.2, 0.75) AS database_valid_after_merge; SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid_after_merge; -------------------------------------------------------------------------------- -- Phase 3: Post-merge inserts (new memtable data + 1 L1 segment) -------------------------------------------------------------------------------- INSERT INTO merge_test (content) VALUES ('hello new insertion'); INSERT INTO merge_test (content) VALUES ('database transaction log'); SELECT 'Phase 3: post-merge inserts (memtable + 1 L1 segment)' AS phase; -- Should find 3 documents with 'hello' (2 in L1, 1 in memtable) SELECT COUNT(*) AS hello_count_with_new_inserts FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('hello', 'merge_test_idx') LIMIT 100 ) t; -- Should find 5 documents with 'database' (4 in L1, 1 in memtable) SELECT COUNT(*) AS database_count_with_new_inserts FROM ( SELECT id FROM merge_test ORDER BY content <@> to_bm25query('database', 'merge_test_idx') LIMIT 100 ) t; -- Validate BM25 scoring with mixed sources (memtable + L1 segment) SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid_mixed; SELECT validate_bm25_scoring('merge_test', 'content', 'merge_test_idx', 'database', 'english', 1.2, 0.75) AS database_valid_mixed; -------------------------------------------------------------------------------- -- Phase 4: Verify total document count -------------------------------------------------------------------------------- SELECT 'Phase 4: final verification' AS phase; SELECT COUNT(*) AS total_documents FROM merge_test; -- Show final scores for reference SELECT id, content, ROUND((content <@> to_bm25query('database', 'merge_test_idx'))::numeric, 4) AS database_score FROM merge_test ORDER BY content <@> to_bm25query('database', 'merge_test_idx'), id; -------------------------------------------------------------------------------- -- Phase 7: Bound ordinary compaction outputs -------------------------------------------------------------------------------- SET pg_textsearch.segments_per_level = 4; SET pg_textsearch.max_segment_size = '1MB'; CREATE TABLE merge_bounded (id bigint PRIMARY KEY, content text); CREATE INDEX merge_bounded_idx ON merge_bounded USING bm25(content) WITH (text_config='simple'); DO $$ DECLARE batch integer; BEGIN FOR batch IN 0..3 LOOP INSERT INTO merge_bounded SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 4) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('merge_bounded_idx'); END LOOP; END $$; DO $$ DECLARE summary text := bm25_summarize_index('merge_bounded_idx'); BEGIN IF regexp_count(summary, 'L1 Segment [0-9]+:') <> 2 OR summary ~ 'L0 Segment' THEN RAISE EXCEPTION 'ordinary bounded compaction layout is wrong: %', summary; END IF; END $$; SELECT count(*) = 8000 AS bounded_merge_docs_preserved FROM ( SELECT 1 FROM merge_bounded ORDER BY content <@> to_bm25query('common', 'merge_bounded_idx') LIMIT 8000 ) ranked; RESET pg_textsearch.max_segment_size; RESET pg_textsearch.segments_per_level; DROP TABLE merge_bounded CASCADE; -- A level with no legal count reduction must not starve a higher -- level that has one. Explicit spills only, so the layout below is -- deterministic. SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; SET pg_textsearch.max_segment_size = '1MB'; SET pg_textsearch.segments_per_level = 2; CREATE TABLE merge_starve (id bigint PRIMARY KEY, content text); CREATE INDEX merge_starve_idx ON merge_starve USING bm25(content) WITH (text_config='simple', compaction='manual'); DO $$ DECLARE batch integer; BEGIN -- Eight pairable segments compact into a four-segment L1 run. FOR batch IN 0..7 LOOP INSERT INTO merge_starve SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 4) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('merge_starve_idx'); IF batch % 2 = 1 AND NOT bm25_compact_step('merge_starve_idx') THEN RAISE EXCEPTION 'failed to construct an L1 segment'; END IF; END LOOP; PERFORM set_config('pg_textsearch.segments_per_level', '8', false); -- Each of these alone exceeds the budget, so L0 becomes an -- uncombinable run while staying below the level threshold. FOR batch IN 8..11 LOOP INSERT INTO merge_starve SELECT batch * 2000 + gs, 'common ' || repeat(md5((batch * 2000 + gs)::text), 32) FROM generate_series(1, 2000) gs; PERFORM bm25_spill_index('merge_starve_idx'); END LOOP; END $$; -- Widening the budget makes L1's run legally mergeable (4 x 0.8MB fits) -- while L0's 2.1MB segments still cannot pair. Lowering the threshold -- triggers both levels at once. SET pg_textsearch.max_segment_size = '4MB'; SET pg_textsearch.segments_per_level = 4; INSERT INTO merge_starve SELECT 24000 + gs, 'common ' || repeat(md5((24000 + gs)::text), 32) FROM generate_series(1, 2000) gs; SELECT bm25_spill_index('merge_starve_idx') > 0 AS starve_spilled; DO $$ DECLARE summary text := bm25_summarize_index('merge_starve_idx'); BEGIN IF NOT bm25_compact_step('merge_starve_idx') THEN RAISE EXCEPTION 'higher-level compaction did not run'; END IF; summary := bm25_summarize_index('merge_starve_idx'); IF regexp_count(summary, 'L1 Segment [0-9]+:') <> 1 THEN RAISE EXCEPTION 'uncombinable L0 starved a compactable L1: %', summary; END IF; END $$; SELECT count(*) = 26000 AS starve_docs_preserved FROM ( SELECT 1 FROM merge_starve ORDER BY content <@> to_bm25query('common', 'merge_starve_idx') LIMIT 26000 ) ranked; RESET pg_textsearch.max_segment_size; RESET pg_textsearch.segments_per_level; RESET pg_textsearch.bulk_load_threshold; RESET pg_textsearch.memtable_pages_threshold; DROP TABLE merge_starve CASCADE; -- Cleanup DROP TABLE merge_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/merge_injection.sql000066400000000000000000000110171525762047700244210ustar00rootroot00000000000000SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.segments_per_level = 3; SELECT pg_textsearch_test_attach_segment_limit(2); CREATE TABLE merge_capacity (id serial PRIMARY KEY, content text); CREATE INDEX merge_capacity_idx ON merge_capacity USING bm25(content) WITH (text_config='english', compaction='manual'); DO $$ DECLARE n integer; BEGIN FOR n IN 1..2 LOOP INSERT INTO merge_capacity (content) VALUES (format('capacity document %s filler', n)); PERFORM bm25_spill_index('merge_capacity_idx'); END LOOP; END $$; INSERT INTO merge_capacity (content) VALUES ('capacity pending document filler'); DO $$ BEGIN BEGIN PERFORM bm25_spill_index('merge_capacity_idx'); RAISE EXCEPTION 'expected segment count limit error'; EXCEPTION WHEN program_limit_exceeded THEN IF SQLERRM <> 'bm25 segment count limit reached at level 0' THEN RAISE; END IF; END; END $$; DO $$ DECLARE summary text := bm25_summarize_index('merge_capacity_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 2 OR summary !~ 'L0 Segment 2:' OR summary !~ E'Memtable:\n terms: 0\n documents: 1' THEN RAISE EXCEPTION 'segment capacity failure mutated the index: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM merge_capacity ORDER BY content <@> to_bm25query('capacity', 'merge_capacity_idx') ) ranked) <> 3 THEN RAISE EXCEPTION 'segment capacity failure lost documents'; END IF; END $$; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); DROP TABLE merge_capacity CASCADE; CREATE TABLE merge_nested_full (id serial PRIMARY KEY, content text); CREATE INDEX merge_nested_full_idx ON merge_nested_full USING bm25(content) WITH (text_config='english', compaction='manual'); -- Construct two full destination levels without relying on tiny outputs -- being promoted repeatedly. DO $$ DECLARE n integer; BEGIN FOR level IN REVERSE 2..1 LOOP FOR n IN 1..2 LOOP INSERT INTO merge_nested_full (content) SELECT format('nested full document %s filler', gs) FROM generate_series(1, 6) gs; PERFORM bm25_spill_index('merge_nested_full_idx'); END LOOP; PERFORM pg_textsearch_test_move_level( 'merge_nested_full_idx', 0, level); END LOOP; END $$; DO $$ DECLARE summary text := bm25_summarize_index('merge_nested_full_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 4 OR summary !~ 'L1 Segment 2:' OR summary !~ 'L2 Segment 2:' OR summary ~ 'L0 Segment 1:' THEN RAISE EXCEPTION 'nested capacity layout was not constructed: %', summary; END IF; END $$; SELECT pg_textsearch_test_attach_segment_limit(2); SET pg_textsearch.segments_per_level = 2; DO $$ DECLARE n integer; BEGIN FOR n IN 25..26 LOOP INSERT INTO merge_nested_full (content) VALUES (format('nested full document %s filler', n)); PERFORM bm25_spill_index('merge_nested_full_idx'); END LOOP; END $$; SELECT bm25_compact_step('merge_nested_full_idx'); DO $$ DECLARE summary text := bm25_summarize_index('merge_nested_full_idx'); BEGIN IF regexp_count(summary, 'L[0-7] Segment [0-9]+:') <> 4 OR summary !~ 'L1 Segment 2:' OR summary !~ 'L2 Segment 2:' OR summary ~ 'L0 Segment 1:' THEN RAISE EXCEPTION 'nested capacity blockers were not drained: %', summary; END IF; IF (SELECT count(*) FROM ( SELECT 1 FROM merge_nested_full ORDER BY content <@> to_bm25query('nested', 'merge_nested_full_idx') ) ranked) <> 26 THEN RAISE EXCEPTION 'nested capacity compaction lost documents'; END IF; END $$; SELECT bm25_compact('merge_nested_full_idx'); SELECT bm25_level_counts('merge_nested_full_idx') = ARRAY[0, 1, 0, 0, 0, 0, 0, 0] AND NOT bm25_needs_compaction('merge_nested_full_idx') AS capacity_recourse_converges_without_promotion; SELECT injection_points_detach( 'pg-textsearch-segment-count-limit'); RESET pg_textsearch.segments_per_level; DROP TABLE merge_nested_full CASCADE; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/mixed.sql000066400000000000000000000200141525762047700223630ustar00rootroot00000000000000-- Test concurrent operations on pg_textsearch indexes -- This test verifies that concurrent access to shared memory structures is safe -- and that operations like inserts, searches, and index building work correctly CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; -- Clean up from any previous tests DROP TABLE IF EXISTS concurrent_test_docs CASCADE; DROP TABLE IF EXISTS concurrent_test_docs2 CASCADE; -- Create test tables for concurrent operations CREATE TABLE concurrent_test_docs ( id SERIAL PRIMARY KEY, content TEXT NOT NULL, category VARCHAR(50), created_at TIMESTAMP DEFAULT NOW() ); CREATE TABLE concurrent_test_docs2 ( id SERIAL PRIMARY KEY, content TEXT NOT NULL, priority INTEGER DEFAULT 1 ); -- Test 1: Sequential inserts to verify basic functionality \echo 'Test 1: Sequential baseline test' INSERT INTO concurrent_test_docs (content, category) VALUES ('database query optimization techniques', 'tech'), ('concurrent programming patterns', 'tech'), ('shared memory management systems', 'tech'), ('lock-free data structures', 'tech'), ('database indexing strategies', 'tech'); -- Create index after initial data CREATE INDEX concurrent_idx1 ON concurrent_test_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Verify basic search works (only matching documents via index) SELECT id, content, ROUND((content <@> to_bm25query('database concurrent', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY content <@> to_bm25query('database concurrent', 'concurrent_idx1') LIMIT 10; -- Test 2: Simulate concurrent inserts with same terms \echo 'Test 2: Concurrent inserts with overlapping terms' -- Insert documents that will compete for same posting lists INSERT INTO concurrent_test_docs (content, category) VALUES ('database systems require concurrent access control', 'tech'), ('concurrent database operations need careful synchronization', 'tech'), ('database concurrent processing improves performance', 'tech'); -- Search to verify posting lists updated correctly SELECT id, content, ROUND((content <@> to_bm25query('database concurrent', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 5; -- Test 3: Multiple index creation (simulates concurrent index builds) \echo 'Test 3: Multiple index creation' -- Create second table and index to test multiple indexes INSERT INTO concurrent_test_docs2 (content, priority) VALUES ('high priority database tasks', 3), ('concurrent processing systems', 2), ('shared memory data structures', 1), ('database performance optimization', 3), ('concurrent system design patterns', 2); CREATE INDEX concurrent_idx2 ON concurrent_test_docs2 USING bm25(content) WITH (text_config='english', k1=1.5, b=0.8); -- Verify both indexes work independently SELECT 'Table 1' as source, id, content FROM concurrent_test_docs UNION ALL SELECT 'Table 2' as source, id, content FROM concurrent_test_docs2 ORDER BY source, id; -- Test 4: Stress test with many inserts \echo 'Test 4: Bulk insert stress test' -- Generate series of documents with overlapping terms INSERT INTO concurrent_test_docs (content, category) SELECT 'test document number ' || i || ' contains database and concurrent terms', 'stress' FROM generate_series(1, 50) i; -- Verify search still works with larger dataset SELECT COUNT(*) as total_matches FROM concurrent_test_docs; -- Test top results still make sense SELECT id, substring(content, 1, 50) || '...' as content_preview, ROUND((content <@> to_bm25query('database concurrent', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 8; -- Test 5: Mixed operations (insert while searching) \echo 'Test 5: Mixed insert and search operations' -- This simulates a session inserting while another searches BEGIN; INSERT INTO concurrent_test_docs (content, category) VALUES ('new concurrent database research paper', 'research'); -- Search within same transaction SELECT id, content, ROUND((content <@> to_bm25query('research database', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 3; COMMIT; -- Verify the insert is visible after commit SELECT id, content, ROUND((content <@> to_bm25query('research database', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY score LIMIT 3; -- Test 6: Index integrity under updates \echo 'Test 6: Update operations' -- Update some documents to test posting list maintenance UPDATE concurrent_test_docs SET content = 'updated database system with enhanced concurrent features' WHERE id IN (1, 2); -- Verify search finds updated documents SELECT * FROM ( SELECT id, content, ROUND((content <@> to_bm25query('enhanced database', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY content <@> to_bm25query('enhanced database', 'concurrent_idx1') LIMIT 10 ) sub ORDER BY score, id; -- Test 7: Delete operations \echo 'Test 7: Delete operations' -- Count before delete SELECT COUNT(*) as count_before_delete FROM concurrent_test_docs; -- Delete some stress test documents DELETE FROM concurrent_test_docs WHERE content LIKE 'test document number%' AND id % 5 = 0; -- Count after delete to verify cleanup SELECT COUNT(*) as count_after_delete FROM concurrent_test_docs; -- Test 8: Verify string interning consistency \echo 'Test 8: String interning consistency' -- Insert documents with exact same terms to test string interning INSERT INTO concurrent_test_docs (content, category) VALUES ('exact same terms for testing consistency', 'test'), ('exact same terms for testing consistency', 'test'), ('testing consistency with exact same terms', 'test'); -- Search should find all variants SELECT id, content, ROUND((content <@> to_bm25query('exact same terms', 'concurrent_idx1'))::numeric, 4) as score FROM concurrent_test_docs ORDER BY content <@> to_bm25query('exact same terms', 'concurrent_idx1'), id LIMIT 10; -- Test 9: Multiple indexes on same table \echo 'Test 9: Multiple indexes on same table' -- Create a test table for multiple indexes CREATE TABLE multi_idx_test ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); -- Insert initial data INSERT INTO multi_idx_test (content) VALUES ('hello world from the test'), ('the quick brown fox jumps'), ('hello from another document'); -- Create English index first (only BM25 index, planner must use it) CREATE INDEX multi_idx_english ON multi_idx_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert more data after index creation INSERT INTO multi_idx_test (content) VALUES ('world of databases and indexes'), ('hello database world'); -- Query English index while it's the only BM25 index SET pg_textsearch.log_scores = false; SELECT * FROM ( SELECT id, content, ROUND((content <@> to_bm25query('hello world', 'multi_idx_english'))::numeric, 4) as english_score FROM multi_idx_test ORDER BY content <@> to_bm25query('hello world', 'multi_idx_english') LIMIT 10 ) sub ORDER BY english_score, id; -- Now create Simple index too CREATE INDEX multi_idx_simple ON multi_idx_test USING bm25(content) WITH (text_config='simple', k1=1.5, b=0.8); -- Verify both indexes exist and function independently SELECT i.indexrelid::regclass as index_name, pg_size_pretty(pg_relation_size(i.indexrelid)) as index_size FROM pg_index i JOIN pg_class c ON c.oid = i.indrelid WHERE c.relname = 'multi_idx_test' ORDER BY index_name; -- Clean up test table DROP TABLE multi_idx_test CASCADE; -- Final verification \echo 'Final verification: Index statistics' SELECT 'concurrent_test_docs' as table_name, COUNT(*) as total_docs, COUNT(DISTINCT content) as unique_docs FROM concurrent_test_docs UNION ALL SELECT 'concurrent_test_docs2' as table_name, COUNT(*) as total_docs, COUNT(DISTINCT content) as unique_docs FROM concurrent_test_docs2; -- Clean up DROP TABLE concurrent_test_docs CASCADE; DROP TABLE concurrent_test_docs2 CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/parallel_bmw.sql000066400000000000000000000062471525762047700237320ustar00rootroot00000000000000-- Test case: parallel_bmw -- Validates that parallel index build computes correct block_max_norm -- (MIN fieldnorm) for BMW skip entries, matching the serial build path. -- -- The bug: build_parallel.c computed MAX fieldnorm instead of MIN, -- causing BMW to underestimate block score upper bounds and incorrectly -- skip blocks containing high-scoring short documents. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; SET min_parallel_table_scan_size = 0; SET maintenance_work_mem = '256MB'; -------------------------------------------------------------------------------- -- Setup: 3-tier document design to expose block_max_norm bug -------------------------------------------------------------------------------- -- The posting list for "target" is ordered by CTID (insertion order): -- -- Rows 1-5000: MEDIUM docs (50 words with "target") -> set threshold -- Rows 5001-7000: SHORT + LONG docs (alternating, both with "target") -- Rows 7001-150000: FILLER docs (no "target", for parallel build sizing) -- -- BMW processes blocks sequentially. Medium-doc blocks are processed first, -- establishing a threshold. Then mixed short+long blocks follow: -- -- Correct (MIN fieldnorm): upper bound from short doc -> above threshold -- Buggy (MAX fieldnorm): upper bound from long doc -> below threshold -- -- With the bug, BMW skips the mixed blocks entirely, missing the short -- docs that should rank highest in top-k results. SET max_parallel_maintenance_workers = 2; CREATE TABLE parallel_bmw_test ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_bmw_test (content) SELECT CASE -- Medium docs: "target" + 49 filler words = 50 total WHEN i <= 5000 THEN 'target ' || repeat('alpha ', 49) -- Short docs (odd): just "target" = 1 word WHEN i <= 7000 AND i % 2 = 1 THEN 'target' -- Long docs (even): "target" + 199 filler words = 200 total WHEN i <= 7000 THEN 'target ' || repeat('beta ', 199) -- Filler: no "target" term ELSE 'gamma delta' END FROM generate_series(1, 150000) AS i; ANALYZE parallel_bmw_test; CREATE INDEX parallel_bmw_test_idx ON parallel_bmw_test USING bm25(content) WITH (text_config='english'); -------------------------------------------------------------------------------- -- Test 1: BMW top-k must find short docs despite them being in later blocks -------------------------------------------------------------------------------- -- Short docs (1 word) score highest due to BM25 length normalization. -- With the bug, their blocks are skipped because the upper bound is -- computed from the long docs (200 words) sharing those blocks, and that -- upper bound falls below the threshold set by medium docs (50 words). SELECT COUNT(*) AS short_docs_in_top10 FROM ( SELECT content FROM parallel_bmw_test ORDER BY content <@> to_bm25query('target', 'parallel_bmw_test_idx') LIMIT 10 ) sub WHERE content = 'target'; -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- DROP TABLE parallel_bmw_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/parallel_build.sql000066400000000000000000000563731525762047700242510ustar00rootroot00000000000000-- Test case: parallel_build -- Tests parallel index build with various worker counts and corner cases CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -- Lower threshold so our test tables qualify for parallel build SET min_parallel_table_scan_size = 0; -- Sufficient memory for parallel workers (32MB per worker + leader) SET maintenance_work_mem = '256MB'; -------------------------------------------------------------------------------- -- Test 1: Serial build (0 workers) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 0; CREATE TABLE parallel_test_serial ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_serial (content) SELECT 'Document ' || i || ' contains words like apple banana cherry' FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_serial; -- Should NOT show "parallel index build" notice (serial build) CREATE INDEX parallel_test_serial_idx ON parallel_test_serial USING bm25(content) WITH (text_config='english'); -- Verify queries work SELECT COUNT(*) AS apple_count FROM (SELECT 1 FROM parallel_test_serial ORDER BY content <@> to_bm25query('apple', 'parallel_test_serial_idx')) sub; -------------------------------------------------------------------------------- -- Test 2: Single worker (edge case - minimum parallelism) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 1; CREATE TABLE parallel_test_1worker ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_1worker (content) SELECT 'Document ' || i || ' with database query optimization terms' FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_1worker; -- Should show "parallel index build: launched 1 of 1 requested workers" CREATE INDEX parallel_test_1worker_idx ON parallel_test_1worker USING bm25(content) WITH (text_config='english'); SELECT COUNT(*) AS database_count FROM (SELECT 1 FROM parallel_test_1worker ORDER BY content <@> to_bm25query('database', 'parallel_test_1worker_idx')) sub; REINDEX INDEX parallel_test_1worker_idx; -- Parallel workers must flatten array-valued index expressions too. CREATE INDEX parallel_test_array_expr_idx ON parallel_test_1worker USING bm25 ((ARRAY[content, 'arrayneedle'])) WITH (text_config='english'); SELECT count(*) AS array_expression_hits FROM ( SELECT id FROM parallel_test_1worker ORDER BY ARRAY[content, 'arrayneedle'] <@> to_bm25query('arrayneedle', 'parallel_test_array_expr_idx') LIMIT 5 ) hits; DROP INDEX parallel_test_array_expr_idx; -------------------------------------------------------------------------------- -- Test 3: Two workers (common case) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; CREATE TABLE parallel_test_2workers ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_2workers (content) SELECT 'Document number ' || i || ' contains words like ' || CASE i % 5 WHEN 0 THEN 'apple banana cherry' WHEN 1 THEN 'database query optimization' WHEN 2 THEN 'machine learning algorithm' WHEN 3 THEN 'distributed systems design' ELSE 'network protocol security' END || ' and additional text to make it longer ' || CASE i % 3 WHEN 0 THEN 'with some extra content here' WHEN 1 THEN 'including more varied terms' ELSE 'plus different phrases altogether' END FROM generate_series(1, 150000) AS i; ANALYZE parallel_test_2workers; -- Should show "parallel index build: launched 2 of 2 requested workers" CREATE INDEX parallel_test_2workers_idx ON parallel_test_2workers USING bm25(content) WITH (text_config='english'); -- Verify all document groups are searchable SELECT COUNT(*) AS apple_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('apple', 'parallel_test_2workers_idx')) sub; SELECT COUNT(*) AS database_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('database', 'parallel_test_2workers_idx')) sub; SELECT COUNT(*) AS machine_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('machine', 'parallel_test_2workers_idx')) sub; SELECT COUNT(*) AS distributed_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('distributed', 'parallel_test_2workers_idx')) sub; SELECT COUNT(*) AS network_count FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('network', 'parallel_test_2workers_idx')) sub; -- Verify top-k ordering (limit to deterministic results) SELECT id, ROUND((content <@> to_bm25query('machine learning', 'parallel_test_2workers_idx'))::numeric, 4) AS score FROM parallel_test_2workers WHERE id <= 100 ORDER BY content <@> to_bm25query('machine learning', 'parallel_test_2workers_idx'), id LIMIT 5; -------------------------------------------------------------------------------- -- Test 4: Four workers (higher parallelism) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 4; CREATE TABLE parallel_test_4workers ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_4workers (content) SELECT 'Document ' || i || ' with search engine indexing text ' || CASE i % 7 WHEN 0 THEN 'quick brown fox' WHEN 1 THEN 'lazy sleeping dog' WHEN 2 THEN 'postgres full text' WHEN 3 THEN 'inverted index structure' WHEN 4 THEN 'bm25 ranking algorithm' WHEN 5 THEN 'term frequency idf' ELSE 'document relevance score' END FROM generate_series(1, 200000) AS i; ANALYZE parallel_test_4workers; -- Should show "parallel index build: launched N of 4 requested workers" -- (may launch fewer if system limits) CREATE INDEX parallel_test_4workers_idx ON parallel_test_4workers USING bm25(content) WITH (text_config='english'); SELECT COUNT(*) AS fox_count FROM (SELECT 1 FROM parallel_test_4workers ORDER BY content <@> to_bm25query('fox', 'parallel_test_4workers_idx')) sub; SELECT COUNT(*) AS postgres_count FROM (SELECT 1 FROM parallel_test_4workers ORDER BY content <@> to_bm25query('postgres', 'parallel_test_4workers_idx')) sub; -------------------------------------------------------------------------------- -- Test 5: Corner case - very short documents (single term) -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; CREATE TABLE parallel_test_short ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_short (content) SELECT CASE i % 10 WHEN 0 THEN 'alpha' WHEN 1 THEN 'beta' WHEN 2 THEN 'gamma' WHEN 3 THEN 'delta' WHEN 4 THEN 'epsilon' WHEN 5 THEN 'zeta' WHEN 6 THEN 'eta' WHEN 7 THEN 'theta' WHEN 8 THEN 'iota' ELSE 'kappa' END FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_short; CREATE INDEX parallel_test_short_idx ON parallel_test_short USING bm25(content) WITH (text_config='english'); -- Each term should appear in 10% of documents SELECT COUNT(*) AS alpha_count FROM (SELECT 1 FROM parallel_test_short ORDER BY content <@> to_bm25query('alpha', 'parallel_test_short_idx')) sub; SELECT COUNT(*) AS beta_count FROM (SELECT 1 FROM parallel_test_short ORDER BY content <@> to_bm25query('beta', 'parallel_test_short_idx')) sub; -------------------------------------------------------------------------------- -- Test 6: Corner case - documents with many duplicate terms -------------------------------------------------------------------------------- CREATE TABLE parallel_test_dupes ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_dupes (content) SELECT repeat('repeat ', (i % 50) + 1) || 'unique' || i FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_dupes; CREATE INDEX parallel_test_dupes_idx ON parallel_test_dupes USING bm25(content) WITH (text_config='english'); -- "repeat" appears in all documents with varying term frequency SELECT COUNT(*) AS repeat_count FROM (SELECT 1 FROM parallel_test_dupes ORDER BY content <@> to_bm25query('repeat', 'parallel_test_dupes_idx')) sub; -- Each "unique" term appears exactly once SELECT COUNT(*) AS unique_count FROM (SELECT 1 FROM parallel_test_dupes ORDER BY content <@> to_bm25query('unique1', 'parallel_test_dupes_idx')) sub; -------------------------------------------------------------------------------- -- Test 7: Corner case - NULL content (should be skipped) -------------------------------------------------------------------------------- CREATE TABLE parallel_test_nulls ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_nulls (content) SELECT CASE WHEN i % 10 = 0 THEN NULL ELSE 'document ' || i || ' has content' END FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_nulls; CREATE INDEX parallel_test_nulls_idx ON parallel_test_nulls USING bm25(content) WITH (text_config='english'); -- Only non-NULL documents should be indexed (90%) SELECT COUNT(*) AS document_count FROM (SELECT 1 FROM parallel_test_nulls ORDER BY content <@> to_bm25query('document', 'parallel_test_nulls_idx')) sub; -------------------------------------------------------------------------------- -- Test 8: Post-build operations work correctly -------------------------------------------------------------------------------- -- Use the 2-worker table for post-build tests -- Additional inserts after parallel build INSERT INTO parallel_test_2workers (content) VALUES ('completely new unique document inserted after parallel build'); -- Search for newly inserted content SELECT COUNT(*) AS new_doc_found FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('completely', 'parallel_test_2workers_idx')) sub; -- VACUUM should work VACUUM parallel_test_2workers; -- Queries still work after VACUUM SELECT COUNT(*) AS apple_after_vacuum FROM (SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query('apple', 'parallel_test_2workers_idx')) sub; -- Index stats still available SELECT bm25_summarize_index('parallel_test_2workers_idx') IS NOT NULL AS has_summary; -------------------------------------------------------------------------------- -- Test 9: Custom BM25 parameters with parallel build -------------------------------------------------------------------------------- CREATE TABLE parallel_test_custom ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_custom (content) SELECT 'Testing custom parameters with document ' || i || ' and some text' FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_custom; -- Use non-default k1 and b parameters CREATE INDEX parallel_test_custom_idx ON parallel_test_custom USING bm25(content) WITH (text_config='english', k1='1.5', b='0.5'); SELECT COUNT(*) AS custom_count FROM (SELECT 1 FROM parallel_test_custom ORDER BY content <@> to_bm25query('custom', 'parallel_test_custom_idx')) sub; -------------------------------------------------------------------------------- -- Test 10: Below threshold - should use serial even with workers configured -------------------------------------------------------------------------------- -- Table with fewer than 100K rows should use serial build CREATE TABLE parallel_test_below_threshold ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO parallel_test_below_threshold (content) SELECT 'Small table document ' || i FROM generate_series(1, 50000) AS i; ANALYZE parallel_test_below_threshold; -- Should NOT show "parallel index build" (below 100K threshold) CREATE INDEX parallel_test_below_idx ON parallel_test_below_threshold USING bm25(content) WITH (text_config='english'); SELECT COUNT(*) AS small_count FROM (SELECT 1 FROM parallel_test_below_threshold ORDER BY content <@> to_bm25query('small', 'parallel_test_below_idx')) sub; -------------------------------------------------------------------------------- -- Test 11: Parallel build with documents that produce no lexemes -------------------------------------------------------------------------------- CREATE TABLE parallel_test_empty ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO parallel_test_empty (content) SELECT CASE WHEN i % 2 = 0 THEN '' ELSE 'the' END FROM generate_series(1, 100000) AS i; ANALYZE parallel_test_empty; CREATE INDEX parallel_test_empty_idx ON parallel_test_empty USING bm25(content) WITH (text_config='english'); SELECT COUNT(*) AS empty_count FROM parallel_test_empty WHERE content @@ to_tsquery('english', '!missing'); -------------------------------------------------------------------------------- -- Test 12: Zero-lexeme documents respect the serial build memory budget -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 0; SET maintenance_work_mem = '1MB'; CREATE TABLE serial_test_empty ( id SERIAL PRIMARY KEY, content TEXT NOT NULL ); INSERT INTO serial_test_empty (content) SELECT '' FROM generate_series(1, 140000); CREATE INDEX serial_test_empty_idx ON serial_test_empty USING bm25(content) WITH (text_config='english'); SELECT bm25_summarize_index('serial_test_empty_idx') ~ 'Total: 2 segments,.*0 terms, 140000 docs' AS document_arrays_flushed; -------------------------------------------------------------------------------- -- Test 13: Low parallel budgets are divided across actual workers -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; SET maintenance_work_mem = '64MB'; INSERT INTO parallel_test_2workers (content) VALUES (repeat('large ', 20000)), (''); ANALYZE parallel_test_2workers; CREATE INDEX parallel_test_low_budget_idx ON parallel_test_2workers USING bm25(content) WITH (text_config='english'); SELECT COUNT(*) AS low_budget_documents FROM ( SELECT 1 FROM parallel_test_2workers ORDER BY content <@> to_bm25query( 'database', 'parallel_test_low_budget_idx') ) ranked; SELECT id AS parallel_large_document_id FROM parallel_test_2workers ORDER BY content <@> to_bm25query( 'large', 'parallel_test_low_budget_idx'), id LIMIT 1; SELECT bm25_summarize_index('parallel_test_low_budget_idx') ~ E'total_docs: 150003\n' AS parallel_tokenless_row_counted; -------------------------------------------------------------------------------- -- Test 14: Serial and parallel streaming builds produce identical results -------------------------------------------------------------------------------- SET maintenance_work_mem = '256MB'; CREATE TABLE parallel_stream_test AS SELECT i AS id, CASE WHEN i % 100 = 0 THEN '' ELSE 'anchor shared t' || i::text END AS content FROM generate_series(1, 100000) i; ALTER TABLE parallel_stream_test SET (parallel_workers=2); ANALYZE parallel_stream_test; CREATE TYPE parallel_stream_result AS ( id integer, score double precision ); SET max_parallel_maintenance_workers=0; CREATE INDEX parallel_stream_serial ON parallel_stream_test USING bm25(content) WITH (text_config='simple'); CREATE TEMP TABLE parallel_stream_expected AS SELECT ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'anchor shared', 'parallel_stream_serial') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'anchor shared', 'parallel_stream_serial'), id LIMIT 100000 ) ranked ) AS shared_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 't99999', 'parallel_stream_serial') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 't99999', 'parallel_stream_serial'), id LIMIT 100000 ) ranked ) AS unique_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'missing', 'parallel_stream_serial') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'missing', 'parallel_stream_serial'), id LIMIT 100000 ) ranked ) AS missing_results; SELECT cardinality(shared_results) AS shared_count, ARRAY(SELECT (result).id FROM unnest(unique_results) result) AS unique_ids, cardinality(missing_results) AS missing_count FROM parallel_stream_expected; SELECT bm25_summarize_index('parallel_stream_serial') ~ E'total_docs: 100000\n total_len: 297000\n' AS serial_corpus_totals; DROP INDEX parallel_stream_serial; SET max_parallel_maintenance_workers=2; CREATE INDEX parallel_stream_parallel ON parallel_stream_test USING bm25(content) WITH (text_config='simple'); WITH actual AS ( SELECT ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS shared_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 't99999', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 't99999', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS unique_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'missing', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'missing', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS missing_results ) SELECT actual.shared_results = expected.shared_results AS shared_matches, actual.unique_results = expected.unique_results AS unique_matches, actual.missing_results = expected.missing_results AS missing_matches FROM actual, parallel_stream_expected expected; SELECT bm25_summarize_index('parallel_stream_parallel') ~ E'total_docs: 100000\n total_len: 297000\n' AS parallel_corpus_totals; DROP INDEX parallel_stream_parallel; SET pg_textsearch.compress_segments=off; CREATE INDEX parallel_stream_parallel ON parallel_stream_test USING bm25(content) WITH (text_config='simple'); WITH actual AS ( SELECT ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'anchor shared', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS shared_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 't99999', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 't99999', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS unique_results, ARRAY( SELECT ROW(id, score)::parallel_stream_result FROM ( SELECT id, content <@> to_bm25query( 'missing', 'parallel_stream_parallel') AS score FROM parallel_stream_test ORDER BY content <@> to_bm25query( 'missing', 'parallel_stream_parallel'), id LIMIT 100000 ) ranked ) AS missing_results ) SELECT actual.shared_results = expected.shared_results AS shared_matches, actual.unique_results = expected.unique_results AS unique_matches, actual.missing_results = expected.missing_results AS missing_matches FROM actual, parallel_stream_expected expected; SELECT bm25_summarize_index('parallel_stream_parallel') ~ E'total_docs: 100000\n total_len: 297000\n' AS uncompressed_corpus_totals; RESET pg_textsearch.compress_segments; -------------------------------------------------------------------------------- -- Test 15: Small worker budgets can produce more than 64 segments per worker -------------------------------------------------------------------------------- SET max_parallel_maintenance_workers = 2; SET maintenance_work_mem = '4MB'; CREATE TABLE parallel_many_segments AS SELECT i AS id, 'anchor ' || ( SELECT string_agg('t' || lpad(((i - 1) * 10 + j)::text, 9, '0'), ' ' ORDER BY j) FROM generate_series(1, 10) j ) AS content FROM generate_series(1, 100000) i; ALTER TABLE parallel_many_segments SET (parallel_workers=2); ANALYZE parallel_many_segments; CREATE INDEX parallel_many_segments_idx ON parallel_many_segments USING bm25(content) WITH (text_config='simple'); REINDEX INDEX parallel_many_segments_idx; SELECT count(*) AS all_worker_segments_indexed FROM ( SELECT id FROM parallel_many_segments ORDER BY content <@> to_bm25query('anchor', 'parallel_many_segments_idx') LIMIT 100000 ) ranked; SELECT id AS last_worker_document FROM parallel_many_segments ORDER BY content <@> to_bm25query('t001000000', 'parallel_many_segments_idx') LIMIT 10; -------------------------------------------------------------------------------- -- Cleanup -------------------------------------------------------------------------------- DROP TABLE parallel_test_serial CASCADE; DROP TABLE parallel_test_1worker CASCADE; DROP TABLE parallel_test_2workers CASCADE; DROP TABLE parallel_test_4workers CASCADE; DROP TABLE parallel_test_short CASCADE; DROP TABLE parallel_test_dupes CASCADE; DROP TABLE parallel_test_nulls CASCADE; DROP TABLE parallel_test_custom CASCADE; DROP TABLE parallel_test_below_threshold CASCADE; DROP TABLE parallel_test_empty CASCADE; DROP TABLE serial_test_empty CASCADE; DROP TABLE parallel_stream_test CASCADE; DROP TABLE parallel_many_segments CASCADE; DROP TABLE parallel_stream_expected; DROP TYPE parallel_stream_result; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/partial_index.sql000066400000000000000000000160361525762047700241110ustar00rootroot00000000000000-- Partial index tests for pg_textsearch CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================ -- Basic partial index -- ============================================================ CREATE TABLE partial_docs ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO partial_docs (content, category) VALUES ('postgresql database management', 'tech'), ('machine learning algorithms', 'tech'), ('cooking recipes for beginners', 'lifestyle'), ('full text search and retrieval', 'tech'), ('gardening tips and tricks', 'lifestyle'); CREATE INDEX partial_tech_idx ON partial_docs USING bm25 (content) WITH (text_config='english', compaction='manual') WHERE category = 'tech'; SET enable_seqscan = off; -- Explicit query against partial index SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; -- Lifestyle rows should not appear SELECT id, content, ROUND((content <@> to_bm25query('cooking', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('cooking', 'partial_tech_idx') LIMIT 5; -- ============================================================ -- Multi-language use case -- ============================================================ CREATE TABLE multilang ( id SERIAL PRIMARY KEY, content TEXT, lang CHAR(2) NOT NULL ); INSERT INTO multilang (content, lang) VALUES ('the quick brown fox jumps over the lazy dog', 'en'), ('postgresql database management system', 'en'), ('full text searching and information retrieval', 'en'), ('simple words without stemming needed', 'xx'), ('another simple document for testing', 'xx'); CREATE INDEX multilang_en_idx ON multilang USING bm25 (content) WITH (text_config='english') WHERE lang = 'en'; CREATE INDEX multilang_xx_idx ON multilang USING bm25 (content) WITH (text_config='simple') WHERE lang = 'xx'; -- English stemming: "databases" matches "database" SELECT id, content, ROUND((content <@> to_bm25query('databases', 'multilang_en_idx'))::numeric, 4) AS score FROM multilang WHERE lang = 'en' ORDER BY content <@> to_bm25query('databases', 'multilang_en_idx') LIMIT 5; -- Simple: no stemming SELECT id, content, ROUND((content <@> to_bm25query('simple', 'multilang_xx_idx'))::numeric, 4) AS score FROM multilang WHERE lang = 'xx' ORDER BY content <@> to_bm25query('simple', 'multilang_xx_idx') LIMIT 5; -- ============================================================ -- Expression + partial combined -- ============================================================ CREATE TABLE expr_partial ( id SERIAL PRIMARY KEY, data JSONB ); INSERT INTO expr_partial (data) VALUES ('{"text": "postgresql database management", "type": "article"}'), ('{"text": "machine learning algorithms", "type": "article"}'), ('{"text": "random user comment", "type": "comment"}'), ('{"text": "full text search retrieval", "type": "article"}'); CREATE INDEX expr_partial_idx ON expr_partial USING bm25 ((data->>'text')) WITH (text_config='english') WHERE (data->>'type') = 'article'; SELECT id, ROUND(((data->>'text') <@> to_bm25query('database', 'expr_partial_idx'))::numeric, 4) AS score FROM expr_partial WHERE (data->>'type') = 'article' ORDER BY (data->>'text') <@> to_bm25query('database', 'expr_partial_idx') LIMIT 5; -- ============================================================ -- Insert into partial index -- predicate filtering -- ============================================================ -- Insert a tech row that matches the predicate and query INSERT INTO partial_docs (content, category) VALUES ('advanced database optimization techniques', 'tech'); -- New tech row should appear in results SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; -- Insert a row that does NOT match the predicate INSERT INTO partial_docs (content, category) VALUES ('database administration basics', 'lifestyle'); -- It should not appear in partial index results SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; -- ============================================================ -- VACUUM on partial index -- ============================================================ DELETE FROM partial_docs WHERE id = 1; VACUUM partial_docs; -- Verify partial index still works after vacuum SELECT id, content, ROUND((content <@> to_bm25query('database', 'partial_tech_idx'))::numeric, 4) AS score FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; -- ============================================================ -- Verify partial index scan is used -- ============================================================ EXPLAIN (COSTS OFF) SELECT id FROM partial_docs WHERE category = 'tech' ORDER BY content <@> to_bm25query('database', 'partial_tech_idx') LIMIT 5; -- ============================================================ -- Implicit resolution skips partial indexes -- ============================================================ -- Table with ONLY a partial index (no non-partial BM25 index) CREATE TABLE partial_only ( id SERIAL PRIMARY KEY, content TEXT, category TEXT ); INSERT INTO partial_only (content, category) VALUES ('database management', 'tech'), ('cooking recipes', 'lifestyle'); CREATE INDEX partial_only_idx ON partial_only USING bm25 (content) WITH (text_config='english') WHERE category = 'tech'; -- Implicit resolution should ERROR because only partial -- indexes exist on this column. \set ON_ERROR_STOP off SELECT id, ROUND((content <@> 'database')::numeric, 4) AS score FROM partial_only ORDER BY content <@> 'database' LIMIT 5; \set ON_ERROR_STOP on DROP TABLE partial_only CASCADE; -- ============================================================ -- Cleanup -- ============================================================ DROP TABLE partial_docs CASCADE; DROP TABLE multilang CASCADE; DROP TABLE expr_partial CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/partitioned.sql000066400000000000000000000305171525762047700236100ustar00rootroot00000000000000-- Test BM25 index behavior with partitioned tables -- -- IMPORTANT: BM25 indexes on partitioned tables use PARTITION-LOCAL STATISTICS. -- Each partition maintains its own: -- - Document count (total_docs) -- - Average document length (avg_doc_len) -- - Per-term document frequencies for IDF calculation -- -- This means: -- - Queries targeting a single partition compute accurate BM25 scores using -- that partition's statistics -- - Queries spanning multiple partitions return scores computed independently -- per partition, which may not be directly comparable across partitions -- -- This test validates that per-partition scores are computed correctly using -- partition-local statistics. -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Load validation functions \set ECHO none \i test/sql/validation.sql \set ECHO all SET enable_seqscan = off; -- ============================================================================= -- Test 1: Basic partitioned table with BM25 index -- When creating an index on a partitioned table, PostgreSQL automatically -- creates indexes on all existing partitions. The parent partitioned table -- itself doesn't call our index build functions, only the partitions do. -- ============================================================================= CREATE TABLE partitioned_docs ( id SERIAL, content TEXT, created_at TIMESTAMP ) PARTITION BY RANGE (created_at); -- Create a partition first CREATE TABLE partition_2024 PARTITION OF partitioned_docs FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); -- Insert data into partition INSERT INTO partition_2024 (content, created_at) VALUES ('Data in the partition', '2024-06-15'); -- Creating index on partitioned table will automatically create it on partitions -- This creates an index with auto-generated name like partition_2024_content_idx CREATE INDEX partitioned_bm25_idx ON partitioned_docs USING bm25(content) WITH (text_config='english'); -- Verify the auto-created partition index works (using the generated name) SELECT indexrelid::regclass as index_name FROM pg_index WHERE indrelid = 'partition_2024'::regclass AND indexrelid::regclass::text LIKE '%content%'; -- Create another partition after index exists CREATE TABLE partition_2025 PARTITION OF partitioned_docs FOR VALUES FROM ('2025-01-01') TO ('2026-01-01'); -- Insert data into new partition INSERT INTO partition_2025 (content, created_at) VALUES ('Data in the 2025 partition', '2025-06-15'); -- Verify the auto-created index on new partition SELECT COUNT(*) as num_partition_indexes FROM pg_index WHERE indrelid IN ('partition_2024'::regclass, 'partition_2025'::regclass); -- ============================================================================= -- Test 2: Implicit index resolution with <@> operator on partitioned table -- The planner hook should resolve partitioned_bm25_idx, and the executor -- should map it to the correct partition index at scan time -- ============================================================================= -- Query partition_2024 - should use partition_2024_content_idx EXPLAIN (COSTS OFF) SELECT content FROM partitioned_docs WHERE created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; -- Actually execute to verify the query works SELECT content FROM partitioned_docs WHERE created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; -- Query partition_2025 - should use partition_2025_content_idx SELECT content FROM partitioned_docs WHERE created_at >= '2025-01-01' AND created_at < '2026-01-01' ORDER BY content <@> 'partition' LIMIT 1; -- Cleanup first test DROP TABLE partitioned_docs CASCADE; -- ============================================================================= -- Test 3: Sub-partitioned table (multi-level partition hierarchy) -- Tests that is_partition_of() correctly walks up the inheritance chain -- ============================================================================= CREATE TABLE subpart_docs ( id SERIAL, content TEXT, region TEXT, created_at TIMESTAMP ) PARTITION BY LIST (region); -- Create sub-partitioned partition CREATE TABLE subpart_docs_us PARTITION OF subpart_docs FOR VALUES IN ('us') PARTITION BY RANGE (created_at); -- Create leaf partition CREATE TABLE subpart_docs_us_2024 PARTITION OF subpart_docs_us FOR VALUES FROM ('2024-01-01') TO ('2025-01-01'); -- Create another region partition (not sub-partitioned) CREATE TABLE subpart_docs_eu PARTITION OF subpart_docs FOR VALUES IN ('eu'); -- Insert test data INSERT INTO subpart_docs_us_2024 (content, region, created_at) VALUES ('US data from 2024', 'us', '2024-06-15'); INSERT INTO subpart_docs_eu (content, region, created_at) VALUES ('EU data', 'eu', '2024-06-15'); -- Create index on top-level partitioned table CREATE INDEX subpart_bm25_idx ON subpart_docs USING bm25(content) WITH (text_config='english'); -- Verify indexes were created at all levels SELECT c.relname, c.relkind FROM pg_class c JOIN pg_index i ON c.oid = i.indexrelid WHERE i.indrelid IN ( 'subpart_docs'::regclass, 'subpart_docs_us'::regclass, 'subpart_docs_us_2024'::regclass, 'subpart_docs_eu'::regclass ) ORDER BY c.relname; -- Query that hits leaf partition (3-level: top -> sub -> leaf) -- This tests is_partition_of() walking up multiple levels SELECT content FROM subpart_docs WHERE region = 'us' AND created_at >= '2024-01-01' AND created_at < '2025-01-01' ORDER BY content <@> 'data' LIMIT 1; -- Query that hits non-sub-partitioned partition (2-level: top -> leaf) SELECT content FROM subpart_docs WHERE region = 'eu' ORDER BY content <@> 'data' LIMIT 1; -- Cleanup DROP TABLE subpart_docs CASCADE; -- ============================================================================= -- Test 4: Partition-local BM25 score validation -- -- This test validates that BM25 scores are computed correctly using -- PARTITION-LOCAL statistics, not global statistics across all partitions. -- -- We create two partitions with different corpus characteristics: -- - partition_small: 3 documents -- - partition_large: 10 documents -- -- The same term will have different IDF values in each partition because -- IDF = ln(1 + (N - df + 0.5) / (df + 0.5)) depends on partition-local N and df. -- -- We validate scores by querying each partition's table directly and comparing -- against pure SQL BM25 computation using that partition's statistics. -- ============================================================================= CREATE TABLE partitioned_scoring ( id SERIAL, content TEXT, partition_key INT ) PARTITION BY LIST (partition_key); -- Create small partition (3 docs) CREATE TABLE partition_small PARTITION OF partitioned_scoring FOR VALUES IN (1); -- Create large partition (10 docs) CREATE TABLE partition_large PARTITION OF partitioned_scoring FOR VALUES IN (2); -- Insert documents into small partition INSERT INTO partition_small (content, partition_key) VALUES ('database search query', 1), ('full text search engine', 1), ('postgresql database system', 1); -- Insert documents into large partition (with different term distribution) INSERT INTO partition_large (content, partition_key) VALUES ('database management system', 2), ('search engine optimization', 2), ('web application development', 2), ('cloud computing services', 2), ('machine learning algorithms', 2), ('data analysis tools', 2), ('software engineering practices', 2), ('network security protocols', 2), ('mobile app development', 2), ('database query optimization', 2); -- Create index on partitioned table CREATE INDEX scoring_bm25_idx ON partitioned_scoring USING bm25(content) WITH (text_config='english'); -- Get the auto-generated partition index names SELECT i.indrelid::regclass as partition_table, i.indexrelid::regclass as partition_index FROM pg_index i JOIN pg_class c ON i.indexrelid = c.oid WHERE i.indrelid IN ('partition_small'::regclass, 'partition_large'::regclass) ORDER BY i.indrelid::regclass::text; -- Validate scoring on SMALL partition (3 docs) -- The validation function computes expected BM25 using partition-local statistics SELECT validate_bm25_scoring( 'partition_small', 'content', 'partition_small_content_idx', 'database', 'english', 1.2, 0.75 ) as small_partition_database_valid; SELECT validate_bm25_scoring( 'partition_small', 'content', 'partition_small_content_idx', 'search', 'english', 1.2, 0.75 ) as small_partition_search_valid; -- Validate scoring on LARGE partition (10 docs) SELECT validate_bm25_scoring( 'partition_large', 'content', 'partition_large_content_idx', 'database', 'english', 1.2, 0.75 ) as large_partition_database_valid; SELECT validate_bm25_scoring( 'partition_large', 'content', 'partition_large_content_idx', 'development', 'english', 1.2, 0.75 ) as large_partition_development_valid; -- Show that the same query produces DIFFERENT IDF values in each partition -- because N (total docs) differs: 3 vs 10 -- For 'database': df=2 in small (2/3 docs), df=2 in large (2/10 docs) -- Small IDF = ln(1 + (3 - 2 + 0.5) / (2 + 0.5)) = ln(1 + 1.5/2.5) = ln(1.6) ≈ 0.47 -- Large IDF = ln(1 + (10 - 2 + 0.5) / (2 + 0.5)) = ln(1 + 8.5/2.5) = ln(4.4) ≈ 1.48 -- -- This demonstrates partition-local statistics in action: -- Show actual scores from small partition (only matching documents) SELECT 'small' as partition, content, ROUND((content <@> to_bm25query('database', 'partition_small_content_idx'))::numeric, 4) as score FROM partition_small ORDER BY content <@> to_bm25query('database', 'partition_small_content_idx') LIMIT 10; -- Show actual scores from large partition (only matching documents) SELECT 'large' as partition, content, ROUND((content <@> to_bm25query('database', 'partition_large_content_idx'))::numeric, 4) as score FROM partition_large ORDER BY content <@> to_bm25query('database', 'partition_large_content_idx') LIMIT 10; -- Validate index scan vs standalone scoring consistency per partition SELECT validate_index_vs_standalone( 'partition_small', 'content', 'partition_small_content_idx', 'database' ) as small_partition_modes_match; SELECT validate_index_vs_standalone( 'partition_large', 'content', 'partition_large_content_idx', 'database' ) as large_partition_modes_match; -- ============================================================================= -- Test 5: Score expression replacement via parent table (MergeAppend) -- -- This tests that the planner hook correctly replaces score expressions when -- querying through the parent partitioned table. The query plan uses MergeAppend -- to combine results from partition index scans. The planner hook must: -- 1. Detect BM25 index scans inside MergeAppend children -- 2. Replace score expressions in SELECT with stub functions that retrieve -- cached scores from the index scan -- -- Without proper handling, the SELECT score would use standalone scoring -- (which looks up the parent table's index, not partition indexes), resulting -- in incorrect or zero scores. -- ============================================================================= -- Query across all partitions via parent table with score in both SELECT and ORDER BY -- The planner should use MergeAppend with partition index scans EXPLAIN (COSTS OFF) SELECT content, -(content <@> to_bm25query('database', 'scoring_bm25_idx')) as score FROM partitioned_scoring ORDER BY content <@> to_bm25query('database', 'scoring_bm25_idx') LIMIT 5; -- Verify scores are non-zero when querying via parent table -- If the planner hook fails to replace the score expression, we'd get 0 scores SELECT content, ROUND(-(content <@> to_bm25query('database', 'scoring_bm25_idx'))::numeric, 4) as score FROM partitioned_scoring ORDER BY content <@> to_bm25query('database', 'scoring_bm25_idx') LIMIT 5; -- Verify the returned scores are all non-zero (would fail if planner hook broken) SELECT COUNT(*) as nonzero_scores FROM ( SELECT content, -(content <@> to_bm25query('database', 'scoring_bm25_idx')) as score FROM partitioned_scoring ORDER BY content <@> to_bm25query('database', 'scoring_bm25_idx') LIMIT 5 ) t; -- Cleanup SET enable_seqscan = on; DROP TABLE partitioned_scoring CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/partitioned_many.sql000066400000000000000000000032211525762047700246240ustar00rootroot00000000000000-- Test BM25 index creation on partitioned table with many partitions -- This tests for the "too many LWLocks taken" error that occurs when -- creating indexes on tables with many partitions containing data. -- -- The bug triggers during document processing when per-index locks accumulate. -- With 200 partitions and data, the old code would exceed MAX_SIMUL_LWLOCKS. CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create a partitioned table with 200 partitions -- (200 is enough to trigger the bug - old limit was ~142 partitions) CREATE TABLE docs_many_parts ( id serial, content text, category int ) PARTITION BY RANGE (category); -- Create 200 range partitions DO $$ DECLARE i int; BEGIN FOR i IN 0..199 LOOP EXECUTE format( 'CREATE TABLE docs_many_parts_%s PARTITION OF docs_many_parts FOR VALUES FROM (%s) TO (%s)', i, i, i + 1 ); END LOOP; END $$; -- Insert one row per partition - data is required to trigger the bug -- (the lock is acquired during document processing, not index creation) INSERT INTO docs_many_parts (content, category) SELECT 'test document ' || i, i FROM generate_series(0, 199) AS i; -- This should not fail with "too many LWLocks taken" -- Previously failed at around partition 142 when documents were processed CREATE INDEX docs_many_parts_idx ON docs_many_parts USING bm25(content) WITH (text_config='english'); -- Verify all 200 partition indexes were created SELECT COUNT(*) AS partition_index_count FROM pg_indexes WHERE indexname LIKE 'docs_many_parts_%_content_idx'; -- Clean up DROP TABLE docs_many_parts CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/pgstats.sql000066400000000000000000000026501525762047700227500ustar00rootroot00000000000000-- Test that BM25 index scans report statistics to pg_stat_user_indexes CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create a test table with BM25 index CREATE TABLE test_pgstats (id serial PRIMARY KEY, content text); CREATE INDEX test_pgstats_idx ON test_pgstats USING bm25(content) WITH (text_config='english'); INSERT INTO test_pgstats (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz judge my vow'); -- Reset statistics for our index SELECT pg_stat_reset_single_table_counters('test_pgstats_idx'::regclass); -- Run a query that uses the BM25 index scan SELECT content FROM test_pgstats ORDER BY content <@> to_bm25query('fox', 'test_pgstats_idx') LIMIT 5; -- Check that idx_scan was incremented -- (pg_stat counters are updated asynchronously, so force a stats flush) SELECT pg_stat_force_next_flush(); SELECT idx_scan > 0 AS has_scan_stats, idx_tup_read > 0 AS has_tup_read_stats FROM pg_stat_user_indexes WHERE indexrelname = 'test_pgstats_idx'; -- Run a second query and verify the count increases SELECT content FROM test_pgstats ORDER BY content <@> to_bm25query('dog', 'test_pgstats_idx') LIMIT 5; SELECT pg_stat_force_next_flush(); SELECT idx_scan > 1 AS multiple_scans FROM pg_stat_user_indexes WHERE indexrelname = 'test_pgstats_idx'; -- Cleanup DROP TABLE test_pgstats; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/preload.sql000066400000000000000000000032741525762047700227140ustar00rootroot00000000000000-- Preloaded utility hooks must work without CREATE EXTENSION in this database. SELECT EXISTS (SELECT FROM pg_am WHERE amname = 'bm25') AS bm25_installed; CREATE SCHEMA preload_test; CREATE TABLE preload_test.docs (id integer PRIMARY KEY, content text); INSERT INTO preload_test.docs VALUES (1, 'alpha'), (2, 'bravo'); ALTER TABLE preload_test.docs CLUSTER ON docs_pkey; REINDEX DATABASE :"DBNAME"; REINDEX SCHEMA preload_test; REINDEX TABLE preload_test.docs; REINDEX INDEX preload_test.docs_pkey; VACUUM FULL; VACUUM (FULL, SKIP_LOCKED); CLUSTER; -- Ownership changes use the same database-wide BM25 discovery. CREATE ROLE preload_owner; ALTER TABLE preload_test.docs OWNER TO preload_owner; REASSIGN OWNED BY preload_owner TO CURRENT_USER; DROP ROLE preload_owner; SELECT * FROM preload_test.docs ORDER BY id; -- Creating and dropping the extension in this backend must not leave stale -- access-method state in the preloaded hooks. SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE INDEX docs_bm25 ON preload_test.docs USING bm25 (content) WITH (text_config = 'english'); REINDEX DATABASE :"DBNAME"; REINDEX SCHEMA preload_test; VACUUM FULL; CLUSTER; SET enable_seqscan = off; SELECT id FROM preload_test.docs ORDER BY content <@> to_bm25query('alpha', 'preload_test.docs_bm25') LIMIT 1; RESET enable_seqscan; DROP INDEX preload_test.docs_bm25; DROP EXTENSION pg_textsearch; RESET client_min_messages; SELECT EXISTS (SELECT FROM pg_am WHERE amname = 'bm25') AS bm25_installed; REINDEX DATABASE :"DBNAME"; REINDEX SCHEMA preload_test; VACUUM FULL; VACUUM (FULL, SKIP_LOCKED); CLUSTER; SELECT * FROM preload_test.docs ORDER BY id; DROP TABLE preload_test.docs; DROP SCHEMA preload_test; timescale-pg_textsearch-b63fe58/test/sql/queries.sql000066400000000000000000000135341525762047700227430ustar00rootroot00000000000000-- This test demonstrates top-k pg_textsearch query patterns for efficient text search -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; -- Setup test data with realistic documents CREATE TABLE articles ( id SERIAL PRIMARY KEY, title TEXT, content TEXT, category TEXT ); INSERT INTO articles (title, content, category) VALUES ('Introduction to Databases', 'postgresql database management system with advanced indexing capabilities for fast query processing', 'technology'), ('Machine Learning Fundamentals', 'machine learning algorithms and artificial intelligence techniques for data analysis and prediction', 'technology'), ('Text Search Algorithms', 'full text search with tapir ranking and relevance scoring for information retrieval systems', 'technology'), ('Information Retrieval', 'information retrieval and search engines using vector space models and term frequency analysis', 'technology'), ('Natural Language Processing', 'natural language processing techniques including parsing stemming and semantic analysis', 'technology'), ('Database Optimization', 'database indexing and query optimization strategies for improving performance in large datasets', 'technology'), ('Document Classification', 'text mining and document classification using machine learning and statistical methods', 'technology'), ('Search Engine Design', 'search relevance and ranking algorithms with focus on user experience and result quality', 'technology'), ('Distributed Computing', 'distributed systems and scalability challenges in modern cloud computing environments', 'technology'), ('Data Mining Techniques', 'data mining algorithms for pattern discovery and knowledge extraction from large databases', 'technology'), ('Web Search Evolution', 'evolution of web search from simple keyword matching to semantic understanding', 'history'), ('Big Data Analytics', 'big data analytics platforms and tools for processing massive datasets efficiently', 'technology'); -- Create pg_textsearch index with text_config CREATE INDEX articles_tapir_idx ON articles USING bm25(content) WITH (text_config='english'); -- Disable sequential scans to force index usage SET enable_seqscan = off; -- Test 1: Basic text similarity search with LIMIT (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('database', 'articles_tapir_idx') as score FROM articles ORDER BY content <@> 'database' LIMIT 5; SELECT title, content, ROUND((content <@> to_bm25query('database', 'articles_tapir_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> 'database' LIMIT 5; -- Test 2: Multi-term search with ranking (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('machine learning', 'articles_tapir_idx') as score FROM articles ORDER BY content <@> 'machine learning' LIMIT 3; SELECT title, content, ROUND((content <@> to_bm25query('machine learning', 'articles_tapir_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> 'machine learning' LIMIT 3; -- Test 3: Category-filtered search (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('search algorithms', 'articles_tapir_idx') as score FROM articles WHERE category = 'technology' ORDER BY content <@> 'search algorithms' LIMIT 10; SELECT title, content, ROUND((content <@> to_bm25query('search algorithms', 'articles_tapir_idx'))::numeric, 4) as score FROM articles WHERE category = 'technology' ORDER BY content <@> 'search algorithms' LIMIT 10; -- Test 4: Find similar articles to a specific one (standalone scoring with explicit corpus) -- Note: This query uses explicit index name because of dynamic query text SELECT a2.title, a2.content, ROUND((a2.content <@> to_bm25query(a1.content, 'articles_tapir_idx'))::numeric, 4) as score FROM articles a1, articles a2 WHERE a1.id = 3 -- "Text Search Algorithms" article AND a2.id != a1.id ORDER BY a2.content <@> to_bm25query(a1.content, 'articles_tapir_idx') LIMIT 5; -- Test 5: Top results with scoring (implicit index resolution) EXPLAIN (COSTS OFF) SELECT title, content, content <@> to_bm25query('database optimization', 'articles_tapir_idx') as score FROM articles ORDER BY content <@> 'database optimization' LIMIT 10; SELECT title, content, ROUND((content <@> to_bm25query('database optimization', 'articles_tapir_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> 'database optimization' LIMIT 10; -- Test 6: Batch search with different queries -- Note: Uses explicit index name because query text comes from a subquery -- Note: Query plan varies by PG version so EXPLAIN is skipped -- Test scoring consistency for multi-term query (implicit index resolution) -- Note: standalone_results uses explicit index name since it has a WHERE clause \echo 'Testing ORDER BY vs standalone scoring for multi-term queries' WITH order_by_results AS ( SELECT id, title, ROUND((content <@> to_bm25query('machine learning algorithm', 'articles_tapir_idx'))::numeric, 6) as order_by_score FROM articles ORDER BY content <@> 'machine learning algorithm' LIMIT 2 ), standalone_results AS ( SELECT id, title, ROUND((content <@> to_bm25query('machine learning algorithm', 'articles_tapir_idx'))::numeric, 6) as standalone_score FROM articles WHERE id IN (SELECT id FROM order_by_results) ) SELECT o.id, o.title, o.order_by_score, s.standalone_score, CASE WHEN abs(o.order_by_score - s.standalone_score) < 0.000001 THEN '✓ CONSISTENT' ELSE '✗ INCONSISTENT' END as consistency_check FROM order_by_results o JOIN standalone_results s ON o.id = s.id ORDER BY o.id; -- Cleanup DROP TABLE articles CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/query_hint_roundtrip.sql000066400000000000000000000026601525762047700255610ustar00rootroot00000000000000-- Uses the existing test module installed in injection-enabled CI builds. -- This case itself does not require injection points. SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION pg_textsearch_test; CREATE TABLE hint_copy_docs (id int, body text); INSERT INTO hint_copy_docs SELECT g, 'common' FROM generate_series(1, 1000) g; CREATE INDEX hint_copy_idx ON hint_copy_docs USING bm25(body) WITH (text_config='english'); ANALYZE hint_copy_docs; SET enable_seqscan = off; -- Validate zeroed padding, full datum copying and plan serialization. -- Use both implicit index resolution and an explicitly named index. SELECT pg_textsearch_test_query_hint_roundtrip($q$ SELECT id FROM hint_copy_docs WHERE id < 100 ORDER BY body <@> 'common'::bm25query LIMIT 10 OFFSET 2 $q$, 12) AS typed_query_roundtrip; SELECT pg_textsearch_test_query_hint_roundtrip($q$ SELECT id FROM hint_copy_docs WHERE id < 100 ORDER BY body <@> to_bm25query('common', 'hint_copy_idx') LIMIT 20 $q$, 20) AS explicit_query_roundtrip; -- Exercise a large query value, where redundant datum copies cost more. SELECT pg_textsearch_test_query_hint_roundtrip(format($q$ SELECT id FROM hint_copy_docs WHERE id < 100 ORDER BY body <@> %L::bm25query LIMIT 10 $q$, repeat('common ', 10000)), 10) AS large_query_roundtrip; RESET enable_seqscan; DROP TABLE hint_copy_docs; DROP EXTENSION pg_textsearch_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/quoted_identifiers.sql000066400000000000000000000064011525762047700251470ustar00rootroot00000000000000-- Test case: quoted_identifiers -- Tests that BM25 indexes work correctly with quoted (case-sensitive) -- identifiers for both index names and schema names. -- Regression test for https://github.com/timescale/pg_textsearch/issues/285 CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Test 1: Uppercase index name (the reported bug) CREATE TABLE qi_test1 ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX "IX_qi_test1_content" ON qi_test1 USING bm25(content) WITH (text_config='english'); INSERT INTO qi_test1 (content) VALUES ('lorem ipsum dolor sit amet'); INSERT INTO qi_test1 (content) VALUES ('quick brown fox jumps'); INSERT INTO qi_test1 (content) VALUES ('lorem ipsum again'); SELECT id, content FROM qi_test1 ORDER BY id; -- Test 2: Query with uppercase index name SET enable_seqscan = off; SELECT id, content, ROUND((content <@> to_bm25query('lorem', 'IX_qi_test1_content'))::numeric, 4) AS score FROM qi_test1 ORDER BY content <@> to_bm25query('lorem', 'IX_qi_test1_content'), id; -- Test 3: Uppercase schema name with lowercase index CREATE SCHEMA "MySchema"; CREATE TABLE "MySchema".docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX idx_myschema_docs ON "MySchema".docs USING bm25(content) WITH (text_config='english'); INSERT INTO "MySchema".docs (content) VALUES ('hello world'); INSERT INTO "MySchema".docs (content) VALUES ('goodbye world'); SELECT id, content FROM "MySchema".docs ORDER BY id; -- Query using schema-qualified index SELECT id, content, ROUND((content <@> to_bm25query('hello', '"MySchema".idx_myschema_docs'))::numeric, 4) AS score FROM "MySchema".docs ORDER BY content <@> to_bm25query('hello', '"MySchema".idx_myschema_docs'), id; -- Test 4: Both uppercase schema and uppercase index CREATE TABLE "MySchema".docs2 ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX "IX_MyDocs" ON "MySchema".docs2 USING bm25(content) WITH (text_config='english'); INSERT INTO "MySchema".docs2 (content) VALUES ('alpha beta gamma'); INSERT INTO "MySchema".docs2 (content) VALUES ('delta epsilon'); SELECT id, content FROM "MySchema".docs2 ORDER BY id; -- Query using both-uppercase schema-qualified index SELECT id, content, ROUND((content <@> to_bm25query('alpha', '"MySchema"."IX_MyDocs"'))::numeric, 4) AS score FROM "MySchema".docs2 ORDER BY content <@> to_bm25query('alpha', '"MySchema"."IX_MyDocs"'), id; -- Test 5: UPDATE with uppercase index name UPDATE qi_test1 SET content = 'updated lorem ipsum' WHERE id = 1; SELECT id, content FROM qi_test1 ORDER BY id; -- Test 6: DELETE with uppercase index name DELETE FROM qi_test1 WHERE id = 2; SELECT id, content FROM qi_test1 ORDER BY id; -- Test 7: VACUUM with uppercase index name VACUUM qi_test1; SELECT id, content FROM qi_test1 ORDER BY id; -- Test 8: bm25_dump_index with schema-qualified uppercase index SELECT bm25_dump_index('"MySchema"."IX_MyDocs"')::text ~ 'Tapir Index Debug' AS dump_works; -- Test 9: bm25_summarize_index with schema-qualified uppercase index SELECT bm25_summarize_index('"MySchema"."IX_MyDocs"')::text ~ 'total_docs: 2' AS summarize_works; -- Cleanup DROP TABLE qi_test1 CASCADE; DROP TABLE "MySchema".docs CASCADE; DROP TABLE "MySchema".docs2 CASCADE; DROP SCHEMA "MySchema" CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/rescan.sql000066400000000000000000000251741525762047700225440ustar00rootroot00000000000000-- Test: iterative index scan (rescan with increasing limit) -- -- Demonstrates and tests the interaction between LIMIT, WHERE clause -- post-filtering, and the index's internal result limit. -- -- The core problem: the index computes top-k results in one shot. -- When a WHERE clause filters out rows, fewer than LIMIT rows may be -- returned even though qualifying rows exist in the table. -- -- The fix: when the executor exhausts the index's result set without -- satisfying LIMIT, the index rescans with a larger internal limit -- (exponential backoff: 2x, 4x, ...) until enough rows are found. SET log_duration = off; CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET client_min_messages = NOTICE; SET enable_seqscan = false; ------------------------------------------------------------------------ -- Setup: table with two categories, deliberately skewed BM25 scores ------------------------------------------------------------------------ CREATE TABLE rescan_test ( id SERIAL PRIMARY KEY, category TEXT, content TEXT ); -- 30 'common' rows: high BM25 score for 'database' (term appears 3x, -- short document) INSERT INTO rescan_test (category, content) SELECT 'common', 'database database database management optimization' FROM generate_series(1, 30); -- 20 'rare' rows: lower BM25 score for 'database' (term appears 1x, -- longer document, diluted by other terms) INSERT INTO rescan_test (category, content) SELECT 'rare', 'the modern application framework provides a database ' 'connection pooling layer for enterprise deployments' FROM generate_series(1, 20); CREATE INDEX rescan_idx ON rescan_test USING bm25(content) WITH (text_config='english'); ------------------------------------------------------------------------ -- Baseline: without WHERE, LIMIT works correctly ------------------------------------------------------------------------ -- Should return exactly 5 rows SELECT COUNT(*) AS without_where FROM ( SELECT id FROM rescan_test ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 5 ) q; ------------------------------------------------------------------------ -- Test 1: LIMIT + WHERE — complete miss -- -- LIMIT 5 is pushed down to the index (no indexclauses to prevent it). -- The index returns top-5 results, all 'common' rows (higher score). -- WHERE category='rare' filters out all 5 — returns 0 rows. -- -- After fix: index rescans with larger limit until 5 'rare' rows found. ------------------------------------------------------------------------ SELECT COUNT(*) AS with_where_complete_miss FROM ( SELECT id, category FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 5 ) q; ------------------------------------------------------------------------ -- Test 2: default_limit + WHERE — same problem without explicit LIMIT -- -- When no LIMIT clause, the default_limit (set low here) controls -- how many results the index computes. WHERE can still starve results. ------------------------------------------------------------------------ SET pg_textsearch.default_limit = 10; SELECT COUNT(*) AS default_limit_miss FROM ( SELECT id, category FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') ) q; -- Reset to normal SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 3: partial miss — some results pass, but fewer than LIMIT -- -- Mix categories so that some top-k results pass the filter. ------------------------------------------------------------------------ INSERT INTO rescan_test (category, content) VALUES ('mixed', 'database systems and database optimization techniques'), ('mixed', 'database management in enterprise systems'); -- Refresh index knowledge REINDEX INDEX rescan_idx; -- Ask for 10 'mixed' rows but only 2 exist. With a small internal -- limit, the index may not even find those 2 among the top-k results. SET pg_textsearch.default_limit = 5; SELECT COUNT(*) AS partial_miss FROM ( SELECT id, category FROM rescan_test WHERE category = 'mixed' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 10 ) q; SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 4: verify result ordering after rescan -- -- Even when the index rescans with a larger limit, results must still -- be returned in correct BM25 score order. ------------------------------------------------------------------------ -- All 20 'rare' rows should be returned, in score order (they all -- have the same score, so order by id is acceptable as tiebreaker) SELECT id, category, ROUND((content <@> to_bm25query('database', 'rescan_idx'))::numeric, 4) AS score FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 5; ------------------------------------------------------------------------ -- Test 5: large table with aggressive filtering -- -- Scale up to make the problem more realistic. With 500 total rows -- and only 10 matching the filter, the index needs multiple rescans. ------------------------------------------------------------------------ CREATE TABLE rescan_large ( id SERIAL PRIMARY KEY, tag INT, content TEXT ); -- 490 rows with tag=0, high score for 'server' INSERT INTO rescan_large (tag, content) SELECT 0, 'server cluster server configuration server deployment' FROM generate_series(1, 490); -- 10 rows with tag=1, lower score for 'server' INSERT INTO rescan_large (tag, content) SELECT 1, 'the application connects to a server through a ' 'load balanced proxy for high availability' FROM generate_series(1, 10); CREATE INDEX rescan_large_idx ON rescan_large USING bm25(content) WITH (text_config='english'); -- Ask for 8 rows with tag=1. Only 10 exist out of 500 total. -- The index must scan deep enough to find them. SELECT COUNT(*) AS large_table_filter FROM ( SELECT id FROM rescan_large WHERE tag = 1 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 8 ) q; ------------------------------------------------------------------------ -- Test 6: WHERE on indexed column's table, not a separate column -- -- Even when the WHERE references the same table, if it's not an -- index qual, the same problem applies. ------------------------------------------------------------------------ SELECT COUNT(*) AS where_on_id FROM ( SELECT id FROM rescan_large WHERE id > 495 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 3 ) q; ------------------------------------------------------------------------ -- Test 7: LIMIT + OFFSET + WHERE -- -- OFFSET makes this harder: we need to skip rows AND satisfy LIMIT, -- so the index needs even more candidates. ------------------------------------------------------------------------ SELECT COUNT(*) AS limit_offset_where FROM ( SELECT id FROM rescan_test WHERE category = 'rare' ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 3 OFFSET 2 ) q; ------------------------------------------------------------------------ -- Test 8: multiple backoffs required -- -- With default_limit=5, finding 3 rows with tag=1 among 500 rows -- requires the limit to grow: 5 -> 10 -> 20 -> 40 -> 80 -> 160 -> -- 320 -> 640. That's 7 doublings. The NOTICE trace messages from -- the rescan code confirm each backoff step. ------------------------------------------------------------------------ SET pg_textsearch.default_limit = 5; SELECT COUNT(*) AS multi_backoff FROM ( SELECT id FROM rescan_large WHERE tag = 1 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 3 ) q; SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 9: backoff terminates when all matching rows exhausted -- -- Ask for 20 rows with tag=1, but only 10 exist. The index should -- keep doubling until it has scanned all 500 docs, then stop. ------------------------------------------------------------------------ SET pg_textsearch.default_limit = 5; SELECT COUNT(*) AS exhausted_backoff FROM ( SELECT id FROM rescan_large WHERE tag = 1 ORDER BY content <@> to_bm25query('server', 'rescan_large_idx') LIMIT 20 ) q; SET pg_textsearch.default_limit = 1000; ------------------------------------------------------------------------ -- Test 10: Postgres-level rescan via correlated LATERAL join -- -- Forces the executor to call the index's rescan function after -- results have already been returned. The correlation (category = -- v.x) prevents Postgres from materializing the subquery, so each -- outer row triggers a true index rescan that must clean up the -- result arrays from the previous iteration. ------------------------------------------------------------------------ SELECT v.x, t.id, t.category FROM (VALUES ('common'), ('rare')) AS v(x), LATERAL ( SELECT id, category FROM rescan_test WHERE category = v.x ORDER BY content <@> to_bm25query('database', 'rescan_idx') LIMIT 2 ) t ORDER BY v.x, t.id; -- Filtered and invisible candidates must not impose a scan-wide limit. CREATE TABLE rescan_deep ( id int, tag int, content text ) WITH (autovacuum_enabled = false); INSERT INTO rescan_deep SELECT g, 0, 'alpha beta' FROM generate_series(1, 100050) g; INSERT INTO rescan_deep SELECT 100050 + g, 1, 'alpha gamma gamma gamma gamma gamma gamma gamma' FROM generate_series(1, 5) g; CREATE INDEX rescan_deep_idx ON rescan_deep USING bm25(content) WITH (text_config = 'simple'); ANALYZE rescan_deep; SELECT array_agg(id ORDER BY id) AS deep_filtered FROM ( SELECT id FROM rescan_deep WHERE tag = 1 ORDER BY content <@> to_bm25query('alpha', 'rescan_deep_idx') LIMIT 10 ) s; DELETE FROM rescan_deep WHERE tag = 0; SELECT array_agg(id ORDER BY id) AS deep_deleted FROM ( SELECT id FROM rescan_deep ORDER BY content <@> to_bm25query('alpha', 'rescan_deep_idx') LIMIT 10 ) s; DROP TABLE rescan_deep; -- Incremental Sort must see a whole tie group, even beyond 100,000 rows. CREATE TABLE rescan_ties (id int, content text); INSERT INTO rescan_ties SELECT g, 'alpha beta' FROM generate_series(100055, 1, -1) g; CREATE INDEX rescan_ties_idx ON rescan_ties USING bm25(content) WITH (text_config = 'simple'); SELECT array_agg(id ORDER BY id) AS deep_ties FROM ( SELECT id, content <@> to_bm25query('alpha', 'rescan_ties_idx') FROM rescan_ties ORDER BY content <@> to_bm25query('alpha', 'rescan_ties_idx'), id LIMIT 10 ) s; DROP TABLE rescan_ties; DROP TABLE rescan_test CASCADE; DROP TABLE rescan_large CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/rls.sql000066400000000000000000000527601525762047700220720ustar00rootroot00000000000000-- BM25/RLS combinations are allowed by default, but administrators can -- disable creation, rebuild, or later RLS enablement. CREATE EXTENSION pg_textsearch; SHOW pg_textsearch.allow_rls; -- RLS policies filter BM25 index results when the combination is allowed. CREATE ROLE rls_visibility_reader; CREATE TABLE rls_visibility (id integer, content text); INSERT INTO rls_visibility VALUES (1, 'target target target target target target target target'), (2, 'target filler filler filler filler filler'); ALTER TABLE rls_visibility ENABLE ROW LEVEL SECURITY; CREATE POLICY rls_visibility_policy ON rls_visibility TO rls_visibility_reader USING (id = 2); CREATE INDEX rls_visibility_idx ON rls_visibility USING bm25(content) WITH (text_config='english'); GRANT SELECT ON rls_visibility TO rls_visibility_reader; CREATE FUNCTION pg_temp.first_plan_child(query text) RETURNS text LANGUAGE plpgsql AS $$ DECLARE plan json; BEGIN EXECUTE 'EXPLAIN (FORMAT JSON, COSTS OFF) ' || query INTO plan; RETURN plan->0->'Plan'->'Plans'->0->>'Node Type'; END $$; SET enable_seqscan = off; \pset format unaligned SELECT id AS unrestricted_top FROM rls_visibility ORDER BY content <@> to_bm25query('target', 'rls_visibility_idx') LIMIT 1; SET ROLE rls_visibility_reader; SELECT pg_temp.first_plan_child($query$ SELECT id FROM rls_visibility ORDER BY content <@> to_bm25query('target', 'rls_visibility_idx') LIMIT 1 $query$) = 'Index Scan' AS rls_uses_bm25_index; SELECT id AS rls_visible_top FROM rls_visibility ORDER BY content <@> to_bm25query('target', 'rls_visibility_idx') LIMIT 1; RESET ROLE; \pset format aligned RESET enable_seqscan; DROP TABLE rls_visibility; DROP ROLE rls_visibility_reader; CREATE TABLE rls_existing (id integer, content text); INSERT INTO rls_existing VALUES (1, 'known secret term'); ALTER TABLE rls_existing ENABLE ROW LEVEL SECURITY; CREATE INDEX rls_existing_idx ON rls_existing USING bm25(content) WITH (text_config='english'); SET pg_textsearch.allow_rls = off; ALTER TABLE IF EXISTS rls_missing ENABLE ROW LEVEL SECURITY; CREATE INDEX IF NOT EXISTS rls_existing_idx ON rls_existing USING bm25(content) WITH (text_config='english'); \pset format unaligned SELECT indisvalid AS existing_index_valid FROM pg_index WHERE indexrelid = 'rls_existing_idx'::regclass; \pset format aligned -- Existing combinations remain usable when the GUC changes. SELECT content FROM rls_existing ORDER BY content <@> to_bm25query('known', 'rls_existing_idx') LIMIT 1; INSERT INTO rls_existing VALUES (2, 'another known term'); \set VERBOSITY terse REINDEX INDEX rls_existing_idx; \set VERBOSITY default CREATE TABLE rls_before_index (id integer, content text); ALTER TABLE rls_before_index ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse BEGIN; CREATE INDEX CONCURRENTLY rls_concurrent_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ROLLBACK; CREATE INDEX CONCURRENTLY rls_concurrent_top_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); \pset format unaligned SELECT to_regclass('rls_concurrent_top_idx') IS NULL AS no_index_catalog_entry; \pset format aligned CREATE INDEX CONCURRENTLY IF NOT EXISTS rls_concurrent_ifne_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); \pset format unaligned SELECT to_regclass('rls_concurrent_ifne_idx') IS NULL AS no_if_not_exists_catalog_entry; \pset format aligned CREATE INDEX rls_before_index_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); \set VERBOSITY default CREATE TABLE index_before_rls (id integer, content text); CREATE INDEX index_before_rls_idx ON index_before_rls USING bm25(content) WITH (text_config='english'); \set VERBOSITY terse ALTER VIEW index_before_rls ENABLE ROW LEVEL SECURITY; ALTER TABLE index_before_rls DROP COLUMN missing, ENABLE ROW LEVEL SECURITY; \pset format unaligned SELECT relrowsecurity FROM pg_class WHERE oid = 'index_before_rls'::regclass; ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY, DISABLE ROW LEVEL SECURITY; SELECT relrowsecurity FROM pg_class WHERE oid = 'index_before_rls'::regclass; ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY; SELECT relrowsecurity FROM pg_class WHERE oid = 'index_before_rls'::regclass; \pset format aligned \set VERBOSITY default -- A child index is protected by RLS enabled on an inheritance ancestor. CREATE TABLE rls_parent (id integer, content text); CREATE TABLE rls_child () INHERITS (rls_parent); ALTER TABLE rls_parent ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_child_idx ON rls_child USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- A direct partition-child index is protected by RLS on its parent. CREATE TABLE rls_direct_partitioned (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE rls_direct_partition PARTITION OF rls_direct_partitioned FOR VALUES FROM (0) TO (10); ALTER TABLE rls_direct_partitioned ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_direct_partition_idx ON rls_direct_partition USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- A multi-level partition descendant is protected by RLS at the top. CREATE TABLE rls_partition_top (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE rls_partition_mid PARTITION OF rls_partition_top FOR VALUES FROM (0) TO (100) PARTITION BY RANGE (id); CREATE TABLE rls_partition_leaf PARTITION OF rls_partition_mid FOR VALUES FROM (0) TO (10); ALTER TABLE rls_partition_top ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_partition_leaf_idx ON rls_partition_leaf USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- A multiple-inheritance child is protected if either parent has RLS. CREATE TABLE rls_multi_parent_a (id integer, content text); CREATE TABLE rls_multi_parent_b (extra text); CREATE TABLE rls_multi_child () INHERITS (rls_multi_parent_a, rls_multi_parent_b); ALTER TABLE rls_multi_parent_b ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_multi_child_idx ON rls_multi_child USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- An existing BM25-indexed table cannot be attached below an RLS ancestor. CREATE TABLE rls_attach_parent (id integer, content text); ALTER TABLE rls_attach_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_attach_child (id integer, content text); CREATE INDEX rls_attach_child_idx ON rls_attach_child USING bm25(content) WITH (text_config='english'); \set VERBOSITY terse ALTER TABLE rls_attach_child INHERIT rls_attach_parent; \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_attach_child'::regclass AND inhparent = 'rls_attach_parent'::regclass ) AS inherit_not_attached; \pset format aligned \set VERBOSITY default -- An existing BM25-indexed table cannot become an RLS parent's partition. CREATE TABLE rls_attach_partitioned (id integer, content text) PARTITION BY RANGE (id); ALTER TABLE rls_attach_partitioned ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_attach_partition (id integer, content text); CREATE INDEX rls_attach_partition_idx ON rls_attach_partition USING bm25(content) WITH (text_config='english'); \set VERBOSITY terse ALTER TABLE rls_attach_partitioned ATTACH PARTITION rls_attach_partition FOR VALUES FROM (0) TO (10); \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_attach_partition'::regclass AND inhparent = 'rls_attach_partitioned'::regclass ) AS partition_not_attached; \pset format aligned \set VERBOSITY default -- Successful checks do not retain locks on traversed relations. CREATE TABLE rls_lock_root (id integer, content text); CREATE TABLE rls_lock_parent () INHERITS (rls_lock_root); CREATE TABLE rls_lock_child () INHERITS (rls_lock_parent); BEGIN; CREATE INDEX rls_lock_child_idx ON rls_lock_child USING bm25(content) WITH (text_config='english'); \pset format unaligned SELECT count(*) = 0 AS no_ancestor_locks_held FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'relation' AND relation IN ('rls_lock_root'::regclass, 'rls_lock_parent'::regclass) AND mode = 'AccessShareLock' AND granted; \pset format aligned ROLLBACK; CREATE TABLE rls_lock_enable_root (id integer, content text); CREATE TABLE rls_lock_enable_child () INHERITS (rls_lock_enable_root); BEGIN; ALTER TABLE rls_lock_enable_root ENABLE ROW LEVEL SECURITY; \pset format unaligned SELECT count(*) = 0 AS no_descendant_locks_held FROM pg_locks WHERE pid = pg_backend_pid() AND locktype = 'relation' AND relation = 'rls_lock_enable_child'::regclass AND mode = 'AccessShareLock' AND granted; \pset format aligned ROLLBACK; -- Enabling RLS on a parent is blocked by a BM25 index on a descendant. CREATE TABLE index_parent (id integer, content text); CREATE TABLE index_child () INHERITS (index_parent); CREATE INDEX index_child_idx ON index_child USING bm25(content) WITH (text_config='english'); \set VERBOSITY terse ALTER TABLE index_parent ENABLE ROW LEVEL SECURITY; \set VERBOSITY default -- A partition build is protected by RLS enabled on the partitioned parent. CREATE TABLE rls_partitioned (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE rls_partition PARTITION OF rls_partitioned FOR VALUES FROM (0) TO (10); ALTER TABLE rls_partitioned ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_partitioned_idx ON rls_partitioned USING bm25(content) WITH (text_config='english'); \set VERBOSITY default -- Enabling RLS on a partitioned parent is blocked by descendant BM25 indexes. CREATE TABLE index_partitioned (id integer, content text) PARTITION BY RANGE (id); CREATE TABLE index_partition PARTITION OF index_partitioned FOR VALUES FROM (0) TO (10); CREATE INDEX index_partition_idx ON index_partition USING bm25(content) WITH (text_config='english'); \set VERBOSITY terse ALTER TABLE index_partitioned ENABLE ROW LEVEL SECURITY; \set VERBOSITY default -- Event-trigger DDL must be attributed to the command that actually creates -- or alters each catalog object, not to a stale pre-trigger name lookup. CREATE FUNCTION rls_ddl_race_start() RETURNS event_trigger AS $$ DECLARE action text := current_setting('rls_test.action', true); BEGIN IF action IS NULL OR action IN ('', 'running', 'done', 'drop_enable', 'drop_inherit', 'drop_attach') THEN RETURN; END IF; PERFORM set_config('rls_test.action', 'running', true); IF action = 'swap_enable' THEN EXECUTE 'ALTER TABLE rls_evt_enable_target ' 'RENAME TO rls_evt_enable_swap'; EXECUTE 'ALTER TABLE rls_evt_enable_guarded ' 'RENAME TO rls_evt_enable_target'; EXECUTE 'ALTER TABLE rls_evt_enable_swap ' 'RENAME TO rls_evt_enable_guarded'; PERFORM set_config('rls_test.action', 'restore_enable', true); ELSIF action = 'swap_inherit' THEN EXECUTE 'ALTER TABLE rls_evt_inherit_target ' 'RENAME TO rls_evt_inherit_swap'; EXECUTE 'ALTER TABLE rls_evt_inherit_guarded ' 'RENAME TO rls_evt_inherit_target'; EXECUTE 'ALTER TABLE rls_evt_inherit_swap ' 'RENAME TO rls_evt_inherit_guarded'; PERFORM set_config('rls_test.action', 'restore_inherit', true); ELSIF action = 'swap_attach' THEN EXECUTE 'ALTER TABLE rls_evt_attach_target ' 'RENAME TO rls_evt_attach_swap'; EXECUTE 'ALTER TABLE rls_evt_attach_guarded ' 'RENAME TO rls_evt_attach_target'; EXECUTE 'ALTER TABLE rls_evt_attach_swap ' 'RENAME TO rls_evt_attach_guarded'; PERFORM set_config('rls_test.action', 'restore_attach', true); ELSIF action = 'drop_ifne_collision' THEN EXECUTE 'DROP INDEX rls_evt_ifne_idx'; PERFORM set_config('rls_test.action', 'done', true); ELSIF action = 'drop_extension' THEN PERFORM set_config('client_min_messages', 'warning', true); EXECUTE 'DROP EXTENSION pg_textsearch CASCADE'; PERFORM set_config('rls_test.action', 'done', true); END IF; END $$ LANGUAGE plpgsql; CREATE FUNCTION rls_ddl_race_end() RETURNS event_trigger AS $$ DECLARE action text := current_setting('rls_test.action', true); BEGIN IF action IS NULL OR action NOT IN ('restore_enable', 'restore_inherit', 'restore_attach', 'drop_enable', 'drop_inherit', 'drop_attach') THEN RETURN; END IF; PERFORM set_config('rls_test.action', 'running', true); IF action = 'restore_enable' THEN EXECUTE 'ALTER TABLE rls_evt_enable_target ' 'RENAME TO rls_evt_enable_swap'; EXECUTE 'ALTER TABLE rls_evt_enable_guarded ' 'RENAME TO rls_evt_enable_target'; EXECUTE 'ALTER TABLE rls_evt_enable_swap ' 'RENAME TO rls_evt_enable_guarded'; ELSIF action = 'restore_inherit' THEN EXECUTE 'ALTER TABLE rls_evt_inherit_target ' 'RENAME TO rls_evt_inherit_swap'; EXECUTE 'ALTER TABLE rls_evt_inherit_guarded ' 'RENAME TO rls_evt_inherit_target'; EXECUTE 'ALTER TABLE rls_evt_inherit_swap ' 'RENAME TO rls_evt_inherit_guarded'; ELSIF action = 'restore_attach' THEN EXECUTE 'ALTER TABLE rls_evt_attach_target ' 'RENAME TO rls_evt_attach_swap'; EXECUTE 'ALTER TABLE rls_evt_attach_guarded ' 'RENAME TO rls_evt_attach_target'; EXECUTE 'ALTER TABLE rls_evt_attach_swap ' 'RENAME TO rls_evt_attach_guarded'; ELSIF action = 'drop_enable' THEN EXECUTE 'DROP TABLE rls_evt_drop_enable_target CASCADE'; ELSIF action = 'drop_inherit' THEN EXECUTE 'DROP TABLE rls_evt_drop_inherit_child CASCADE'; ELSE EXECUTE 'DROP TABLE rls_evt_drop_attach_child CASCADE'; END IF; PERFORM set_config('rls_test.action', 'done', true); END $$ LANGUAGE plpgsql; CREATE EVENT TRIGGER rls_ddl_race_start_trigger ON ddl_command_start WHEN TAG IN ('ALTER TABLE', 'CREATE INDEX') EXECUTE FUNCTION rls_ddl_race_start(); CREATE EVENT TRIGGER rls_ddl_race_end_trigger ON ddl_command_end WHEN TAG IN ('ALTER TABLE') EXECUTE FUNCTION rls_ddl_race_end(); CREATE TABLE rls_evt_enable_target (id integer, content text); CREATE TABLE rls_evt_enable_guarded (id integer, content text); CREATE INDEX rls_evt_enable_idx ON rls_evt_enable_guarded USING bm25(content) WITH (text_config='english'); SET rls_test.action = 'swap_enable'; \set VERBOSITY terse ALTER TABLE rls_evt_enable_target ENABLE ROW LEVEL SECURITY; \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT relrowsecurity AS swapped_enable_rolled_back FROM pg_class WHERE oid = 'rls_evt_enable_guarded'::regclass; \pset format aligned CREATE TABLE rls_evt_inherit_parent (id integer, content text); ALTER TABLE rls_evt_inherit_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_inherit_target (id integer, content text); CREATE TABLE rls_evt_inherit_guarded (id integer, content text); CREATE INDEX rls_evt_inherit_idx ON rls_evt_inherit_guarded USING bm25(content) WITH (text_config='english'); SET rls_test.action = 'swap_inherit'; \set VERBOSITY terse ALTER TABLE rls_evt_inherit_target INHERIT rls_evt_inherit_parent; \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_evt_inherit_guarded'::regclass AND inhparent = 'rls_evt_inherit_parent'::regclass ) AS swapped_inherit_rolled_back; \pset format aligned CREATE TABLE rls_evt_attach_parent (id integer, content text) PARTITION BY RANGE (id); ALTER TABLE rls_evt_attach_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_attach_target (id integer, content text); CREATE TABLE rls_evt_attach_guarded (id integer, content text); CREATE INDEX rls_evt_attach_idx ON rls_evt_attach_guarded USING bm25(content) WITH (text_config='english'); SET rls_test.action = 'swap_attach'; \set VERBOSITY terse ALTER TABLE rls_evt_attach_parent ATTACH PARTITION rls_evt_attach_target FOR VALUES FROM (0) TO (10); \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT NOT EXISTS ( SELECT 1 FROM pg_inherits WHERE inhrelid = 'rls_evt_attach_guarded'::regclass AND inhparent = 'rls_evt_attach_parent'::regclass ) AS swapped_attach_rolled_back; \pset format aligned CREATE TABLE rls_evt_drop_enable_target (id integer, content text); CREATE INDEX rls_evt_drop_enable_idx ON rls_evt_drop_enable_target USING bm25(content) WITH (text_config='english'); SET rls_test.action = 'drop_enable'; ALTER TABLE rls_evt_drop_enable_target ENABLE ROW LEVEL SECURITY; RESET rls_test.action; \pset format unaligned SELECT to_regclass('rls_evt_drop_enable_target') IS NULL AS dropped_enable_target_absent; \pset format aligned CREATE TABLE rls_evt_drop_inherit_parent (id integer, content text); ALTER TABLE rls_evt_drop_inherit_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_drop_inherit_child (id integer, content text); CREATE INDEX rls_evt_drop_inherit_idx ON rls_evt_drop_inherit_child USING bm25(content) WITH (text_config='english'); SET rls_test.action = 'drop_inherit'; ALTER TABLE rls_evt_drop_inherit_child INHERIT rls_evt_drop_inherit_parent; RESET rls_test.action; \pset format unaligned SELECT to_regclass('rls_evt_drop_inherit_child') IS NULL AS dropped_inherit_child_absent; \pset format aligned CREATE TABLE rls_evt_drop_attach_parent (id integer, content text) PARTITION BY RANGE (id); ALTER TABLE rls_evt_drop_attach_parent ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_drop_attach_child (id integer, content text); CREATE INDEX rls_evt_drop_attach_idx ON rls_evt_drop_attach_child USING bm25(content) WITH (text_config='english'); SET rls_test.action = 'drop_attach'; ALTER TABLE rls_evt_drop_attach_parent ATTACH PARTITION rls_evt_drop_attach_child FOR VALUES FROM (0) TO (10); RESET rls_test.action; \pset format unaligned SELECT to_regclass('rls_evt_drop_attach_child') IS NULL AS dropped_attach_child_absent; \pset format aligned CREATE TABLE rls_evt_ifne_heap (id integer, content text); ALTER TABLE rls_evt_ifne_heap ENABLE ROW LEVEL SECURITY; CREATE TABLE rls_evt_ifne_collision (id integer); CREATE INDEX rls_evt_ifne_idx ON rls_evt_ifne_collision(id); SET rls_test.action = 'drop_ifne_collision'; \set VERBOSITY terse CREATE INDEX CONCURRENTLY IF NOT EXISTS rls_evt_ifne_idx ON rls_evt_ifne_heap USING bm25(content) WITH (text_config='english'); \set VERBOSITY default RESET rls_test.action; \pset format unaligned SELECT EXISTS ( SELECT 1 FROM pg_index WHERE indexrelid = 'rls_evt_ifne_idx'::regclass AND indrelid = 'rls_evt_ifne_collision'::regclass AND indisvalid ) AS ifne_collision_restored, NOT EXISTS ( SELECT 1 FROM pg_index WHERE indrelid = 'rls_evt_ifne_heap'::regclass ) AS no_forbidden_ifne_index; \pset format aligned CREATE TABLE rls_evt_progress_heap (id integer, content text); CREATE INDEX rls_evt_progress_idx ON rls_evt_progress_heap USING bm25(content) WITH (text_config='english'); CREATE TABLE rls_evt_missing_am_heap (id integer, content text); SET rls_test.action = 'drop_extension'; \set VERBOSITY terse CREATE INDEX rls_evt_missing_am_idx ON rls_evt_missing_am_heap USING bm25(content) WITH (text_config='english'); \set VERBOSITY default RESET rls_test.action; REINDEX INDEX rls_evt_progress_idx; DROP EVENT TRIGGER rls_ddl_race_start_trigger; DROP EVENT TRIGGER rls_ddl_race_end_trigger; DROP FUNCTION rls_ddl_race_start(); DROP FUNCTION rls_ddl_race_end(); RESET pg_textsearch.allow_rls; ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY; DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'rls_guc_user') THEN EXECUTE 'REASSIGN OWNED BY rls_guc_user TO CURRENT_USER'; EXECUTE 'DROP OWNED BY rls_guc_user CASCADE'; DROP ROLE rls_guc_user; END IF; END $$; CREATE ROLE rls_guc_user; SET pg_textsearch.allow_rls = off; SET ROLE rls_guc_user; \set VERBOSITY terse CREATE INDEX rls_unauthorized_idx ON rls_before_index USING bm25(content) WITH (text_config='english'); ALTER TABLE index_before_rls ENABLE ROW LEVEL SECURITY; SET pg_textsearch.allow_rls = off; \set VERBOSITY default RESET ROLE; DROP OWNED BY rls_guc_user; DROP ROLE rls_guc_user; DROP TABLE rls_existing, rls_before_index, index_before_rls CASCADE; DROP TABLE rls_child, rls_parent, index_child, index_parent CASCADE; DROP TABLE rls_partitioned, index_partitioned CASCADE; DROP TABLE rls_direct_partitioned, rls_partition_top CASCADE; DROP TABLE rls_multi_child, rls_multi_parent_a, rls_multi_parent_b CASCADE; DROP TABLE rls_attach_child, rls_attach_parent CASCADE; DROP TABLE rls_attach_partitioned, rls_attach_partition CASCADE; DROP TABLE rls_lock_child, rls_lock_parent, rls_lock_root CASCADE; DROP TABLE rls_evt_enable_target, rls_evt_enable_guarded CASCADE; DROP TABLE rls_evt_inherit_target, rls_evt_inherit_guarded, rls_evt_inherit_parent CASCADE; DROP TABLE rls_evt_attach_target, rls_evt_attach_guarded, rls_evt_attach_parent CASCADE; DROP TABLE rls_evt_drop_inherit_parent, rls_evt_drop_attach_parent CASCADE; DROP TABLE rls_evt_ifne_heap, rls_evt_ifne_collision CASCADE; DROP TABLE rls_evt_progress_heap, rls_evt_missing_am_heap CASCADE; CREATE TABLE rls_without_extension (id integer); SET pg_textsearch.allow_rls = off; DROP EXTENSION pg_textsearch; ALTER TABLE rls_without_extension ENABLE ROW LEVEL SECURITY; \set VERBOSITY terse CREATE INDEX rls_missing_am_idx ON rls_without_extension USING bm25(id); \set VERBOSITY default DROP TABLE rls_without_extension; timescale-pg_textsearch-b63fe58/test/sql/schema.sql000066400000000000000000000121641525762047700225240ustar00rootroot00000000000000-- Test case: schema -- Tests index operations with schema-qualified tables CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- Create a custom schema CREATE SCHEMA docs; -- Create table in the custom schema CREATE TABLE docs.articles ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO docs.articles (content) VALUES ('hello world'); INSERT INTO docs.articles (content) VALUES ('goodbye cruel world'); INSERT INTO docs.articles (content) VALUES ('hello cruel goodbye'); -- Create index on schema-qualified table CREATE INDEX articles_idx ON docs.articles USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Test 1: Unqualified index name (not in search path - should fail) SELECT id, content, ROUND((content <@> to_bm25query('hello', 'articles_idx'))::numeric, 4) as score FROM docs.articles ORDER BY content <@> to_bm25query('hello', 'articles_idx'), id; -- Test 2: Schema-qualified index name works correctly SELECT id, content, ROUND((content <@> to_bm25query('hello', 'docs.articles_idx'))::numeric, 4) as score FROM docs.articles ORDER BY content <@> to_bm25query('hello', 'docs.articles_idx'), id; -- Test 3: With search_path set, unqualified names work SET search_path = docs, public; SELECT id, content, ROUND((content <@> to_bm25query('hello', 'articles_idx'))::numeric, 4) as score FROM articles ORDER BY content <@> to_bm25query('hello', 'articles_idx'), id; -- Reset search_path SET search_path = public; -- Test 4: Table in public schema works with unqualified names CREATE TABLE public_articles ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO public_articles (content) VALUES ('hello world'); INSERT INTO public_articles (content) VALUES ('goodbye cruel world'); CREATE INDEX public_articles_idx ON public_articles USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Unqualified index name works for public schema tables SELECT id, content, ROUND((content <@> to_bm25query('hello', 'public_articles_idx'))::numeric, 4) as score FROM public_articles ORDER BY content <@> to_bm25query('hello', 'public_articles_idx'), id; -- Test 5: INSERT operations with schema-qualified tables and implicit index -- names (reproduces the reported bug) CREATE SCHEMA bar; CREATE TABLE bar.baz(content TEXT NOT NULL); -- Create index with implicit name (PostgreSQL will name it baz_content_idx) CREATE INDEX ON bar.baz USING bm25(content) WITH (text_config='english'); -- INSERTs work correctly with schema-qualified tables INSERT INTO bar.baz VALUES('really'); INSERT INTO bar.baz VALUES('testing schemas'); INSERT INTO bar.baz VALUES('another document'); -- Verify data was inserted correctly SELECT * FROM bar.baz ORDER BY content; -- Test search functionality SELECT content, ROUND((content <@> to_bm25query('testing', 'bar.baz_content_idx'))::numeric, 4) as score FROM bar.baz ORDER BY content <@> to_bm25query('testing', 'bar.baz_content_idx'), content; -- Test BM25 edge case: With only 2 documents where a term appears in exactly -- 50% of them (1 out of 2), the IDF becomes log(1) = 0, making the entire -- BM25 score 0 regardless of term frequency. This is mathematically correct -- behavior, not a bug. CREATE SCHEMA edge_case; CREATE TABLE edge_case.demo(content TEXT NOT NULL); CREATE INDEX ON edge_case.demo USING bm25(content) WITH (text_config='english'); -- Insert exactly 2 documents, one with "testing" INSERT INTO edge_case.demo VALUES('really'); INSERT INTO edge_case.demo VALUES('testing schemas'); -- With 2 docs and "testing" in 1 (50%), IDF = log((2-1+0.5)/(1+0.5)) = log(1) = 0 SELECT content, ROUND((content <@> to_bm25query('testing', 'edge_case.demo_content_idx'))::numeric, 4) as score FROM edge_case.demo ORDER BY content <@> to_bm25query('testing', 'edge_case.demo_content_idx'), content; DROP TABLE edge_case.demo CASCADE; DROP SCHEMA edge_case CASCADE; -- Test 6: bm25_dump_index with schema-qualified indexes -- Test that the debug function properly handles schema resolution -- Test debug with public schema index (should work) SELECT bm25_dump_index('public_articles_idx')::text ~ 'Tapir Index Debug: public_articles_idx' AS debug_public_works; -- Test debug with schema-qualified index name (should work) SELECT bm25_dump_index('bar.baz_content_idx')::text ~ 'Tapir Index Debug: bar.baz_content_idx' AS debug_schema_qualified; -- Test debug with unqualified name for schema index (should fail when not in search path) SELECT bm25_dump_index('baz_content_idx')::text ~ 'ERROR: Index ''baz_content_idx'' not found' AS debug_unqualified_fails; -- Test debug with search_path set SET search_path = bar, public; SELECT bm25_dump_index('baz_content_idx')::text ~ 'Tapir Index Debug: baz_content_idx' AS debug_with_search_path; -- Reset search_path SET search_path = public; -- Test debug with non-existent index SELECT bm25_dump_index('nonexistent_idx')::text ~ 'ERROR: Index ''nonexistent_idx'' not found' AS debug_nonexistent; -- Cleanup DROP TABLE bar.baz CASCADE; DROP SCHEMA bar CASCADE; DROP TABLE docs.articles CASCADE; DROP TABLE public_articles CASCADE; DROP SCHEMA docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring1.sql000066400000000000000000000065461525762047700230200ustar00rootroot00000000000000-- Test case: scoring1 -- Generated BM25 test with 2 documents and 2 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring1_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring1_bulk (content) VALUES ('hello world'); INSERT INTO scoring1_bulk (content) VALUES ('goodbye cruel world'); -- Create index after data insertion (bulk build) CREATE INDEX scoring1_bulk_idx ON scoring1_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring1_bulk ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_valid; -- Validate index scan vs standalone scoring match for 'hello' SELECT validate_index_vs_standalone('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'hello') as hello_modes_match; -- Bulk mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring1_bulk ORDER BY content <@> 'cruel', id; -- Validate BM25 scoring for 'cruel' SELECT validate_bm25_scoring('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'cruel', 'english', 1.2, 0.75) as cruel_valid; -- Validate index scan vs standalone scoring match for 'cruel' SELECT validate_index_vs_standalone('scoring1_bulk', 'content', 'scoring1_bulk_idx', 'cruel') as cruel_modes_match; -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring1_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring1_incr_idx ON scoring1_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert test documents incrementally INSERT INTO scoring1_incr (content) VALUES ('hello world'); INSERT INTO scoring1_incr (content) VALUES ('goodbye cruel world'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring1_incr ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring1_incr', 'content', 'scoring1_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; -- Validate index scan vs standalone scoring match for 'hello' (incremental) SELECT validate_index_vs_standalone('scoring1_incr', 'content', 'scoring1_incr_idx', 'hello') as hello_incr_modes_match; -- Incremental mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring1_incr ORDER BY content <@> 'cruel', id; -- Validate BM25 scoring for 'cruel' (incremental) SELECT validate_bm25_scoring('scoring1_incr', 'content', 'scoring1_incr_idx', 'cruel', 'english', 1.2, 0.75) as cruel_incr_valid; -- Validate index scan vs standalone scoring match for 'cruel' (incremental) SELECT validate_index_vs_standalone('scoring1_incr', 'content', 'scoring1_incr_idx', 'cruel') as cruel_incr_modes_match; -- Cleanup DROP TABLE scoring1_bulk CASCADE; DROP TABLE scoring1_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring2.sql000066400000000000000000000105361525762047700230130ustar00rootroot00000000000000-- Test case: scoring2 -- Generated BM25 test with 5 documents and 4 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring2_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring2_bulk (content) VALUES ('hello world'); INSERT INTO scoring2_bulk (content) VALUES ('goodbye world'); INSERT INTO scoring2_bulk (content) VALUES ('hello goodbye'); INSERT INTO scoring2_bulk (content) VALUES ('world domination'); INSERT INTO scoring2_bulk (content) VALUES ('hello'); -- Create index after data insertion (bulk build) CREATE INDEX scoring2_bulk_idx ON scoring2_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_bulk_valid; -- Bulk mode query 2: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'world', id; -- Validate BM25 scoring for 'world' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'world', 'english', 1.2, 0.75) as world_bulk_valid; -- Bulk mode query 3: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; -- Bulk mode query 4: 'domination' SELECT id, content, ROUND((content <@> 'domination')::numeric, 4) as score FROM scoring2_bulk ORDER BY content <@> 'domination', id; -- Validate BM25 scoring for 'domination' SELECT validate_bm25_scoring('scoring2_bulk', 'content', 'scoring2_bulk_idx', 'domination', 'english', 1.2, 0.75) as domination_bulk_valid; -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring2_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring2_incr_idx ON scoring2_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert test documents incrementally INSERT INTO scoring2_incr (content) VALUES ('hello world'); INSERT INTO scoring2_incr (content) VALUES ('goodbye world'); INSERT INTO scoring2_incr (content) VALUES ('hello goodbye'); INSERT INTO scoring2_incr (content) VALUES ('world domination'); INSERT INTO scoring2_incr (content) VALUES ('hello'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; -- Incremental mode query 2: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'world', id; -- Validate BM25 scoring for 'world' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'world', 'english', 1.2, 0.75) as world_incr_valid; -- Incremental mode query 3: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; -- Incremental mode query 4: 'domination' SELECT id, content, ROUND((content <@> 'domination')::numeric, 4) as score FROM scoring2_incr ORDER BY content <@> 'domination', id; -- Validate BM25 scoring for 'domination' (incremental) SELECT validate_bm25_scoring('scoring2_incr', 'content', 'scoring2_incr_idx', 'domination', 'english', 1.2, 0.75) as domination_incr_valid; -- Cleanup DROP TABLE scoring2_bulk CASCADE; DROP TABLE scoring2_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring3.sql000066400000000000000000000060761525762047700230200ustar00rootroot00000000000000-- Test case: scoring3 -- Generated BM25 test with 3 documents and 2 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring3_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring3_bulk (content) VALUES ('the quick brown fox jumps over the lazy dog'); INSERT INTO scoring3_bulk (content) VALUES ('a short sentence'); INSERT INTO scoring3_bulk (content) VALUES ('this is a medium length sentence that contains several words'); -- Create index after data insertion (bulk build) CREATE INDEX scoring3_bulk_idx ON scoring3_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Bulk mode query 1: 'quick' SELECT id, content, ROUND((content <@> to_bm25query('quick', 'scoring3_bulk_idx'))::numeric, 4) as score FROM scoring3_bulk ORDER BY content <@> 'quick', id; -- Validate BM25 scoring for 'quick' SELECT validate_bm25_scoring('scoring3_bulk', 'content', 'scoring3_bulk_idx', 'quick', 'english', 1.2, 0.75) as quick_bulk_valid; -- Bulk mode query 2: 'sentence' SELECT id, content, ROUND((content <@> to_bm25query('sentence', 'scoring3_bulk_idx'))::numeric, 4) as score FROM scoring3_bulk ORDER BY content <@> 'sentence', id; -- Validate BM25 scoring for 'sentence' SELECT validate_bm25_scoring('scoring3_bulk', 'content', 'scoring3_bulk_idx', 'sentence', 'english', 1.2, 0.75) as sentence_bulk_valid; -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring3_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring3_incr_idx ON scoring3_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert test documents incrementally INSERT INTO scoring3_incr (content) VALUES ('the quick brown fox jumps over the lazy dog'); INSERT INTO scoring3_incr (content) VALUES ('a short sentence'); INSERT INTO scoring3_incr (content) VALUES ('this is a medium length sentence that contains several words'); -- Incremental mode query 1: 'quick' SELECT id, content, ROUND((content <@> to_bm25query('quick', 'scoring3_incr_idx'))::numeric, 4) as score FROM scoring3_incr ORDER BY content <@> 'quick', id; -- Validate BM25 scoring for 'quick' (incremental) SELECT validate_bm25_scoring('scoring3_incr', 'content', 'scoring3_incr_idx', 'quick', 'english', 1.2, 0.75) as quick_incr_valid; -- Incremental mode query 2: 'sentence' SELECT id, content, ROUND((content <@> to_bm25query('sentence', 'scoring3_incr_idx'))::numeric, 4) as score FROM scoring3_incr ORDER BY content <@> 'sentence', id; -- Validate BM25 scoring for 'sentence' (incremental) SELECT validate_bm25_scoring('scoring3_incr', 'content', 'scoring3_incr_idx', 'sentence', 'english', 1.2, 0.75) as sentence_incr_valid; -- Cleanup DROP TABLE scoring3_bulk CASCADE; DROP TABLE scoring3_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring4.sql000066400000000000000000000037711525762047700230200ustar00rootroot00000000000000-- Test case: scoring4 -- Generated BM25 test with 2 documents and 1 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring4_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring4_bulk (content) VALUES ('goodbye world'); INSERT INTO scoring4_bulk (content) VALUES ('goodbyes are hard'); -- Create index after data insertion (bulk build) CREATE INDEX scoring4_bulk_idx ON scoring4_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Bulk mode query 1: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring4_bulk ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring4_bulk', 'content', 'scoring4_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring4_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring4_incr_idx ON scoring4_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert test documents incrementally INSERT INTO scoring4_incr (content) VALUES ('goodbye world'); INSERT INTO scoring4_incr (content) VALUES ('goodbyes are hard'); -- Incremental mode query 1: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring4_incr ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring4_incr', 'content', 'scoring4_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; -- Cleanup DROP TABLE scoring4_bulk CASCADE; DROP TABLE scoring4_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring5.sql000066400000000000000000000101001525762047700230010ustar00rootroot00000000000000-- Test case: scoring5 -- Generated BM25 test with 3 documents and 4 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring5_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring5_bulk (content) VALUES ('hello world'); INSERT INTO scoring5_bulk (content) VALUES ('goodbye cruel world'); INSERT INTO scoring5_bulk (content) VALUES ('goodbye nerds'); -- Create index after data insertion (bulk build) CREATE INDEX scoring5_bulk_idx ON scoring5_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_bulk_valid; -- Bulk mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'cruel', id; -- Validate BM25 scoring for 'cruel' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'cruel', 'english', 1.2, 0.75) as cruel_bulk_valid; -- Bulk mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'world', id; -- Validate BM25 scoring for 'world' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'world', 'english', 1.2, 0.75) as world_bulk_valid; -- Bulk mode query 4: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring5_bulk ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring5_bulk', 'content', 'scoring5_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring5_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring5_incr_idx ON scoring5_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert test documents incrementally INSERT INTO scoring5_incr (content) VALUES ('hello world'); INSERT INTO scoring5_incr (content) VALUES ('goodbye cruel world'); INSERT INTO scoring5_incr (content) VALUES ('goodbye nerds'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; -- Incremental mode query 2: 'cruel' SELECT id, content, ROUND((content <@> 'cruel')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'cruel', id; -- Validate BM25 scoring for 'cruel' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'cruel', 'english', 1.2, 0.75) as cruel_incr_valid; -- Incremental mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'world', id; -- Validate BM25 scoring for 'world' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'world', 'english', 1.2, 0.75) as world_incr_valid; -- Incremental mode query 4: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring5_incr ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring5_incr', 'content', 'scoring5_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; -- Cleanup DROP TABLE scoring5_bulk CASCADE; DROP TABLE scoring5_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring6.sql000066400000000000000000000064651525762047700230250ustar00rootroot00000000000000-- Test case: scoring6 -- Generated BM25 test with 2 documents and 3 queries -- Testing both bulk build and incremental build modes CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = true; SET enable_seqscan = off; -- MODE 1: Bulk build (insert data, then create index) CREATE TABLE scoring6_bulk ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO scoring6_bulk (content) VALUES ('hello, world!'); INSERT INTO scoring6_bulk (content) VALUES ('goodbye cruel world...'); -- Create index after data insertion (bulk build) CREATE INDEX scoring6_bulk_idx ON scoring6_bulk USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Bulk mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring6_bulk ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' SELECT validate_bm25_scoring('scoring6_bulk', 'content', 'scoring6_bulk_idx', 'hello', 'english', 1.2, 0.75) as hello_bulk_valid; -- Bulk mode query 2: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring6_bulk ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' SELECT validate_bm25_scoring('scoring6_bulk', 'content', 'scoring6_bulk_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_bulk_valid; -- Bulk mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring6_bulk ORDER BY content <@> 'world', id; -- Validate BM25 scoring for 'world' SELECT validate_bm25_scoring('scoring6_bulk', 'content', 'scoring6_bulk_idx', 'world', 'english', 1.2, 0.75) as world_bulk_valid; -- MODE 2: Incremental build (create index, then insert data) CREATE TABLE scoring6_incr ( id SERIAL PRIMARY KEY, content TEXT ); -- Create index before data insertion (incremental build) CREATE INDEX scoring6_incr_idx ON scoring6_incr USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert test documents incrementally INSERT INTO scoring6_incr (content) VALUES ('hello, world!'); INSERT INTO scoring6_incr (content) VALUES ('goodbye cruel world...'); -- Incremental mode query 1: 'hello' SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) as score FROM scoring6_incr ORDER BY content <@> 'hello', id; -- Validate BM25 scoring for 'hello' (incremental) SELECT validate_bm25_scoring('scoring6_incr', 'content', 'scoring6_incr_idx', 'hello', 'english', 1.2, 0.75) as hello_incr_valid; -- Incremental mode query 2: 'goodbye' SELECT id, content, ROUND((content <@> 'goodbye')::numeric, 4) as score FROM scoring6_incr ORDER BY content <@> 'goodbye', id; -- Validate BM25 scoring for 'goodbye' (incremental) SELECT validate_bm25_scoring('scoring6_incr', 'content', 'scoring6_incr_idx', 'goodbye', 'english', 1.2, 0.75) as goodbye_incr_valid; -- Incremental mode query 3: 'world' SELECT id, content, ROUND((content <@> 'world')::numeric, 4) as score FROM scoring6_incr ORDER BY content <@> 'world', id; -- Validate BM25 scoring for 'world' (incremental) SELECT validate_bm25_scoring('scoring6_incr', 'content', 'scoring6_incr_idx', 'world', 'english', 1.2, 0.75) as world_incr_valid; -- Cleanup DROP TABLE scoring6_bulk CASCADE; DROP TABLE scoring6_incr CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/scoring_injection.sql000066400000000000000000000076231525762047700247760ustar00rootroot00000000000000\pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION dblink; CREATE TEMP TABLE scoring_sessions (name text PRIMARY KEY, pid int); CREATE TABLE scoring_readers (id int, body text) WITH (autovacuum_enabled = false); CREATE INDEX scoring_readers_idx ON scoring_readers USING bm25(body) WITH (text_config = 'simple'); INSERT INTO scoring_readers VALUES (1, 'alpha'), (2, 'needle'); SET enable_seqscan = off; SELECT id FROM scoring_readers ORDER BY body <@> to_bm25query('needle', 'scoring_readers_idx') LIMIT 1; SELECT dblink_connect('reader', format( 'host=%s port=%s dbname=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database())); INSERT INTO scoring_sessions SELECT 'reader', pid FROM dblink('reader', 'SELECT pg_backend_pid()') AS t(pid int); SELECT dblink_exec('reader', 'SET enable_seqscan = off'); SELECT * FROM dblink('reader', 'SELECT injection_points_set_local()') AS t(result text); SELECT * FROM dblink('reader', $$SELECT injection_points_attach('pg-textsearch-scoring-source', 'wait')$$ ) AS t(result text); SELECT dblink_send_query('reader', $q$ SELECT id FROM scoring_readers ORDER BY body <@> to_bm25query('alpha', 'scoring_readers_idx') LIMIT 1 $q$); DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '30 seconds'; BEGIN LOOP PERFORM pg_stat_clear_snapshot(); IF EXISTS ( SELECT 1 FROM pg_stat_activity JOIN scoring_sessions USING (pid) WHERE name = 'reader' AND wait_event_type = 'InjectionPoint' AND wait_event = 'pg-textsearch-scoring-source' ) THEN RETURN; END IF; IF dblink_is_busy('reader') = 0 THEN RAISE EXCEPTION 'reader finished without reaching scoring pause'; END IF; IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'timed out waiting for scoring pause'; END IF; PERFORM pg_sleep(0.01); END LOOP; END $$; -- A reader of the unchanged cache must finish while scoring is paused. SELECT dblink_connect('fast', format( 'host=%s port=%s dbname=%s', current_setting('unix_socket_directories'), current_setting('port'), current_database())); INSERT INTO scoring_sessions SELECT 'fast', pid FROM dblink('fast', 'SELECT pg_backend_pid()') AS t(pid int); SELECT dblink_exec('fast', 'SET enable_seqscan = off'); SELECT dblink_send_query('fast', $q$ SELECT id FROM scoring_readers ORDER BY body <@> to_bm25query('needle', 'scoring_readers_idx') LIMIT 1 $q$); DO $$ DECLARE deadline timestamptz := clock_timestamp() + interval '30 seconds'; wait_type text; wait_name text; BEGIN LOOP EXIT WHEN dblink_is_busy('fast') = 0; PERFORM pg_stat_clear_snapshot(); SELECT wait_event_type, wait_event INTO wait_type, wait_name FROM pg_stat_activity JOIN scoring_sessions USING (pid) WHERE name = 'fast'; -- Detect the regression by its lock wait, not elapsed time. EXIT WHEN wait_type = 'LWLock' AND wait_name = 'tapir_cache_apply_lock'; IF clock_timestamp() >= deadline THEN RAISE EXCEPTION 'fast reader made no progress: %/%', wait_type, wait_name; END IF; PERFORM pg_sleep(0.01); END LOOP; END $$; SELECT dblink_is_busy('fast') AS blocked; SELECT injection_points_wakeup('pg-textsearch-scoring-source'); SELECT * FROM dblink_get_result('fast') AS t(id int); SELECT * FROM dblink_get_result('fast') AS t(id int); SELECT dblink_disconnect('fast'); SELECT * FROM dblink_get_result('reader') AS t(id int); SELECT * FROM dblink_get_result('reader') AS t(id int); SELECT dblink_disconnect('reader'); DROP TABLE scoring_sessions; DROP TABLE scoring_readers; DROP EXTENSION dblink; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/security.sql000066400000000000000000000051361525762047700231340ustar00rootroot00000000000000-- Test security restrictions on debug and admin functions CREATE EXTENSION pg_textsearch; -- Create test data CREATE TABLE security_test (id serial, content text); INSERT INTO security_test (content) VALUES ('test document'); CREATE INDEX security_test_idx ON security_test USING bm25(content) WITH (text_config='english'); -- Create a non-superuser role DO $$ BEGIN IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = 'test_nonsuperuser') THEN EXECUTE 'REASSIGN OWNED BY test_nonsuperuser TO CURRENT_USER'; EXECUTE 'DROP OWNED BY test_nonsuperuser CASCADE'; DROP ROLE test_nonsuperuser; END IF; END $$; CREATE ROLE test_nonsuperuser LOGIN; GRANT ALL ON TABLE security_test TO test_nonsuperuser; GRANT USAGE ON SCHEMA public TO test_nonsuperuser; -- Test as superuser / index owner (should all work) SELECT bm25_dump_index('security_test_idx') IS NOT NULL AS dump_works; SELECT bm25_summarize_index('security_test_idx') IS NOT NULL AS summarize_works; SELECT bm25_spill_index('security_test_idx') IS NULL AS spill_works; SELECT bm25_force_merge('security_test_idx'); -- Switch to non-superuser (not the index owner) SET ROLE test_nonsuperuser; -- Debug functions should fail (REVOKE blocks execution) SELECT bm25_dump_index('security_test_idx'); SELECT bm25_summarize_index('security_test_idx'); -- Admin functions should fail with "must be owner" SELECT bm25_spill_index('security_test_idx'); SELECT bm25_force_merge('security_test_idx'); -- REVOKE should block non-superusers on debug functions SELECT has_function_privilege('test_nonsuperuser', 'bm25_dump_index(text)', 'EXECUTE') AS dump_priv; SELECT has_function_privilege('test_nonsuperuser', 'bm25_summarize_index(text)', 'EXECUTE') AS summarize_priv; -- Resource-control GUCs should require superuser SET pg_textsearch.bulk_load_threshold = 1; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.segments_per_level = 2; SET pg_textsearch.max_segment_size = '1MB'; -- Logging GUCs should require superuser (info disclosure in pooled envs) SET pg_textsearch.log_scores = on; SET pg_textsearch.log_bmw_stats = on; -- User-facing GUCs should still be settable SET pg_textsearch.default_limit = 500; SHOW pg_textsearch.default_limit; SET pg_textsearch.compress_segments = off; SHOW pg_textsearch.compress_segments; -- Reset to superuser RESET ROLE; SHOW pg_textsearch.max_segment_size; SET pg_textsearch.max_segment_size = '1MB'; SHOW pg_textsearch.max_segment_size; RESET pg_textsearch.max_segment_size; -- Cleanup DROP TABLE security_test; DROP OWNED BY test_nonsuperuser CASCADE; DROP ROLE test_nonsuperuser; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/security_acl.sql000066400000000000000000000157551525762047700237630ustar00rootroot00000000000000-- Standalone BM25 scoring must enforce the invoking role's SELECT -- privilege on the indexed relation and column. Opening a caller-named -- index for scoring bypasses the executor's range-table permission -- checks, so the operator function performs the ACL check itself. CREATE EXTENSION pg_textsearch; -- Victim objects live in a schema the attacker cannot use. CREATE SCHEMA victim; CREATE TABLE victim.docs (id serial, secret text, note text); INSERT INTO victim.docs (secret, note) VALUES ('alpha secret token', 'hello world'), ('beta secret token', 'goodbye world'); CREATE INDEX docs_bm25 ON victim.docs USING bm25(secret) WITH (text_config='english'); -- Expression index over two columns not indexed directly. CREATE TABLE victim.expr_docs (id serial, a text, b text); INSERT INTO victim.expr_docs (a, b) VALUES ('alpha apple', 'beta banana'); CREATE INDEX expr_bm25 ON victim.expr_docs USING bm25((a || ' ' || b)) WITH (text_config='english'); -- Partial index whose predicate references a non-indexed column. CREATE TABLE victim.part_docs (id serial, body text, flag boolean); INSERT INTO victim.part_docs (body, flag) VALUES ('alpha apple', true), ('hidden text', false); CREATE INDEX part_bm25 ON victim.part_docs USING bm25(body) WITH (text_config='english') WHERE flag; -- Idempotent role setup DO $$ DECLARE r text; BEGIN FOREACH r IN ARRAY ARRAY['bm25_attacker', 'bm25_reader', 'bm25_colreader', 'bm25_wrongcol', 'bm25_nousage', 'bm25_exprreader', 'bm25_predreader'] LOOP IF EXISTS (SELECT 1 FROM pg_roles WHERE rolname = r) THEN EXECUTE format('REASSIGN OWNED BY %I TO CURRENT_USER', r); EXECUTE format('DROP OWNED BY %I CASCADE', r); EXECUTE format('DROP ROLE %I', r); END IF; END LOOP; END $$; -- Connect-only attacker: no schema usage, no table/column SELECT. CREATE ROLE bm25_attacker LOGIN; -- Reader with full table SELECT. CREATE ROLE bm25_reader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_reader; GRANT SELECT ON victim.docs TO bm25_reader; -- Reader with column-level SELECT on the indexed column only. CREATE ROLE bm25_colreader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_colreader; GRANT SELECT (secret) ON victim.docs TO bm25_colreader; -- Reader with column-level SELECT on a non-indexed column only. CREATE ROLE bm25_wrongcol LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_wrongcol; GRANT SELECT (note) ON victim.docs TO bm25_wrongcol; -- Reader holding table SELECT but lacking schema USAGE: an index scan -- would be denied at the schema, so scoring must be too. CREATE ROLE bm25_nousage LOGIN; GRANT SELECT ON victim.docs TO bm25_nousage; -- Reader used to exercise the expression-index column set. CREATE ROLE bm25_exprreader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_exprreader; GRANT SELECT (a) ON victim.expr_docs TO bm25_exprreader; -- Reader used to exercise the partial-index predicate column set. CREATE ROLE bm25_predreader LOGIN; GRANT USAGE ON SCHEMA victim TO bm25_predreader; GRANT SELECT (body) ON victim.part_docs TO bm25_predreader; -- Owner baseline: scoring works. SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25'))) < 0 AS owner_present; ------------------------------------------------------------------ -- Attacker: every standalone scoring path must be denied. ------------------------------------------------------------------ SET ROLE bm25_attacker; -- Scalar-subquery constructor (hides the constructor from the planner). SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS attacker_present; -- Absent term must be denied too (no zero-vs-negative oracle). SELECT 'zzzabsent'::text <@> (SELECT to_bm25query('zzzabsent', 'victim.docs_bm25')) AS attacker_absent; -- text[] array scoring path. SELECT ARRAY['alpha']::text[] <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS attacker_array; RESET ROLE; ------------------------------------------------------------------ -- Reader with column-level SELECT on a non-indexed column: denied. ------------------------------------------------------------------ SET ROLE bm25_wrongcol; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS wrongcol_present; RESET ROLE; ------------------------------------------------------------------ -- Reader with table SELECT but no schema USAGE: denied at the schema. ------------------------------------------------------------------ SET ROLE bm25_nousage; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25')) AS nousage_present; RESET ROLE; ------------------------------------------------------------------ -- Expression index: a role missing SELECT on any referenced column is -- denied; granting the remaining column allows scoring. ------------------------------------------------------------------ SET ROLE bm25_exprreader; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.expr_bm25')) AS expr_denied; RESET ROLE; GRANT SELECT (b) ON victim.expr_docs TO bm25_exprreader; SET ROLE bm25_exprreader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.expr_bm25'))) < 0 AS expr_allowed; RESET ROLE; ------------------------------------------------------------------ -- Partial index: a role missing SELECT on the predicate column is -- denied; granting it allows scoring. ------------------------------------------------------------------ SET ROLE bm25_predreader; SELECT 'alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.part_bm25')) AS pred_denied; RESET ROLE; GRANT SELECT (flag) ON victim.part_docs TO bm25_predreader; SET ROLE bm25_predreader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.part_bm25'))) < 0 AS pred_allowed; RESET ROLE; ------------------------------------------------------------------ -- Reader with full table SELECT: scoring works. ------------------------------------------------------------------ SET ROLE bm25_reader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25'))) < 0 AS reader_present; SELECT 'zzzabsent'::text <@> (SELECT to_bm25query('zzzabsent', 'victim.docs_bm25')) = 0 AS reader_absent; RESET ROLE; ------------------------------------------------------------------ -- Reader with column-level SELECT on the indexed column: scoring works. ------------------------------------------------------------------ SET ROLE bm25_colreader; SELECT ('alpha'::text <@> (SELECT to_bm25query('alpha', 'victim.docs_bm25'))) < 0 AS colreader_present; RESET ROLE; -- Cleanup DROP SCHEMA victim CASCADE; DROP OWNED BY bm25_attacker, bm25_reader, bm25_colreader, bm25_wrongcol, bm25_nousage, bm25_exprreader, bm25_predreader CASCADE; DROP ROLE bm25_attacker; DROP ROLE bm25_reader; DROP ROLE bm25_colreader; DROP ROLE bm25_wrongcol; DROP ROLE bm25_nousage; DROP ROLE bm25_exprreader; DROP ROLE bm25_predreader; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/segment.sql000066400000000000000000000161101525762047700227210ustar00rootroot00000000000000-- Test case: segment -- Tests segment query functionality with multiple spill cycles and -- post-spill inserts CREATE EXTENSION IF NOT EXISTS pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Create table and index CREATE TABLE segment_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX segment_test_idx ON segment_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -------------------------------------------------------------------------------- -- Phase 1: Initial data in memtable -------------------------------------------------------------------------------- INSERT INTO segment_test (content) VALUES ('hello world'); INSERT INTO segment_test (content) VALUES ('goodbye cruel world'); INSERT INTO segment_test (content) VALUES ('hello goodbye'); INSERT INTO segment_test (content) VALUES ('world peace'); -- Validate BM25 scoring before spill (memtable only) SELECT 'Phase 1: memtable only' AS phase; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; -------------------------------------------------------------------------------- -- Phase 2: First spill to segment -------------------------------------------------------------------------------- SELECT bm25_spill_index('segment_test_idx') IS NOT NULL AS first_spill; -- Validate BM25 scoring after first spill (segment only) SELECT 'Phase 2: after first spill (segment only)' AS phase; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; -------------------------------------------------------------------------------- -- Phase 3: Post-spill inserts (memtable + segment) -------------------------------------------------------------------------------- INSERT INTO segment_test (content) VALUES ('hello again friend'); INSERT INTO segment_test (content) VALUES ('peaceful world harmony'); INSERT INTO segment_test (content) VALUES ('cruel goodbye forever'); -- Validate BM25 scoring with mixed data (segment + memtable) SELECT 'Phase 3: post-spill inserts (segment + memtable)' AS phase; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'cruel', 'english', 1.2, 0.75) AS cruel_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; -------------------------------------------------------------------------------- -- Phase 4: Second spill (two segments now) -------------------------------------------------------------------------------- SELECT bm25_spill_index('segment_test_idx') IS NOT NULL AS second_spill; -- Validate BM25 scoring after second spill (two segments) SELECT 'Phase 4: after second spill (two segments)' AS phase; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'cruel', 'english', 1.2, 0.75) AS cruel_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; -------------------------------------------------------------------------------- -- Phase 5: More inserts after second spill -------------------------------------------------------------------------------- INSERT INTO segment_test (content) VALUES ('hello hello hello'); INSERT INTO segment_test (content) VALUES ('world world'); INSERT INTO segment_test (content) VALUES ('new unique term'); -- Validate with three data sources (two segments + memtable) SELECT 'Phase 5: more inserts (two segments + memtable)' AS phase; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'unique', 'english', 1.2, 0.75) AS unique_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world', 'english', 1.2, 0.75) AS hello_world_valid; -------------------------------------------------------------------------------- -- Phase 6: Third spill and final validation -------------------------------------------------------------------------------- SELECT bm25_spill_index('segment_test_idx') IS NOT NULL AS third_spill; SELECT 'Phase 6: after third spill (three segments)' AS phase; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello', 'english', 1.2, 0.75) AS hello_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'world', 'english', 1.2, 0.75) AS world_valid; SELECT validate_bm25_scoring('segment_test', 'content', 'segment_test_idx', 'hello world cruel peace', 'english', 1.2, 0.75) AS multi_term_valid; -------------------------------------------------------------------------------- -- Verify row counts and final state -------------------------------------------------------------------------------- SELECT COUNT(*) AS total_documents FROM segment_test; -- Show final scores for reference (not validated, just for visibility) SELECT id, content, ROUND((content <@> 'hello')::numeric, 4) AS hello_score FROM segment_test ORDER BY content <@> 'hello', id; -- Cleanup DROP TABLE segment_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/segment_integrity.sql000066400000000000000000000255771525762047700250400ustar00rootroot00000000000000-- Test case: segment_integrity -- Verifies that segment operations preserve exact document scores. -- -- Key insight: BM25 scores legitimately change when documents are added -- (IDF and avgdl change). This test captures scores IMMEDIATELY before -- an operation and verifies they are IDENTICAL after. This catches: -- - Data loss during spill/merge -- - Corruption of posting lists -- - Document length encoding errors -- - Term frequency encoding errors CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.log_scores = false; SET enable_seqscan = off; -- Create table with enough documents to be meaningful CREATE TABLE integrity_test ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX integrity_test_idx ON integrity_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Insert a variety of documents with different term frequencies and lengths INSERT INTO integrity_test (content) VALUES ('hello world'), ('hello hello world'), ('hello hello hello world world'), ('goodbye cruel world'), ('goodbye goodbye'), ('the quick brown fox jumps over the lazy dog'), ('fox fox fox hunting'), ('lazy dog sleeps'), ('world peace and harmony'), ('database indexing and search'), ('full text search with ranking'), ('search search search query'), ('query optimization techniques'), ('hello world database search'), ('unique document here'), ('another unique entry'), ('fox and dog are friends'), ('peace in the world'), ('cruel cruel world'), ('harmony and peace forever'); -------------------------------------------------------------------------------- -- Test 1: Spill integrity (memtable -> segment) -------------------------------------------------------------------------------- -- Capture scores IMMEDIATELY before spill CREATE TEMP TABLE pre_spill_scores AS SELECT id, content, query, score FROM ( SELECT id, content, 'hello'::text AS query, (content <@> to_bm25query('hello', 'integrity_test_idx'))::numeric AS score FROM integrity_test UNION ALL SELECT id, content, 'world'::text, (content <@> to_bm25query('world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'fox'::text, (content <@> to_bm25query('fox', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'search'::text, (content <@> to_bm25query('search', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'hello world'::text, (content <@> to_bm25query('hello world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'peace harmony'::text, (content <@> to_bm25query('peace harmony', 'integrity_test_idx'))::numeric FROM integrity_test ) sub; SELECT 'Test 1: Spill integrity' AS test, COUNT(DISTINCT id) AS docs, COUNT(DISTINCT query) AS queries FROM pre_spill_scores; -- Spill to segment SELECT bm25_spill_index('integrity_test_idx') IS NOT NULL AS spill_ok; -- Verify ALL scores match after spill SELECT CASE WHEN COUNT(*) = (SELECT COUNT(*) FROM pre_spill_scores) THEN 'PASS' ELSE 'FAIL' END AS spill_integrity, COUNT(*) AS matching, (SELECT COUNT(*) FROM pre_spill_scores) AS expected FROM pre_spill_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) <= 0.0001; -- Show any mismatches (should be empty) SELECT 'Spill mismatches:' AS label; SELECT p.id, p.query, ROUND(p.score, 6) AS before, ROUND((p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric, 6) AS after FROM pre_spill_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) > 0.0001 LIMIT 5; DROP TABLE pre_spill_scores; -------------------------------------------------------------------------------- -- Test 2: Second spill integrity (add docs, then spill again) -------------------------------------------------------------------------------- -- Add more documents INSERT INTO integrity_test (content) VALUES ('new document after first spill'), ('hello from the other side'), ('world domination plans'), ('search and rescue mission'), ('fox news network'); -- Capture scores IMMEDIATELY before second spill CREATE TEMP TABLE pre_spill2_scores AS SELECT id, content, query, score FROM ( SELECT id, content, 'hello'::text AS query, (content <@> to_bm25query('hello', 'integrity_test_idx'))::numeric AS score FROM integrity_test UNION ALL SELECT id, content, 'world'::text, (content <@> to_bm25query('world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'search'::text, (content <@> to_bm25query('search', 'integrity_test_idx'))::numeric FROM integrity_test ) sub; SELECT 'Test 2: Second spill integrity' AS test, COUNT(DISTINCT id) AS docs FROM pre_spill2_scores; -- Second spill SELECT bm25_spill_index('integrity_test_idx') IS NOT NULL AS spill2_ok; -- Verify scores match SELECT CASE WHEN COUNT(*) = (SELECT COUNT(*) FROM pre_spill2_scores) THEN 'PASS' ELSE 'FAIL' END AS spill2_integrity, COUNT(*) AS matching, (SELECT COUNT(*) FROM pre_spill2_scores) AS expected FROM pre_spill2_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) <= 0.0001; DROP TABLE pre_spill2_scores; -------------------------------------------------------------------------------- -- Test 3: Merge integrity -------------------------------------------------------------------------------- -- Configure for merge (2 segments triggers merge) SET pg_textsearch.segments_per_level = 2; -- Add more docs and spill to trigger merge INSERT INTO integrity_test (content) VALUES ('merge test document one'), ('merge test hello world'), ('merge fox and dog'), ('search merge optimization'); -- Capture scores IMMEDIATELY before merge-triggering spill CREATE TEMP TABLE pre_merge_scores AS SELECT id, content, query, score FROM ( SELECT id, content, 'hello'::text AS query, (content <@> to_bm25query('hello', 'integrity_test_idx'))::numeric AS score FROM integrity_test UNION ALL SELECT id, content, 'world'::text, (content <@> to_bm25query('world', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'merge'::text, (content <@> to_bm25query('merge', 'integrity_test_idx'))::numeric FROM integrity_test UNION ALL SELECT id, content, 'search'::text, (content <@> to_bm25query('search', 'integrity_test_idx'))::numeric FROM integrity_test ) sub; SELECT 'Test 3: Merge integrity' AS test, COUNT(DISTINCT id) AS docs FROM pre_merge_scores; -- This spill triggers merge SELECT bm25_spill_index('integrity_test_idx') IS NOT NULL AS merge_spill_ok; -- Verify all scores match after merge SELECT CASE WHEN COUNT(*) = (SELECT COUNT(*) FROM pre_merge_scores) THEN 'PASS' ELSE 'FAIL' END AS merge_integrity, COUNT(*) AS matching, (SELECT COUNT(*) FROM pre_merge_scores) AS expected FROM pre_merge_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) <= 0.0001; -- Show any mismatches (should be empty) SELECT 'Merge mismatches:' AS label; SELECT p.id, p.query, ROUND(p.score, 6) AS before, ROUND((p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric, 6) AS after FROM pre_merge_scores p WHERE ABS(p.score - (p.content <@> to_bm25query(p.query, 'integrity_test_idx'))::numeric) > 0.0001 LIMIT 5; DROP TABLE pre_merge_scores; -------------------------------------------------------------------------------- -- Test 4: Document reachability -- Verify documents with matching terms are returned by index scan. -- Note: Documents without any queried terms correctly return score=0 -- and may not appear in ORDER BY results - this is expected behavior. -------------------------------------------------------------------------------- SELECT 'Test 4: Document reachability' AS test; -- Test that documents with 'hello' are found when searching for 'hello' -- Count docs containing 'hello' in table vs docs returned by index SELECT 'hello' AS term, (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'hello')) AS expected, (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('hello', 'integrity_test_idx') ) t) AS found, CASE WHEN (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'hello')) = (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('hello', 'integrity_test_idx') ) t) THEN 'PASS' ELSE 'FAIL' END AS hello_reachable; SELECT 'world' AS term, (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'world')) AS expected, (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('world', 'integrity_test_idx') ) t) AS found, CASE WHEN (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'world')) = (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('world', 'integrity_test_idx') ) t) THEN 'PASS' ELSE 'FAIL' END AS world_reachable; SELECT 'search' AS term, (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'search')) AS expected, (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('search', 'integrity_test_idx') ) t) AS found, CASE WHEN (SELECT COUNT(*) FROM integrity_test WHERE to_tsvector('english', content) @@ to_tsquery('english', 'search')) = (SELECT COUNT(*) FROM ( SELECT id FROM integrity_test ORDER BY content <@> to_bm25query('search', 'integrity_test_idx') ) t) THEN 'PASS' ELSE 'FAIL' END AS search_reachable; -------------------------------------------------------------------------------- -- Summary -------------------------------------------------------------------------------- SELECT 'SUMMARY' AS section, COUNT(*) AS total_documents FROM integrity_test; -- Cleanup DROP TABLE integrity_test CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/segment_reclaim.sql000066400000000000000000000245031525762047700244220ustar00rootroot00000000000000-- Deferred, standby-safe reclaim of displaced segment pages (#380). -- A merge must PARK displaced pages in the tombstone chain rather than -- freeing them immediately; a later VACUUM (once the reclaim horizon -- has advanced past the merge stamp) reclaims them. -- -- Temporary relations isolate the reclaim horizon from other backends -- and standby feedback, and prevent autovacuum from draining the pages. -- WAL and standby safety are covered by standby_reclaim.sh. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TEMP TABLE reclaim_docs (id int, body text); CREATE TEMP TABLE reclaim_control (id int, body text); INSERT INTO reclaim_docs SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; INSERT INTO reclaim_control SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; CREATE INDEX reclaim_idx ON reclaim_docs USING bm25 (body) WITH (text_config = 'english'); CREATE INDEX reclaim_control_idx ON reclaim_control USING bm25 (body) WITH (text_config = 'english'); -- Build wrote a segment directly, so the memtable is empty here; the -- next batch fills it so the following spill produces a second segment. INSERT INTO reclaim_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; INSERT INTO reclaim_control SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS spilled; \pset format unaligned SELECT bm25_spill_index('reclaim_control_idx') > 0 AS control_spilled; -- Merge the L0 segments into one L1 segment; this displaces the source -- segments' pages, which must be parked (not freed). SELECT bm25_force_merge('reclaim_idx'); SELECT bm25_force_merge('reclaim_control_idx'); -- After a merge, displaced pages are parked (> 0), NOT freed. SELECT bm25_pending_free_pages('reclaim_idx') > 0 AS parked_after_merge; -- The control index used identical construction, so its first detached -- batch must have the same size. Drain only that batch before performing -- identical second merges on both indexes; its later parked count then -- measures the second batch alone. SELECT bm25_pending_free_pages('reclaim_idx') AS first_batch_count \gset SELECT bm25_pending_free_pages('reclaim_control_idx') = :first_batch_count AS control_first_batch_matches; SELECT txid_current() IS NOT NULL AS control_t1; SELECT txid_current() IS NOT NULL AS control_t2; VACUUM reclaim_control; SELECT bm25_pending_free_pages('reclaim_control_idx') AS control_after_first_drain; -- Attach another detached batch while the first one is still parked in -- reclaim_idx. If publication replaced the old head, its count would equal -- the control's second batch alone instead of first_batch + control_batch. INSERT INTO reclaim_docs SELECT g, 'alpha beta second merge term' || (g % 50) FROM generate_series(4001, 5000) g; INSERT INTO reclaim_control SELECT g, 'alpha beta second merge term' || (g % 50) FROM generate_series(4001, 5000) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS second_spilled; SELECT bm25_spill_index('reclaim_control_idx') > 0 AS control_second_spilled; SELECT bm25_force_merge('reclaim_idx'); SELECT bm25_force_merge('reclaim_control_idx'); SELECT bm25_pending_free_pages('reclaim_idx') = :first_batch_count + bm25_pending_free_pages('reclaim_control_idx') AS preserved_old_tombstones; \pset format aligned DROP TABLE reclaim_control; -- Advance the session's xid horizon past the merge stamp so the parked -- pages become reclaimable, then VACUUM to drain them. SELECT txid_current() IS NOT NULL AS t1; SELECT txid_current() IS NOT NULL AS t2; VACUUM reclaim_docs; -- All parked pages reclaimed. SELECT bm25_pending_free_pages('reclaim_idx') AS parked_after_vacuum; -- Sanity: queries still work after reclaim. SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM reclaim_docs ORDER BY body <@> to_bm25query('alpha', 'reclaim_idx') LIMIT 10 ) s; -- The drained pages must be genuinely reusable from the FSM, not just -- unlinked from the tombstone chain: allocating a fresh segment must -- pull those freed blocks via GetFreeIndexPage rather than extend the -- index relation. Capture the size, spill a new segment, and assert -- the relation did not grow. A regression that recorded the frees but -- skipped IndexFreeSpaceMapVacuum would still empty the chain above yet -- extend the relation here (the freed blocks would not be discoverable -- by the root-down GetFreeIndexPage search). SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int AS blocks_before_reuse \gset INSERT INTO reclaim_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(5001, 6000) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS reuse_spilled; SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int <= :blocks_before_reuse AS reused_freed_pages_no_extension; -- VACUUM's own segment replacement/drop path must also park displaced -- pages instead of returning them to the FSM immediately (#413). Build a -- fresh index layout so the observation is independent of the merge case -- above. DROP INDEX reclaim_idx; TRUNCATE reclaim_docs; INSERT INTO reclaim_docs SELECT g, 'vacuum alpha beta term' || (g % 50) FROM generate_series(1, 1500) g; CREATE INDEX reclaim_idx ON reclaim_docs USING bm25 (body) WITH (text_config = 'english'); -- Add a second segment through the on-disk memtable, then delete exactly -- those rows so VACUUM drops that all-dead segment. First create an FSM -- free-page pool: unlocked VACUUM preparation may safely allocate its -- detached tombstone pages there instead of extending the relation. INSERT INTO reclaim_docs SELECT g, 'vacuum pool beta term' || (g % 50) FROM generate_series(1501, 3000) g; \pset format unaligned SELECT bm25_spill_index('reclaim_idx') > 0 AS vacuum_pool_spilled; SELECT bm25_force_merge('reclaim_idx'); SELECT txid_current() IS NOT NULL AS vp1; SELECT txid_current() IS NOT NULL AS vp2; VACUUM reclaim_docs; SELECT bm25_pending_free_pages('reclaim_idx') AS pool_after_vacuum; INSERT INTO reclaim_docs SELECT g, 'vacuum dropped beta term' || (g % 50) FROM generate_series(3001, 3020) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS vacuum_spilled; SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int AS blocks_before_vacuum_drop \gset DELETE FROM reclaim_docs WHERE id BETWEEN 3001 AND 3020; VACUUM reclaim_docs; SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int <= :blocks_before_vacuum_drop AS vacuum_tombstone_reused_fsm; -- VACUUM may retain the pages or reclaim them immediately if concurrent -- transactions already advanced the horizon. A later VACUUM drains any -- retained pages. segment_reclaim_injection.sql pins the horizon with -- an injection point to assert the parking behaviour deterministically. SELECT txid_current() IS NOT NULL AS vt1; SELECT txid_current() IS NOT NULL AS vt2; VACUUM reclaim_docs; SELECT bm25_pending_free_pages('reclaim_idx') AS parked_after_vacuum_drain; -- The drained pages must be reusable from the FSM without extending the -- index relation. SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int AS blocks_before_vacuum_reuse \gset INSERT INTO reclaim_docs SELECT g, 'vacuum reuse beta term' || (g % 50) FROM generate_series(3021, 3720) g; SELECT bm25_spill_index('reclaim_idx') > 0 AS vacuum_reuse_spilled; SELECT pg_relation_size('reclaim_idx') / current_setting('block_size')::int <= :blocks_before_vacuum_reuse AS vacuum_reused_freed_pages_no_extension; \pset format aligned -- Force merge may truncate only pages that reclaim has already stamped -- recyclable. Build a free-page pool below a one-page memtable at EOF, then -- let force merge spill into that pool. The retired memtable page remains -- protected by dead_fxid and must survive until VACUUM stamps it -- TP_FREE_PAGE_MAGIC. \pset format unaligned CREATE TEMP TABLE truncate_docs (id int, body text); INSERT INTO truncate_docs SELECT g, 'truncate alpha beta gamma ' || g || ' ' || repeat(md5(g::text), 4) FROM generate_series(1, 2000) g; CREATE INDEX truncate_idx ON truncate_docs USING bm25 (body) WITH (text_config = 'english', compaction = 'off'); INSERT INTO truncate_docs SELECT g, 'truncate alpha beta delta ' || g || ' ' || repeat(md5(g::text), 4) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('truncate_idx') > 0 AS truncate_pool_spilled; SELECT bm25_force_merge('truncate_idx'); SELECT bm25_pending_free_pages('truncate_idx') > 0 AS truncate_pool_parked; INSERT INTO truncate_docs VALUES (4001, 'truncate eof marker'); CREATE TEMP TABLE truncate_suffix AS SELECT max(blkno)::bigint AS blkno FROM bm25_memtable_chain('truncate_idx'); SELECT blkno + 1 = pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AS truncate_memtable_at_eof FROM truncate_suffix; SELECT txid_current() IS NOT NULL AS truncate_t1; SELECT txid_current() IS NOT NULL AS truncate_t2; VACUUM truncate_docs; SELECT bm25_pending_free_pages('truncate_idx') = 0 AS truncate_pool_drained; SELECT bm25_force_merge('truncate_idx'); SELECT EXISTS ( SELECT 1 FROM bm25_memtable_dead_pages('truncate_idx') dead, truncate_suffix suffix WHERE dead.blkno = suffix.blkno AND dead.dead_fxid > 0 ) AND ( SELECT blkno < pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int FROM truncate_suffix ) AS force_merge_preserved_unreclaimed_suffix; SELECT txid_current() IS NOT NULL AS truncate_reclaim_t1; SELECT txid_current() IS NOT NULL AS truncate_reclaim_t2; VACUUM truncate_docs; CREATE TEMP TABLE truncate_recyclable_size AS SELECT pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AS blocks; SELECT bm25_force_merge('truncate_idx'); SELECT before.blocks > pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AND suffix.blkno >= pg_relation_size('truncate_idx'::regclass, 'main') / current_setting('block_size')::int AS force_merge_truncated_recyclable_suffix FROM truncate_recyclable_size before, truncate_suffix suffix; DROP TABLE truncate_docs; \pset format aligned DROP TABLE reclaim_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/segment_reclaim_injection.sql000066400000000000000000000047741525762047700264740ustar00rootroot00000000000000-- Deferred reclaim asserted with a pinned horizon. -- -- segment_reclaim.sql cannot assert that VACUUM parks a dropped -- segment's pages, because a concurrently advancing xid horizon may let -- the same VACUUM reclaim them immediately. Pin the horizon with an -- injection point so parking is observable, then release it and confirm -- a later VACUUM drains the parked pages. -- A temporary table isolates reclaim from unrelated snapshots. Permanent -- table WAL and standby safety are covered by standby_reclaim.sh. \pset format unaligned SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; CREATE TEMP TABLE reclaim_hold_docs (id int, body text); INSERT INTO reclaim_hold_docs SELECT g, 'alpha beta gamma term' || (g % 50) FROM generate_series(1, 1500) g; CREATE INDEX reclaim_hold_idx ON reclaim_hold_docs USING bm25 (body) WITH (text_config = 'english', compaction = 'off'); -- Second segment holding only rows that are about to be deleted, so -- VACUUM drops it wholesale and displaces its pages. INSERT INTO reclaim_hold_docs SELECT g, 'delta epsilon term' || (g % 50) FROM generate_series(3001, 3400) g; SELECT bm25_spill_index('reclaim_hold_idx') > 0 AS second_segment_spilled; SELECT bm25_pending_free_pages('reclaim_hold_idx') AS parked_before_vacuum; SELECT pg_textsearch_test_attach_reclaim_horizon_hold(); DELETE FROM reclaim_hold_docs WHERE id BETWEEN 3001 AND 3400; VACUUM reclaim_hold_docs; -- The horizon is pinned, so the dropped segment's pages must still be -- parked rather than recycled. SELECT bm25_pending_free_pages('reclaim_hold_idx') > 0 AS parked_after_vacuum_drop; -- Repeat VACUUMs must not drain them while the horizon stays pinned. VACUUM reclaim_hold_docs; SELECT bm25_pending_free_pages('reclaim_hold_idx') > 0 AS still_parked_after_second_vacuum; -- Release the horizon; the next VACUUM drains every parked page. SELECT injection_points_detach('pg-textsearch-reclaim-horizon'); SELECT txid_current() IS NOT NULL AS t1; SELECT txid_current() IS NOT NULL AS t2; VACUUM reclaim_hold_docs; SELECT bm25_pending_free_pages('reclaim_hold_idx') AS parked_after_release; -- Queries still work after reclaim. SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM reclaim_hold_docs ORDER BY body <@> to_bm25query('alpha', 'reclaim_hold_idx') LIMIT 10 ) s; DROP TABLE reclaim_hold_docs; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; \pset format aligned timescale-pg_textsearch-b63fe58/test/sql/strings.sql000066400000000000000000000164361525762047700227630ustar00rootroot00000000000000-- Test long string handling including URLs, paths, and long terms -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Setup test table for long strings DROP TABLE IF EXISTS long_string_docs CASCADE; CREATE TABLE long_string_docs ( id SERIAL PRIMARY KEY, content TEXT, category VARCHAR(50) ); -- Insert test data with various long string patterns INSERT INTO long_string_docs (content, category) VALUES -- URLs and web content ('Visit our website at https://www.example-company.com/products/database-solutions/postgresql-extensions/full-text-search?utm_source=docs&utm_medium=referral&utm_campaign=tapir-extension for more information about BM25 indexing', 'web'), ('API endpoint: https://api.search-service.example.org/v2/documents/search/bm25?query=full+text+search&limit=100&offset=0&include_scores=true&config=english', 'api'), ('Documentation available at https://docs.postgresql-extensions.community/tapir/getting-started/installation-guide.html#shared-memory-configuration', 'docs'), -- Long file paths and system content ('Error in file /usr/local/lib/postgresql/extensions/tapir/src/posting-list-management/shared-memory/string-interning-hashtable.c line 1547', 'system'), ('Log file: /var/log/postgresql/tapir-extension/performance-monitoring/bm25-scoring-statistics/daily-reports/2024-08-27-full-text-search-analytics.log', 'logs'), -- Long technical terms and identifiers ('PostgreSQL extension configuration parameter shared_preload_libraries_pgfts_bm25_full_text_search_with_relevance_scoring_enabled', 'config'), ('Function name: tp_calculate_bm25_relevance_score_with_term_frequency_inverse_document_frequency_and_document_length_normalization', 'technical'), -- Mixed long content with URLs embedded ('Our advanced information retrieval system uses BM25 scoring algorithm implementation available at https://github.com/enterprise-search/tapir-postgresql-extension/blob/main/src/algorithms/bm25-relevance-scoring.c for processing full-text search queries', 'mixed'), -- Very long single terms (scientific/chemical names) ('The antidisestablishmentarianism movement was characterized by pneumonoultramicroscopicsilicovolcanoconiosispneumonoultramicroscopicsilicovolcanoconiosis research', 'scientific'), -- Long JSON-like structured content ('Configuration: {"search_engine":"tapir","algorithms":{"primary":"bm25","parameters":{"k1":1.2,"b":0.75}},"text_processing":{"stemming":"english","tokenization":"standard"},"performance":{"shared_memory_mb":64,"max_terms":1000000}}', 'json'); -- Create pg_textsearch index on long strings CREATE INDEX long_strings_idx ON long_string_docs USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- Test 1: Search for URL components SELECT id, LEFT(content, 80) || '...' as content_preview, ROUND((content <@> 'https website')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'https website' LIMIT 5; -- Test 2: Search for technical terms SELECT * FROM ( SELECT id, category, ROUND((content <@> 'postgresql extension')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'postgresql extension' LIMIT 10 ) AS subquery ORDER BY score, id; -- Test 3: Search for long path components SELECT * FROM ( SELECT id, LEFT(content, 60) || '...' as content_preview, ROUND((content <@> 'file log error')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'file log error' LIMIT 10 ) AS subquery ORDER BY score, id; -- Test 4: Test mixed long and short terms SELECT * FROM ( SELECT id, ROUND((content <@> 'algorithm bm25')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'algorithm bm25' LIMIT 10 ) AS subquery ORDER BY score, id; -- Test 5: Performance test with multiple long term queries SELECT * FROM ( SELECT id, category, ROUND((content <@> 'postgresql pg_textsearch extension search')::numeric, 4) as multi_term_score FROM long_string_docs ORDER BY content <@> 'postgresql pg_textsearch extension search' LIMIT 10 ) AS subquery ORDER BY multi_term_score, id; -- Test 6: Test that very long terms don't cause memory issues INSERT INTO long_string_docs (content, category) VALUES ('This document contains a ridiculously long compound word: ' || 'antidisestablishmentarianismpneumonoultramicroscopicsilicovolcanoconiosis' || 'supercalifragilisticexpialidociousfloccinaucinihilipilificationhippopotomonstrosesquippedaliophobia' || ' and should still be indexed correctly', 'stress_test'); -- Verify the stress test document was indexed SELECT * FROM ( SELECT id, LEFT(content, 50) || '...' as preview, ROUND((content <@> 'document ridiculously long')::numeric, 4) as score FROM long_string_docs ORDER BY content <@> 'document ridiculously long' LIMIT 10 ) AS subquery ORDER BY score, id; -- Test 7: Memory usage statistics after indexing long strings SELECT COUNT(*) as total_documents, AVG(LENGTH(content)) as avg_content_length, MAX(LENGTH(content)) as max_content_length, COUNT(CASE WHEN LENGTH(content) > 200 THEN 1 END) as long_documents, COUNT(CASE WHEN LENGTH(content) > 500 THEN 1 END) as very_long_documents FROM long_string_docs; -- Test 8: Test stack/heap threshold boundary (1KB limit for alloca) INSERT INTO long_string_docs (content, category) VALUES -- Just under 1KB threshold (should use stack allocation) ('Short term: ' || REPEAT('stackallocation', 40), 'stack_test'), -- Just over 1KB threshold (should use heap allocation) ('Long term: ' || REPEAT('heapallocationfallback', 50), 'heap_test'), -- Very long content with mixed term lengths ('Mixed content with ' || REPEAT('shortterm ', 20) || ' and ' || REPEAT('verylongtermnameforheapallocationtestingpurposes', 30) || ' to test both stack and heap allocation paths in the same document', 'mixed_allocation'); -- Verify threshold test documents can be searched SELECT id, category, LENGTH(content) as content_length, CASE WHEN LENGTH(content) < 1000 THEN 'Stack allocation likely' ELSE 'Heap allocation likely' END as allocation_type, ROUND((content <@> to_bm25query('term allocation', 'long_strings_idx'))::numeric, 4) as score FROM long_string_docs WHERE category IN ('stack_test', 'heap_test', 'mixed_allocation') ORDER BY content <@> to_bm25query('term allocation', 'long_strings_idx'), content_length; -- Test 9: Extreme length document (tests PostgreSQL's text limit handling) INSERT INTO long_string_docs (content, category) VALUES ('Extreme test: ' || REPEAT('This is a very long repeated phrase for testing PostgreSQL text limits and our hybrid stack heap allocation strategy. ', 200), 'extreme'); -- Verify extreme length document indexing SELECT 'Extreme length test' as test_name, LENGTH(content) as total_length, ROUND((content <@> to_bm25query('extreme testing', 'long_strings_idx'))::numeric, 4) as score FROM long_string_docs WHERE category = 'extreme' ORDER BY content <@> to_bm25query('extreme testing', 'long_strings_idx'); -- Cleanup DROP TABLE long_string_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/temp_table.sql000066400000000000000000000040071525762047700233750ustar00rootroot00000000000000-- -- Tests for BM25 indexes on temporary tables. -- Regression test for GitHub issue #247: crash on rollback after -- scanning a BM25 index on a temp table. -- CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Basic: create, populate, query, drop CREATE TEMP TABLE t_basic (id serial, body text); INSERT INTO t_basic VALUES (DEFAULT, 'hello world'), (DEFAULT, 'foo bar baz'); CREATE INDEX t_basic_idx ON t_basic USING bm25(body) WITH (text_config = 'english'); SELECT id, body <@> 'hello' AS score FROM t_basic ORDER BY score LIMIT 5; DROP TABLE t_basic; -- Rollback to savepoint after scan (original crash from #247) BEGIN; CREATE TEMP TABLE t_sp (id serial, body text); INSERT INTO t_sp VALUES (DEFAULT, 'hello world'), (DEFAULT, 'foo bar baz'); CREATE INDEX t_sp_idx ON t_sp USING bm25(body) WITH (text_config = 'english'); SAVEPOINT sp1; SELECT body <@> 'hello' FROM t_sp LIMIT 1; ROLLBACK TO SAVEPOINT sp1; -- Table and index should still be usable after savepoint rollback SELECT body <@> 'hello' FROM t_sp LIMIT 1; COMMIT; DROP TABLE t_sp; -- Full rollback after scan (second crash case from #247) BEGIN; CREATE TEMP TABLE t_rb (id serial, body text); INSERT INTO t_rb VALUES (DEFAULT, 'hello world'); CREATE INDEX t_rb_idx ON t_rb USING bm25(body) WITH (text_config = 'english'); SELECT body <@> 'hello' FROM t_rb LIMIT 1; ROLLBACK; -- Rollback of CREATE INDEX itself BEGIN; CREATE TEMP TABLE t_rb2 (id serial, body text); INSERT INTO t_rb2 VALUES (DEFAULT, 'hello world'); SAVEPOINT before_idx; CREATE INDEX t_rb2_idx ON t_rb2 USING bm25(body) WITH (text_config = 'english'); ROLLBACK TO SAVEPOINT before_idx; COMMIT; DROP TABLE t_rb2; -- ON COMMIT DROP temp table with index BEGIN; CREATE TEMP TABLE t_ocd (id serial, body text) ON COMMIT DROP; INSERT INTO t_ocd VALUES (DEFAULT, 'on commit drop test'); CREATE INDEX t_ocd_idx ON t_ocd USING bm25(body) WITH (text_config = 'english'); SELECT body <@> 'test' FROM t_ocd LIMIT 1; COMMIT; -- Table should be gone now SELECT * FROM t_ocd; -- expect error DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/text_array.sql000066400000000000000000000100661525762047700234450ustar00rootroot00000000000000-- Test BM25 index on text[] columns CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Basic text array table and index CREATE TABLE docs_array ( id SERIAL PRIMARY KEY, content text[] ); CREATE INDEX docs_array_bm25_idx ON docs_array USING bm25(content) WITH (text_config='english'); INSERT INTO docs_array (content) VALUES (ARRAY['the quick brown fox', 'jumped over the lazy dog']), (ARRAY['database system design', 'query optimization']), (ARRAY['full text search', 'inverted index structure']), (ARRAY['postgres extensions', 'custom access methods']), (ARRAY['information retrieval', 'relevance ranking']); -- BM25 search using explicit bm25query SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('database', 'docs_array_bm25_idx') LIMIT 3; -- Score equivalence: text[] scores should match scalar concatenation -- Create equivalent scalar table CREATE TABLE docs_scalar ( id SERIAL PRIMARY KEY, content text ); CREATE INDEX docs_scalar_bm25_idx ON docs_scalar USING bm25(content) WITH (text_config='english'); INSERT INTO docs_scalar (content) VALUES ('the quick brown fox jumped over the lazy dog'), ('database system design query optimization'), ('full text search inverted index structure'), ('postgres extensions custom access methods'), ('information retrieval relevance ranking'); -- Scores from text[] and text should match for same content SELECT a.id, a.content <@> to_bm25query('database', 'docs_array_bm25_idx') AS array_score, s.content <@> to_bm25query('database', 'docs_scalar_bm25_idx') AS scalar_score FROM docs_array a JOIN docs_scalar s ON a.id = s.id WHERE a.id = 2 ORDER BY a.id; -- NULL handling: NULL elements should be skipped INSERT INTO docs_array (content) VALUES (ARRAY['hello world', NULL, 'test document']); SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('hello', 'docs_array_bm25_idx') LIMIT 1; -- NULL array should be skipped (no crash) INSERT INTO docs_array (content) VALUES (NULL); -- Empty array should be handled gracefully INSERT INTO docs_array (content) VALUES (ARRAY[]::text[]); -- Search still works after NULLs and empties SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('database', 'docs_array_bm25_idx') LIMIT 1; -- UPDATE a text[] row and verify search picks up changes UPDATE docs_array SET content = ARRAY['updated database', 'new content'] WHERE id = 1; SELECT id, content FROM docs_array ORDER BY content <@> to_bm25query('updated', 'docs_array_bm25_idx') LIMIT 1; -- DELETE and verify DELETE FROM docs_array WHERE id = 2; SELECT count(*) FROM docs_array WHERE id = 2; -- Implicit text[] <@> text resolution SELECT id, content FROM docs_array ORDER BY content <@> 'postgres' LIMIT 1; -- Domains over text[] use the array opclass and must be flattened. CREATE DOMAIN text_array_domain AS text[]; CREATE TABLE docs_domain (id int, content text_array_domain); INSERT INTO docs_domain VALUES (1, ARRAY['domainbuild', 'common']); CREATE INDEX docs_domain_idx ON docs_domain USING bm25 (content) WITH (text_config='simple'); SELECT id, round((content <@> to_bm25query('domainbuild', 'docs_domain_idx'))::numeric, 4) AS score FROM docs_domain ORDER BY content <@> to_bm25query('domainbuild', 'docs_domain_idx') LIMIT 5; INSERT INTO docs_domain VALUES (2, ARRAY['domaininsert', 'common']); SELECT id, round((content <@> to_bm25query('domaininsert', 'docs_domain_idx'))::numeric, 4) AS score FROM docs_domain ORDER BY content <@> to_bm25query('domaininsert', 'docs_domain_idx') LIMIT 5; UPDATE docs_domain SET content = ARRAY['domainupdate'] WHERE id = 1; SELECT id, round((content <@> to_bm25query('domainupdate', 'docs_domain_idx'))::numeric, 4) AS score FROM docs_domain ORDER BY content <@> to_bm25query('domainupdate', 'docs_domain_idx') LIMIT 5; -- Clean up DROP TABLE docs_domain; DROP DOMAIN text_array_domain; DROP TABLE docs_array CASCADE; DROP TABLE docs_scalar CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/text_config.sql000066400000000000000000000010151525762047700235660ustar00rootroot00000000000000-- Test parameter "text_config" accepts qualified names CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create table CREATE TABLE text_config_tbl ( id SERIAL PRIMARY KEY, content TEXT ); -- Verify that we can use qualified names when specifying "text_config" -- and the text configuration can be found by Postgres CREATE INDEX text_config_idx1 ON text_config_tbl USING bm25 (content) WITH (text_config = 'pg_catalog.english', k1=1.2, b=0.75); DROP TABLE text_config_tbl; DROP EXTENSION pg_textsearch CASCADE;timescale-pg_textsearch-b63fe58/test/sql/tombstone_bounds_injection.sql000066400000000000000000000077371525762047700267240ustar00rootroot00000000000000-- Invalid head/next links must use corruption handling, not fail in I/O. \pset format unaligned SET client_min_messages = error; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; -- Keep the valid prefix parked so recovery must unlink its bad next link. SELECT pg_textsearch_test_attach_reclaim_horizon_hold(); CREATE TEMP TABLE expected_pending (n bigint); CREATE FUNCTION pg_temp.corrupt_link(at_head boolean, target text) RETURNS void LANGUAGE plpgsql AS $$ DECLARE parked bigint; bad_block bigint; BEGIN CREATE TEMP TABLE bounds_docs (id int, body text); INSERT INTO bounds_docs SELECT g, 'alpha beta' FROM generate_series(1, 40) g; CREATE INDEX bounds_idx ON bounds_docs USING bm25(body) WITH (text_config = 'simple', compaction = 'manual'); INSERT INTO bounds_docs SELECT g, 'alpha gamma' FROM generate_series(41, 80) g; PERFORM bm25_spill_index('bounds_idx'); PERFORM bm25_force_merge('bounds_idx'); parked := bm25_pending_free_pages('bounds_idx'); IF parked <= 0 THEN RAISE EXCEPTION 'expected parked source pages'; END IF; INSERT INTO expected_pending VALUES (CASE WHEN at_head THEN 0 ELSE parked END); bad_block := CASE target WHEN 'zero' THEN 0 WHEN 'eof' THEN pg_relation_size('bounds_idx') / current_setting('block_size')::bigint WHEN 'past_eof' THEN pg_relation_size('bounds_idx') / current_setting('block_size')::bigint + 100 ELSE NULL END; IF bad_block IS NULL THEN RAISE EXCEPTION 'unknown corruption target'; END IF; PERFORM pg_textsearch_test_set_tombstone_link( 'bounds_idx', at_head, bad_block); END $$; CREATE FUNCTION pg_temp.check_diagnostic() RETURNS void LANGUAGE plpgsql AS $$ BEGIN -- Repeating the diagnostic must still error: it cannot repair the chain. FOR attempt IN 1..2 LOOP BEGIN PERFORM bm25_pending_free_pages('bounds_idx'); RAISE EXCEPTION 'corrupt link was accepted'; EXCEPTION WHEN data_corrupted THEN IF SQLERRM NOT LIKE 'pg_textsearch: corrupt tombstone page %' THEN RAISE; END IF; END; END LOOP; END $$; CREATE FUNCTION pg_temp.check_recovery() RETURNS void LANGUAGE plpgsql AS $$ DECLARE hits int[]; BEGIN IF bm25_pending_free_pages('bounds_idx') <> (SELECT n FROM expected_pending) THEN RAISE EXCEPTION 'recovery lost the valid prefix or kept the bad link'; END IF; INSERT INTO bounds_docs VALUES (81, 'probe'); SELECT array_agg(id) INTO hits FROM ( SELECT id FROM bounds_docs ORDER BY body <@> to_bm25query('probe', 'bounds_idx') LIMIT 1 ) ranked; IF hits IS DISTINCT FROM ARRAY[81] THEN RAISE EXCEPTION 'index unusable after recovery: %', hits; END IF; DROP TABLE bounds_docs; TRUNCATE expected_pending; END $$; SELECT pg_temp.corrupt_link(true, 'zero'); SELECT pg_temp.check_diagnostic(); VACUUM bounds_docs; SELECT pg_temp.check_recovery(); SELECT pg_temp.corrupt_link(true, 'eof'); SELECT pg_temp.check_diagnostic(); VACUUM bounds_docs; SELECT pg_temp.check_recovery(); SELECT pg_temp.corrupt_link(true, 'past_eof'); SELECT pg_temp.check_diagnostic(); VACUUM bounds_docs; SELECT pg_temp.check_recovery(); SELECT pg_temp.corrupt_link(false, 'zero'); SELECT pg_temp.check_diagnostic(); VACUUM bounds_docs; SELECT pg_temp.check_recovery(); SELECT pg_temp.corrupt_link(false, 'eof'); SELECT pg_temp.check_diagnostic(); VACUUM bounds_docs; SELECT pg_temp.check_recovery(); SELECT pg_temp.corrupt_link(false, 'past_eof'); SELECT pg_temp.check_diagnostic(); VACUUM bounds_docs; SELECT pg_temp.check_recovery(); SELECT injection_points_detach('pg-textsearch-reclaim-horizon'); DROP TABLE expected_pending; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/tombstone_recover.sql000066400000000000000000000043131525762047700250200ustar00rootroot00000000000000-- Self-healing recovery of a corrupt deferred-free tombstone chain -- (issue #427). -- -- Prevention (the allocator refusing to reuse a live page) stops NEW -- corruption, but an index already corrupted by an older binary stays -- wedged: every write path triggers a merge/vacuum tombstone drain that -- raises "corrupt tombstone page ..." and aborts. The drain must -- instead recover — drop the unverifiable tail of the chain, warn, and -- let writes proceed — so operators are not forced to REINDEX. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; -- Keep the recovery WARNING out of the golden output (it names a -- block number); the test asserts the post-recovery state instead. SET client_min_messages = error; CREATE TABLE recover_docs (id int, body text) WITH (autovacuum_enabled = false); INSERT INTO recover_docs SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; CREATE INDEX recover_idx ON recover_docs USING bm25 (body) WITH (text_config = 'english'); INSERT INTO recover_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('recover_idx') > 0 AS spilled; SELECT bm25_force_merge('recover_idx'); SELECT bm25_pending_free_pages('recover_idx') > 0 AS parked; -- Simulate an already-corrupt chain node, as an older page-reuse bug -- would have left behind. SELECT bm25_test_corrupt_tombstone_head('recover_idx') > 0 AS corrupted; -- A VACUUM drives the tombstone drain. It must self-heal (drop the -- corrupt tail with a WARNING) instead of raising and wedging the -- index for every future write. VACUUM recover_docs; -- The chain was reset past the corruption, so the drain no longer -- errors and reports a clean (empty) pending set. SELECT bm25_pending_free_pages('recover_idx') AS pending_after_recovery; -- Writes and queries proceed normally after recovery. INSERT INTO recover_docs VALUES (999001, 'alpha beta gamma probe'); SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM recover_docs ORDER BY body <@> to_bm25query('probe', 'recover_idx') LIMIT 10 ) s; DROP TABLE recover_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/tombstone_reuse.sql000066400000000000000000000070101525762047700244730ustar00rootroot00000000000000-- Recycled-page corruption of the deferred-free tombstone chain -- (issues #426, #427). -- -- The deferred-free tombstone chain (metap.pending_free_head) parks -- displaced segment pages until a later VACUUM returns them to the FSM. -- The chain links are WAL-logged, but the index FSM that guards those -- pages is NOT crash-safe and its GetFreeIndexPage() claim is not -- atomic. Either can offer a block that is still referenced by the -- tombstone chain. If an allocator then reuses that block, it -- overwrites a live tombstone page, and the next tombstone drain -- fails with "corrupt tombstone page ..." — permanently wedging every -- write path for the index (issue #427). -- -- This test reproduces that hazard deterministically with an -- internal-only scaffold that returns the live head tombstone page to -- the FSM, then drives a normal allocation. A correct allocator must -- refuse to reuse a block that is still a live pg_textsearch structure -- page. -- -- autovacuum + auto-spill are disabled so the only allocations and -- drains are the explicit ones below, keeping FSM state deterministic. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET pg_textsearch.memtable_pages_threshold = 0; SET pg_textsearch.bulk_load_threshold = 0; CREATE TABLE reuse_docs (id int, body text) WITH (autovacuum_enabled = false); INSERT INTO reuse_docs SELECT g, 'alpha beta gamma delta term' || (g % 50) FROM generate_series(1, 2000) g; CREATE INDEX reuse_idx ON reuse_docs USING bm25 (body) WITH (text_config = 'english'); -- Build wrote a segment directly, so the memtable is empty here; the -- next batch fills it so the following spill produces a second segment. INSERT INTO reuse_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(2001, 4000) g; SELECT bm25_spill_index('reuse_idx') > 0 AS spilled; -- Merge the L0 segments into one L1 segment; this displaces the source -- segments' pages, which are parked in the tombstone chain (not freed). SELECT bm25_force_merge('reuse_idx'); SELECT bm25_pending_free_pages('reuse_idx') > 0 AS parked_after_merge; -- Drain any incidental FSM free pages left by build/merge by allocating -- a fresh segment, so the block recycled below is the single -- deterministic allocation target. Parked pages are not in the FSM, so -- the tombstone chain is untouched by this step. INSERT INTO reuse_docs SELECT g, 'alpha beta term' || (g % 50) FROM generate_series(4001, 6000) g; SELECT bm25_spill_index('reuse_idx') > 0 AS drain_spilled; SELECT bm25_pending_free_pages('reuse_idx') > 0 AS still_parked; -- Hand the still-live head tombstone page back to the (now empty) FSM. SELECT bm25_test_recycle_tombstone_head('reuse_idx') > 0 AS recycled; -- The memtable is empty (last op was a spill), so this single insert -- bootstraps a fresh memtable page and allocates it from the FSM — -- picking up the recycled block. A correct allocator must NOT reuse -- that block, because it is still a live tombstone page. INSERT INTO reuse_docs VALUES (999001, 'alpha beta gamma probe'); -- Walking the tombstone chain must still succeed. With a broken -- allocator the insert above overwrote the live tombstone page, so -- this raises: ERROR: pg_textsearch: corrupt tombstone page ... SELECT bm25_pending_free_pages('reuse_idx') >= 0 AS chain_intact; -- Sanity: the index still answers queries and the probe row is found. SELECT count(*) > 0 AS has_hits FROM ( SELECT 1 FROM reuse_docs ORDER BY body <@> to_bm25query('probe', 'reuse_idx') LIMIT 10 ) s; DROP TABLE reuse_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/unlogged_index.sql000066400000000000000000000341131525762047700242550ustar00rootroot00000000000000-- Test unlogged index behavior and initialization fork creation CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Create UNLOGGED table and index CREATE UNLOGGED TABLE unlogged_test ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO unlogged_test (content) VALUES ('test document'); CREATE INDEX unlogged_idx ON unlogged_test USING bm25(content) WITH (text_config='english', k1=1.2, b=0.75); -- VERIFICATION: Check if _init fork exists and has valid size -- -- We verify that the init fork contains at least the meta page SELECT relname, -- Main fork should have data (Metapage + DocID page + etc) pg_relation_size(indexrelid, 'main') > current_setting('block_size')::int as main_has_data, -- Init fork must exist and be at least one page size CASE WHEN pg_relation_size(indexrelid, 'init') >= current_setting('block_size')::int THEN 'PASS' ELSE 'FAIL' END as init_check FROM pg_index i JOIN pg_class c ON c.oid = i.indexrelid WHERE c.relname = 'unlogged_idx'; -- ---------------------------------------------------------------- -- Coverage for level-N → level-(N+1) merge on an UNLOGGED index -- -- merge.c uses GenericXLog for the linkage update on both -- WAL-logged and UNLOGGED / TEMP indexes (issue #374 unified -- the paths). The UNLOGGED case is otherwise unexercised -- by the regression suite -- almost every other test runs against -- WAL-logged indexes -- so drive a real compaction here. -- -- Set segments_per_level = 2 so 3 spills cross the threshold and -- the third spill triggers a level-0 → level-1 merge. -- ---------------------------------------------------------------- CREATE UNLOGGED TABLE unlogged_merge ( id SERIAL PRIMARY KEY, content TEXT ); CREATE INDEX unlogged_merge_idx ON unlogged_merge USING bm25(content) WITH (text_config='english'); SET pg_textsearch.segments_per_level = 2; -- Three explicit spills, all routed through the GenericXLog -- linkage path because the index is UNLOGGED. INSERT INTO unlogged_merge (content) SELECT 'alpha bravo doc ' || i FROM generate_series(1, 50) i; SELECT bm25_spill_index('unlogged_merge_idx') > 0 AS spill1_wrote; INSERT INTO unlogged_merge (content) SELECT 'charlie delta doc ' || i FROM generate_series(51, 100) i; SELECT bm25_spill_index('unlogged_merge_idx') > 0 AS spill2_wrote; -- Third spill should trigger compaction of L0 → L1 via the -- GenericXLog merge fallback (covers merge.c's else-branch). INSERT INTO unlogged_merge (content) SELECT 'echo foxtrot doc ' || i FROM generate_series(101, 150) i; SELECT bm25_spill_index('unlogged_merge_idx') > 0 AS spill3_wrote; -- Verify the merge happened: bm25_summarize_index should report -- at least one segment at level 1 (post-merge target). If the -- merge didn't run, all segments would still be at level 0. SELECT bm25_summarize_index('unlogged_merge_idx') ~ 'L1 Segment' AS l1_has_segment; -- Sanity: a query against the merged corpus still returns rows. SELECT count(*) > 0 AS query_returns_rows FROM ( SELECT id FROM unlogged_merge ORDER BY content <@> to_bm25query('alpha', 'unlogged_merge_idx') LIMIT 10000 ) t; RESET pg_textsearch.segments_per_level; -- ---------------------------------------------------------------- -- Coverage for shrinkage in the GenericXLog merge fallback -- -- The else-branch of merge.c also has to subtract dead-doc count -- and total_tokens from the in-memory atomics and the on-disk -- metapage when V5 alive-bitset shrinkage drops docs at merge -- time. Without a delete+vacuum sequence the shrinkage values -- are zero and the `if (docs_shrinkage > 0)` / `if -- (tokens_shrinkage > 0)` branches stay uncovered. Drive a -- clean delete + VACUUM + force-merge here to exercise them. -- -- Run with the default segments_per_level (= 8) so the L0 spills -- here don't auto-compact: we want the merge to run only via the -- explicit force-merge call after the alive-bitset bits have been -- flipped dead by VACUUM. -- -- HISTORY / FIX (issue #397). -- -- The post-merge docs_persisted assertion has flaked on the PG17/18 -- Sanitizer jobs twice now: -- * first against the legacy `total_docs:` shared-memory atomic -- (fixed in 65b77c52 by switching the assertion to the metapage -- value), -- * then against the new `docs_persisted:` metapage assertion -- itself (issue #397). -- -- Root cause of the #397 flake: every iteration `DROP TABLE` + -- `CREATE UNLOGGED TABLE` + `CREATE INDEX` churns pg_catalog -- (pg_class, pg_attribute, pg_shdepend, ...). That triggers -- autovacuum workers whose `backend_xmin` holds back the cluster's -- `OldestNonRemovableTransactionId`. If our `VACUUM -- unlogged_shrink` runs while such a worker is active, the index -- ambulkdelete callback classifies our DELETE'd tuples as "dead but -- not yet removable", the per-segment alive_bitset never shrinks, -- and `bm25_force_merge` produces a merged segment with all 200 -- source docs instead of the expected 100. Symptom: a clean 2x -- drift (`docs_persisted: 200` and merged-segment `docs=200`). -- -- The diagnostic loop captured exactly this signature when the -- flake fired locally, with the wait DO block below reporting -- `[pid=N,type=autovacuum worker,query=autovacuum: VACUUM ANALYZE -- pg_catalog.pg_shdepend]` as the xmin holder. -- -- Fix: run the post-DELETE workload with a guarded VACUUM cycle: -- -- (a) Wait briefly for any backend with a held xmin to finish. -- Soft timeout (no error) so the loop keeps making forward -- progress even on a heavily-churning cluster. -- (b) Run `VACUUM unlogged_shrink`. -- (c) Wait again, then run `VACUUM` a second time. This closes -- the residual race where a new autovacuum worker spawns -- between the (a) wait-exit and the (b) VACUUM's OldestXmin -- computation. When (b) already shrank the bitset, the -- second VACUUM is a cheap no-op. -- -- The loop count is 8 -- small enough to keep CI wall time short -- and to limit our own catalog churn, but large enough to amplify -- any residual race a single-iteration assertion would miss. Per -- iteration we capture three diagnostic columns: -- -- pre_merge_segs -- scrubbed segment summary AFTER VACUUM, -- BEFORE force_merge. Happy path shows -- both L0 segments annotated `(alive=50, -- dead=50)`; on a stale-bitset flake both -- would still report alive=100. -- docs_persisted -- metapage total_docs after merge (must -- be `docs_persisted: 100`). -- post_merge_segs -- scrubbed segment summary AFTER merge -- (must be one L1 segment with docs=100). -- ---------------------------------------------------------------- CREATE TABLE unlogged_shrink_diag ( iter int PRIMARY KEY, pre_merge_segs text, docs_persisted text, post_merge_segs text ); -- Drive the per-iteration workload via \gexec. Each iteration is -- emitted as multiple rows (one statement per row) so \gexec sends -- each statement as a separate PQexec call -- otherwise psql would -- batch them into one simple-query, which implicitly wraps the -- block in a transaction and rejects VACUUM. Output of the -- generator query and of every executed iteration is suppressed so -- the regression .out file stays focused on the final diagnostic -- SELECTs. \set ECHO none \o /dev/null SET client_min_messages = warning; WITH steps(stepno, sql_template) AS (VALUES (1, $$DROP TABLE IF EXISTS unlogged_shrink CASCADE$$), (2, $$CREATE UNLOGGED TABLE unlogged_shrink ( id SERIAL PRIMARY KEY, content TEXT ) WITH (autovacuum_enabled = false, toast.autovacuum_enabled = false)$$), (3, $$CREATE INDEX unlogged_shrink_idx ON unlogged_shrink USING bm25(content) WITH (text_config='english')$$), (4, $$INSERT INTO unlogged_shrink (content) SELECT 'foo doc ' || i FROM generate_series(1, 100) i$$), (5, $$SELECT bm25_spill_index('unlogged_shrink_idx')$$), (6, $$INSERT INTO unlogged_shrink (content) SELECT 'bar doc ' || i FROM generate_series(101, 200) i$$), (7, $$SELECT bm25_spill_index('unlogged_shrink_idx')$$), (8, $$DELETE FROM unlogged_shrink WHERE id <= 50 OR (id > 100 AND id <= 150)$$), -- Issue #397 workaround: wait for any backend with a held -- backend_xmin to finish, then VACUUM, then wait+VACUUM a -- second time to close the residual race where a fresh -- autovacuum worker spawns between wait-exit and our VACUUM's -- OldestXmin computation. Both waits use a soft 30 s cap -- (RAISE NOTICE on timeout, not RAISE EXCEPTION) so that on a -- heavily-churning dev box the loop keeps making forward -- progress and the final assertion is the source of truth. -- See the comment block above for full root-cause notes. (9, $$DO $WAIT$ DECLARE start_time timestamptz := clock_timestamp(); blockers int; BEGIN LOOP SELECT count(*) INTO blockers FROM pg_stat_activity WHERE backend_xmin IS NOT NULL AND pid <> pg_backend_pid() AND backend_type <> 'walsender'; EXIT WHEN blockers = 0; IF clock_timestamp() - start_time > interval '30 seconds' THEN RAISE NOTICE 'wait for backend_xmin timed out after 30 s'; EXIT; END IF; PERFORM pg_sleep(0.02); END LOOP; END $WAIT$ $$), (10, $$VACUUM unlogged_shrink$$), (11, $$DO $WAIT$ DECLARE start_time timestamptz := clock_timestamp(); blockers int; BEGIN LOOP SELECT count(*) INTO blockers FROM pg_stat_activity WHERE backend_xmin IS NOT NULL AND pid <> pg_backend_pid() AND backend_type <> 'walsender'; EXIT WHEN blockers = 0; IF clock_timestamp() - start_time > interval '30 seconds' THEN RAISE NOTICE 'wait for backend_xmin timed out after 30 s'; EXIT; END IF; PERFORM pg_sleep(0.02); END LOOP; END $WAIT$ $$), (12, $$VACUUM unlogged_shrink$$), (13, $$INSERT INTO unlogged_shrink_diag (iter, pre_merge_segs) SELECT %1$s, COALESCE(NULLIF( array_to_string( ARRAY( SELECT regexp_replace( regexp_replace(line, 'block=\d+, pages=\d+, size=[0-9.]+MB, ', '', 'g'), '\s+', ' ', 'g') FROM unnest(string_to_array( bm25_summarize_index('unlogged_shrink_idx'), E'\n')) AS t(line) WHERE line ~ '^\s*L\d+ Segment ' ORDER BY line ), E'\n'), ''), '')$$), (14, $$SELECT bm25_force_merge('unlogged_shrink_idx')$$), (15, $$UPDATE unlogged_shrink_diag SET docs_persisted = COALESCE( substring(s from 'docs_persisted: \d+'), ''), post_merge_segs = COALESCE(NULLIF( array_to_string( ARRAY( SELECT regexp_replace( regexp_replace(line, 'block=\d+, pages=\d+, size=[0-9.]+MB, ', '', 'g'), '\s+', ' ', 'g') FROM unnest(string_to_array(s, E'\n')) AS t(line) WHERE line ~ '^\s*L\d+ Segment ' ORDER BY line ), E'\n'), ''), '') FROM (SELECT bm25_summarize_index('unlogged_shrink_idx') AS s) sub WHERE iter = %1$s$$) ) SELECT format(sql_template, n) FROM generate_series(1, 8) n, steps ORDER BY n, stepno \gexec RESET client_min_messages; \o \set ECHO queries -- The merge-time alive-bitset shrinkage path only runs when VACUUM -- marked the deleted docs dead, which requires OldestXmin to have -- advanced past our DELETE. If a concurrent backend holds -- OldestXmin back (e.g. autovacuum on the catalog or a parallel -- regression session), ambulkdelete classifies the tuples as "dead -- but not yet removable", the alive_bitset never shrinks (its -- post-VACUUM summary keeps docs=100 with no `dead=`), and the -- merge legitimately keeps all 200 source docs. That is the setup -- failing its precondition (the #397 flake), not a regression. -- -- So assert correctness only for iterations where VACUUM did shrink -- the bitset (pre_merge_segs annotated `dead=`): each must report -- `docs_persisted: 100` and a merged L1 segment with `docs=100`. -- Iterations where OldestXmin was pinned are skipped, leaving the -- test deterministic under concurrent load while still catching a -- broken shrinkage path. -- Regression detector: iterations whose bitset shrank at VACUUM but -- whose merge did NOT drop the dead docs (the #397 "2x drift"). -- Must be empty; the segment columns are included so a failure -- shows the shrinkage ran (`(alive=50, dead=50)` after VACUUM). SELECT iter, pre_merge_segs, docs_persisted, post_merge_segs FROM unlogged_shrink_diag WHERE pre_merge_segs ~ 'dead=' AND (docs_persisted IS DISTINCT FROM 'docs_persisted: 100' OR post_merge_segs !~ 'docs=100') ORDER BY iter; -- Cleanup DROP TABLE unlogged_shrink_diag; DROP TABLE unlogged_shrink; DROP TABLE unlogged_merge; DROP TABLE unlogged_test; DROP EXTENSION pg_textsearch CASCADE;timescale-pg_textsearch-b63fe58/test/sql/unsupported.sql000066400000000000000000000101361525762047700236510ustar00rootroot00000000000000-- Test cases for queries that are NOT YET fully supported -- These document known limitations of the current implicit index resolution CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Setup test tables CREATE TABLE docs ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE categories ( id SERIAL PRIMARY KEY, doc_id INT REFERENCES docs(id), category TEXT ); INSERT INTO docs (content) VALUES ('postgresql database management'), ('machine learning algorithms'), ('text search and retrieval'); INSERT INTO categories (doc_id, category) VALUES (1, 'technology'), (2, 'science'), (3, 'technology'); CREATE INDEX docs_bm25_idx ON docs USING bm25(content) WITH (text_config='english'); SET enable_seqscan = off; -- ============================================================================= -- LIMITATION 1: JOINs with implicit scoring don't use the index -- The query works but falls back to seq scan + standalone scoring. -- This means it returns all documents (non-matching get score 0) vs explicit -- to_bm25query() which uses the index and only returns matching documents. -- ============================================================================= \echo 'Test: JOIN with implicit score - works but returns all docs (no index)' SELECT d.id, d.content, c.category, ROUND((d.content <@> 'database')::numeric, 4) as score FROM docs d JOIN categories c ON d.id = c.doc_id ORDER BY d.content <@> 'database' LIMIT 5; -- The workaround is to use explicit to_bm25query() for score projection \echo 'Test: JOIN with explicit to_bm25query - works correctly' SELECT d.id, d.content, c.category, ROUND((d.content <@> to_bm25query('database', 'docs_bm25_idx'))::numeric, 4) as score FROM docs d JOIN categories c ON d.id = c.doc_id ORDER BY d.content <@> to_bm25query('database', 'docs_bm25_idx') LIMIT 5; -- ============================================================================= -- LIMITATION 2: Multiple BM25 indexes on same column -- Implicit resolution warns and uses one of the indexes; name one -- explicitly with to_bm25query() for control. -- ============================================================================= \echo 'Test: Multiple BM25 indexes on same column' CREATE INDEX docs_bm25_simple_idx ON docs USING bm25(content) WITH (text_config='simple'); -- Implicit resolution warns; the matched row is the same whichever tied -- index the planner picks. SELECT id FROM docs ORDER BY content <@> 'database' LIMIT 3; -- Explicit naming pins a specific index. EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('database', 'docs_bm25_simple_idx') as score FROM docs ORDER BY content <@> to_bm25query('database', 'docs_bm25_simple_idx') LIMIT 3; DROP INDEX docs_bm25_simple_idx; -- ============================================================================= -- LIMITATION 3: Aggregate functions on scores require explicit to_bm25query() -- ============================================================================= \echo 'Test: Aggregate on scores with explicit index' SELECT COUNT(*) as matching_docs, ROUND(AVG((content <@> to_bm25query('database', 'docs_bm25_idx'))::numeric), 4) as avg_score FROM docs; -- ============================================================================= -- LIMITATION 4: text <@> text operator doesn't use index scans -- The planner hook transforms text <@> text to text <@> bm25query, but the -- transformed expression doesn't match for index ordered scans. -- Workaround: Use to_bm25query() function (with or without index name). -- ============================================================================= \echo 'Test: text <@> text operator vs to_bm25query()' -- text <@> text: planner transforms but doesn't use index scan EXPLAIN (COSTS OFF) SELECT id, content <@> 'database' as score FROM docs ORDER BY content <@> 'database' LIMIT 3; -- to_bm25query(): properly uses index scan EXPLAIN (COSTS OFF) SELECT id, content <@> to_bm25query('database') as score FROM docs ORDER BY content <@> to_bm25query('database') LIMIT 3; -- Cleanup DROP TABLE categories CASCADE; DROP TABLE docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/updates.sql000066400000000000000000000073051525762047700227320ustar00rootroot00000000000000-- Test UPDATE operations with BM25 indexes -- This test reproduces the NULL index_state crash reported in production -- Install the extension if not already installed CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Test basic UPDATE operations CREATE TABLE update_test ( id SERIAL PRIMARY KEY, content TEXT, meta JSONB ); -- Create BM25 index CREATE INDEX idx_update_test_content ON update_test USING bm25 (content) WITH (text_config = 'english'); -- Insert initial data INSERT INTO update_test (content, meta) VALUES ('The quick brown fox jumps over the lazy dog', '{"type": "test", "version": 1}'::jsonb), ('PostgreSQL is a powerful database system', '{"type": "database", "version": 1}'::jsonb), ('Full text search with BM25 ranking is efficient', '{"type": "search", "version": 1}'::jsonb); -- Test simple UPDATE UPDATE update_test SET content = 'The quick brown fox runs fast' WHERE id = 1; -- Test UPDATE with RETURNING UPDATE update_test SET content = 'PostgreSQL is an advanced relational database', meta = meta || '{"last_updated": "2025-01-01", "version": 2}'::jsonb WHERE id = 2 RETURNING id, content; -- Test UPDATE matching multiple rows UPDATE update_test SET meta = meta || '{"updated": true}'::jsonb WHERE content ILIKE '%database%'; -- Test UPDATE with complex WHERE clause (similar to production crash) UPDATE update_test SET content = 'Advanced full text search with BM25 scoring', meta = meta || '{"last_updated": "2025-01-15", "num_updates": 3}'::jsonb WHERE meta @> '{"type": "search"}'::jsonb RETURNING id; -- Test UPDATE with NULL content UPDATE update_test SET content = NULL WHERE id = 1; -- Test UPDATE from NULL to non-NULL UPDATE update_test SET content = 'Previously NULL content now has text' WHERE id = 1; -- Test bulk UPDATE UPDATE update_test SET content = content || ' - updated in bulk' WHERE id IN (2, 3); -- Test UPDATE with subquery UPDATE update_test SET content = (SELECT 'Content from subquery for id ' || u.id::text FROM update_test u WHERE u.id = update_test.id) WHERE id = 2; -- Verify data after updates SELECT id, substring(content, 1, 50) as content_preview, meta->>'type' as type FROM update_test ORDER BY id; -- Test UPDATE in transaction with ROLLBACK BEGIN; UPDATE update_test SET content = 'Transaction test' WHERE id = 1; SELECT id, substring(content, 1, 30) as preview FROM update_test WHERE id = 1; ROLLBACK; -- Verify content was not changed SELECT id, substring(content, 1, 30) as preview FROM update_test WHERE id = 1; -- Test UPDATE in transaction with COMMIT BEGIN; UPDATE update_test SET content = 'Committed transaction update' WHERE id = 1; COMMIT; -- Verify content was changed SELECT id, substring(content, 1, 30) as preview FROM update_test WHERE id = 1; -- Test concurrent UPDATE scenario (serialized for single-session test) -- In production, this would be multiple concurrent sessions UPDATE update_test SET content = 'Concurrent update 1' WHERE id = 2; UPDATE update_test SET content = 'Concurrent update 2' WHERE id = 2; -- Test UPDATE with inheritance (should fail gracefully) CREATE TABLE update_parent ( id SERIAL PRIMARY KEY, content TEXT ); CREATE TABLE update_child () INHERITS (update_parent); -- This should fail with a clear error about inheritance not being supported -- We use DO block to catch the error and show it DO $$ BEGIN CREATE INDEX idx_update_parent_content ON update_parent USING bm25 (content) WITH (text_config = 'english'); EXCEPTION WHEN OTHERS THEN RAISE NOTICE 'Expected error: %', SQLERRM; END; $$; -- Clean up DROP TABLE IF EXISTS update_child; DROP TABLE IF EXISTS update_parent; DROP TABLE IF EXISTS update_test CASCADE; DROP EXTENSION IF EXISTS pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/vacuum.sql000066400000000000000000000046341525762047700225670ustar00rootroot00000000000000-- Test VACUUM behavior with BM25 indexes -- Ensure extension is loaded CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Isolate physical cleanup from other backends and standby feedback. CREATE TEMP TABLE vacuum_test ( id SERIAL PRIMARY KEY, content TEXT ); -- Insert test documents INSERT INTO vacuum_test (content) VALUES ('document one with test'), ('document two without keyword'), ('document three with test word'), ('document four plain text'), ('document five has test'); -- Create index CREATE INDEX vacuum_idx ON vacuum_test USING bm25 (content) WITH (text_config = 'english'); -- Force index scan (small table would otherwise prefer seq scan) SET enable_seqscan = off; -- Check index statistics before deletions -- Extract just the total_docs line from debug output SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_before; -- Delete some documents DELETE FROM vacuum_test WHERE id IN ( SELECT id FROM vacuum_test WHERE content LIKE '%test%' LIMIT 2 ); -- Check that index statistics haven't changed yet (deletions not processed) SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_after_delete; -- Run VACUUM to clean up deleted tuples VACUUM vacuum_test; -- Check index statistics after VACUUM -- Note: Our current implementation may not update stats until rebuild SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_after_vacuum; -- Search should still work correctly after VACUUM SELECT id, substring(content, 1, 30) as content_preview FROM vacuum_test ORDER BY content <@> to_bm25query('test', 'vacuum_idx') LIMIT 5; -- Test VACUUM FULL (more aggressive cleanup) -- VACUUM FULL rebuilds indexes DELETE FROM vacuum_test WHERE content NOT LIKE '%test%'; -- We use \set VERBOSITY terse to avoid OID-specific error messages \set VERBOSITY terse VACUUM FULL vacuum_test; \set VERBOSITY default -- Check statistics after VACUUM FULL SELECT split_part( split_part(bm25_dump_index('vacuum_idx')::text, 'total_docs: ', 2), E'\n', 1 ) AS total_docs_after_vacuum_full; -- Verify search still works SELECT id, substring(content, 1, 30) as content_preview FROM vacuum_test ORDER BY content <@> to_bm25query('test', 'vacuum_idx'); -- Clean up DROP TABLE vacuum_test; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/vacuum_bitmap.sql000066400000000000000000000263721525762047700241260ustar00rootroot00000000000000-- Test alive bitset tombstone tracking during VACUUM CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Temporary relations keep VACUUM's horizon independent of other -- backends and standby feedback, so deleted tuples can be removed. -- ============================================================= -- Test 1: Basic VACUUM with bitmap marking -- ============================================================= CREATE TEMP TABLE vb_test (id serial PRIMARY KEY, content text); INSERT INTO vb_test (content) SELECT 'document number ' || i || ' about databases' FROM generate_series(1, 100) i; CREATE INDEX vb_idx ON vb_test USING bm25 (content) WITH (text_config = 'english'); -- Verify initial query works SELECT count(*) FROM ( SELECT id FROM vb_test ORDER BY content <@> to_bm25query('databases', 'vb_idx') LIMIT 1000 ) q; -- Delete some rows DELETE FROM vb_test WHERE id <= 20; -- VACUUM to apply bitmap marking VACUUM vb_test; -- Query should return only live rows SELECT count(*) FROM ( SELECT id FROM vb_test ORDER BY content <@> to_bm25query('databases', 'vb_idx') LIMIT 1000 ) q; -- Verify no deleted rows appear SELECT count(*) FROM ( SELECT id FROM vb_test ORDER BY content <@> to_bm25query('databases', 'vb_idx') LIMIT 1000 ) q WHERE id <= 20; DROP TABLE vb_test; -- ============================================================= -- Test 2: Full segment delete — segment should be dropped -- ============================================================= CREATE TEMP TABLE vb_full (id serial PRIMARY KEY, content text); INSERT INTO vb_full (content) SELECT 'fulldelete document ' || i FROM generate_series(1, 50) i; CREATE INDEX vb_full_idx ON vb_full USING bm25 (content) WITH (text_config = 'english'); DELETE FROM vb_full; VACUUM vb_full; -- Should return 0 results SELECT count(*) FROM ( SELECT id FROM vb_full ORDER BY content <@> to_bm25query('fulldelete', 'vb_full_idx') LIMIT 1000 ) q; -- Index should still be usable after inserting new rows INSERT INTO vb_full (content) VALUES ('fresh document about topics'); SELECT count(*) FROM ( SELECT id FROM vb_full ORDER BY content <@> to_bm25query('fresh', 'vb_full_idx') LIMIT 1000 ) q; DROP TABLE vb_full; -- ============================================================= -- Test 3: CTID reuse correctness -- ============================================================= CREATE TEMP TABLE vb_reuse (id serial PRIMARY KEY, content text); INSERT INTO vb_reuse (content) SELECT 'alpha keyword document ' || i FROM generate_series(1, 50) i; CREATE INDEX vb_reuse_idx ON vb_reuse USING bm25 (content) WITH (text_config = 'english'); -- Verify alpha matches SELECT count(*) > 0 AS has_alpha FROM ( SELECT id FROM vb_reuse ORDER BY content <@> to_bm25query('alpha', 'vb_reuse_idx') LIMIT 1000 ) q; -- Delete all, VACUUM (frees CTIDs for reuse) DELETE FROM vb_reuse; VACUUM vb_reuse; -- Insert new rows (Postgres may reuse CTIDs) INSERT INTO vb_reuse (content) SELECT 'beta keyword document ' || i FROM generate_series(1, 50) i; -- alpha should return nothing SELECT count(*) FROM ( SELECT id FROM vb_reuse ORDER BY content <@> to_bm25query('alpha', 'vb_reuse_idx') LIMIT 1000 ) q; -- beta should return results SELECT count(*) > 0 AS has_beta FROM ( SELECT id FROM vb_reuse ORDER BY content <@> to_bm25query('beta', 'vb_reuse_idx') LIMIT 1000 ) q; DROP TABLE vb_reuse; -- ============================================================= -- Test 4: Multiple segments with mixed deletes -- ============================================================= CREATE TEMP TABLE vb_multi (id serial PRIMARY KEY, content text); INSERT INTO vb_multi (content) SELECT 'first batch document ' || i || ' with search term' FROM generate_series(1, 50) i; CREATE INDEX vb_multi_idx ON vb_multi USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); -- Force first batch to segment SELECT bm25_spill_index('vb_multi_idx'); -- Insert second batch INSERT INTO vb_multi (content) SELECT 'second batch document ' || i || ' with search term' FROM generate_series(1, 50) i; -- Force second batch to segment SELECT bm25_spill_index('vb_multi_idx'); -- Delete from first batch only DELETE FROM vb_multi WHERE id <= 25; VACUUM vb_multi; -- Should find docs from both batches (minus deleted) SELECT count(*) FROM ( SELECT id FROM vb_multi ORDER BY content <@> to_bm25query('search', 'vb_multi_idx') LIMIT 1000 ) q; DROP TABLE vb_multi; -- ============================================================= -- Test 5: Merge after VACUUM cleans up dead docs -- ============================================================= CREATE TEMP TABLE vb_merge (id serial PRIMARY KEY, content text); INSERT INTO vb_merge (content) SELECT 'merge test document ' || i FROM generate_series(1, 50) i; CREATE INDEX vb_merge_idx ON vb_merge USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); -- Delete half DELETE FROM vb_merge WHERE id <= 25; VACUUM vb_merge; -- Force merge to clean up dead docs SELECT bm25_force_merge('vb_merge_idx'); -- Query should still work correctly SELECT count(*) FROM ( SELECT id FROM vb_merge ORDER BY content <@> to_bm25query('merge', 'vb_merge_idx') LIMIT 1000 ) q; DROP TABLE vb_merge; -- ============================================================= -- Test 6: Score correctness after deletes (multi-term BMW) -- ============================================================= CREATE TEMP TABLE vb_scores (id serial PRIMARY KEY, content text); INSERT INTO vb_scores (content) VALUES ('database systems performance tuning'), ('performance optimization techniques'), ('database indexing strategies'), ('query optimization in databases'), ('systems programming fundamentals'), ('advanced database concepts'), ('delete me not relevant'), ('also delete me irrelevant'), ('performance database benchmark'), ('indexing performance metrics'); CREATE INDEX vb_scores_idx ON vb_scores USING bm25 (content) WITH (text_config = 'english'); -- Delete some rows DELETE FROM vb_scores WHERE content LIKE '%delete me%'; VACUUM vb_scores; -- Multi-term query via BMW index scan SELECT id, content <@> to_bm25query('performance database', 'vb_scores_idx') AS score FROM vb_scores ORDER BY content <@> to_bm25query('performance database', 'vb_scores_idx') LIMIT 5; DROP TABLE vb_scores; -- ============================================================= -- Test 7: Many deletes (>64 per segment, exercises realloc) -- ============================================================= CREATE TEMP TABLE vb_many (id serial PRIMARY KEY, content text); INSERT INTO vb_many (content) SELECT 'manydelete document ' || i || ' searchable' FROM generate_series(1, 200) i; CREATE INDEX vb_many_idx ON vb_many USING bm25 (content) WITH (text_config = 'english'); -- Delete 150 of 200 rows (well past the initial 64-element array) DELETE FROM vb_many WHERE id <= 150; VACUUM vb_many; SELECT count(*) FROM ( SELECT id FROM vb_many ORDER BY content <@> to_bm25query('searchable', 'vb_many_idx') LIMIT 1000 ) q; DROP TABLE vb_many; -- ============================================================= -- Test 8: Multi-segment merge with dead docs -- ============================================================= CREATE TEMP TABLE vb_multimerge (id serial PRIMARY KEY, content text); INSERT INTO vb_multimerge (content) SELECT 'batch1 doc ' || i || ' target' FROM generate_series(1, 50) i; CREATE INDEX vb_mm_idx ON vb_multimerge USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); SELECT bm25_spill_index('vb_mm_idx'); INSERT INTO vb_multimerge (content) SELECT 'batch2 doc ' || i || ' target' FROM generate_series(1, 50) i; SELECT bm25_spill_index('vb_mm_idx'); INSERT INTO vb_multimerge (content) SELECT 'batch3 doc ' || i || ' target' FROM generate_series(1, 50) i; SELECT bm25_spill_index('vb_mm_idx'); -- Delete from each batch DELETE FROM vb_multimerge WHERE id <= 10; DELETE FROM vb_multimerge WHERE id > 50 AND id <= 60; DELETE FROM vb_multimerge WHERE id > 100 AND id <= 110; VACUUM vb_multimerge; -- Merge all segments — exercises N-way merge with dead docs SELECT bm25_force_merge('vb_mm_idx'); -- 150 total - 30 deleted = 120 live SELECT count(*) FROM ( SELECT id FROM vb_multimerge ORDER BY content <@> to_bm25query('target', 'vb_mm_idx') LIMIT 1000 ) q; DROP TABLE vb_multimerge; -- ============================================================= -- Test 9: All docs dead across segments then merge -- ============================================================= CREATE TEMP TABLE vb_alldead (id serial PRIMARY KEY, content text); INSERT INTO vb_alldead (content) SELECT 'alldead doc ' || i FROM generate_series(1, 30) i; CREATE INDEX vb_alldead_idx ON vb_alldead USING bm25 (content) WITH (text_config = 'english'); SELECT bm25_spill_index('vb_alldead_idx'); INSERT INTO vb_alldead (content) SELECT 'alldead second ' || i FROM generate_series(1, 30) i; SELECT bm25_spill_index('vb_alldead_idx'); DELETE FROM vb_alldead; VACUUM vb_alldead; -- Both segments should be dropped — merge is a no-op SELECT bm25_force_merge('vb_alldead_idx'); DO $$ DECLARE summary text := bm25_summarize_index('vb_alldead_idx'); BEGIN IF summary !~ E'Segments:\n \\(none\\)' THEN RAISE EXCEPTION 'all-dead force merge left a segment: %', summary; END IF; END $$; SELECT count(*) FROM ( SELECT id FROM vb_alldead ORDER BY content <@> to_bm25query('alldead', 'vb_alldead_idx') LIMIT 1000 ) q; -- Insert fresh data to verify index still works INSERT INTO vb_alldead (content) VALUES ('revival document here'); SELECT count(*) FROM ( SELECT id FROM vb_alldead ORDER BY content <@> to_bm25query('revival', 'vb_alldead_idx') LIMIT 1000 ) q; DROP TABLE vb_alldead; -- ============================================================= -- Test 10: Dump shows alive bitset info -- ============================================================= CREATE TEMP TABLE vb_dump (id serial PRIMARY KEY, content text); INSERT INTO vb_dump (content) SELECT 'dump test ' || i FROM generate_series(1, 10) i; CREATE INDEX vb_dump_idx ON vb_dump USING bm25 (content) WITH (text_config = 'english'); -- Verify dump output includes alive bitset info SELECT bm25_dump_index('vb_dump_idx') LIKE '%Alive: 10 / 10 docs%' AS has_alive_info; DROP TABLE vb_dump; -- ============================================================= -- Test 11: Batched CTID reads across pages and segments -- ============================================================= CREATE TEMP TABLE vb_batched (id serial PRIMARY KEY, content text); INSERT INTO vb_batched (content) SELECT 'bounded batch ' || i FROM generate_series(1, 3000) i; CREATE INDEX vb_batched_idx ON vb_batched USING bm25 (content) WITH (text_config = 'english', compaction = 'manual'); INSERT INTO vb_batched (content) SELECT 'bounded batch ' || i FROM generate_series(3001, 6000) i; SELECT bm25_spill_index('vb_batched_idx') > 0 AS spilled; DELETE FROM vb_batched WHERE id % 3 = 0; VACUUM vb_batched; SELECT count(*) FROM ( SELECT id FROM vb_batched ORDER BY content <@> to_bm25query('bounded', 'vb_batched_idx') LIMIT 6000 ) q; DROP TABLE vb_batched; DROP EXTENSION pg_textsearch; timescale-pg_textsearch-b63fe58/test/sql/vacuum_extended.sql000066400000000000000000000445721525762047700244540ustar00rootroot00000000000000-- Extended vacuum tests for BM25 indexes -- Exercises vacuum paths with segments, bulk deletes, and empty indexes CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- ============================================================================= -- Test 1: Vacuum with segments present -- ============================================================================= CREATE TABLE vacuum_seg_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_seg_idx ON vacuum_seg_test USING bm25(content) WITH (text_config='english'); -- Insert data and force segment creation INSERT INTO vacuum_seg_test (content) SELECT 'vacuum segment test document number ' || i FROM generate_series(1, 200) AS i; SELECT bm25_spill_index('vacuum_seg_idx'); -- Delete half the documents DELETE FROM vacuum_seg_test WHERE id <= 100; -- VACUUM with segments on disk VACUUM vacuum_seg_test; -- Verify search still works after vacuum with segments SELECT count(*) FROM ( SELECT 1 FROM vacuum_seg_test ORDER BY content <@> to_bm25query('vacuum', 'vacuum_seg_idx') ) sub; -- ============================================================================= -- Test 2: VACUUM FULL with segments (forces index rebuild) -- ============================================================================= SET client_min_messages = warning; VACUUM FULL vacuum_seg_test; RESET client_min_messages; -- Verify search works after VACUUM FULL rebuild SELECT count(*) FROM ( SELECT 1 FROM vacuum_seg_test ORDER BY content <@> to_bm25query('vacuum', 'vacuum_seg_idx') ) sub; -- The rebuild must also produce complete corpus accounting. The -- notice suppression above hides the build-count line, which is the -- only place the rebuild reported it, so assert it here instead. -- 100 is the live-only rebuild; 200 appears when a concurrent -- snapshot keeps the 100 deleted rows alive through VACUUM FULL. -- Every doc is exactly 6 tokens, so avg_doc_len pins the token side -- of the accounting for either doc count: a rebuild that dropped or -- double-counted tokens moves avg_doc_len off 6.00. SELECT bm25_summarize_index('vacuum_seg_idx') ~ E'total_docs: (100|200)\n' AND bm25_summarize_index('vacuum_seg_idx') ~ E'avg_doc_len: 6\\.00\n' AS rebuild_accounting_valid; DROP TABLE vacuum_seg_test; -- ============================================================================= -- Test 3: Vacuum on empty index -- ============================================================================= CREATE TABLE vacuum_empty_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_empty_idx ON vacuum_empty_test USING bm25(content) WITH (text_config='english'); -- Vacuum empty index (should be a no-op) VACUUM vacuum_empty_test; -- Insert one doc, delete it, vacuum INSERT INTO vacuum_empty_test (content) VALUES ('single document'); DELETE FROM vacuum_empty_test; VACUUM vacuum_empty_test; DROP TABLE vacuum_empty_test; -- ============================================================================= -- Test 4: Bulk delete from memtable (exercises tp_bulkdelete_memtable_ctids) -- ============================================================================= CREATE TABLE vacuum_bulk_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_bulk_idx ON vacuum_bulk_test USING bm25(content) WITH (text_config='english'); -- Insert docs (stays in memtable) INSERT INTO vacuum_bulk_test (content) SELECT 'bulk delete test term' || (i % 10) || ' document ' || i FROM generate_series(1, 50) AS i; -- Delete most of them DELETE FROM vacuum_bulk_test WHERE id > 5; -- Vacuum to clean up memtable entries VACUUM vacuum_bulk_test; -- Verify remaining docs are searchable SELECT count(*) FROM ( SELECT 1 FROM vacuum_bulk_test ORDER BY content <@> to_bm25query('bulk', 'vacuum_bulk_idx') ) sub; DROP TABLE vacuum_bulk_test; -- ============================================================================= -- Test 5: REINDEX with unflushed memtable data -- -- Verifies that REINDEX correctly rebuilds from the heap even when -- the memtable has unflushed entries from recent INSERTs. -- ============================================================================= CREATE TABLE reindex_memtable_test ( id serial PRIMARY KEY, content text ); CREATE INDEX reindex_memtable_idx ON reindex_memtable_test USING bm25(content) WITH (text_config='english'); -- Phase 1: Initial data via CREATE INDEX (already built above) INSERT INTO reindex_memtable_test (content) SELECT 'original document about databases number ' || i FROM generate_series(1, 50) AS i; -- Force a spill so we have segments on disk SELECT bm25_spill_index('reindex_memtable_idx'); -- Verify baseline: all 50 docs are searchable SELECT count(*) AS pre_reindex_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; -- Phase 2: Insert more rows that stay in memtable (unflushed) INSERT INTO reindex_memtable_test (content) SELECT 'additional document about databases number ' || i FROM generate_series(51, 75) AS i; -- Verify memtable scan finds all 75 docs SELECT count(*) AS with_memtable_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; -- Phase 3: REINDEX discards memtable, rebuilds from heap \set VERBOSITY terse REINDEX INDEX reindex_memtable_idx; \set VERBOSITY default -- All 75 docs must still be found (rebuilt from heap, not memtable) SELECT count(*) AS post_reindex_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; -- Phase 4: Verify INSERTs still work after REINDEX INSERT INTO reindex_memtable_test (content) VALUES ('final document about databases after reindex'); SELECT count(*) AS final_count FROM ( SELECT 1 FROM reindex_memtable_test ORDER BY content <@> to_bm25query('databases', 'reindex_memtable_idx') ) sub; DROP TABLE reindex_memtable_test; -- ============================================================================= -- Test 6: Bulk load with unflushed memtable data -- -- Verifies that a large INSERT into an already-indexed table correctly -- triggers auto-spills from the memtable while preserving earlier -- unflushed memtable entries. Uses a low spill threshold so the test -- triggers multiple spills without needing millions of rows. -- ============================================================================= -- Lower spill threshold BEFORE creating the table so it applies to inserts SET pg_textsearch.memtable_pages_threshold = 1; CREATE TABLE bulk_memtable_test ( id serial PRIMARY KEY, content text ); CREATE INDEX bulk_memtable_idx ON bulk_memtable_test USING bm25(content) WITH (text_config='english'); -- Phase 1: Small insert that stays in memtable INSERT INTO bulk_memtable_test (content) SELECT 'initial document about networking topic ' || i FROM generate_series(1, 10) AS i; SELECT count(*) AS initial_count FROM ( SELECT 1 FROM bulk_memtable_test ORDER BY content <@> to_bm25query('networking', 'bulk_memtable_idx') ) sub; -- Phase 2: Bulk insert triggers multiple auto-spills INSERT INTO bulk_memtable_test (content) SELECT 'bulk loaded document about networking and databases number ' || i FROM generate_series(11, 5000) AS i; -- All 5000 docs must be found (initial memtable + spilled segments + tail) SELECT count(*) AS after_bulk_count FROM ( SELECT 1 FROM bulk_memtable_test ORDER BY content <@> to_bm25query('networking', 'bulk_memtable_idx') ) sub; -- Verify segments were created by the auto-spill SELECT bm25_summarize_index('bulk_memtable_idx') ~ 'Total:.*segments' AS has_segments; -- Phase 3: More inserts after bulk load still work INSERT INTO bulk_memtable_test (content) VALUES ('final document about networking after bulk load'); SELECT count(*) AS final_count FROM ( SELECT 1 FROM bulk_memtable_test ORDER BY content <@> to_bm25query('networking', 'bulk_memtable_idx') ) sub; RESET pg_textsearch.memtable_pages_threshold; DROP TABLE bulk_memtable_test; -- ============================================================================= -- Test 7: VACUUM spills un-spilled memtable on insert-only tables (issue #333) -- -- On insert-only workloads ambulkdelete is skipped (no dead tuples), so the -- memtable spill that used to live only there never ran. The memtable -- chain pages then grew unbounded and queries had to walk the entire -- chain. This test exercises the amvacuumcleanup path that now performs -- the spill. -- ============================================================================= CREATE TABLE vacuum_insert_only_test ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_insert_only_idx ON vacuum_insert_only_test USING bm25(content) WITH (text_config='english'); -- Populate the memtable chain above TP_MIN_SPILL_POSTINGS (1000) but -- below pg_textsearch.bulk_load_threshold so the spill only happens -- via amvacuumcleanup. 2000 docs * ~4 terms each ≈ 8000 postings. INSERT INTO vacuum_insert_only_test (content) SELECT 'insert only document number ' || i FROM generate_series(1, 2000) AS i; -- Chain should be populated before VACUUM. SELECT count(*) > 0 AS chain_populated_before_vacuum FROM bm25_memtable_chain('vacuum_insert_only_idx'); -- No dead tuples here, so ambulkdelete is not invoked; only -- amvacuumcleanup runs. It must still drain the memtable chain. VACUUM vacuum_insert_only_test; -- Chain must be fully drained and the memtable contents spilled to a segment. SELECT count(*) = 0 AS chain_empty_after_vacuum FROM bm25_memtable_chain('vacuum_insert_only_idx'); -- Search still finds all 2000 docs. SELECT count(*) AS post_vacuum_count FROM ( SELECT 1 FROM vacuum_insert_only_test ORDER BY content <@> to_bm25query('document', 'vacuum_insert_only_idx') ) sub; DROP TABLE vacuum_insert_only_test; -- ============================================================================= -- Test 8: tiny memtable stays un-spilled across VACUUM (issue #333 guard) -- ============================================================================= -- -- Below TP_MIN_SPILL_POSTINGS the VACUUM-cleanup spill is a no-op, -- because writing a runt L0 segment would cost more in subsequent -- compaction than chain replay saves. CREATE TABLE vacuum_tiny_test (id serial PRIMARY KEY, content text); CREATE INDEX vacuum_tiny_idx ON vacuum_tiny_test USING bm25(content) WITH (text_config='english'); -- ~50 docs * ~4 terms = ~200 postings, well below the guard. INSERT INTO vacuum_tiny_test (content) SELECT 'tiny doc ' || i FROM generate_series(1, 50) AS i; VACUUM vacuum_tiny_test; -- Chain is still present; no runt L0 segment was produced. SELECT count(*) > 0 AS chain_preserved_after_vacuum FROM bm25_memtable_chain('vacuum_tiny_idx'); DROP TABLE vacuum_tiny_test; -- ============================================================================= -- Test 9: dropping a memtable-spilled L0 segment keeps total_len sane -- ============================================================================= -- -- Regression test. tp_write_segment used to leave header.total_tokens -- at the cumulative shared-atomic value instead of the per-segment -- sum. When VACUUM later dropped that segment (all docs dead) the -- inflated header.total_tokens was subtracted from metap->total_len -- and the atomic, clamping them to 0 (or wrapping the atomic) and -- breaking avgdl in BM25 scoring for the surviving docs. -- -- We populate two L0 segments with distinct id ranges, delete every -- row that landed in the second one, and then run VACUUM. Afterwards -- a search over the surviving corpus must return real matches. CREATE TABLE l0_total_len_test (id serial PRIMARY KEY, content text); CREATE INDEX l0_total_len_idx ON l0_total_len_test USING bm25(content) WITH (text_config='english'); -- Segment 1: ids 1..50 (the survivors). INSERT INTO l0_total_len_test (content) SELECT 'alpha common term doc ' || i FROM generate_series(1, 50) AS i; SELECT bm25_spill_index('l0_total_len_idx'); -- Segment 2: ids 51..100 (to be fully deleted). INSERT INTO l0_total_len_test (content) SELECT 'beta common term doc ' || i FROM generate_series(51, 100) AS i; SELECT bm25_spill_index('l0_total_len_idx'); -- Wipe every row that went into segment 2, then VACUUM to drop it. DELETE FROM l0_total_len_test WHERE id > 50; VACUUM l0_total_len_test; -- total_len must reflect only the surviving segment's tokens. The -- pre-fix path inflated header.total_tokens to the cumulative atomic -- so the subtraction clamped total_len to 0. A healthy avg_doc_len -- is the sharpest regression signal. SELECT bm25_summarize_index('l0_total_len_idx') ~ E'total_len: (250|500)\n' AND bm25_summarize_index('l0_total_len_idx') ~ E'avg_doc_len: 5\\.00\n' AS l0_accounting_valid; DROP TABLE l0_total_len_test; -- ============================================================================= -- Test 10: merged segments carry raw total_tokens (long docs + VACUUM) -- ============================================================================= -- -- Regression test. merge.c used to recompute header.total_tokens as -- Σ decode_fieldnorm(fieldnorms[d]), which is exponentially quantized -- for doc_length > 39 (e.g. 100→96). metap->total_len stayed raw -- (set from the atomic), so the drift was latent until a VACUUM drop -- of the merged segment fed the quantized header value back into -- metap via tp_apply_vacuum_shrinkage, leaving phantom tokens. -- -- Fix: merge starts from Σ source.header.total_tokens (raw) minus a -- dead-only fieldnorm approximation. For all-live sources the merged -- total_tokens is exact, so the VACUUM subtraction cancels to zero. SET pg_textsearch.segments_per_level = 2; CREATE TABLE merge_long_docs (id serial PRIMARY KEY, content text); CREATE INDEX merge_long_idx ON merge_long_docs USING bm25(content) WITH (text_config='english'); -- Each doc has 100 unique tokens. decode(encode(100)) = 96. INSERT INTO merge_long_docs (content) SELECT string_agg('term' || (j + i*1000), ' ') FROM generate_series(1, 30) i, generate_series(1, 100) j GROUP BY i; SELECT bm25_spill_index('merge_long_idx'); INSERT INTO merge_long_docs (content) SELECT string_agg('other' || (j + i*1000), ' ') FROM generate_series(1, 30) i, generate_series(1, 100) j GROUP BY i; SELECT bm25_spill_index('merge_long_idx'); -- Sanity: both batches merged into a single L1 segment. SELECT bm25_summarize_index('merge_long_idx') ~ E'L1 Segment' AS has_l1_segment; -- Delete every row so VACUUM drops the merged segment. With the fix -- the L1 header carries raw 6000 tokens; subtracting that from the -- atomic/metapage total_len (also 6000) leaves 0. Pre-fix the L1 -- header was 60 × 96 = 5760 (quantized), leaving 240 phantom tokens -- that would bias avg_doc_len for any surviving docs and persist -- across the next tp_sync_metapage_stats. DELETE FROM merge_long_docs; VACUUM merge_long_docs; WITH summary AS ( SELECT bm25_summarize_index('merge_long_idx') AS value ) SELECT (value ~ E'total_len: 0\n' AND value ~ E'total_docs: 0\n') OR (value ~ E'total_len: 6000\n' AND value ~ E'total_docs: 60\n') AS merged_drop_accounting_valid FROM summary; RESET pg_textsearch.segments_per_level; DROP TABLE merge_long_docs; -- ============================================================================= -- Test 11: merge drops V5-bitset-dead docs and applies shrinkage -- ============================================================================= -- -- Regression test. tp_merge_level_segments used to update -- level_heads/level_counts but leave metap->total_docs / total_len -- at the pre-merge value when the merged segment dropped docs that -- VACUUM had previously marked dead in the source bitsets. After -- the merge, Σ segment.num_docs (= live count) was strictly less -- than metap->total_docs, violating the primary invariant. -- -- Setup: two L0 spills below segments_per_level so no compaction -- fires yet, VACUUM to flip dead bits, then a third spill that -- triggers an L0→L1 compaction. The merged L1 segment excludes the -- dead docs; metap must shrink to match. SET pg_textsearch.segments_per_level = 3; CREATE TABLE merge_drop_dead (id serial PRIMARY KEY, content text); CREATE INDEX merge_drop_idx ON merge_drop_dead USING bm25(content) WITH (text_config='english'); -- Spills 1 and 2. INSERT INTO merge_drop_dead (content) SELECT 'one alpha common term doc ' || i FROM generate_series(1, 30) AS i; SELECT bm25_spill_index('merge_drop_idx'); INSERT INTO merge_drop_dead (content) SELECT 'two beta common term doc ' || i FROM generate_series(31, 60) AS i; SELECT bm25_spill_index('merge_drop_idx'); -- Delete half the rows and VACUUM to flip dead bits in the two L0s. DELETE FROM merge_drop_dead WHERE id % 2 = 0; VACUUM merge_drop_dead; -- Third spill triggers compaction (segments_per_level=3); merge -- drops the bitset-dead docs. INSERT INTO merge_drop_dead (content) SELECT 'three gamma common term doc ' || i FROM generate_series(61, 90) AS i; SELECT bm25_spill_index('merge_drop_idx') > 0 AS third_spill_ok; -- 30 + 30 + 30 = 90 inserted; 30 deleted. Which state is reachable -- depends on whether VACUUM could reclaim the deleted rows: -- -- * no competing snapshot: dead bits flip, the merge drops them, -- and both metap->total_docs and Σ segment.num_docs are 60; -- * a held snapshot pins the deleted rows: no dead bits, the merge -- keeps everything, and both counts are 90. -- -- Either way the invariant total_docs = Σ segment.num_docs must -- hold. The pre-fix bug produced neither state: metap->total_docs -- stayed at 90 (summed sources' num_docs) while the merged segment -- carried only the 60 live docs. Comparing the two counts against -- each other -- not just the metapage count against a set of -- allowed values -- is what rejects that. WITH summary AS ( SELECT bm25_summarize_index('merge_drop_idx') AS value ), parsed AS ( SELECT (regexp_match(value, 'docs_persisted: (\d+)'))[1]::bigint AS metap_docs, (regexp_match(value, 'Total: \d+ segments, \d+ pages \([0-9.]+MB\), ' '\d+ terms, (\d+) docs'))[1]::bigint AS segment_docs, value ~ 'L1 Segment' AS compacted FROM summary ) SELECT compacted AS compaction_occurred, metap_docs = segment_docs AS metap_matches_segments, metap_docs IN (60, 90) AS docs_in_expected_states FROM parsed; RESET pg_textsearch.segments_per_level; DROP TABLE merge_drop_dead; -- Clean up DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/vacuum_rebuild.sql000066400000000000000000000117711525762047700242750ustar00rootroot00000000000000-- Test: VACUUM correctly removes dead index entries so CTID reuse -- does not produce false positives. -- -- The bug only manifests via ORDER BY <@> LIMIT queries, which use -- the index scan path. Seq scans re-evaluate the <@> operator on -- the heap tuple and catch stale results. We use enable_seqscan=off -- to ensure the index scan is used even for small tables. CREATE EXTENSION IF NOT EXISTS pg_textsearch; SET enable_seqscan = off; -- ================================================================ -- Test 1: Basic CTID reuse after VACUUM -- ================================================================ CREATE TABLE vacuum_rebuild_test ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_rebuild_idx ON vacuum_rebuild_test USING bm25(content) WITH (text_config = 'english'); -- Insert documents with distinctive terms INSERT INTO vacuum_rebuild_test (content) VALUES ('alpha bravo charlie'), ('delta echo foxtrot'), ('golf hotel india'); -- Force to segments so VACUUM has something to rebuild SELECT bm25_spill_index('vacuum_rebuild_idx'); -- Verify baseline: index scan for 'alpha' returns the right row SELECT id, content FROM vacuum_rebuild_test ORDER BY content <@> to_bm25query('alpha', 'vacuum_rebuild_idx') LIMIT 1; -- Delete rows containing 'alpha' and 'delta', then VACUUM to free -- their CTIDs for reuse. DELETE FROM vacuum_rebuild_test WHERE id <= 2; VACUUM vacuum_rebuild_test; -- Insert new rows. Postgres will reuse the freed CTIDs (0,1) and -- (0,2) for these new rows. INSERT INTO vacuum_rebuild_test (content) VALUES ('juliet kilo lima'), ('mike november oscar'); -- Index scan for 'alpha' must return no rows. Without the fix, -- the stale segment entry for 'alpha' points to reused CTID (0,1) -- and the index scan returns 'juliet kilo lima' as a false match. SELECT id, content FROM vacuum_rebuild_test ORDER BY content <@> to_bm25query('alpha', 'vacuum_rebuild_idx') LIMIT 10; -- Search for new terms must work via index scan SELECT id, content FROM vacuum_rebuild_test ORDER BY content <@> to_bm25query('juliet', 'vacuum_rebuild_idx') LIMIT 1; DROP TABLE vacuum_rebuild_test; -- ================================================================ -- Test 2: All docs in segment deleted (segment removed from chain) -- ================================================================ CREATE TABLE vacuum_allgone ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_allgone_idx ON vacuum_allgone USING bm25(content) WITH (text_config = 'english'); INSERT INTO vacuum_allgone (content) VALUES ('papaya mango'), ('papaya kiwi'); SELECT bm25_spill_index('vacuum_allgone_idx'); -- Delete everything, vacuum DELETE FROM vacuum_allgone; VACUUM vacuum_allgone; -- Insert fresh data INSERT INTO vacuum_allgone (content) VALUES ('strawberry banana'); -- Must NOT find phantom 'papaya' results via index scan SELECT id, content FROM vacuum_allgone ORDER BY content <@> to_bm25query('papaya', 'vacuum_allgone_idx') LIMIT 10; -- Must find the new row SELECT id, content FROM vacuum_allgone ORDER BY content <@> to_bm25query('strawberry', 'vacuum_allgone_idx') LIMIT 1; DROP TABLE vacuum_allgone; -- ================================================================ -- Test 3: Multi-level segments with deletions -- ================================================================ SET pg_textsearch.memtable_pages_threshold = 1; CREATE TABLE vacuum_multilevel ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_multilevel_idx ON vacuum_multilevel USING bm25(content) WITH (text_config = 'english'); -- Insert enough to create L0 segments and trigger compaction to L1 INSERT INTO vacuum_multilevel (content) SELECT 'multilevel test document number ' || i || ' with keyword searchterm' FROM generate_series(1, 2000) AS i; -- Verify we have segments at multiple levels SELECT bm25_summarize_index('vacuum_multilevel_idx') ~ 'L1' AS has_l1_segments; -- Delete half DELETE FROM vacuum_multilevel WHERE id <= 1000; VACUUM vacuum_multilevel; -- Verify remaining docs are searchable via index scan SELECT count(*) AS remaining FROM ( SELECT id FROM vacuum_multilevel ORDER BY content <@> to_bm25query('searchterm', 'vacuum_multilevel_idx') LIMIT 2000 ) sub; RESET pg_textsearch.memtable_pages_threshold; DROP TABLE vacuum_multilevel; -- ================================================================ -- Test 4: VACUUM with no dead tuples (no-op) -- ================================================================ CREATE TABLE vacuum_noop ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_noop_idx ON vacuum_noop USING bm25(content) WITH (text_config = 'english'); INSERT INTO vacuum_noop (content) VALUES ('nothing deleted here'); SELECT bm25_spill_index('vacuum_noop_idx'); VACUUM vacuum_noop; SELECT id, content FROM vacuum_noop ORDER BY content <@> to_bm25query('deleted', 'vacuum_noop_idx') LIMIT 1; DROP TABLE vacuum_noop; RESET enable_seqscan; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/vacuum_rebuild_injection.sql000066400000000000000000000233601525762047700263340ustar00rootroot00000000000000\pset format unaligned \set VERBOSITY terse SET client_min_messages = warning; CREATE EXTENSION pg_textsearch; CREATE EXTENSION injection_points; CREATE EXTENSION pg_textsearch_test; SET enable_seqscan = off; -- Temporary fixtures isolate immediate cleanup from other backends' xmin. -- Legacy token totals are rebuilt from heap data. CREATE TEMP TABLE vacuum_legacy_stats ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_legacy_stats_idx ON vacuum_legacy_stats USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_legacy_stats (content) SELECT 'legacy inflated token document ' || i FROM generate_series(1, 10) AS i; SELECT pg_textsearch_test_attach_legacy_segment(1000000); SELECT bm25_spill_index('vacuum_legacy_stats_idx') > 0 AS legacy_header_inflated; SELECT injection_points_detach('pg-textsearch-legacy-segment'); DELETE FROM vacuum_legacy_stats WHERE id <= 5; VACUUM vacuum_legacy_stats; SELECT bm25_level_counts('vacuum_legacy_stats_idx'::regclass) AS levels_after_legacy_vacuum; SELECT bm25_summarize_index('vacuum_legacy_stats_idx') ~ E'total_docs: 5\n' AND bm25_summarize_index('vacuum_legacy_stats_idx') ~ E'total_len: 25\n' AS legacy_totals_rebuilt; SELECT bm25_dump_index('vacuum_legacy_stats_idx') LIKE '%Alive: 5 / 5 docs%' AS legacy_replacement_published; DROP TABLE vacuum_legacy_stats; -- Legacy replacement rebases a changed text configuration. CREATE TEXT SEARCH CONFIGURATION public.vacuum_mutable_cfg (COPY = english); CREATE TEMP TABLE vacuum_mutable_config ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_mutable_config_idx ON vacuum_mutable_config USING bm25(content) WITH (text_config = 'public.vacuum_mutable_cfg', compaction = 'off'); INSERT INTO vacuum_mutable_config (content) VALUES ('the the the alpha'), ('the the the alpha'); SELECT pg_textsearch_test_attach_legacy_segment(2); SELECT bm25_spill_index('vacuum_mutable_config_idx') > 0 AS mutable_legacy_created; SELECT injection_points_detach('pg-textsearch-legacy-segment'); ALTER TEXT SEARCH CONFIGURATION public.vacuum_mutable_cfg ALTER MAPPING FOR asciiword WITH simple; DELETE FROM vacuum_mutable_config WHERE id = 1; VACUUM vacuum_mutable_config; SELECT bm25_summarize_index('vacuum_mutable_config_idx') ~ E'total_docs: 1\n' AND bm25_summarize_index('vacuum_mutable_config_idx') ~ E'total_len: 4\n' AS mutable_config_totals_rebased; DROP TABLE vacuum_mutable_config; DROP TEXT SEARCH CONFIGURATION public.vacuum_mutable_cfg; -- Inconsistent multi-segment legacy totals are fully rebased. CREATE TEMP TABLE vacuum_multi_legacy ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_multi_legacy_idx ON vacuum_multi_legacy USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_multi_legacy (content) SELECT 'multi legacy token document ' || i FROM generate_series(1, 5) AS i; SELECT pg_textsearch_test_attach_legacy_segment(1000000); SELECT bm25_spill_index('vacuum_multi_legacy_idx') > 0 AS first_legacy_created; SELECT injection_points_detach('pg-textsearch-legacy-segment'); INSERT INTO vacuum_multi_legacy (content) SELECT 'multi legacy token document ' || i FROM generate_series(6, 10) AS i; SELECT pg_textsearch_test_attach_legacy_segment(2000000); SELECT bm25_spill_index('vacuum_multi_legacy_idx') > 0 AS second_legacy_created; SELECT injection_points_detach('pg-textsearch-legacy-segment'); SELECT pg_textsearch_test_attach_vacuum_total_len(4000000); DELETE FROM vacuum_multi_legacy WHERE id = 1; VACUUM vacuum_multi_legacy; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); SELECT bm25_summarize_index('vacuum_multi_legacy_idx') ~ E'total_docs: 9\n' AND bm25_summarize_index('vacuum_multi_legacy_idx') ~ E'total_len: 45\n' AS multi_legacy_totals_rebased; SELECT bm25_dump_index('vacuum_multi_legacy_idx') NOT LIKE '%Version: 4%' AS all_legacy_segments_rebuilt; DROP TABLE vacuum_multi_legacy; -- Impossible current-format totals fail closed. CREATE TEMP TABLE vacuum_current_corrupt ( id serial PRIMARY KEY, content text ); INSERT INTO vacuum_current_corrupt (content) VALUES ('current token document one'), ('current token document two'); CREATE INDEX vacuum_current_corrupt_idx ON vacuum_current_corrupt USING bm25(content) WITH (text_config = 'english'); SELECT pg_textsearch_test_attach_vacuum_total_len(0); DELETE FROM vacuum_current_corrupt WHERE id = 1; VACUUM vacuum_current_corrupt; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); DROP TABLE vacuum_current_corrupt; CREATE TEMP TABLE vacuum_current_inflated ( id serial PRIMARY KEY, content text ); INSERT INTO vacuum_current_inflated (content) VALUES ('inflated token document one'), ('inflated token document two'); CREATE INDEX vacuum_current_inflated_idx ON vacuum_current_inflated USING bm25(content) WITH (text_config = 'english'); SELECT pg_textsearch_test_attach_vacuum_total_len(1000); VACUUM vacuum_current_inflated; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); DROP TABLE vacuum_current_inflated; -- Mixed graphs fail closed when a current-format header is corrupt. CREATE TEMP TABLE vacuum_mixed_current_corrupt ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_mixed_current_corrupt_idx ON vacuum_mixed_current_corrupt USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_mixed_current_corrupt (content) VALUES ('legacy mixed document one'), ('legacy mixed document two'); SELECT pg_textsearch_test_attach_legacy_segment(8); SELECT bm25_spill_index('vacuum_mixed_current_corrupt_idx') > 0 AS mixed_legacy_created; SELECT injection_points_detach('pg-textsearch-legacy-segment'); INSERT INTO vacuum_mixed_current_corrupt (content) VALUES ('current mixed document three'), ('current mixed document four'); SELECT pg_textsearch_test_attach_v5_segment_total_len(1); SELECT bm25_spill_index('vacuum_mixed_current_corrupt_idx') > 0 AS mixed_current_created; SELECT injection_points_detach('pg-textsearch-v5-segment-total-len'); VACUUM vacuum_mixed_current_corrupt; SELECT bm25_summarize_index('vacuum_mixed_current_corrupt_idx') ~ E'total_docs: 4\n' AND bm25_summarize_index('vacuum_mixed_current_corrupt_idx') ~ E'total_len: 16\n' AS mixed_corruption_preserved_totals; SELECT bm25_dump_index('vacuum_mixed_current_corrupt_idx') LIKE '%Version: 4%' AS mixed_corruption_preserved_legacy; DROP TABLE vacuum_mixed_current_corrupt; CREATE TEMP TABLE vacuum_mixed_no_dead ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_mixed_no_dead_idx ON vacuum_mixed_no_dead USING bm25(content) WITH (text_config = 'english', compaction = 'off'); INSERT INTO vacuum_mixed_no_dead (content) VALUES ('mixed legacy document one'), ('mixed legacy document two'); SELECT pg_textsearch_test_attach_legacy_segment(1000); SELECT bm25_spill_index('vacuum_mixed_no_dead_idx') > 0 AS no_dead_legacy_created; SELECT injection_points_detach('pg-textsearch-legacy-segment'); INSERT INTO vacuum_mixed_no_dead (content) VALUES ('mixed current document three'), ('mixed current document four'); SELECT bm25_spill_index('vacuum_mixed_no_dead_idx') > 0 AS no_dead_current_spilled; SELECT pg_textsearch_test_attach_vacuum_total_len(100); VACUUM vacuum_mixed_no_dead; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); SELECT bm25_summarize_index('vacuum_mixed_no_dead_idx') ~ E'total_docs: 4\n' AND bm25_summarize_index('vacuum_mixed_no_dead_idx') ~ E'total_len: 16\n' AS no_dead_mixed_totals_rebased; SELECT bm25_dump_index('vacuum_mixed_no_dead_idx') NOT LIKE '%Version: 4%' AS no_dead_legacy_rebuilt; DROP TABLE vacuum_mixed_no_dead; -- Spill cannot compact corrupt legacy totals before VACUUM repairs them. CREATE TEMP TABLE vacuum_spill_legacy ( id serial PRIMARY KEY, content text ); CREATE INDEX vacuum_spill_legacy_idx ON vacuum_spill_legacy USING bm25(content) WITH (text_config = 'english', compaction = 'off'); DO $$ BEGIN FOR batch IN 1..4 LOOP INSERT INTO vacuum_spill_legacy (content) SELECT 'alpha beta gamma delta' FROM generate_series(1, 10); PERFORM pg_textsearch_test_attach_legacy_segment(batch * 1000); PERFORM bm25_spill_index('vacuum_spill_legacy_idx'); PERFORM injection_points_detach('pg-textsearch-legacy-segment'); END LOOP; END $$; SELECT pg_textsearch_test_attach_vacuum_total_len(5000); -- ALTER's managed lifecycle rejects temporary indexes; test storage here. UPDATE pg_class SET reloptions = array_replace(reloptions, 'compaction=off', 'compaction=inline') WHERE oid = 'vacuum_spill_legacy_idx'::regclass; SET pg_textsearch.segments_per_level = 2; INSERT INTO vacuum_spill_legacy (content) VALUES ('alpha beta gamma delta'); DELETE FROM vacuum_spill_legacy WHERE id = 1; VACUUM vacuum_spill_legacy; SELECT injection_points_detach('pg-textsearch-vacuum-total-len'); SELECT bm25_summarize_index('vacuum_spill_legacy_idx') ~ E'total_docs: 40\n' AND bm25_summarize_index('vacuum_spill_legacy_idx') ~ E'total_len: 160\n' AS spill_legacy_totals_rebased; SELECT bm25_dump_index('vacuum_spill_legacy_idx') NOT LIKE '%Version: 4%' AS spill_legacy_segments_rebuilt; SELECT (bm25_level_counts( 'vacuum_spill_legacy_idx'::regclass))[2] > 0 AS deferred_inline_compaction_ran; RESET pg_textsearch.segments_per_level; DROP TABLE vacuum_spill_legacy; RESET enable_seqscan; DROP EXTENSION pg_textsearch_test; DROP EXTENSION injection_points; DROP EXTENSION pg_textsearch; \set VERBOSITY default \pset format aligned timescale-pg_textsearch-b63fe58/test/sql/validation.sql000066400000000000000000000652621525762047700234250ustar00rootroot00000000000000-- BM25 Score Validation Functions -- -- Formula: BM25(q,d) = Σ IDF(qi) * (tf(qi,d) * (k1 + 1)) / (tf(qi,d) + k1 * (1 - b + b * |d| / avgdl)) -- Where: -- IDF(qi) = ln(1 + (N - df(qi) + 0.5) / (df(qi) + 0.5)) -- tf(qi,d) = term frequency of qi in document d -- |d| = document length (quantized via fieldnorm), avgdl = average document length -- k1 = term frequency saturation, b = length normalization -- -- Note: Document lengths are quantized using Lucene's SmallFloat encoding scheme -- (fieldnorm) which maps lengths to 256 buckets. This matches Tapir's internal -- representation for consistent scoring between index scans and operator evaluation. -- -- Usage Example: -- SELECT validate_bm25_scoring('mytable', 'content_column', 'my_bm25_idx', 'search query', 'english', 1.2, 0.75) -- Fieldnorm decode table (Lucene SmallFloat.byte4ToInt) -- Values 0-39 are exact, larger values use exponential encoding CREATE OR REPLACE FUNCTION fieldnorm_quantize(doc_length integer) RETURNS integer AS $$ DECLARE decode_table integer[] := ARRAY[ 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 42, 44, 46, 48, 50, 52, 54, 56, 60, 64, 68, 72, 76, 80, 84, 88, 96, 104, 112, 120, 128, 136, 144, 152, 168, 184, 200, 216, 232, 248, 264, 280, 312, 344, 376, 408, 440, 472, 504, 536, 600, 664, 728, 792, 856, 920, 984, 1048, 1176, 1304, 1432, 1560, 1688, 1816, 1944, 2072, 2328, 2584, 2840, 3096, 3352, 3608, 3864, 4120, 4632, 5144, 5656, 6168, 6680, 7192, 7704, 8216, 9240, 10264, 11288, 12312, 13336, 14360, 15384, 16408, 18456, 20504, 22552, 24600, 26648, 28696, 30744, 32792, 36888, 40984, 45080, 49176, 53272, 57368, 61464, 65560, 73752, 81944, 90136, 98328, 106520, 114712, 122904, 131096, 147480, 163864, 180248, 196632, 213016, 229400, 245784, 262168, 294936, 327704, 360472, 393240, 426008, 458776, 491544, 524312, 589848, 655384, 720920, 786456, 851992, 917528, 983064, 1048600, 1179672, 1310744, 1441816, 1572888, 1703960, 1835032, 1966104, 2097176, 2359320, 2621464, 2883608, 3145752, 3407896, 3670040, 3932184, 4194328, 4718616, 5242904, 5767192, 6291480, 6815768, 7340056, 7864344, 8388632, 9437208, 10485784, 11534360, 12582936, 13631512, 14680088, 15728664, 16777240, 18874392, 20971544, 23068696, 25165848, 27263000, 29360152, 31457304, 33554456, 37748760, 41943064, 46137368, 50331672, 54525976, 58720280, 62914584, 67108888, 75497496, 83886104, 92274712, 100663320, 109051928, 117440536, 125829144, 134217752, 150994968, 167772184, 184549400, 201326616, 218103832, 234881048, 251658264, 268435480, 301989912, 335544344, 369098776, 402653208, 436207640, 469762072, 503316504, 536870936, 603979800, 671088664, 738197528, 805306392, 872415256, 939524120, 1006632984, 1073741848, 1207959576, 1342177304, 1476395032, 1610612760, 1744830488, 1879048216, 2013265944 ]; lo integer := 1; -- PostgreSQL arrays are 1-indexed hi integer := 256; mid integer; BEGIN -- Binary search to find largest index where decode_table[i] <= doc_length WHILE lo < hi LOOP mid := (lo + hi + 1) / 2; IF decode_table[mid] <= doc_length THEN lo := mid; ELSE hi := mid - 1; END IF; END LOOP; -- Return the quantized (decoded) value RETURN decode_table[lo]; END; $$ LANGUAGE plpgsql IMMUTABLE STRICT; CREATE OR REPLACE FUNCTION validate_bm25_scoring( p_table_name text, p_text_column text, p_index_name text, p_query text, p_text_config text, p_k1 float, p_b float ) RETURNS boolean AS $$ DECLARE v_total_docs integer; v_avg_doc_length float8; v_all_match boolean; v_idx_match_10 boolean; v_idx_match_100 boolean; v_sql text; v_idx_sql text; v_saved_log_scores text; BEGIN -- Suppress per-row BM25 NOTICEs from this helper's internal index -- queries so output stays stable regardless of caller's log_scores -- setting, and so the cold+warm doubled execution does not duplicate -- NOTICE lines. log_scores is registered PGC_SUSET; non-superuser -- callers cannot change it, so skip the set_config when it's already -- off and tolerate insufficient_privilege errors otherwise. v_saved_log_scores := current_setting('pg_textsearch.log_scores', true); IF v_saved_log_scores IS DISTINCT FROM 'off' THEN BEGIN PERFORM set_config('pg_textsearch.log_scores', 'off', true); EXCEPTION WHEN insufficient_privilege THEN -- Caller lacks privilege to mutate the GUC; leave it as-is. -- Restore step will also be skipped (v_saved_log_scores -> NULL). v_saved_log_scores := NULL; END; END IF; -- Step 1: Tokenize documents with proper term frequency counting EXECUTE format($sql$ CREATE TEMP TABLE temp_docs ON COMMIT DROP AS WITH numbered AS ( SELECT ROW_NUMBER() OVER () as doc_id, %I as content FROM %I WHERE %I IS NOT NULL ), doc_terms AS ( SELECT doc_id, content, lexeme, array_length(string_to_array(positions::text, ','), 1) as term_count FROM numbered, LATERAL unnest(to_tsvector(%L, content)) as t(lexeme, positions, weights) ) SELECT doc_id, content, array_agg(lexeme ORDER BY lexeme) as unique_tokens, fieldnorm_quantize(SUM(term_count)::integer) as doc_length FROM doc_terms GROUP BY doc_id, content $sql$, p_text_column, p_table_name, p_text_column, p_text_config); -- Step 2: Calculate corpus statistics SELECT COUNT(*), AVG(td.doc_length)::float8 INTO v_total_docs, v_avg_doc_length FROM temp_docs td; -- Step 3: Build term frequency table with actual counts from positions EXECUTE format($sql$ CREATE TEMP TABLE temp_tf ON COMMIT DROP AS WITH numbered AS ( SELECT ROW_NUMBER() OVER () as doc_id, %I as content FROM %I WHERE %I IS NOT NULL ) SELECT n.doc_id, t.lexeme as term, array_length(string_to_array(t.positions::text, ','), 1) as term_freq, (SELECT doc_length FROM temp_docs td WHERE td.doc_id = n.doc_id) as doc_length FROM numbered n, LATERAL unnest(to_tsvector(%L, n.content)) as t(lexeme, positions, weights) $sql$, p_text_column, p_table_name, p_text_column, p_text_config); -- Step 4: Build document frequency table CREATE TEMP TABLE temp_df ON COMMIT DROP AS SELECT term, COUNT(DISTINCT temp_tf.doc_id) as doc_freq FROM temp_tf GROUP BY term; -- Step 5: Tokenize query EXECUTE format($sql$ CREATE TEMP TABLE temp_query ON COMMIT DROP AS SELECT DISTINCT lexeme as term FROM unnest(to_tsvector(%L, %L)) $sql$, p_text_config, p_query); -- Step 6: Get Tapir scores v_sql := format($sql$ CREATE TEMP TABLE temp_tapir_scores ON COMMIT DROP AS SELECT ROW_NUMBER() OVER () as doc_id, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL ORDER BY ROW_NUMBER() OVER () $sql$, p_text_column, p_query, p_index_name, p_table_name, p_text_column); EXECUTE v_sql; -- Step 7: Materialize SQL ground-truth scores and compare to standalone CREATE TEMP TABLE temp_sql_scores ON COMMIT DROP AS WITH sql_scores AS ( SELECT d.doc_id, d.content, COALESCE(SUM( -- IDF: ln(1 + (N - df + 0.5) / (df + 0.5)) ln(1 + (v_total_docs - df.doc_freq + 0.5)::float8 / (df.doc_freq + 0.5)) * -- TF component: (tf * (k1 + 1)) / (tf + k1 * (1 - b + b * |D|/avgdl)) (tf.term_freq * (p_k1 + 1)) / (tf.term_freq + p_k1 * (1 - p_b + p_b * tf.doc_length / v_avg_doc_length)) ), 0) as bm25_score FROM temp_docs d CROSS JOIN temp_query q LEFT JOIN temp_tf tf ON tf.doc_id = d.doc_id AND tf.term = q.term LEFT JOIN temp_df df ON df.term = q.term GROUP BY d.doc_id, d.content ) SELECT doc_id, content, bm25_score FROM sql_scores; SELECT bool_and( -- Compare to 4 decimal places ROUND(s.bm25_score::numeric, 4) = ROUND((-t.score)::numeric, 4) ) INTO v_all_match FROM temp_sql_scores s JOIN temp_tapir_scores t ON s.doc_id = t.doc_id; -- Step 8: Exercise the index-scan path (top-k ORDER BY + LIMIT) twice -- for each of two cap sizes. With memtable_cache_enabled=on, the first -- run exercises the cold/lazy-build path and the second the warm cache; -- both runs must match the SQL ground-truth scores from Step 7. With -- the cache disabled the two runs are trivially identical so existing -- behavior is unchanged. v_idx_sql := format($sql$ SELECT %I as content, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL ORDER BY %I <@> to_bm25query(%L, %L) LIMIT %s $sql$, p_text_column, p_text_column, p_query, p_index_name, p_table_name, p_text_column, p_text_column, p_query, p_index_name, 10); EXECUTE format( 'CREATE TEMP TABLE temp_idx_cold_10 ON COMMIT DROP AS %s', v_idx_sql); EXECUTE format( 'CREATE TEMP TABLE temp_idx_warm_10 ON COMMIT DROP AS %s', v_idx_sql); SELECT bool_and( -- Cold and warm must produce the same ranked content+score; both -- must match the SQL ground-truth score for that content. Use -- IS NOT DISTINCT FROM so one-sided rows (NULL from FULL OUTER -- JOIN) are detected instead of being silently masked by NULL -- propagation through `=`. c.content IS NOT DISTINCT FROM w.content AND ROUND(c.score::numeric, 4) IS NOT DISTINCT FROM ROUND(w.score::numeric, 4) AND EXISTS ( SELECT 1 FROM temp_sql_scores s WHERE s.content = c.content AND ROUND(s.bm25_score::numeric, 4) = ROUND((-c.score)::numeric, 4) ) ) INTO v_idx_match_10 FROM (SELECT row_number() OVER (ORDER BY score, content) AS rn, content, score FROM temp_idx_cold_10) c FULL OUTER JOIN (SELECT row_number() OVER (ORDER BY score, content) AS rn, content, score FROM temp_idx_warm_10) w USING (rn); v_idx_sql := format($sql$ SELECT %I as content, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL ORDER BY %I <@> to_bm25query(%L, %L) LIMIT %s $sql$, p_text_column, p_text_column, p_query, p_index_name, p_table_name, p_text_column, p_text_column, p_query, p_index_name, 100); EXECUTE format( 'CREATE TEMP TABLE temp_idx_cold_100 ON COMMIT DROP AS %s', v_idx_sql); EXECUTE format( 'CREATE TEMP TABLE temp_idx_warm_100 ON COMMIT DROP AS %s', v_idx_sql); SELECT bool_and( c.content IS NOT DISTINCT FROM w.content AND ROUND(c.score::numeric, 4) IS NOT DISTINCT FROM ROUND(w.score::numeric, 4) AND EXISTS ( SELECT 1 FROM temp_sql_scores s WHERE s.content = c.content AND ROUND(s.bm25_score::numeric, 4) = ROUND((-c.score)::numeric, 4) ) ) INTO v_idx_match_100 FROM (SELECT row_number() OVER (ORDER BY score, content) AS rn, content, score FROM temp_idx_cold_100) c FULL OUTER JOIN (SELECT row_number() OVER (ORDER BY score, content) AS rn, content, score FROM temp_idx_warm_100) w USING (rn); -- Clean up DROP TABLE IF EXISTS temp_docs; DROP TABLE IF EXISTS temp_tf; DROP TABLE IF EXISTS temp_df; DROP TABLE IF EXISTS temp_query; DROP TABLE IF EXISTS temp_tapir_scores; DROP TABLE IF EXISTS temp_sql_scores; DROP TABLE IF EXISTS temp_idx_cold_10; DROP TABLE IF EXISTS temp_idx_warm_10; DROP TABLE IF EXISTS temp_idx_cold_100; DROP TABLE IF EXISTS temp_idx_warm_100; -- Restore caller's log_scores setting. Skipped when v_saved_log_scores -- is NULL, which means we either didn't change the GUC (already 'off') -- or couldn't (non-superuser caller). IF v_saved_log_scores IS NOT NULL THEN BEGIN PERFORM set_config('pg_textsearch.log_scores', v_saved_log_scores, true); EXCEPTION WHEN insufficient_privilege THEN -- Best-effort: caller lost privilege mid-call (unlikely) NULL; END; END IF; RETURN COALESCE(v_all_match, false) AND COALESCE(v_idx_match_10, true) AND COALESCE(v_idx_match_100, true); END; $$ LANGUAGE plpgsql; -- Debug function to show detailed score computation CREATE OR REPLACE FUNCTION debug_bm25_computation( p_table_name text, p_text_column text, p_index_name text, p_query text, p_text_config text, p_k1 float, p_b float ) RETURNS TABLE( doc_id integer, content text, doc_length integer, sql_bm25 float8, tapir_bm25 float8, difference float8, matches_4dp boolean, term_details jsonb ) AS $$ DECLARE v_total_docs integer; v_avg_doc_length float8; v_sql text; BEGIN -- Tokenize documents with proper position counting EXECUTE format($sql$ CREATE TEMP TABLE debug_docs ON COMMIT DROP AS WITH numbered AS ( SELECT ROW_NUMBER() OVER () as doc_id, %I as content FROM %I WHERE %I IS NOT NULL ), doc_terms AS ( SELECT doc_id, content, lexeme, array_length(string_to_array(positions::text, ','), 1) as term_count FROM numbered, LATERAL unnest(to_tsvector(%L, content)) as t(lexeme, positions, weights) ) SELECT doc_id, content, array_agg(DISTINCT lexeme) as unique_tokens, fieldnorm_quantize(SUM(term_count)::integer) as doc_length FROM doc_terms GROUP BY doc_id, content $sql$, p_text_column, p_table_name, p_text_column, p_text_config); -- Get statistics (using quantized doc lengths) SELECT COUNT(*), AVG(dd.doc_length)::float8 INTO v_total_docs, v_avg_doc_length FROM debug_docs dd; -- Compute term frequencies with actual position counts EXECUTE format($sql$ CREATE TEMP TABLE debug_tf ON COMMIT DROP AS WITH numbered AS ( SELECT ROW_NUMBER() OVER () as doc_id, %I as content FROM %I WHERE %I IS NOT NULL ) SELECT n.doc_id, t.lexeme as term, array_length(string_to_array(t.positions::text, ','), 1) as tf, (SELECT doc_length::int FROM debug_docs d WHERE d.doc_id = n.doc_id) as doc_len FROM numbered n, LATERAL unnest(to_tsvector(%L, n.content)) as t(lexeme, positions, weights) $sql$, p_text_column, p_table_name, p_text_column, p_text_config); CREATE TEMP TABLE debug_df ON COMMIT DROP AS SELECT term, COUNT(DISTINCT debug_tf.doc_id) as df FROM debug_tf GROUP BY term; -- Query terms EXECUTE format($sql$ CREATE TEMP TABLE debug_query ON COMMIT DROP AS SELECT DISTINCT lexeme as term FROM unnest(to_tsvector(%L, %L)) $sql$, p_text_config, p_query); -- Get Tapir scores v_sql := format($sql$ CREATE TEMP TABLE debug_tapir_scores ON COMMIT DROP AS SELECT ROW_NUMBER() OVER () as doc_id, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL ORDER BY ROW_NUMBER() OVER () $sql$, p_text_column, p_query, p_index_name, p_table_name, p_text_column); EXECUTE v_sql; -- Compute and return scores with details RETURN QUERY WITH score_details AS ( SELECT d.doc_id, d.content::text, d.doc_length::int as doc_length, COALESCE(SUM( ln(1 + (v_total_docs - df.df + 0.5)::float8 / (df.df + 0.5)) * (tf.tf * (p_k1 + 1)) / (tf.tf + p_k1 * (1 - p_b + p_b * tf.doc_len / v_avg_doc_length)) ), 0) as sql_score, jsonb_agg( CASE WHEN tf.tf IS NOT NULL THEN jsonb_build_object( 'term', q.term, 'tf', tf.tf, 'df', df.df, 'idf', ln(1 + (v_total_docs - df.df + 0.5)::float8 / (df.df + 0.5)), 'score', ln(1 + (v_total_docs - df.df + 0.5)::float8 / (df.df + 0.5)) * (tf.tf * (p_k1 + 1)) / (tf.tf + p_k1 * (1 - p_b + p_b * tf.doc_len / v_avg_doc_length)) ) ELSE NULL END ) as term_details FROM debug_docs d CROSS JOIN debug_query q LEFT JOIN debug_tf tf ON tf.doc_id = d.doc_id AND tf.term = q.term LEFT JOIN debug_df df ON df.term = q.term GROUP BY d.doc_id, d.content, d.doc_length ) SELECT s.doc_id::integer, s.content, s.doc_length::integer, s.sql_score, -t.score as tapir_score, ABS(s.sql_score - (-t.score)) as difference, ROUND(s.sql_score::numeric, 4) = ROUND((-t.score)::numeric, 4) as matches_4dp, s.term_details FROM score_details s JOIN debug_tapir_scores t ON s.doc_id = t.doc_id ORDER BY s.sql_score DESC; -- Clean up DROP TABLE IF EXISTS debug_docs; DROP TABLE IF EXISTS debug_tf; DROP TABLE IF EXISTS debug_df; DROP TABLE IF EXISTS debug_query; DROP TABLE IF EXISTS debug_tapir_scores; END; $$ LANGUAGE plpgsql; -- Validate that index scan and standalone scoring produce identical results. -- Index scan is triggered by ORDER BY score LIMIT; standalone runs on all -- rows. The index query is executed TWICE so that, when the memtable cache -- is enabled, the first run exercises the cold/lazy-build path and the -- second run exercises the warm path; both runs must agree with standalone -- scoring and with each other. CREATE OR REPLACE FUNCTION validate_index_vs_standalone( p_table_name text, p_text_column text, p_index_name text, p_query text, p_limit integer DEFAULT 100 ) RETURNS boolean AS $$ DECLARE v_cold_match boolean; v_warm_match boolean; v_cold_eq_warm boolean; v_idx_sql text; v_sql text; v_saved_log_scores text; BEGIN -- Suppress per-row BM25 NOTICEs from the helper's internal queries so -- output stays stable whether the caller has set log_scores or not, and -- so the cold+warm doubled execution does not duplicate NOTICE lines. -- log_scores is PGC_SUSET; skip the set_config when it's already off -- and tolerate insufficient_privilege for non-superuser callers. v_saved_log_scores := current_setting('pg_textsearch.log_scores', true); IF v_saved_log_scores IS DISTINCT FROM 'off' THEN BEGIN PERFORM set_config('pg_textsearch.log_scores', 'off', true); EXCEPTION WHEN insufficient_privilege THEN v_saved_log_scores := NULL; END; END IF; -- Index scan query reused for cold and warm runs v_idx_sql := format($sql$ SELECT %I as content, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL ORDER BY %I <@> to_bm25query(%L, %L) LIMIT %s $sql$, p_text_column, p_text_column, p_query, p_index_name, p_table_name, p_text_column, p_text_column, p_query, p_index_name, p_limit); -- Cold run (first touch of the index after any prior state) EXECUTE format( 'CREATE TEMP TABLE idx_scan_scores_cold ON COMMIT DROP AS %s', v_idx_sql); -- Warm run (re-execute identical query; exercises warm cache when enabled) EXECUTE format( 'CREATE TEMP TABLE idx_scan_scores_warm ON COMMIT DROP AS %s', v_idx_sql); -- Get scores via standalone scoring mode (no ORDER BY score) v_sql := format($sql$ CREATE TEMP TABLE standalone_scores ON COMMIT DROP AS SELECT %I as content, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL AND (%I <@> to_bm25query(%L, %L)) < 0 $sql$, p_text_column, p_text_column, p_query, p_index_name, p_table_name, p_text_column, p_text_column, p_query, p_index_name); EXECUTE v_sql; -- Verify cold run matches standalone SELECT bool_and( EXISTS ( SELECT 1 FROM standalone_scores s WHERE s.content = i.content AND ROUND(s.score::numeric, 6) = ROUND(i.score::numeric, 6) ) ) INTO v_cold_match FROM idx_scan_scores_cold i; -- Verify warm run matches standalone SELECT bool_and( EXISTS ( SELECT 1 FROM standalone_scores s WHERE s.content = i.content AND ROUND(s.score::numeric, 6) = ROUND(i.score::numeric, 6) ) ) INTO v_warm_match FROM idx_scan_scores_warm i; -- Verify cold and warm produce identical ranked results. Use -- IS NOT DISTINCT FROM so one-sided rows from FULL OUTER JOIN fail -- the comparison instead of being masked by NULL propagation, and -- give row_number() a deterministic ordering so tied scores rank -- the same in both subqueries. SELECT bool_and( c.content IS NOT DISTINCT FROM w.content AND ROUND(c.score::numeric, 6) IS NOT DISTINCT FROM ROUND(w.score::numeric, 6) ) INTO v_cold_eq_warm FROM (SELECT row_number() OVER (ORDER BY score, content) AS rn, content, score FROM idx_scan_scores_cold) c FULL OUTER JOIN (SELECT row_number() OVER (ORDER BY score, content) AS rn, content, score FROM idx_scan_scores_warm) w USING (rn); -- Cleanup DROP TABLE IF EXISTS idx_scan_scores_cold; DROP TABLE IF EXISTS idx_scan_scores_warm; DROP TABLE IF EXISTS standalone_scores; -- Restore caller's log_scores setting. Skipped when v_saved_log_scores -- is NULL (we didn't change it, or couldn't as a non-superuser). IF v_saved_log_scores IS NOT NULL THEN BEGIN PERFORM set_config('pg_textsearch.log_scores', v_saved_log_scores, true); EXCEPTION WHEN insufficient_privilege THEN NULL; END; END IF; RETURN COALESCE(v_cold_match, true) AND COALESCE(v_warm_match, true) AND COALESCE(v_cold_eq_warm, true); END; $$ LANGUAGE plpgsql; -- Simpler comparison function CREATE OR REPLACE FUNCTION compare_bm25_scores( p_table_name text, p_text_column text, p_index_name text, p_query text, p_text_config text, p_k1 float, p_b float ) RETURNS TABLE( doc_id integer, content text, sql_score float8, tapir_score float8, difference float8, matches_4dp boolean ) AS $$ DECLARE v_total_docs integer; v_avg_doc_length float8; v_sql text; BEGIN -- Tokenize documents with position counting EXECUTE format($sql$ CREATE TEMP TABLE comp_docs ON COMMIT DROP AS WITH numbered AS ( SELECT ROW_NUMBER() OVER () as doc_id, %I as content FROM %I WHERE %I IS NOT NULL ), doc_terms AS ( SELECT doc_id, content, lexeme, array_length(string_to_array(positions::text, ','), 1) as term_count FROM numbered, LATERAL unnest(to_tsvector(%L, content)) as t(lexeme, positions, weights) ) SELECT doc_id, content, fieldnorm_quantize(SUM(term_count)::integer) as doc_length FROM doc_terms GROUP BY doc_id, content $sql$, p_text_column, p_table_name, p_text_column, p_text_config); -- Get statistics SELECT COUNT(*), AVG(c.doc_length)::float8 INTO v_total_docs, v_avg_doc_length FROM comp_docs c; -- Build term frequencies with position counts EXECUTE format($sql$ CREATE TEMP TABLE comp_tf ON COMMIT DROP AS WITH numbered AS ( SELECT ROW_NUMBER() OVER () as doc_id, %I as content FROM %I WHERE %I IS NOT NULL ) SELECT n.doc_id, t.lexeme as term, array_length(string_to_array(t.positions::text, ','), 1) as tf, (SELECT doc_length::int FROM comp_docs c WHERE c.doc_id = n.doc_id) as doc_len FROM numbered n, LATERAL unnest(to_tsvector(%L, n.content)) as t(lexeme, positions, weights) $sql$, p_text_column, p_table_name, p_text_column, p_text_config); -- Document frequencies CREATE TEMP TABLE comp_df ON COMMIT DROP AS SELECT term, COUNT(DISTINCT comp_tf.doc_id) as df FROM comp_tf GROUP BY term; -- Query terms EXECUTE format($sql$ CREATE TEMP TABLE comp_query ON COMMIT DROP AS SELECT DISTINCT lexeme as term FROM unnest(to_tsvector(%L, %L)) $sql$, p_text_config, p_query); -- Tapir scores v_sql := format($sql$ CREATE TEMP TABLE comp_tapir ON COMMIT DROP AS SELECT ROW_NUMBER() OVER () as doc_id, (%I <@> to_bm25query(%L, %L))::float8 as score FROM %I WHERE %I IS NOT NULL ORDER BY ROW_NUMBER() OVER () $sql$, p_text_column, p_query, p_index_name, p_table_name, p_text_column); EXECUTE v_sql; -- Compute and compare RETURN QUERY WITH sql_scores AS ( SELECT d.doc_id, d.content::text, COALESCE(SUM( ln(1 + (v_total_docs - df.df + 0.5)::float8 / (df.df + 0.5)) * (tf.tf * (p_k1 + 1)) / (tf.tf + p_k1 * (1 - p_b + p_b * tf.doc_len / v_avg_doc_length)) ), 0) as bm25_score FROM comp_docs d CROSS JOIN comp_query q LEFT JOIN comp_tf tf ON tf.doc_id = d.doc_id AND tf.term = q.term LEFT JOIN comp_df df ON df.term = q.term GROUP BY d.doc_id, d.content ) SELECT s.doc_id::integer, s.content, s.bm25_score as sql_score, -t.score as tapir_score, ABS(s.bm25_score - (-t.score)) as difference, ROUND(s.bm25_score::numeric, 4) = ROUND((-t.score)::numeric, 4) as matches_4dp FROM sql_scores s JOIN comp_tapir t ON s.doc_id = t.doc_id ORDER BY s.bm25_score DESC; -- Clean up DROP TABLE IF EXISTS comp_docs; DROP TABLE IF EXISTS comp_tf; DROP TABLE IF EXISTS comp_df; DROP TABLE IF EXISTS comp_query; DROP TABLE IF EXISTS comp_tapir; END; $$ LANGUAGE plpgsql; timescale-pg_textsearch-b63fe58/test/sql/vector.sql000066400000000000000000000063731525762047700225730ustar00rootroot00000000000000-- Test bm25vector type and operators functionality -- Load pg_textsearch extension CREATE EXTENSION IF NOT EXISTS pg_textsearch; -- Enable score logging for testing SET pg_textsearch.log_scores = true; SET client_min_messages = NOTICE; SET enable_seqscan = false; -- Cleanup any existing state first DROP TABLE IF EXISTS test_docs CASCADE; -- Setup test table CREATE TABLE test_docs ( id SERIAL PRIMARY KEY, content TEXT ); INSERT INTO test_docs (content) VALUES ('the quick brown fox'), ('jumped over the lazy dog'), ('sphinx of black quartz'), ('hello world example'), ('postgresql full text search'); -- Create pg_textsearch index CREATE INDEX docs_vector_idx ON test_docs USING bm25(content) WITH (text_config='english'); -- Test bm25vector I/O functions -- Test input/output with index name format SELECT 'docs_vector_idx:{hello:2,world:1}'::bm25vector; SELECT 'docs_vector_idx:{}'::bm25vector; -- Test pg_textsearch scoring using text <@> bm25query operations SELECT content, ROUND((content <@> to_bm25query('hello world', 'docs_vector_idx'))::numeric, 4) as score FROM test_docs ORDER BY content <@> to_bm25query('hello world', 'docs_vector_idx') LIMIT 10; -- Test different query terms with multi-term query SELECT content, ROUND((content <@> to_bm25query('postgresql search', 'docs_vector_idx'))::numeric, 4) as score FROM test_docs ORDER BY content <@> to_bm25query('postgresql search', 'docs_vector_idx') LIMIT 10; -- Test bm25vector equality operator SELECT 'docs_vector_idx:{hello:1,world:2}'::bm25vector = 'docs_vector_idx:{hello:1,world:2}'::bm25vector; SELECT 'docs_vector_idx:{hello:1,world:2}'::bm25vector = 'docs_vector_idx:{world:2,hello:1}'::bm25vector; -- Test scoring with real documents SELECT id, content, ROUND((content <@> to_bm25query('quick fox', 'docs_vector_idx'))::numeric, 4) as relevance FROM test_docs ORDER BY content <@> to_bm25query('quick fox', 'docs_vector_idx') -- BM25 scoring for ranking LIMIT 3; -- Test with another index using simple config CREATE INDEX docs_simple_idx ON test_docs USING bm25(content) WITH (text_config='simple'); -- Test scoring consistency: ORDER BY vs standalone scoring \echo 'Testing ORDER BY vs standalone scoring consistency' WITH order_by_scores AS ( SELECT id, content, ROUND((content <@> to_bm25query('fox dog', 'docs_simple_idx'))::numeric, 6) as order_by_score FROM test_docs ORDER BY content <@> to_bm25query('fox dog', 'docs_simple_idx') LIMIT 3 ), standalone_scores AS ( -- Force standalone scoring by using all documents and calculating scores explicitly SELECT id, content, ROUND((content <@> to_bm25query('fox dog', 'docs_simple_idx'))::numeric, 6) as standalone_score FROM test_docs -- No WHERE clause to force sequential scan and standalone scoring for all rows ) SELECT o.id, o.order_by_score, s.standalone_score, CASE WHEN abs(o.order_by_score - s.standalone_score) < 0.000001 THEN '✓ SCORES MATCH' ELSE '✗ SCORES DIFFER' END as score_consistency FROM order_by_scores o JOIN standalone_scores s ON o.id = s.id ORDER BY o.id; -- Cleanup DROP INDEX docs_vector_idx; DROP INDEX docs_simple_idx; DROP TABLE test_docs; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/vector_v1_rejected.sql000066400000000000000000000132241525762047700250370ustar00rootroot00000000000000-- Test that legacy v1 bm25vector binary input is detected and -- rejected with a clear error (rather than silently misinterpreted -- as v2 and producing garbage results or crashes). -- -- pg_textsearch < 1.2.0 used a different on-disk bm25vector layout. -- bm25vector values are almost always transient (built by -- to_bm25vector for a query, consumed during scan) — actually -- persisting them in user tables was uncommon — so 1.2.0 dropped -- v1 read support. Anyone with persisted v1 data must recompute -- via to_bm25vector(text, index_name) from the source text. -- -- This test pins the rejection behavior: the error message must -- direct users to the correct recovery path. -- -- Also covers the varint multi-byte encode/decode paths in v2 at -- byte-width boundaries (1/2/3/4-byte freq, 1/2-byte lex_len) — -- these aren't exercised by typical tokenized text where almost -- all frequencies are 1. CREATE EXTENSION IF NOT EXISTS pg_textsearch; CREATE TABLE compat_docs (id SERIAL PRIMARY KEY, content TEXT); INSERT INTO compat_docs (content) VALUES ('seed document for index'); CREATE INDEX compat_idx ON compat_docs USING bm25(content) WITH (text_config='english'); -- ======================================================================== -- v1 input rejection -- ======================================================================== CREATE TABLE v1_reject (id int, v bm25vector); -- Should fail with a clear error pointing at the version mismatch. \set VERBOSITY terse \copy v1_reject FROM PROGRAM 'python3 test/scripts/gen_v1_bm25vector.py' WITH (FORMAT binary) \set VERBOSITY default -- Confirm nothing was inserted. SELECT count(*) AS rows_after_rejection FROM v1_reject; -- ======================================================================== -- Embedded-NUL lexeme rejection -- -- Lexemes are rendered verbatim into a cstring by tpvector_out, so an -- embedded NUL byte would truncate the text output at that byte and -- drop every following entry. A NUL can never come from tokenized text -- or the text input parser (both operate on NUL-terminated cstrings), -- so such a value is only reachable via crafted binary input. It must -- be rejected at the boundary, not silently accepted. -- ======================================================================== CREATE TABLE nul_reject (id int, v bm25vector); -- Should fail: entry 0 lexeme contains an embedded NUL byte. \set VERBOSITY terse \copy nul_reject FROM PROGRAM 'python3 test/scripts/gen_nul_lexeme_bm25vector.py' WITH (FORMAT binary) \set VERBOSITY default -- Confirm nothing was inserted. SELECT count(*) AS rows_after_nul_rejection FROM nul_reject; -- ======================================================================== -- Varint encoding boundary coverage -- -- The v2 per-entry header uses LEB128 varint for both `frequency` -- and `lexeme_len`. The 1-byte path covers values 0..127; values -- ≥ 128 hit the multi-byte encode/decode path. Almost all -- naturally-occurring frequencies are 1 (POSDATALEN of a tsvector -- word), so without explicit fixtures the multi-byte varint path -- isn't exercised. These pin both the encode side (tpvector_in → -- create_tpvector_from_strings → tpvector_varint_encode) and the -- decode side (tpvector_out → tpvector_entry_decode → -- tpvector_varint_decode) at every byte-width transition. -- ======================================================================== -- Single-entry boundaries SELECT 'compat_idx:{lex:0}'::bm25vector::text AS freq_0; -- 1 byte SELECT 'compat_idx:{lex:1}'::bm25vector::text AS freq_1; -- 1 byte SELECT 'compat_idx:{lex:127}'::bm25vector::text AS freq_127; -- 1 byte (max) SELECT 'compat_idx:{lex:128}'::bm25vector::text AS freq_128; -- 2 bytes (min) SELECT 'compat_idx:{lex:16383}'::bm25vector::text AS freq_16383; -- 2 bytes (max) SELECT 'compat_idx:{lex:16384}'::bm25vector::text AS freq_16384; -- 3 bytes (min) SELECT 'compat_idx:{lex:2097151}'::bm25vector::text AS freq_2097151; -- 3 bytes (max) SELECT 'compat_idx:{lex:2097152}'::bm25vector::text AS freq_2097152; -- 4 bytes (min) -- Multi-entry walk across mixed-width frequencies. Verifies -- get_tpvector_next_entry correctly advances past 1-, 2-, and -- 3-byte freq encodings. SELECT 'compat_idx:{a:1,b:127,c:128,d:16383,e:16384}'::bm25vector::text AS mixed_freq_widths; -- Equality through varint boundaries. WITH v AS ( SELECT 'compat_idx:{x:50000,y:1}'::bm25vector AS orig ) SELECT orig::text AS rendered, orig = orig::text::bm25vector AS round_trip_eq FROM v; -- Long-lexeme path (multi-byte lex_len varint). WITH s AS ( SELECT 'compat_idx:{' || repeat('a', 200) || ':1}' AS literal ) SELECT length(literal) AS literal_bytes, literal::bm25vector::text = literal AS roundtrip_200char_lex FROM s; -- ======================================================================== -- Large entry counts -- -- entry_count is caller-controlled and bounded only by the varlena -- size, so tpvector_out must not consume stack per entry: a -- per-entry alloca() is released when the function returns, not at -- end of block, and grew the frame until the backend took SIGSEGV. -- Lexemes are zero-padded so input order matches the canonical -- (byte-sorted) entry order. -- ======================================================================== SELECT ('compat_idx:{' || s || '}')::bm25vector::text = 'compat_idx:{' || s || '}' AS roundtrip_1m_entries FROM (SELECT string_agg('l' || lpad(g::text, 7, '0') || ':1', ',' ORDER BY g) FROM generate_series(1, 1000000) g) t(s); -- Cleanup DROP TABLE v1_reject; DROP TABLE nul_reject; DROP TABLE compat_docs CASCADE; DROP EXTENSION pg_textsearch CASCADE; timescale-pg_textsearch-b63fe58/test/sql/wand.sql000066400000000000000000000120101525762047700222030ustar00rootroot00000000000000-- BMW Multi-term WAND Traversal Test -- -- Regression test for doc-ID ordered traversal in multi-term queries. -- Ensures documents at different block positions across posting lists -- are correctly scored. Uses validation.sql to verify scores match -- the reference BM25 implementation. CREATE EXTENSION pg_textsearch; \set ECHO none \i test/sql/validation.sql \set ECHO all -- ============================================================ -- TEST: Multi-term documents score correctly across block boundaries -- ============================================================ CREATE TABLE bmw_bug (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX bmw_bug_idx ON bmw_bug USING bm25(content) WITH (text_config='english'); -- Insert in order that puts multi-term doc at different block positions: -- 5 alpha-only docs, then 1 multi-term, then 200 beta-only docs INSERT INTO bmw_bug (content) SELECT 'alpha only ' || i FROM generate_series(1, 5) i; INSERT INTO bmw_bug (content) VALUES ('alpha beta both terms here'); -- id=6 INSERT INTO bmw_bug (content) SELECT 'beta only ' || i FROM generate_series(7, 206) i; -- Spill to segment (creates single segment with all 206 docs) SELECT bm25_spill_index('bmw_bug_idx'); -- Posting list layout in segment: -- "alpha": 6 postings (docs 1-5, 6) - fits in single block -- "beta": 201 postings (doc 6, docs 7-206) -- - Block 0: docs 6-133 -- - Block 1: docs 134-206 -- -- Doc 6 is the only multi-term document. -- Validate BMW produces correct scores for multi-term query SELECT validate_bm25_scoring('bmw_bug', 'content', 'bmw_bug_idx', 'alpha beta', 'english', 1.2, 0.75) as two_term_valid; DROP TABLE bmw_bug; -- ============================================================ -- TEST: 3-term query with multiple blocks (buffer management) -- ============================================================ -- This tests a bug where zero-copy buffer pins were shared with the -- reader's cached buffer. When initializing multiple term iterators, -- reading the dictionary for term N could release the buffer that -- term N-1's iterator was still using. CREATE TABLE three_term (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX three_term_idx ON three_term USING bm25(content) WITH (text_config='english'); -- Create layout to test multi-term scoring INSERT INTO three_term (content) SELECT 'alpha only ' || i FROM generate_series(1, 3) i; INSERT INTO three_term (content) VALUES ('alpha beta two terms'); -- id=4 INSERT INTO three_term (content) SELECT 'beta only ' || i FROM generate_series(5, 7) i; INSERT INTO three_term (content) VALUES ('alpha beta gamma three terms here'); -- id=8 INSERT INTO three_term (content) SELECT 'gamma only ' || i FROM generate_series(9, 11) i; SELECT bm25_spill_index('three_term_idx'); -- Validate BMW produces correct scores for 3-term query SELECT validate_bm25_scoring('three_term', 'content', 'three_term_idx', 'alpha beta gamma', 'english', 1.2, 0.75) as three_term_valid; DROP TABLE three_term; -- ============================================================ -- TEST: 8+ term queries with WAND pivot selection -- ============================================================ -- Validates that WAND pivot selection produces correct results -- for many-term queries across multiple blocks. CREATE TABLE wand_many_terms (id SERIAL PRIMARY KEY, content TEXT); CREATE INDEX wand_many_terms_idx ON wand_many_terms USING bm25(content) WITH (text_config='english'); -- Create 500 docs with varying term coverage INSERT INTO wand_many_terms (content) SELECT CASE WHEN i % 100 = 0 THEN 'alpha beta gamma delta epsilon zeta eta theta' WHEN i % 50 = 0 THEN 'alpha beta gamma delta epsilon zeta' WHEN i % 10 = 0 THEN 'alpha beta gamma delta' WHEN i % 5 = 0 THEN 'alpha beta gamma' WHEN i % 2 = 0 THEN 'alpha beta' ELSE 'alpha' END || ' document ' || i FROM generate_series(1, 500) i; SELECT bm25_spill_index('wand_many_terms_idx'); -- 8-term query: WAND vs exhaustive (LIMIT 10 avoids tie boundary) WITH bmw AS ( SELECT id, content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query as score FROM wand_many_terms ORDER BY content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query LIMIT 10 ), exhaustive AS ( SELECT id, score FROM ( SELECT id, content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query as score FROM wand_many_terms ORDER BY content <@> 'alpha beta gamma delta epsilon zeta eta theta'::bm25query ) x LIMIT 10 ) SELECT 'wand-8-term' as test, CASE WHEN COUNT(*) = 0 THEN 'PASS' ELSE 'FAIL' END as result FROM (SELECT * FROM bmw EXCEPT SELECT * FROM exhaustive) diff; -- Validate scores match reference BM25 computation SELECT validate_bm25_scoring('wand_many_terms', 'content', 'wand_many_terms_idx', 'alpha beta gamma delta epsilon zeta eta theta', 'english', 1.2, 0.75) as eight_term_valid; DROP TABLE wand_many_terms; DROP EXTENSION pg_textsearch CASCADE;